diff --git a/.dockerignore b/.dockerignore
index a22e22e..0d5cde7 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,5 +1,43 @@
.git
.env
+**/.env
+**/.env.*
+**/*.key
+**/*.pem
+**/*.crt
+**/*.cer
+**/*.der
+**/*.p12
+**/*.pfx
+**/*.p7b
+**/*.p8
+**/*.jks
+**/*.keystore
+**/id_rsa*
+**/id_ed25519*
+**/id_ecdsa*
+**/id_dsa*
+**/private_key*
+**/private-key*
+**/secrets/**
+**/.secrets/**
+**/credentials/**
+**/secrets.yaml
+**/secrets.yml
+**/secrets.json
+**/credentials.json
+**/service-account*.json
+**/.npmrc
+**/.pypirc
+**/.netrc
+**/.git-credentials
+**/.htpasswd
+**/pip.conf
+**/pip.ini
+**/.ssh/**
+**/.aws/**
+**/.config/gcloud/**
+**/.state/**
**/__pycache__
**/.pytest_cache
**/.mypy_cache
@@ -10,4 +48,35 @@ frontend/dist
frontend/tsconfig.tsbuildinfo
backend/tests
node-agent/tests
+runtime-worker/tests
docs
+
+# Runtime state and downloaded model files must never enter a build context. Keep these paths
+# anchored so config/models/initial-candidates.yaml remains available to source packaging.
+/models/
+/model-artifacts/
+/artifacts/
+/data/
+/.data/
+/.state/
+/state/
+/hf-cache/
+/huggingface/
+/.cache/huggingface/
+/.quarantine/
+/quarantine/
+/downloads/
+/container-state/
+/postgres-data/
+/redis-data/
+/runtime-traces/
+**/*.safetensors
+**/*.gguf
+**/*.ckpt
+**/*.onnx
+**/*.pth
+**/*.pt
+**/*.bin
+**/*.h5
+**/*.pb
+**/*.model
diff --git a/.env.example b/.env.example
index 34c752e..41b1dc1 100644
--- a/.env.example
+++ b/.env.example
@@ -1,4 +1,4 @@
-# ITWorx ModelForge 1.2.1 — configuration example
+# ITWorx ModelForge 1.3.0-rc.1 — configuration example
#
# Generated by scripts/generate_configuration_docs.py. Do not edit by hand.
# Copy to .env and fill in the values marked REQUIRED. See docs/CONFIGURATION.md.
@@ -36,12 +36,24 @@ MODELFORGE_REDIS_URL=redis://localhost:6379/0
# Comma-separated exact origins allowed to call the API from a browser. A wildcard is refused in production because requests are credentialed.
MODELFORGE_CORS_ORIGINS=http://localhost:3000
-# Operator API key guarding every admin route. Generate at least 32 random characters; ModelForge never mints one for you. (REQUIRED in production)
+# Optional JSON array of named human identities with role, credential ID, SHA-256 key digest and timezone-aware expiry. Provision with scripts/operator_identity.py. Replace this deployment setting and recreate the API to rotate or revoke access.
+MODELFORGE_OPERATOR_IDENTITIES=
+
+# Optional salted scrypt verifier generated by scripts/operator_password.py. Enables password login with 15-minute sessions; requires the separate strong operator API key. Changing either value invalidates password sessions.
+MODELFORGE_OPERATOR_PASSWORD_HASH=
+
+# Legacy bootstrap administrator key. Generate at least 32 random characters. Production requires this key or an unexpired named administrator in operator_identities.
MODELFORGE_OPERATOR_API_KEY=
-# Optional Hugging Face token, used only for acquiring gated repositories. It is never passed to a runtime and never leaves the control plane.
+# Optional deployment Hugging Face token for control-plane metadata. Compose may also provision it independently to node-agent downloads; API secret activation does not update nodes.
MODELFORGE_HF_TOKEN=
+# External base64 32-byte AES-256 key for the managed Hugging Face control-plane slot. Keep independent of backup and operator keys; retain it externally for historical restore.
+MODELFORGE_SECRET_SLOT_ROOT_KEY=
+
+# External key identifier bound to encrypted HF slot versions. Replacing a key does not rewrap history.
+MODELFORGE_SECRET_SLOT_ROOT_KEY_ID=hf-control-plane-1
+
# Base64 AES-256 key for backup encryption. Without it no backup can be produced, and without the same key no backup can be restored — store it outside this deployment. (REQUIRED in production)
MODELFORGE_BACKUP_ENCRYPTION_KEY=
@@ -183,6 +195,39 @@ MODELFORGE_OBSERVABILITY_MONITOR_ENABLED=false
# Interval between observability evaluation passes.
MODELFORGE_OBSERVABILITY_POLL_INTERVAL_SECONDS=60
+# Queue and dispatch the deployment-managed operational alert webhook.
+MODELFORGE_ALERT_NOTIFICATIONS_ENABLED=false
+
+# Deployment-managed HTTPS notification destination. Its exact hostname must also appear in alert_webhook_allowed_hosts; the URL is never returned by the API.
+MODELFORGE_ALERT_WEBHOOK_URL=
+
+# Comma-separated exact destination host allowlist for the alert webhook. Wildcards and redirects are not accepted.
+MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS=
+
+# Minimum alert severity sent to the single webhook route: INFO, WARNING or CRITICAL.
+MODELFORGE_ALERT_WEBHOOK_MIN_SEVERITY=WARNING
+
+# Total timeout for one webhook delivery.
+MODELFORGE_ALERT_NOTIFICATION_TIMEOUT_SECONDS=5.0
+
+# Maximum webhook attempts before a delivery fails permanently.
+MODELFORGE_ALERT_NOTIFICATION_MAX_ATTEMPTS=8
+
+# Maximum outbox rows claimed by one notification dispatch pass.
+MODELFORGE_ALERT_NOTIFICATION_BATCH_SIZE=20
+
+# Lease duration before an interrupted delivery can be reclaimed.
+MODELFORGE_ALERT_NOTIFICATION_LEASE_SECONDS=60
+
+# Base delay for bounded exponential webhook retries.
+MODELFORGE_ALERT_NOTIFICATION_RETRY_BASE_SECONDS=15
+
+# In production, check explicitly followed model references for changed revisions. Metadata only; never downloads, tests or promotes a model.
+MODELFORGE_MODEL_UPDATE_MONITOR_ENABLED=true
+
+# Minimum interval between automatic checks of each followed model reference; default one day. Each API process checks at most one due reference per minute.
+MODELFORGE_MODEL_UPDATE_CHECK_INTERVAL_SECONDS=86400
+
# Reconcile interrupted backups and restores at startup.
MODELFORGE_RECOVERY_RECONCILIATION_ENABLED=false
diff --git a/.gitea/tests/test_production_deploy_policy.py b/.gitea/tests/test_production_deploy_policy.py
index faac2a7..8f5e0da 100644
--- a/.gitea/tests/test_production_deploy_policy.py
+++ b/.gitea/tests/test_production_deploy_policy.py
@@ -14,7 +14,6 @@ from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
-DEPLOY_WORKFLOW = REPOSITORY_ROOT / ".gitea" / "workflows" / "unraid-deploy.yml"
VALIDATION_WORKFLOW = REPOSITORY_ROOT / ".gitea" / "workflows" / "managed-validation.yml"
@@ -62,110 +61,15 @@ def _run_script(step: str) -> str:
class ProductionDeployPolicyTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
- cls.deploy = DEPLOY_WORKFLOW.read_text(encoding="utf-8")
cls.validation = VALIDATION_WORKFLOW.read_text(encoding="utf-8")
- def test_deployment_has_no_automatic_trigger(self) -> None:
- trigger = _indented_block(self.deploy, "on:")
- trigger_keys = re.findall(r"^ ([a-z_]+):", trigger, flags=re.MULTILINE)
- self.assertEqual(trigger_keys, ["workflow_dispatch"])
- self.assertNotRegex(trigger, r"(?m)^\s+push:")
-
- def test_dispatch_inputs_are_required_and_fail_safe(self) -> None:
- inputs = _indented_block(self.deploy, " inputs:")
- input_names = re.findall(r"^ ([a-z_]+):", inputs, flags=re.MULTILINE)
- self.assertEqual(input_names, ["release_tag", "release_commit", "action"])
-
- for input_name in ("release_tag", "release_commit"):
- block = _indented_block(self.deploy, f" {input_name}:")
- self.assertRegex(block, r"(?m)^ required: true$")
- self.assertRegex(block, r"(?m)^ type: string$")
-
- action = _indented_block(self.deploy, " action:")
- self.assertRegex(action, r"(?m)^ required: true$")
- self.assertRegex(action, r"(?m)^ default: VERIFY_ONLY$")
- self.assertRegex(action, r"(?m)^ type: choice$")
- options = re.findall(r"^ - (\S+)$", action, flags=re.MULTILINE)
- self.assertEqual(options, ["VERIFY_ONLY", "DEPLOY_STABLE_TO_PRODUCTION"])
-
- def test_inputs_are_validated_before_checkout(self) -> None:
- validation = _named_step(self.deploy, "Validate immutable dispatch inputs")
- checkout = _named_step(self.deploy, "Check out the exact release commit with full history")
- self.assertLess(self.deploy.index(validation), self.deploy.index(checkout))
- self.assertIn('"refs/heads/master"', validation)
- self.assertIn(
- r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$",
- validation,
+ def test_legacy_production_delivery_workflow_is_retired(self) -> None:
+ self.assertFalse(
+ (REPOSITORY_ROOT / ".gitea" / "workflows" / "unraid-deploy.yml").exists()
)
- self.assertIn(r"^[0-9a-f]{40}$", validation)
- self.assertIn("VERIFY_ONLY|DEPLOY_STABLE_TO_PRODUCTION", validation)
- tag_pattern = re.search(r'RELEASE_TAG\}" =~ (\^.*\$) \]\]', validation)
- commit_pattern = re.search(r'RELEASE_COMMIT\}" =~ (\^.*\$) \]\]', validation)
- self.assertIsNotNone(tag_pattern)
- self.assertIsNotNone(commit_pattern)
- assert tag_pattern is not None and commit_pattern is not None
- for valid_tag in ("v0.0.0", "v1.2.1", "v10.20.30"):
- self.assertRegex(valid_tag, tag_pattern.group(1))
- for invalid_tag in ("1.2.1", "v01.2.1", "v1.2.1-rc.1", "v1.2.1+build", "master"):
- self.assertNotRegex(invalid_tag, tag_pattern.group(1))
- self.assertRegex("0" * 40, commit_pattern.group(1))
- for invalid_commit in ("0" * 39, "A" * 40, "g" * 40, "master"):
- self.assertNotRegex(invalid_commit, commit_pattern.group(1))
-
- def test_checkout_is_pinned_to_the_validated_commit_with_full_history(self) -> None:
- checkout = _named_step(self.deploy, "Check out the exact release commit with full history")
- self.assertRegex(checkout, r"actions/checkout@[0-9a-f]{40}")
- self.assertIn("ref: ${{ inputs.release_commit }}", checkout)
- self.assertIn("fetch-depth: 0", checkout)
- self.assertIn("persist-credentials: true", checkout)
-
- def test_release_provenance_is_verified_and_exported(self) -> None:
- provenance = _named_step(self.deploy, "Verify stable release provenance")
- required_fragments = (
- "git fetch --force --no-recurse-submodules origin",
- 'git cat-file -t "${tag_ref}"',
- 'git rev-parse "${tag_ref}^{tag}"',
- 'git rev-parse "${tag_ref}^{commit}"',
- "git rev-parse HEAD",
- "< VERSION",
- "git merge-base --is-ancestor",
- "refs/remotes/origin/master",
- "DEPLOY_COMMIT=%s",
- '>> "${GITHUB_ENV}"',
- )
- for fragment in required_fragments:
- with self.subTest(fragment=fragment):
- self.assertIn(fragment, provenance)
- self.assertNotIn("GITHUB_SHA", self.deploy)
-
- def test_all_production_shell_blocks_parse(self) -> None:
- names = (
- "Validate immutable dispatch inputs",
- "Verify stable release provenance",
- "Verification-only result",
- "Deploy verified stable release to production",
- )
- for name in names:
- script = _run_script(_named_step(self.deploy, name))
- result = subprocess.run(
- ["bash", "-n"],
- input=script.encode("utf-8"),
- capture_output=True,
- check=False,
- )
- with self.subTest(step=name):
- self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace"))
-
- def test_deployment_requires_the_explicit_action_and_verified_commit(self) -> None:
- deploy = _named_step(self.deploy, "Deploy verified stable release to production")
- self.assertIn("inputs.action == 'DEPLOY_STABLE_TO_PRODUCTION'", deploy)
- self.assertIn('test -n "${DEPLOY_COMMIT:-}"', deploy)
- self.assertIn('"${GITHUB_REPOSITORY}" "${DEPLOY_COMMIT}"', deploy)
- self.assertEqual(self.deploy.count("/opt/gitea-deploy/deploy.py deploy"), 1)
-
- verify_only = _named_step(self.deploy, "Verification-only result")
- self.assertIn("inputs.action == 'VERIFY_ONLY'", verify_only)
+ def test_managed_validation_uses_the_shared_validation_pool(self) -> None:
+ self.assertRegex(self.validation, r"(?m)^\s*runs-on:\s*linux-validation\s*$")
def test_managed_validation_runs_policy_before_profile_work(self) -> None:
policy_step = _named_step(self.validation, "Production delivery policy")
@@ -176,39 +80,83 @@ class ProductionDeployPolicyTests(unittest.TestCase):
self.validation.index(" - name: Validate the requested profile"),
)
- def test_security_profile_runs_fail_closed_scanners(self) -> None:
- step = _named_step(self.validation, "Security — secrets and vulnerable dependencies")
- self.assertIn("env.PROFILE == 'security'", step)
- self.assertIn("env.PROFILE == 'full'", step)
- script = _run_script(step)
- self.assertIn("gitleaks git .", script)
- self.assertIn("pip_audit --strict --skip-editable", script)
- self.assertIn("npm audit --audit-level=high --omit=dev", script)
- self.assertNotRegex(script, r"(?m)(?:\|\|\s*true|continue-on-error)")
+ def test_full_validation_uses_a_disposable_managed_postgres_boundary(self) -> None:
+ job = _indented_block(self.validation, " full:")
+ boundary = _named_step(self.validation, "Managed PostgreSQL boundary")
+ cleanup = _named_step(self.validation, "Remove managed PostgreSQL boundary")
- toolchain = _named_step(self.validation, "Toolchain")
- toolchain_script = _run_script(toolchain)
- self.assertIn("command -v gitleaks", toolchain_script)
- self.assertIn("exit 1", toolchain_script)
+ for variable in ("VALIDATION_COMPOSE_PROJECT", "MODELFORGE_POSTGRES_PORT"):
+ self.assertRegex(job, rf"(?m)^ {variable}: .+$")
+ self.assertIn("modelforge-validation-${{ gitea.run_id }}", job)
+ self.assertIn('MODELFORGE_POSTGRES_PORT: "0"', job)
+ self.assertIn("env.PROFILE == 'full'", boundary)
+ self.assertIn("export MODELFORGE_MIGRATION_DATABASE_URL=", boundary)
+ self.assertIn("export MODELFORGE_RUNTIME_DATABASE_URL=", boundary)
+ self.assertIn("up --detach --wait postgres", boundary)
+ self.assertIn('validation_network="${VALIDATION_COMPOSE_PROJECT}_default"', boundary)
+ self.assertIn('job_container="$(hostname)"', boundary)
+ self.assertIn('docker network connect "${validation_network}" "${job_container}"', boundary)
+ self.assertIn('socket.create_connection(("postgres", 5432)', boundary)
+ self.assertNotIn("port postgres 5432", boundary)
+ for variable in (
+ "MODELFORGE_TEST_POSTGRES_URL",
+ "MODELFORGE_TEST_POSTGRES_OWNER_URL",
+ "MODELFORGE_TEST_POSTGRES_RUNTIME_URL",
+ ):
+ self.assertIn(f'echo "{variable}=', boundary)
+ self.assertIn('>> "${GITHUB_ENV}"', boundary)
+ self.assertNotIn("SKIPPED", boundary)
+ self.assertIn("always() && env.PROFILE == 'full'", cleanup)
+ self.assertIn("docker network disconnect", cleanup)
+ self.assertIn('"$(hostname)"', cleanup)
+ self.assertIn("down --volumes --remove-orphans", cleanup)
+ self.assertIn("export MODELFORGE_MIGRATION_DATABASE_URL=", cleanup)
+ self.assertIn("export MODELFORGE_RUNTIME_DATABASE_URL=", cleanup)
- def test_compose_validation_renders_the_production_projection(self) -> None:
- step = _named_step(self.validation, "Compose projections")
- script = _run_script(step)
- self.assertIn(
- "for overlay in backup dr gpu node-agent node-recovery production runtime-worker",
- script,
+ def test_full_validation_traverses_the_live_schema_and_tests_the_built_console(self) -> None:
+ backend = _named_step(self.validation, "Backend — install, lint, typecheck, test")
+ console = _named_step(self.validation, "Console — install frozen, typecheck, test, build")
+ boundary = _named_step(self.validation, "Managed PostgreSQL boundary")
+
+ self.assertIn('MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}"', backend)
+ upgrade_commands = re.findall(
+ r'"\$\{BACKEND_VENV\}/bin/python" -m alembic ([^\n]+)',
+ backend,
)
+ self.assertEqual(
+ upgrade_commands,
+ ["upgrade head", "downgrade 20260828_0022", "upgrade head"],
+ )
+ self.assertLess(self.validation.index(console), self.validation.index(boundary))
+ self.assertLess(self.validation.index(boundary), self.validation.index(backend))
+
+ def test_managed_postgres_shell_blocks_parse(self) -> None:
+ for name in (
+ "Managed PostgreSQL boundary",
+ "Backend — install, lint, typecheck, test",
+ "Remove managed PostgreSQL boundary",
+ ):
+ script = _run_script(_named_step(self.validation, name))
+ result = subprocess.run(
+ ["bash", "-n"],
+ input=script.encode("utf-8"),
+ capture_output=True,
+ check=False,
+ )
+ with self.subTest(step=name):
+ self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace"))
+
+ def test_compose_projection_uses_explicit_non_secret_validation_inputs(self) -> None:
+ projection = _named_step(self.validation, "Compose projections")
for variable in (
"MODELFORGE_POSTGRES_ADMIN_PASSWORD",
- "MODELFORGE_MIGRATION_DATABASE_URL",
+ "MODELFORGE_MIGRATION_DB_PASSWORD",
+ "MODELFORGE_RUNTIME_DB_PASSWORD",
"MODELFORGE_RUNTIME_DATABASE_URL",
- "MODELFORGE_OPERATOR_API_KEY",
- "MODELFORGE_BACKUP_ENCRYPTION_KEY",
- "MODELFORGE_CORS_ORIGINS",
- "VITE_API_BASE_URL",
+ "MODELFORGE_MIGRATION_DATABASE_URL",
):
- with self.subTest(variable=variable):
- self.assertIn(f"export {variable}=", script)
+ self.assertIn(f"export {variable}=", projection)
+ self.assertIn("docker compose --env-file .env.example", projection)
if __name__ == "__main__":
diff --git a/.gitea/tests/test_public_source_export.py b/.gitea/tests/test_public_source_export.py
index 39b3e0f..2afbcd2 100644
--- a/.gitea/tests/test_public_source_export.py
+++ b/.gitea/tests/test_public_source_export.py
@@ -2,12 +2,13 @@
from __future__ import annotations
+import hashlib
+import json
import subprocess
import tempfile
import unittest
from pathlib import Path
-
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
EXPORTER = REPOSITORY_ROOT / "scripts" / "export-public-source.mjs"
VALIDATOR = REPOSITORY_ROOT / "scripts" / "validate-public-source.mjs"
@@ -28,8 +29,36 @@ class PublicSourceExportTests(unittest.TestCase):
"SECURITY.md": "# Security\n",
"CONTRIBUTING.md": "# Contributing\n",
"docker-compose.yml": "services: {}\n",
+ "docker-compose.validation-tls.yml": "services: {}\n",
+ "deploy/validation/Caddyfile": "https://modelforge.example.test:8443 {}\n",
+ "backend/Dockerfile": (
+ 'LABEL org.opencontainers.image.source="https://git.example.com/'
+ 'Jens/ITWorx-ModelForge"\n'
+ ),
+ "frontend/Dockerfile": (
+ 'LABEL org.opencontainers.image.source="https://git.example.com/'
+ 'Jens/ITWorx-ModelForge"\n'
+ ),
+ "node-agent/Dockerfile": (
+ 'LABEL org.opencontainers.image.source="https://git.example.com/'
+ 'Jens/ITWorx-ModelForge"\n'
+ ),
+ "runtime-worker/Dockerfile": (
+ 'LABEL org.opencontainers.image.source="https://git.example.com/'
+ 'Jens/ITWorx-ModelForge"\n'
+ ),
"VERSION": "1.2.1\n",
"backend/pyproject.toml": "[project]\nname = 'fixture-api'\n",
+ "clients/README.md": "# Client usage\n",
+ "clients/python/modelforge_client.py": "# Python client\n",
+ "clients/typescript/modelforge-client.mjs": "// JavaScript client\n",
+ "clients/typescript/modelforge-client.d.mts": "// TypeScript declarations\n",
+ "scripts/m16_chaos.py": "# Chaos scenarios\n",
+ "scripts/m16_invariants.py": "# Invariant checks\n",
+ "scripts/m16_soak.py": "# Soak checks\n",
+ "scripts/operator_identity.py": "# Operator identities\n",
+ "scripts/rc_isolation_preflight.py": "# RC isolation\n",
+ "scripts/rc_stack_retention.py": "# RC retention\n",
"frontend/package.json": '{"name":"fixture-web"}\n',
"node-agent/pyproject.toml": "[project]\nname = 'fixture-agent'\n",
"runtime-worker/pyproject.toml": "[project]\nname = 'fixture-worker'\n",
@@ -104,6 +133,59 @@ class PublicSourceExportTests(unittest.TestCase):
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
validation = self._validate(output)
self.assertEqual(validation.returncode, 0, validation.stdout + validation.stderr)
+ self.assertTrue((output / "clients/python/modelforge_client.py").is_file())
+ self.assertTrue((output / "clients/typescript/modelforge-client.mjs").is_file())
+ manifest = json.loads((output / "PUBLIC_SOURCE_MANIFEST.json").read_text())
+ self.assertEqual(
+ manifest["sourceRepository"], "https://git.example.com/example/modelforge"
+ )
+ bound_paths = {entry["path"] for entry in manifest["files"]}
+ for relative_path in (
+ "backend/Dockerfile",
+ "frontend/Dockerfile",
+ "node-agent/Dockerfile",
+ "runtime-worker/Dockerfile",
+ ):
+ self.assertIn(relative_path, bound_paths)
+ self.assertIn(
+ 'org.opencontainers.image.source="https://git.example.com/example/modelforge"',
+ (output / relative_path).read_text(),
+ )
+
+ dockerfile_path = output / "backend/Dockerfile"
+ original_dockerfile = dockerfile_path.read_bytes()
+ dockerfile_entry = next(
+ entry for entry in manifest["files"] if entry["path"] == "backend/Dockerfile"
+ )
+ forged_dockerfile = original_dockerfile.replace(
+ b"https://git.example.com/example/modelforge",
+ b"https://attacker.invalid/repository",
+ )
+ dockerfile_path.write_bytes(forged_dockerfile)
+ dockerfile_entry["bytes"] = len(forged_dockerfile)
+ dockerfile_entry["sha256"] = hashlib.sha256(forged_dockerfile).hexdigest()
+ (output / "PUBLIC_SOURCE_MANIFEST.json").write_text(
+ json.dumps(manifest), encoding="utf-8"
+ )
+ identity_tampering = self._validate(output)
+ self.assertNotEqual(identity_tampering.returncode, 0)
+ self.assertIn("source label does not match", identity_tampering.stderr)
+
+ dockerfile_path.write_bytes(original_dockerfile)
+ original_manifest = json.loads(
+ (output / "PUBLIC_SOURCE_MANIFEST.json").read_text(encoding="utf-8")
+ )
+ original_manifest["files"] = [
+ {
+ "path": entry["path"],
+ "bytes": len((output / entry["path"]).read_bytes()),
+ "sha256": hashlib.sha256((output / entry["path"]).read_bytes()).hexdigest(),
+ }
+ for entry in manifest["files"]
+ ]
+ (output / "PUBLIC_SOURCE_MANIFEST.json").write_text(
+ json.dumps(original_manifest), encoding="utf-8"
+ )
original_readme = (output / "README.md").read_bytes()
(output / "README.md").write_text("tampered\n", encoding="utf-8")
@@ -127,6 +209,41 @@ class PublicSourceExportTests(unittest.TestCase):
self.assertNotEqual(result.returncode, 0)
self.assertIn("commit the exact source tree", result.stderr)
+ def test_private_owner_inventory_is_excluded_from_public_source(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ repository = self._repository(root)
+ private_files = {
+ "scripts/portfolio_gitea_snapshot.py": "ExampleRAG owner inventory\n",
+ "scripts/portfolio_gitea_repository_ids.json": '{"private-repo": 12}\n',
+ "backend/tests/test_portfolio_gitea_snapshot.py": "192." + "168.1.40\n",
+ "docs/operations/PORTFOLIO_GITEA_SNAPSHOT.md": "Private owner audit\n",
+ }
+ for relative_path, contents in private_files.items():
+ destination = repository / relative_path
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_text(contents, encoding="utf-8")
+ public_script = repository / "scripts/public.py"
+ public_script.write_text("print('public')\n", encoding="utf-8")
+ with (repository / "public-source.allowlist").open("a", encoding="utf-8") as allowlist:
+ allowlist.write("scripts/**\nbackend/**\ndocs/operations/**\n")
+ subprocess.run(["git", "add", "."], cwd=repository, check=True)
+ subprocess.run(
+ [
+ "git", "-c", "user.name=ModelForge test", "-c", "user.email=test@example.invalid",
+ "commit", "-qm", "private inventory fixture",
+ ],
+ cwd=repository,
+ check=True,
+ )
+ output = root / "public"
+ result = self._export(repository, output)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(self._validate(output).returncode, 0)
+ self.assertTrue((output / "scripts/public.py").is_file())
+ for relative_path in private_files:
+ self.assertFalse((output / relative_path).exists())
+
def test_sanitized_destination_collisions_fail_before_writing(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
diff --git a/.gitea/tests/test_rc_acceptance_policy.py b/.gitea/tests/test_rc_acceptance_policy.py
index 650388a..1aca4ff 100644
--- a/.gitea/tests/test_rc_acceptance_policy.py
+++ b/.gitea/tests/test_rc_acceptance_policy.py
@@ -2,13 +2,15 @@
from __future__ import annotations
-import subprocess
import importlib.util
import json
+import os
+import signal
+import subprocess
import tempfile
import unittest
from pathlib import Path
-
+from unittest import mock
ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".gitea" / "workflows" / "public-candidate-acceptance.yml"
@@ -39,9 +41,23 @@ class RcAcceptancePolicyTests(unittest.TestCase):
def test_exact_source_and_docker_are_required(self) -> None:
self.assertIn(r"^[0-9a-f]{40}$", self.workflow)
self.assertIn("ref: ${{ inputs.source_commit }}", self.workflow)
+ self.assertIn("--canonical-source-commit", self.workflow)
+ self.assertIn('"--canonical-source-commit", canonical_commit', self.acceptance)
+ self.assertIn('"source_commit": canonical_commit', self.acceptance)
self.assertIn("docker version", self.workflow)
self.assertNotIn("SKIPPED: no docker", self.workflow)
+ def test_optional_candidate_freeze_retains_only_a_verified_transport_bundle(self) -> None:
+ self.assertIn("retain_image_bundle:", self.workflow)
+ self.assertIn("RETAIN_IMAGE_BUNDLE: ${{ inputs.retain_image_bundle }}", self.workflow)
+ self.assertIn("--retain-image-bundle", self.workflow)
+ self.assertIn("--image-bundle", self.acceptance)
+ self.assertIn("acceptance-evidence/release/*.tar", self.workflow)
+ self.assertIn("acceptance-evidence/release/*.tar.gz", self.workflow)
+ self.assertIn("atexit.register(cleanup_candidate_images", self.acceptance)
+ self.assertIn('canonical_commit="${{ inputs.source_commit }}"', self.workflow)
+ self.assertNotIn("candidate_commit=", self.workflow)
+
def test_security_tools_are_downloaded_with_exact_checksums(self) -> None:
self.assertIn("GITLEAKS_VERSION: 8.30.1", self.workflow)
self.assertIn(
@@ -78,18 +94,38 @@ class RcAcceptancePolicyTests(unittest.TestCase):
self.assertIn("apt-get purge --yes linux-libc-dev", self.runtime_dockerfile)
self.assertNotIn("apt-get autoremove", self.runtime_dockerfile)
self.assertIn("pip check", self.runtime_dockerfile)
+ self.assertIn("python -m pip uninstall --yes pip setuptools wheel", self.runtime_dockerfile)
for dependency in (
+ "filelock==3.20.3",
+ "h2==4.4.1",
+ "idna==3.15",
+ "lief==0.17.2",
"Pillow==12.3.0",
+ "Pygments==2.20.0",
"protobuf==5.29.6",
+ "requests==2.33.0",
"sentencepiece==0.2.2",
"sentence-transformers==6.0.1",
"transformers==5.16.1",
"urllib3==2.7.0",
):
self.assertIn(dependency, self.runtime_pyproject)
+ self.assertNotIn("'filelock==3.20.3'", self.runtime_dockerfile)
+
+ def test_runtime_worker_has_only_group_read_access_to_the_agent_credential(self) -> None:
+ self.assertIn(
+ "groupadd --system --gid 101 modelforge-agent", self.runtime_dockerfile
+ )
+ self.assertIn(
+ "--groups modelforge-agent --home /app modelforge-runtime",
+ self.runtime_dockerfile,
+ )
def test_compose_project_is_isolated_and_always_removed(self) -> None:
- self.assertIn('project = f"modelforge-rc-', self.acceptance)
+ self.assertIn('project = f"modelforge-validation-', self.acceptance)
+ self.assertIn('"MODELFORGE_SOURCE_REFERENCE": "validation-acceptance"', self.acceptance)
+ self.assertIn('project="modelforge-validation-${suffix}"', self.workflow)
+ self.assertNotIn('project = f"modelforge-rc-', self.acceptance)
self.assertIn('"MODELFORGE_API_BIND": "127.0.0.1"', self.acceptance)
self.assertIn('"MODELFORGE_API_PUBLISHED_PORT": "0"', self.acceptance)
self.assertIn('"production_changed": False', self.acceptance)
@@ -100,6 +136,123 @@ class RcAcceptancePolicyTests(unittest.TestCase):
self.assertNotIn("docker-compose.runtime-worker.yml", self.acceptance)
self.assertNotIn("docker-compose.node-agent.yml", self.acceptance)
+ def test_acceptance_override_disables_restarts_for_every_service(self) -> None:
+ spec = importlib.util.spec_from_file_location("rc_server_acceptance", ACCEPTANCE)
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ write_override = getattr(module, "write_acceptance_compose_override", None)
+ self.assertIsNotNone(
+ write_override,
+ "acceptance must expose the same override writer used by the RC run",
+ )
+
+ with tempfile.TemporaryDirectory() as directory:
+ override = Path(directory) / "acceptance-compose.override.yml"
+ write_override(override)
+ env = os.environ.copy()
+ env.update(
+ {
+ "MODELFORGE_VERSION": "9.9.9-test",
+ "MODELFORGE_COMMIT": "a" * 40,
+ "MODELFORGE_BUILT_AT": "2099-01-01T00:00:00+00:00",
+ "MODELFORGE_POSTGRES_DB": "modelforge_validation_test",
+ "MODELFORGE_POSTGRES_ADMIN_PASSWORD": "admin-test-password",
+ "MODELFORGE_MIGRATION_DB_PASSWORD": "owner-test-password",
+ "MODELFORGE_RUNTIME_DB_PASSWORD": "runtime-test-password",
+ "MODELFORGE_MIGRATION_DATABASE_URL": (
+ "postgresql+psycopg://modelforge:owner-test-password@"
+ "postgres:5432/modelforge_validation_test"
+ ),
+ "MODELFORGE_RUNTIME_DATABASE_URL": (
+ "postgresql+psycopg://modelforge_runtime:runtime-test-password@"
+ "postgres:5432/modelforge_validation_test"
+ ),
+ "MODELFORGE_BACKUP_ENCRYPTION_KEY": "acceptance-test-key",
+ "MODELFORGE_CORS_ORIGINS": "https://modelforge.example.test",
+ "VITE_API_BASE_URL": "https://modelforge.example.test",
+ }
+ )
+ result = subprocess.run(
+ [
+ "docker",
+ "compose",
+ "-f",
+ str(ROOT / "docker-compose.yml"),
+ "-f",
+ str(ROOT / "docker-compose.production.yml"),
+ "-f",
+ str(override),
+ "config",
+ "--format",
+ "json",
+ ],
+ cwd=ROOT,
+ env=env,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ services = json.loads(result.stdout)["services"]
+ self.assertTrue(services)
+ self.assertEqual(
+ {name: service.get("restart") for name, service in services.items()},
+ {name: "no" for name in services},
+ )
+
+ def test_compose_cleanup_runs_when_acceptance_receives_sigterm(self) -> None:
+ spec = importlib.util.spec_from_file_location("rc_server_acceptance_cleanup", ACCEPTANCE)
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ managed_cleanup = getattr(module, "managed_compose_cleanup", None)
+ self.assertIsNotNone(
+ managed_cleanup,
+ "acceptance must guard its RC project with interruption-safe cleanup",
+ )
+
+ compose = ("docker", "compose", "-p", "modelforge-validation-test")
+ env = {"COMPOSE_PROJECT_NAME": "modelforge-validation-test"}
+ commands: list[tuple[tuple[str, ...], dict[str, object]]] = []
+ registered: list[object] = []
+ unregistered: list[object] = []
+ handlers: dict[int, object] = {}
+
+ def fake_run(*args: str, **kwargs: object) -> str:
+ commands.append((args, kwargs))
+ return ""
+
+ def fake_signal(signum: int, handler: object) -> object:
+ previous = handlers.get(signum, signal.SIG_DFL)
+ handlers[signum] = handler
+ return previous
+
+ with (
+ mock.patch.object(module, "run", side_effect=fake_run),
+ mock.patch.object(module.atexit, "register", side_effect=registered.append),
+ mock.patch.object(module.atexit, "unregister", side_effect=unregistered.append),
+ mock.patch.object(module.signal, "signal", side_effect=fake_signal),
+ self.assertRaises(SystemExit) as raised,
+ managed_cleanup(compose, env),
+ ):
+ handler = handlers[signal.SIGTERM]
+ assert callable(handler)
+ handler(signal.SIGTERM, None)
+
+ self.assertEqual(raised.exception.code, 128 + signal.SIGTERM)
+ self.assertEqual(
+ commands,
+ [
+ (
+ compose + ("down", "--volumes", "--remove-orphans"),
+ {"env": env, "check": False},
+ )
+ ],
+ )
+ self.assertEqual(len(registered), 1)
+ self.assertEqual(unregistered, registered)
+
def test_clean_install_bootstraps_roles_across_a_remote_docker_daemon(self) -> None:
self.assertIn("def provision_database_roles(", self.acceptance)
self.assertIn('run("docker", "cp", str(bootstrap)', self.acceptance)
@@ -152,7 +305,7 @@ class RcAcceptancePolicyTests(unittest.TestCase):
self.assertNotIn("docker system prune", cleanup)
def test_python_entrypoints_compile(self) -> None:
- for path in (ACCEPTANCE, TRIVY_VALIDATOR):
+ for path in (ACCEPTANCE, TRIVY_VALIDATOR, ROOT / "scripts" / "release_build.py"):
result = subprocess.run(
["python", "-m", "py_compile", str(path)],
capture_output=True,
diff --git a/.gitea/tests/test_rc_isolation_preflight.py b/.gitea/tests/test_rc_isolation_preflight.py
new file mode 100644
index 0000000..775082a
--- /dev/null
+++ b/.gitea/tests/test_rc_isolation_preflight.py
@@ -0,0 +1,215 @@
+"""Unit tests for the fail-closed P12 RC isolation preflight."""
+
+from __future__ import annotations
+
+import importlib.util
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("rc_isolation_preflight", ROOT / "scripts" / "rc_isolation_preflight.py")
+assert SPEC is not None and SPEC.loader is not None
+preflight = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(preflight)
+
+
+PROJECT = "modelforge-validation-p12a"
+
+
+def compose_plan(root: Path) -> dict[str, object]:
+ return {
+ "services": {
+ "api": {
+ "networks": ["default"],
+ "volumes": [
+ {"type": "volume", "source": "postgres-data", "target": "/data"},
+ {"type": "bind", "source": str(root / "config"), "target": "/app/config", "read_only": True},
+ ],
+ "ports": [{"host_ip": "127.0.0.1", "published": "0", "target": 8000}],
+ },
+ "node-agent": {
+ "networks": ["default"],
+ "volumes": [{"type": "volume", "source": "node-state", "target": "/data/state"}],
+ },
+ },
+ "volumes": {
+ "postgres-data": {"name": f"{PROJECT}_postgres-data"},
+ "node-state": {"name": f"{PROJECT}_node-state"},
+ },
+ "networks": {"default": {"name": f"{PROJECT}_default"}},
+ }
+
+
+class RcIsolationPreflightTests(unittest.TestCase):
+ def plan_root(self, *, with_manifest: bool = True) -> tempfile.TemporaryDirectory[str]:
+ temporary = tempfile.TemporaryDirectory()
+ config = Path(temporary.name, "config")
+ config.mkdir()
+ if with_manifest:
+ manifest = config / "models" / "initial-candidates.yaml"
+ manifest.parent.mkdir()
+ manifest.write_text("candidates: []\n", encoding="utf-8")
+ self.addCleanup(temporary.cleanup)
+ return temporary
+
+ def validate(
+ self,
+ plan: dict[str, object],
+ root: Path,
+ *,
+ ports: set[int] | None = None,
+ names: set[str] | None = None,
+ ) -> dict[str, object]:
+ return preflight.validate_compose(
+ plan,
+ project=PROJECT,
+ candidate_root=root,
+ protected_ports=ports or set(),
+ protected_resource_names=names or set(),
+ )
+
+ def test_valid_validation_compose_plan_is_accepted(self) -> None:
+ temporary = self.plan_root()
+ root = Path(temporary.name)
+ result = self.validate(
+ compose_plan(root), root, ports={18000, 13000}, names={"modelforge-postgres-1"}
+ )
+ self.assertEqual(result["project"], PROJECT)
+ self.assertEqual(result["published_ports"], [])
+
+ def test_project_validation_rejects_production_and_legacy_rc_names(self) -> None:
+ for project in (
+ "modelforge",
+ "modelforge-rc-p12a",
+ "modelforge-validation-",
+ "modelforge-validation-prod-",
+ ):
+ with self.subTest(project=project), self.assertRaises(preflight.IsolationError):
+ preflight.validate_project(project)
+
+ def test_missing_bind_source_is_rejected(self) -> None:
+ temporary = self.plan_root(with_manifest=False)
+ root = Path(temporary.name)
+ plan = compose_plan(root)
+ (root / "config").rmdir()
+ with self.assertRaisesRegex(preflight.IsolationError, "missing bind source"):
+ self.validate(plan, root)
+
+ def test_config_manifest_must_exist_under_exact_bind_source(self) -> None:
+ temporary = self.plan_root(with_manifest=False)
+ root = Path(temporary.name)
+ plan = compose_plan(root)
+ # A manifest elsewhere in the candidate tree cannot satisfy this bind's contract.
+ manifest = root / "models" / "initial-candidates.yaml"
+ manifest.parent.mkdir()
+ manifest.write_text("candidates: []\n", encoding="utf-8")
+ with self.assertRaisesRegex(preflight.IsolationError, "missing models/initial-candidates.yaml under its source"):
+ self.validate(plan, root)
+
+ def test_rendered_compose_network_mapping_is_accepted(self) -> None:
+ temporary = self.plan_root()
+ root = Path(temporary.name)
+ plan = compose_plan(root)
+ plan["services"]["api"]["networks"] = {"default": None} # type: ignore[index]
+ self.assertEqual(self.validate(plan, root)["declared_networks"], ["default"])
+
+ def test_unsafe_compose_plans_are_rejected(self) -> None:
+ cases = [
+ (lambda plan: plan["services"]["api"].update(network_mode="host"), "host network"),
+ (lambda plan: plan["services"]["api"].update(container_name="modelforge-api-1"), "global container name"),
+ (lambda plan: plan["services"]["api"]["ports"].__setitem__(0, {"host_ip": "0.0.0.0", "published": "19000", "target": 8000}), "non-loopback"),
+ (lambda plan: plan["services"]["api"]["ports"].__setitem__(0, {"host_ip": "127.0.0.1", "published": "18000", "target": 8000}), "protected host port"),
+ (lambda plan: plan["services"]["api"]["volumes"].__setitem__(1, {"type": "bind", "source": "/mnt/user/appdata/modelforge", "target": "/data", "read_only": True}), "escapes the candidate root"),
+ (lambda plan: plan["services"]["api"]["volumes"].__setitem__(1, {"type": "bind", "source": "config", "target": "/app/config", "read_only": False}), "writable bind mount"),
+ (lambda plan: plan["volumes"]["postgres-data"].update(external=True), "is external"),
+ (lambda plan: plan["networks"]["default"].update(name="modelforge_default"), "non-project name"),
+ ]
+ for mutate, message in cases:
+ with self.subTest(message=message):
+ temporary = self.plan_root()
+ root = Path(temporary.name)
+ plan = compose_plan(root)
+ mutate(plan)
+ with self.assertRaisesRegex(preflight.IsolationError, message):
+ self.validate(plan, root, ports={18000})
+
+ def test_protected_resource_reference_is_rejected(self) -> None:
+ temporary = self.plan_root()
+ root = Path(temporary.name)
+ plan = compose_plan(root)
+ plan["services"]["api"]["environment"] = {"UNSAFE": "modelforge-postgres-1"} # type: ignore[index]
+ with self.assertRaisesRegex(preflight.IsolationError, "protected resources"):
+ self.validate(plan, root, names={"modelforge-postgres-1"})
+
+ def test_only_isolated_tls_service_can_publish_test_port(self) -> None:
+ temporary = self.plan_root()
+ root = Path(temporary.name)
+ source = root / "source" / "deploy" / "validation"
+ source.mkdir(parents=True)
+ (source / "Caddyfile").write_text("https://modelforge.example.com:8443 {}\n", encoding="utf-8")
+ certificates = root / "tls"
+ certificates.mkdir()
+ (certificates / "fullchain.pem").write_text("certificate", encoding="utf-8")
+ (certificates / "privkey.pem").write_text("key", encoding="utf-8")
+ plan = compose_plan(root)
+ plan["networks"]["ingress"] = {"name": f"{PROJECT}_ingress"}
+ plan["services"]["tls"] = {
+ "image": "caddy:2.10-alpine@sha256:4c6e91c6ed0e2fa03efd5b44747b625fec79bc9cd06ac5235a779726618e530d",
+ "networks": ["ingress"], "read_only": True, "cap_drop": ["ALL"],
+ "security_opt": ["no-new-privileges:true"],
+ "volumes": [
+ {"type": "bind", "source": str(source / "Caddyfile"), "target": "/etc/caddy/Caddyfile", "read_only": True},
+ {"type": "bind", "source": str(certificates / "fullchain.pem"), "target": "/run/tls/fullchain.pem", "read_only": True},
+ {"type": "bind", "source": str(certificates / "privkey.pem"), "target": "/run/tls/privkey.pem", "read_only": True},
+ ],
+ "ports": [{"host_ip": "0.0.0.0", "published": "18444", "target": 8443, "protocol": "tcp"}],
+ }
+ with self.assertRaisesRegex(preflight.IsolationError, "non-loopback"):
+ self.validate(plan, root)
+ self.assertEqual(preflight.validate_compose(
+ plan, project=PROJECT, candidate_root=root,
+ protected_ports={443, 8443}, protected_resource_names=set(), public_tls_port=18444,
+ )["published_ports"], [18444])
+ plan["services"]["api"]["ports"][0] = {"host_ip": "0.0.0.0", "published": "18444", "target": 8000}
+ with self.assertRaisesRegex(preflight.IsolationError, "non-loopback"):
+ preflight.validate_compose(
+ plan, project=PROJECT, candidate_root=root,
+ protected_ports=set(), protected_resource_names=set(), public_tls_port=18444,
+ )
+
+ def test_observed_resources_require_exact_labels_and_project_names(self) -> None:
+ observed = {
+ "containers": [{"name": f"{PROJECT}-api-1", "labels": {"com.docker.compose.project": PROJECT}}],
+ "volumes": [{"name": f"{PROJECT}_postgres-data", "labels": {"com.docker.compose.project": PROJECT}}],
+ "networks": [{"name": f"{PROJECT}_default", "labels": {"com.docker.compose.project": PROJECT}}],
+ }
+ self.assertEqual(preflight.validate_observed_resources(observed, project=PROJECT)["containers"], [f"{PROJECT}-api-1"])
+ observed["volumes"][0]["labels"] = {}
+ with self.assertRaisesRegex(preflight.IsolationError, "no exact project label"):
+ preflight.validate_observed_resources(observed, project=PROJECT)
+
+ def test_docker_resource_collection_uses_exact_inspection(self) -> None:
+ label = {"com.docker.compose.project": PROJECT}
+ responses = {
+ ("container", "ls", "--all", "--quiet", "--filter", f"label=com.docker.compose.project={PROJECT}"): "container-id\n",
+ ("volume", "ls", "--quiet", "--filter", f"label=com.docker.compose.project={PROJECT}"): "volume-id\n",
+ ("network", "ls", "--quiet", "--filter", f"label=com.docker.compose.project={PROJECT}"): "network-id\n",
+ ("container", "inspect", "container-id"): '[{"Name":"/' + PROJECT + '-api-1","Config":{"Labels":{"com.docker.compose.project":"' + PROJECT + '"}}}]',
+ ("volume", "inspect", "volume-id"): '[{"Name":"' + PROJECT + '_data","Labels":{"com.docker.compose.project":"' + PROJECT + '"}}]',
+ ("network", "inspect", "network-id"): '[{"Name":"' + PROJECT + '_default","Labels":{"com.docker.compose.project":"' + PROJECT + '"}}]',
+ }
+
+ def run(command, **_kwargs):
+ self.assertEqual(command[0], "docker")
+ return type("Result", (), {"returncode": 0, "stdout": responses[tuple(command[1:])]})()
+
+ with patch.object(preflight.subprocess, "run", run):
+ observed = preflight.collect_observed_resources(PROJECT)
+ self.assertEqual(observed["containers"][0]["labels"], label)
+ self.assertEqual(
+ preflight.validate_observed_resources(observed, project=PROJECT)["networks"],
+ [f"{PROJECT}_default"],
+ )
diff --git a/.gitea/tests/test_rc_stack_retention.py b/.gitea/tests/test_rc_stack_retention.py
new file mode 100644
index 0000000..684886e
--- /dev/null
+++ b/.gitea/tests/test_rc_stack_retention.py
@@ -0,0 +1,316 @@
+"""Focused tests for RC stack inventory and stop-only retention guard."""
+
+from __future__ import annotations
+
+import importlib.util
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location(
+ "rc_stack_retention", ROOT / "scripts" / "rc_stack_retention.py"
+)
+assert SPEC is not None and SPEC.loader is not None
+retention = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(retention)
+
+
+PROJECT = "modelforge-validation-123271"
+NOW = datetime(2026, 9, 23, 10, 0, tzinfo=UTC)
+CREATED = "2026-09-20T10:00:00Z"
+
+
+class MockDocker:
+ def __init__(self, containers: dict[str, dict[str, object]]) -> None:
+ self.containers = containers
+ self.calls: list[tuple[str, ...]] = []
+
+ def run(self, command: tuple[str, ...], **_kwargs: object) -> SimpleNamespace:
+ self.assert_docker(command)
+ args = tuple(command[1:])
+ self.calls.append(args)
+ if args[:4] == (
+ "container", "ls", "--all", "--quiet"
+ ):
+ return SimpleNamespace(
+ returncode=0,
+ stdout="\n".join(self.containers) + "\n",
+ stderr="",
+ )
+ if args[:2] == ("container", "inspect"):
+ identifier = args[2]
+ return SimpleNamespace(
+ returncode=0,
+ stdout=json.dumps([self.containers[identifier]]),
+ stderr="",
+ )
+ if args[:1] == ("stop",):
+ identifiers = args[3:]
+ for identifier in identifiers:
+ state = self.containers[identifier]["State"]
+ assert isinstance(state, dict)
+ state["Status"] = "exited"
+ return SimpleNamespace(returncode=0, stdout="\n".join(identifiers), stderr="")
+ raise AssertionError(f"unexpected Docker command: {args}")
+
+ @staticmethod
+ def assert_docker(command: tuple[str, ...]) -> None:
+ if command[0] != "docker":
+ raise AssertionError(f"unexpected executable: {command[0]}")
+
+
+def docker_container(
+ identifier: str,
+ project: str,
+ service: str,
+ *,
+ status: str = "exited",
+ source: str | None = None,
+ created: str = CREATED,
+ oneoff: str = "False",
+) -> dict[str, object]:
+ mounts: list[dict[str, str]] = []
+ if source is not None:
+ mounts.append({"Type": "bind", "Source": source, "Destination": "/app/config"})
+ return {
+ "Id": identifier,
+ "Name": f"/{project}-{service}-1",
+ "Created": created,
+ "Config": {
+ "Labels": {
+ retention.PROJECT_LABEL: project,
+ retention.SERVICE_LABEL: service,
+ retention.ONEOFF_LABEL: oneoff,
+ }
+ },
+ "State": {"Status": status, "Running": status == "running"},
+ "Mounts": mounts,
+ }
+
+
+class RcStackRetentionTests(unittest.TestCase):
+ def fixture(self, *, api_status: str = "exited", service: str = "postgres", oneoff: str = "False") -> tuple[MockDocker, tempfile.TemporaryDirectory[str]]:
+ temporary = tempfile.TemporaryDirectory()
+ config = Path(temporary.name) / "config"
+ config.mkdir()
+ containers = {
+ "validation-api-id": docker_container("validation-api-id", PROJECT, "api", source=str(config)),
+ "validation-db-id": docker_container(
+ "validation-db-id", PROJECT, service, status="running", oneoff=oneoff
+ ),
+ "prod-api-id": docker_container("prod-api-id", "modelforge", "api", status="running"),
+ "other-api-id": docker_container(
+ "other-api-id", "staging", "api", status="running"
+ ),
+ }
+ if api_status != "exited":
+ containers["validation-api-id"]["State"] = {"Status": api_status, "Running": True}
+ return MockDocker(containers), temporary
+
+ @staticmethod
+ def write_evidence(path: Path, project: str = PROJECT) -> None:
+ path.write_text(
+ json.dumps(
+ {
+ "schema_version": 1,
+ "project": project,
+ "retained": True,
+ "evidence_reference": "acceptance-run-2026-09-20",
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ def test_inventory_scopes_validation_projects_and_flags_manifest_and_age(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ with patch.object(retention.subprocess, "run", docker.run):
+ projects = retention.collect_projects(now=NOW, stale_after_hours=24)
+
+ self.assertEqual([row["project"] for row in projects], [PROJECT])
+ self.assertTrue(projects[0]["stale"])
+ self.assertEqual(len(projects[0]["missing_manifests"]), 1)
+ self.assertEqual(projects[0]["missing_bind_sources"], [])
+ self.assertNotIn("volumes", projects[0])
+ self.assertNotIn("images", projects[0])
+ self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
+ self.assertTrue(all(call[:1] != ("rm",) for call in docker.calls))
+
+ def test_inventory_ignores_non_validation_partial_compose_labels(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ docker.containers["unrelated-id"] = {
+ "Id": "unrelated-id",
+ "Name": "/projectbrain-worker-1",
+ "Config": {"Labels": {retention.PROJECT_LABEL: "projectbrain"}},
+ "State": {"Status": "exited"},
+ }
+
+ with patch.object(retention.subprocess, "run", docker.run):
+ projects = retention.collect_projects(now=NOW, stale_after_hours=24)
+
+ self.assertEqual([row["project"] for row in projects], [PROJECT])
+
+ def test_inventory_fails_closed_for_validation_container_missing_service_label(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ labels = docker.containers["validation-api-id"]["Config"]["Labels"]
+ assert isinstance(labels, dict)
+ del labels[retention.SERVICE_LABEL]
+
+ with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+ retention.RetentionError, "Compose service label is missing"
+ ):
+ retention.collect_projects(now=NOW, stale_after_hours=24)
+
+ def test_default_cli_is_read_only(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ output = io.StringIO()
+ with patch.object(retention.subprocess, "run", docker.run), redirect_stdout(output):
+ status = retention.main([])
+
+ result = json.loads(output.getvalue())
+ self.assertEqual(status, 0)
+ self.assertEqual(result["mode"], "read-only")
+ self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
+
+ def test_docker_errors_and_ambiguous_inspection_fail_closed(self) -> None:
+ failed = SimpleNamespace(returncode=1, stdout="", stderr="daemon unavailable")
+ with patch.object(retention.subprocess, "run", return_value=failed), self.assertRaisesRegex(
+ retention.RetentionError, "Docker command failed"
+ ):
+ retention.collect_projects(now=NOW)
+
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ docker.containers["validation-api-id"]["Id"] = "different-id"
+ with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+ retention.RetentionError, "different container"
+ ):
+ retention.collect_projects(now=NOW)
+
+ def test_project_validation_rejects_production_and_ambiguous_names(self) -> None:
+ for project in (
+ "modelforge",
+ "modelforge-rc-prod-",
+ "modelforge-validation-",
+ "modelforge-validation-prod-",
+ ):
+ with self.subTest(project=project), self.assertRaises(retention.RetentionError):
+ retention.validate_project(project)
+
+ def test_stop_requires_explicit_exact_project_confirmation_and_evidence(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ empty_record = Path(temporary.name) / "empty.json"
+ empty_record.write_text("", encoding="utf-8")
+ with patch.object(retention.subprocess, "run", docker.run):
+ with self.assertRaisesRegex(retention.RetentionError, "exact project name"):
+ retention.stop_project(
+ PROJECT,
+ confirm_stop=None,
+ confirm_no_active_jobs=True,
+ evidence_record=Path("evidence.json"),
+ stale_after_hours=24,
+ now=NOW,
+ )
+ with self.assertRaisesRegex(retention.RetentionError, "nonempty regular file"):
+ retention.stop_project(
+ PROJECT,
+ confirm_stop=PROJECT,
+ confirm_no_active_jobs=True,
+ evidence_record=empty_record,
+ stale_after_hours=24,
+ now=NOW,
+ )
+ mismatch = Path(temporary.name) / "mismatch.json"
+ self.write_evidence(mismatch, project="modelforge-validation-someone-else")
+ with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+ retention.RetentionError, "identify this project"
+ ):
+ retention.stop_project(
+ PROJECT,
+ confirm_stop=PROJECT,
+ confirm_no_active_jobs=True,
+ evidence_record=mismatch,
+ stale_after_hours=24,
+ now=NOW,
+ )
+ self.assertEqual(docker.calls, [])
+
+ def test_stop_only_targets_inspected_project_container_ids(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ evidence = Path(temporary.name) / "evidence.json"
+ self.write_evidence(evidence)
+ with patch.object(retention.subprocess, "run", docker.run):
+ result = retention.stop_project(
+ PROJECT,
+ confirm_stop=PROJECT,
+ confirm_no_active_jobs=True,
+ evidence_record=evidence,
+ stale_after_hours=24,
+ now=NOW,
+ )
+
+ self.assertEqual(result["stopped_container_ids"], ["validation-db-id"])
+ self.assertFalse(result["volumes_or_images_removed"])
+ stop_calls = [call for call in docker.calls if call[:1] == ("stop",)]
+ self.assertEqual(stop_calls, [("stop", "--time", "30", "validation-db-id")])
+ self.assertTrue(all("prod-api-id" not in call for call in stop_calls))
+ self.assertTrue(all(call[:1] not in {("rm",), ("rmi",), ("volume",)} for call in docker.calls))
+
+ def test_stop_fails_if_api_or_oneoff_job_is_running(self) -> None:
+ for options, message in (
+ ({"api_status": "running"}, "API is running"),
+ ({"api_status": "restarting"}, "API is running"),
+ ({"oneoff": "True"}, "one-off job container"),
+ ({"service": "migrate"}, "service activity is unknown"),
+ ):
+ with self.subTest(message=message):
+ docker, temporary = self.fixture(**options)
+ self.addCleanup(temporary.cleanup)
+ evidence = Path(temporary.name) / "evidence.json"
+ self.write_evidence(evidence)
+ with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+ retention.RetentionError, message
+ ):
+ retention.stop_project(
+ PROJECT,
+ confirm_stop=PROJECT,
+ confirm_no_active_jobs=True,
+ evidence_record=evidence,
+ stale_after_hours=24,
+ now=NOW,
+ )
+ self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
+
+ def test_stop_requires_job_attestation(self) -> None:
+ docker, temporary = self.fixture()
+ self.addCleanup(temporary.cleanup)
+ evidence = Path(temporary.name) / "evidence.json"
+ self.write_evidence(evidence)
+ with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
+ retention.RetentionError, "--confirm-no-active-jobs"
+ ):
+ retention.stop_project(
+ PROJECT,
+ confirm_stop=PROJECT,
+ confirm_no_active_jobs=False,
+ evidence_record=evidence,
+ stale_after_hours=24,
+ now=NOW,
+ )
+ self.assertEqual(docker.calls, [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.gitea/tests/test_security_origin_policy.py b/.gitea/tests/test_security_origin_policy.py
new file mode 100644
index 0000000..02f0e84
--- /dev/null
+++ b/.gitea/tests/test_security_origin_policy.py
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import unittest
+from pathlib import Path
+from urllib.parse import urlsplit
+
+
+ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml"
+CANONICAL_ORIGIN = "https://git.example.com"
+
+
+def _origin_case_script() -> str:
+ text = WORKFLOW.read_text(encoding="utf-8")
+ match = re.search(
+ r'(?ms)^ case "\$\{GITHUB_SERVER_URL%/\}" in\n'
+ r'.*?^ esac$',
+ text,
+ )
+ if match is None:
+ raise AssertionError("managed validation has no GITHUB_SERVER_URL case policy")
+ # Remove workflow indentation before executing only the bounded origin policy.
+ return "\n".join(line[10:] for line in match.group(0).splitlines())
+
+
+def _evaluate_origin(origin: str) -> subprocess.CompletedProcess[str]:
+ script = (
+ "set -euo pipefail\n"
+ + _origin_case_script()
+ + "\nprintf '%s\\n' \"${SOURCE_SERVER_URL}\"\n"
+ )
+ env = os.environ.copy()
+ env["GITHUB_SERVER_URL"] = origin
+ bash = shutil.which("bash") or "bash"
+ if os.name == "nt":
+ # System32/bash.exe starts WSL, which does not inherit arbitrary Windows environment
+ # variables. Git Bash runs natively and tests the exact environment used by this fixture.
+ git = shutil.which("git")
+ git_bash = Path(git).resolve().parents[1] / "bin" / "bash.exe" if git else None
+ if git_bash is None or not git_bash.is_file():
+ raise RuntimeError("Git Bash is required for Windows origin policy tests")
+ bash = str(git_bash)
+ return subprocess.run(
+ [bash, "--noprofile", "--norc", "-c", script],
+ cwd=ROOT,
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+
+def _safe_origin(value: str) -> str:
+ parsed = urlsplit(value)
+ hostname = parsed.hostname or "(missing-host)"
+ port = f":{parsed.port}" if parsed.port is not None else ""
+ return f"{parsed.scheme or '(missing-scheme)'}://{hostname}{port}"
+
+
+class SecurityOriginPolicyTests(unittest.TestCase):
+ def test_known_runner_and_public_origins_canonicalize_to_https(self) -> None:
+ accepted = (
+ "https://git.example.com",
+ "https://git.example.com/",
+ "http://git.example.com",
+ "http://git.example.com/",
+ "http://192.0.2.10:3000",
+ "http://192.0.2.10:3000/",
+ "http://gitea:3000",
+ "http://gitea:3000/",
+ "http://Gitea:3000",
+ "http://Gitea:3000/",
+ "http://gitea-runner-internal:3000",
+ "http://gitea-runner-internal:3000/",
+ )
+ for origin in accepted:
+ with self.subTest(origin=origin):
+ result = _evaluate_origin(origin)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout.strip(), CANONICAL_ORIGIN)
+
+ def test_unknown_http_and_https_origins_fail_closed(self) -> None:
+ for origin in (
+ "http://untrusted.invalid",
+ "https://untrusted.invalid",
+ "http://192.0.2.151:3000",
+ "http://gitea:3001",
+ ):
+ with self.subTest(origin=origin):
+ result = _evaluate_origin(origin)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Untrusted Gitea server URL", result.stderr)
+
+ def test_live_runner_origin_is_explicitly_covered_when_present(self) -> None:
+ origin = os.environ.get("GITHUB_SERVER_URL")
+ if not origin:
+ self.skipTest("GITHUB_SERVER_URL is only expected inside the managed runner")
+ result = _evaluate_origin(origin)
+ self.assertEqual(
+ result.returncode,
+ 0,
+ f"managed runner origin is not covered: {_safe_origin(origin)}; {result.stderr.strip()}",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.gitea/tests/test_security_validation_policy.py b/.gitea/tests/test_security_validation_policy.py
new file mode 100644
index 0000000..c9008b9
--- /dev/null
+++ b/.gitea/tests/test_security_validation_policy.py
@@ -0,0 +1,2529 @@
+"""Policy and synthetic evaluator tests for the fail-closed managed security gate."""
+
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+import re
+import shutil
+import stat
+import subprocess
+import tempfile
+import unittest
+from datetime import UTC, date, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, ClassVar
+from unittest import mock
+
+import tomllib
+
+from scripts import security_gate as gate
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW_PATH = REPOSITORY_ROOT / ".gitea" / "workflows" / "managed-validation.yml"
+GITLEAKS_CONFIG = REPOSITORY_ROOT / ".gitleaks.toml"
+PIP_AUDIT_LOCK = REPOSITORY_ROOT / "config" / "security" / "pip-audit-toolchain.lock"
+ACCEPTED_RISKS = REPOSITORY_ROOT / "config" / "security" / "accepted-risks.json"
+GITLEAKS_FALSE_POSITIVES = (
+ REPOSITORY_ROOT / "config" / "security" / "gitleaks-false-positives.json"
+)
+PRODUCT_VERSION = (REPOSITORY_ROOT / "VERSION").read_text(encoding="utf-8").strip()
+
+
+def _indented_block(text: str, header: str) -> str:
+ lines = text.splitlines()
+ try:
+ start = lines.index(header)
+ except ValueError as exc:
+ raise AssertionError(f"Missing YAML header: {header!r}") from exc
+ indent = len(header) - len(header.lstrip())
+ end = len(lines)
+ for index in range(start + 1, len(lines)):
+ line = lines[index]
+ if not line.strip() or line.lstrip().startswith("#"):
+ continue
+ current_indent = len(line) - len(line.lstrip())
+ if current_indent <= indent:
+ end = index
+ break
+ return "\n".join(lines[start:end])
+
+
+def _named_step(text: str, name: str) -> str:
+ return _indented_block(text, f" - name: {name}")
+
+
+def _run_script(step: str) -> str:
+ lines = step.splitlines()
+ if " run: |" not in lines:
+ inline = next((line for line in lines if line.startswith(" run: ")), None)
+ if inline is None:
+ raise AssertionError("Named step does not contain a run command")
+ return inline.removeprefix(" run: ") + "\n"
+ start = lines.index(" run: |") + 1
+ script_lines = lines[start:]
+ if any(line and not line.startswith(" ") for line in script_lines):
+ raise AssertionError("Unexpected indentation in literal run block")
+ return "\n".join(line[10:] if line else "" for line in script_lines) + "\n"
+
+
+def _timeout_minutes(block: str) -> int:
+ match = re.search(r"(?m)^\s+timeout-minutes: ([0-9]+)$", block)
+ if match is None:
+ raise AssertionError("YAML block has no explicit timeout-minutes")
+ return int(match.group(1))
+
+
+def _write_json(path: Path, value: Any) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8")
+
+
+def _git(repository: Path, *arguments: str) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ ["git", "-C", str(repository), *arguments],
+ capture_output=True,
+ check=True,
+ text=True,
+ timeout=30,
+ )
+
+
+def _copy_source_gate_inputs(destination: Path) -> None:
+ relative_paths = [
+ *(manifest for manifest, _ in gate.SOURCE_COMPONENTS.values()),
+ gate.BUILD_TOOLCHAIN_LOCK,
+ *gate.IMAGE_DOCKERFILES.values(),
+ ]
+ for relative_path in relative_paths:
+ source = REPOSITORY_ROOT / relative_path
+ target = destination / relative_path
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(source, target)
+
+
+class SecurityWorkflowPolicyTests(unittest.TestCase):
+ workflow: ClassVar[str]
+ security_job: ClassVar[str]
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
+ cls.security_job = _indented_block(cls.workflow, " security:")
+
+ def test_all_checkouts_are_full_history_without_persisted_credentials(self) -> None:
+ checkouts = [
+ _named_step(self.workflow, "Check out validation source with full history"),
+ _named_step(self.workflow, "Check out security source with full history"),
+ ]
+ self.assertEqual(self.workflow.count("uses: actions/checkout@"), 2)
+ for checkout in checkouts:
+ with self.subTest(checkout=checkout.splitlines()[0]):
+ self.assertRegex(checkout, r"actions/checkout@[0-9a-f]{40}")
+ self.assertIn("fetch-depth: 0", checkout)
+ self.assertIn("persist-credentials: false", checkout)
+ security_checkout = _named_step(
+ self.workflow, "Check out security source with full history"
+ )
+ self.assertIn("fetch-tags: true", security_checkout)
+ history = _run_script(_named_step(self.workflow, "Prove complete non-shallow history"))
+ self.assertIn("+refs/heads/*:refs/remotes/origin/*", history)
+ self.assertIn("+refs/tags/*:refs/tags/*", history)
+ self.assertIn("git fetch --force --prune", history)
+ self.assertIn("git-history.json", history)
+ self.assertIn('git remote set-url origin "${SOURCE_REPOSITORY}"', history)
+ self.assertIn("x-access-token:%s", history)
+ self.assertIn("git config --local http.extraheader", history)
+ self.assertIn("git config --local --unset-all http.extraheader", history)
+ history_step = _named_step(self.workflow, "Prove complete non-shallow history")
+ self.assertIn("GITHUB_TOKEN: ${{ github.token }}", history_step)
+
+ def test_profiles_are_source_mandatory_and_only_explicit_security_builds_images(self) -> None:
+ self.assertIn("inputs.profile == 'security'", self.security_job)
+ self.assertIn("inputs.profile == 'full'", self.security_job)
+ self.assertIn("!inputs.profile", self.security_job)
+ self.assertIn("SECURITY_PROFILE:", self.security_job)
+ images = _named_step(self.workflow, "Build and scan exact shipping images")
+ self.assertIn("gitea.event_name == 'workflow_dispatch'", images)
+ self.assertIn("inputs.profile == 'security'", images)
+ source = _named_step(self.workflow, "Collect source security reports")
+ self.assertNotIn("if:", source)
+
+ def test_time_budget_arithmetic_reserves_evaluator_and_upload(self) -> None:
+ names = (
+ "Prepare fail-closed security evidence",
+ "Check out security source with full history",
+ "Prove complete non-shallow history",
+ "Install checksum-pinned security scanners",
+ "Collect source security reports",
+ "Build and scan exact shipping images",
+ "Remove exact security images",
+ "Evaluate the complete security gate",
+ "Upload immutable security evidence",
+ )
+ bounds = {name: _timeout_minutes(_named_step(self.workflow, name)) for name in names}
+ job_timeout = _timeout_minutes(self.security_job.split(" steps:", maxsplit=1)[0])
+ self.assertGreaterEqual(bounds["Evaluate the complete security gate"], 5)
+ self.assertGreaterEqual(bounds["Upload immutable security evidence"], 5)
+ self.assertGreaterEqual(job_timeout - sum(bounds.values()), 10)
+
+ def test_scanner_artifacts_are_version_and_checksum_pinned(self) -> None:
+ expected = {
+ "PIP_AUDIT_VERSION: 2.10.1",
+ "99ef3f600a317c1945f1e89e227ef26e1c2d618429b8bd3fa6f4f7c440c4611a",
+ "GITLEAKS_VERSION: 8.30.1",
+ "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb",
+ "TRIVY_VERSION: 0.74.0",
+ "2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a",
+ }
+ for fragment in expected:
+ with self.subTest(fragment=fragment):
+ self.assertIn(fragment, self.security_job)
+ self.assertNotRegex(self.security_job, r"curl[^\n]*\|\s*(?:ba)?sh")
+ self.assertNotIn("trivy-action", self.security_job)
+
+ def test_release_version_gate_accepts_only_stable_or_numbered_rc_semver(self) -> None:
+ history = _run_script(_named_step(self.workflow, "Prove complete non-shallow history"))
+ self.assertIn(
+ '^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)'
+ '(-rc\\.(0|[1-9][0-9]*))?$',
+ history,
+ )
+ self.assertNotIn('^[0-9]+\\.[0-9]+\\.[0-9]+$', history)
+ self.assertNotIn("gitleaks-action", self.security_job)
+
+ def test_pip_audit_toolchain_is_fully_resolved_and_hash_locked(self) -> None:
+ entries = [
+ line.strip()
+ for line in PIP_AUDIT_LOCK.read_text(encoding="utf-8").splitlines()
+ if line.strip() and not line.lstrip().startswith("#")
+ ]
+ self.assertGreaterEqual(len(entries), 20)
+ for entry in entries:
+ with self.subTest(entry=entry):
+ self.assertRegex(
+ entry,
+ r"^[A-Za-z0-9._-]+==[^ ]+ --hash=sha256:[0-9a-f]{64}$",
+ )
+ self.assertEqual(sum(entry.startswith("pip-audit==2.10.1 ") for entry in entries), 1)
+ install = _named_step(self.workflow, "Install checksum-pinned security scanners")
+ self.assertTrue(_run_script(install).startswith("set -euo pipefail\n"))
+ self.assertIn("--index-url https://pypi.org/simple", install)
+ self.assertGreaterEqual(install.count("--require-hashes"), 2)
+ self.assertIn("--no-index", install)
+ self.assertIn("config/security/pip-audit-toolchain.lock", install)
+
+ def test_source_scans_are_complete_and_do_not_suppress_findings(self) -> None:
+ source = _named_step(self.workflow, "Collect source security reports")
+ self.assertTrue(_run_script(source).startswith("set -euo pipefail\n"))
+ required = (
+ "source-inventory",
+ "--log-opts=--all",
+ "--redact=100",
+ "sanitize-gitleaks",
+ "pip-source-backend.json",
+ "pip-source-node-agent.json",
+ "pip-source-runtime-worker.json",
+ '"${PIP_AUDIT}" --strict',
+ "npm audit --json",
+ "npm audit --omit=dev --json",
+ "trivy-fs.json",
+ "--scanners vuln,misconfig",
+ "--severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
+ '"${TRIVY}" version --format json',
+ )
+ for fragment in required:
+ with self.subTest(fragment=fragment):
+ self.assertIn(fragment, source)
+ for forbidden in ("--ignore-vuln", "--ignore-unfixed", "--exit-code", "|| true"):
+ self.assertNotIn(forbidden, source)
+ self.assertNotIn("--no-deps", source)
+ self.assertNotIn("--disable-pip", source)
+ self.assertNotIn("${GITHUB_WORKSPACE}/${REPORTS}", source)
+
+ def test_exactly_four_commit_bound_non_push_images_are_built_and_scanned(self) -> None:
+ images = _named_step(self.workflow, "Build and scan exact shipping images")
+ self.assertTrue(_run_script(images).startswith("set -euo pipefail\n"))
+ self.assertEqual(images.count(" docker build --pull"), 4)
+ self.assertEqual(images.count("docker build --pull --no-cache"), 4)
+ self.assertIn("modelforge_runtime_worker.main", images)
+ self.assertIn("sentence_transformers", images)
+ self.assertIn('{"pip", "setuptools", "wheel"} & packages', images)
+ self.assertEqual(images.count("scan_image "), 4)
+ self.assertIn("bind-trivy-image", images)
+ self.assertIn('--expected-tag "modelforge-security-${target}:${GITHUB_SHA}"', images)
+ self.assertIn('${SECURITY_TOOLS}/trivy-image-${target}.raw.json', images)
+ # Four image tags plus the exact expected tag passed to the Trivy binder.
+ self.assertEqual(images.count(":${GITHUB_SHA}"), 5)
+ for target in ("api", "web", "node-agent", "runtime-worker"):
+ self.assertIn(f"image-build-{target}", images)
+ self.assertIn(f"scan_image {target} ", images)
+ self.assertNotIn("docker push", images)
+ self.assertNotIn("--push", images)
+ self.assertEqual(images.count("image-identity --target"), 1)
+ self.assertIn('"${target}"', images)
+ self.assertIn('"${API_ID}"', images)
+ self.assertIn('"${WEB_ID}"', images)
+ self.assertIn("org.opencontainers.image.revision=${GITHUB_SHA}", images)
+ self.assertEqual(images.count("org.opencontainers.image.source=${SOURCE_REPOSITORY}"), 4)
+ self.assertEqual(images.count("org.opencontainers.image.version=${MODELFORGE_VERSION}"), 4)
+ self.assertIn("io.modelforge.security.bundle-path=/usr/share/nginx/html", images)
+ self.assertIn('--expected-source "${SOURCE_REPOSITORY}"', images)
+ self.assertIn('--expected-version "${MODELFORGE_VERSION}"', images)
+ self.assertIn('--bundle-path "${WEB_BUNDLE_PATH}"', images)
+ self.assertIn("VITE_API_BASE_URL=https://security-scan.invalid", images)
+ self.assertIn("web-bundle-evidence", images)
+
+ cleanup = _named_step(self.workflow, "Remove exact security images")
+ self.assertIn("always()", cleanup)
+ self.assertIn("docker image rm", cleanup)
+ self.assertIn("docker image inspect", cleanup)
+ self.assertIn("status/image-cleanup.exit", cleanup)
+ self.assertNotIn("docker image prune", cleanup)
+
+ def test_image_source_uses_the_canonical_https_repository(self) -> None:
+ source = _run_script(_named_step(self.workflow, "Prepare fail-closed security evidence"))
+ self.assertIn(
+ '[[ "${GITHUB_REPOSITORY}" == "Jens/ITWorx-ModelForge" ]]', source
+ )
+ self.assertIn(
+ 'SOURCE_REPOSITORY}" == "https://git.example.com/example/modelforge"',
+ source,
+ )
+ for relative_path in gate.IMAGE_DOCKERFILES.values():
+ dockerfile = (REPOSITORY_ROOT / relative_path).read_text(encoding="utf-8")
+ self.assertEqual(
+ dockerfile.count(
+ f'LABEL org.opencontainers.image.source="{gate.CANONICAL_SOURCE_REPOSITORY}"'
+ ),
+ 1,
+ relative_path,
+ )
+
+ def test_api_release_builds_include_the_root_toolchain_lock(self) -> None:
+ image_build = _run_script(
+ _named_step(self.workflow, "Build and scan exact shipping images")
+ )
+ self.assertIn("--file backend/Dockerfile .", image_build)
+ release_builder = (REPOSITORY_ROOT / "scripts" / "release_build.py").read_text(
+ encoding="utf-8"
+ )
+ self.assertIn('"modelforge-api": (".", "backend/Dockerfile")', release_builder)
+ for relative_path, services in (
+ ("docker-compose.yml", ("api", "migrate")),
+ ("docker-compose.production.yml", ("api",)),
+ ("docker-compose.dr.yml", ("dr-api",)),
+ ):
+ compose = (REPOSITORY_ROOT / relative_path).read_text(encoding="utf-8")
+ for service in services:
+ block = re.search(
+ rf"(?ms)^ {re.escape(service)}:\n(.*?)(?=^ [A-Za-z0-9_-]+:|\Z)",
+ compose,
+ )
+ self.assertIsNotNone(block, f"{relative_path} is missing {service}")
+ self.assertRegex(
+ block.group(1),
+ r"(?m)^ build:\n context: \.\n dockerfile: backend/Dockerfile$",
+ )
+
+ def test_repository_root_image_context_excludes_local_secrets_and_model_data(self) -> None:
+ dockerignore = set(
+ (REPOSITORY_ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines()
+ )
+ self.assertTrue({".env", "**/.env", "**/.env.*"}.issubset(dockerignore))
+ self.assertTrue(
+ {
+ "**/*.key",
+ "**/*.pem",
+ "**/*.crt",
+ "**/*.p12",
+ "**/*.pfx",
+ "**/id_rsa*",
+ "**/id_ed25519*",
+ "**/private_key*",
+ "**/secrets/**",
+ "**/credentials/**",
+ "**/.npmrc",
+ "**/.pypirc",
+ "**/.netrc",
+ "**/.git-credentials",
+ "**/.htpasswd",
+ "**/pip.conf",
+ "**/.state/**",
+ "**/.ssh/**",
+ "**/.aws/**",
+ }.issubset(dockerignore)
+ )
+ self.assertTrue(
+ {
+ "/models/",
+ "/model-artifacts/",
+ "/artifacts/",
+ "/data/",
+ "/.data/",
+ "/.state/",
+ "/state/",
+ "/hf-cache/",
+ "/huggingface/",
+ "**/*.safetensors",
+ "**/*.gguf",
+ "**/*.ckpt",
+ "**/*.onnx",
+ "**/*.bin",
+ "**/*.h5",
+ "**/*.pb",
+ "**/*.model",
+ }.issubset(dockerignore)
+ )
+ self.assertNotIn("models/", dockerignore)
+ self.assertTrue((REPOSITORY_ROOT / "config/models/initial-candidates.yaml").is_file())
+ self.assertIn(
+ ".dockerignore",
+ (REPOSITORY_ROOT / "public-source.allowlist").read_text(encoding="utf-8").splitlines(),
+ )
+
+ def test_console_network_policy_admits_only_the_build_bound_api_origin(self) -> None:
+ headers = (REPOSITORY_ROOT / "frontend" / "security-headers.inc.template").read_text(
+ encoding="utf-8"
+ )
+ dockerfile = (REPOSITORY_ROOT / "frontend" / "Dockerfile").read_text(encoding="utf-8")
+ self.assertIn("connect-src __API_ORIGIN__;", headers)
+ self.assertNotIn("connect-src 'self'", headers)
+ self.assertIn('grep -q "connect-src ${API_ORIGIN};"', dockerfile)
+
+ def test_shipping_python_inventory_is_extracted_and_strictly_audited(self) -> None:
+ images = _named_step(self.workflow, "Build and scan exact shipping images")
+ self.assertEqual(images.count("collect_inventory "), 3)
+ self.assertEqual(images.count("pip-image-") // 2, 3)
+ for target in ("api", "node-agent", "runtime-worker"):
+ self.assertIn(f'collect_inventory {target} "', images)
+ self.assertIn(f"pip-image-{target}.json", images)
+ self.assertIn("importlib.metadata", images)
+ self.assertIn("--strict --no-deps", images)
+ self.assertIn("--disable-pip", images)
+
+ def test_python_validation_environments_are_not_shared(self) -> None:
+ self.assertIn("BACKEND_VENV:", self.workflow)
+ self.assertIn("NODE_AGENT_VENV:", self.workflow)
+ self.assertIn("RUNTIME_WORKER_VENV:", self.workflow)
+ self.assertNotIn("VENV: /tmp/modelforge-validation-venv", self.workflow)
+
+ def test_artifact_upload_is_immutable_retained_and_fail_closed(self) -> None:
+ upload = _named_step(self.workflow, "Upload immutable security evidence")
+ self.assertIn("if: always()", upload)
+ self.assertIn(
+ "https://gitea.com/actions/gitea-upload-artifact@"
+ "81f940d004763f986ba3582c007fd842dd5cb0d7",
+ upload,
+ )
+ self.assertIn(
+ "name: security-gate-${{ gitea.run_id }}-${{ gitea.run_attempt }}-"
+ "${{ gitea.sha }}",
+ upload,
+ )
+ self.assertIn("if-no-files-found: error", upload)
+ self.assertIn("retention-days: 90", upload)
+ self.assertIn("path: |", upload)
+ self.assertNotIn("path: ${{ env.REPORTS }}\n", upload)
+ uploaded_paths = set(
+ re.findall(r"\$\{\{ env\.REPORTS \}\}/([^\r\n]+)", upload)
+ )
+ self.assertEqual(
+ uploaded_paths,
+ {"security-gate-result.json", *gate.expected_evidence_paths("security")},
+ )
+ self.assertNotIn("continue-on-error", upload)
+
+ def test_gitleaks_scanner_config_has_narrow_current_github_rules_and_no_allowlist(self) -> None:
+ with GITLEAKS_CONFIG.open("rb") as handle:
+ config = tomllib.load(handle)
+ self.assertNotIn("extend", config)
+ rules = {rule["id"]: rule for rule in config["rules"]}
+ self.assertEqual(
+ set(rules),
+ {
+ "modelforge-github-opaque-token",
+ "modelforge-github-installation-token",
+ "modelforge-github-fine-grained-pat",
+ },
+ )
+ serialized = json.dumps(config).lower()
+ self.assertNotIn("allowlist", serialized)
+ self.assertTrue(all(rule.get("secretGroup") == 1 for rule in rules.values()))
+ source = _run_script(_named_step(self.workflow, "Collect source security reports"))
+ self.assertIn(
+ 'DEFAULT_GITLEAKS_REPOSITORY="${GITLEAKS_TEMP}/default-worktree"', source
+ )
+ self.assertIn('rm -f -- "${DEFAULT_GITLEAKS_REPOSITORY}/.gitleaks.toml"', source)
+ self.assertIn("--config .gitleaks.toml", source)
+ self.assertIn("RAW_GITLEAKS_DEFAULT", source)
+ self.assertIn("RAW_GITLEAKS_CUSTOM", source)
+ self.assertIn("gitleaks-merge.exit", source)
+ classic = "".join(("gh", "p_", "aBcDeFgHiJkLmNoPqRsTuVwXyZ", "0123456789"))
+ self.assertIsNotNone(re.search(rules["modelforge-github-opaque-token"]["regex"], classic))
+ opaque_body = classic.removeprefix("ghp_")
+ for prefix in ("ghp_", "gho_", "ghu_", "ghr_"):
+ with self.subTest(prefix=prefix):
+ self.assertIsNotNone(
+ re.search(
+ rules["modelforge-github-opaque-token"]["regex"],
+ prefix + opaque_body,
+ )
+ )
+ installation_rule = rules["modelforge-github-installation-token"]["regex"]
+ self.assertIsNotNone(re.search(installation_rule, "ghs_" + opaque_body))
+ self.assertIsNotNone(
+ re.search(installation_rule, "ghs_" + "A" * 40 + "." + "B" * 40 + "." + "C" * 40)
+ )
+ self.assertIsNotNone(
+ re.search(
+ rules["modelforge-github-fine-grained-pat"]["regex"],
+ "github_pat_" + "A1" * 41,
+ )
+ )
+ self.assertIsNone(
+ re.search(
+ rules["modelforge-github-opaque-token"]["regex"],
+ "example-not-a-credential",
+ )
+ )
+ install = _run_script(
+ _named_step(self.workflow, "Install checksum-pinned security scanners")
+ )
+ for fragment in (
+ "custom_formats=(ghp gho ghu ghs ghr github-pat)",
+ "custom-${format}",
+ "default-trigger",
+ "custom-control",
+ "default-control",
+ '"AK", "IA", "A1B2C3D4E5F6G7H8"',
+ "custom_trigger_code",
+ "default_trigger_code",
+ "trap cleanup_self_test EXIT",
+ 'rm -r -- "${test_root}"',
+ ):
+ self.assertIn(fragment, install)
+ self.assertNotIn("|| true", install)
+
+ def test_pinned_gitleaks_binary_detects_temporary_history_and_benign_control(self) -> None:
+ binary = os.environ.get("GITLEAKS_BIN") or shutil.which("gitleaks")
+ if binary is None:
+ self.skipTest("a Gitleaks binary is not available locally")
+ version = subprocess.run(
+ [binary, "version"], capture_output=True, check=False, text=True
+ )
+ if "8.30.1" not in version.stdout + version.stderr:
+ self.skipTest("available Gitleaks binary is not the pinned 8.30.1 build")
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ custom_names = ("ghp", "gho", "ghu", "ghs", "ghr", "github-pat")
+ repositories = {
+ name: root / name
+ for name in (*custom_names, "default-trigger", "control")
+ }
+ for repository in repositories.values():
+ repository.mkdir()
+ subprocess.run(["git", "-C", str(repository), "init", "-q"], check=True)
+ subprocess.run(
+ ["git", "-C", str(repository), "config", "user.name", "security-gate"],
+ check=True,
+ )
+ subprocess.run(
+ [
+ "git",
+ "-C",
+ str(repository),
+ "config",
+ "user.email",
+ "security-gate@invalid.example",
+ ],
+ check=True,
+ )
+ body = "".join(("aBcDeFgHiJkLmNoPqRsTuVwXyZ", "0123456789"))
+ aws_key = "".join(("AK", "IA", "A1B2C3D4E5F6G7H8"))
+ custom_tokens = {
+ name: "".join(("gh", f"{name[-1]}_", body))
+ for name in custom_names
+ if name != "github-pat"
+ }
+ custom_tokens["github-pat"] = "".join(("github", "_pat_", "A" * 82))
+ trigger_values = {
+ **{
+ name: f'token = "{token}"\n'
+ for name, token in custom_tokens.items()
+ },
+ "default-trigger": f'key = "{aws_key}"\n',
+ }
+ for name, value in trigger_values.items():
+ trigger_path = repositories[name] / "credential.txt"
+ trigger_path.write_text(value, encoding="utf-8")
+ _git(repositories[name], "add", ".")
+ _git(repositories[name], "commit", "-qm", "trigger")
+ trigger_path.write_text("credential removed\n", encoding="utf-8")
+ _git(repositories[name], "add", ".")
+ _git(repositories[name], "commit", "-qm", "remove")
+ (repositories["control"] / "control.txt").write_text(
+ 'token = "example-not-a-credential"\n', encoding="utf-8"
+ )
+ subprocess.run(["git", "-C", str(repositories["control"]), "add", "."], check=True)
+ subprocess.run(
+ ["git", "-C", str(repositories["control"]), "commit", "-qm", "control"],
+ check=True,
+ )
+
+ def detect(name: str, *, custom: bool) -> subprocess.CompletedProcess[str]:
+ command = [binary, "git"]
+ if custom:
+ command.extend(("--config", str(GITLEAKS_CONFIG)))
+ command.extend(
+ (
+ "--log-opts=--all",
+ "--redact=100",
+ "--report-format",
+ "json",
+ "--report-path",
+ str(root / f"{name}-{'custom' if custom else 'default'}.json"),
+ "--no-banner",
+ str(repositories[name]),
+ )
+ )
+ return subprocess.run(
+ command,
+ capture_output=True,
+ check=False,
+ text=True,
+ timeout=60,
+ )
+
+ trigger_custom = [detect(name, custom=True) for name in custom_names]
+ trigger_default = detect("default-trigger", custom=False)
+ control_default = detect("control", custom=False)
+ control_custom = detect("control", custom=True)
+ self.assertEqual([result.returncode for result in trigger_custom], [1] * 6)
+ self.assertEqual(trigger_default.returncode, 1)
+ self.assertEqual(control_default.returncode, 0)
+ self.assertEqual(control_custom.returncode, 0)
+
+ def test_dual_pass_gitleaks_matches_exact_historical_false_positive_hashes(self) -> None:
+ binary = os.environ.get("GITLEAKS_BIN") or shutil.which("gitleaks")
+ if binary is None:
+ self.skipTest("a Gitleaks binary is not available locally")
+ version = subprocess.run(
+ [binary, "version"], capture_output=True, check=False, text=True
+ )
+ if "8.30.1" not in version.stdout + version.stderr:
+ self.skipTest("available Gitleaks binary is not the pinned 8.30.1 build")
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ repository = root / "config-free-history"
+ subprocess.run(
+ ["git", "clone", "--quiet", "--no-local", str(REPOSITORY_ROOT), str(repository)],
+ check=True,
+ timeout=60,
+ )
+ (repository / ".gitleaks.toml").unlink(missing_ok=True)
+
+ def scan(*, custom: bool) -> tuple[subprocess.CompletedProcess[str], Path]:
+ output = root / f"repository-{'custom' if custom else 'default'}.json"
+ command = [binary, "git"]
+ if custom:
+ command.extend(("--config", str(GITLEAKS_CONFIG)))
+ command.extend(
+ (
+ "--log-opts=--all",
+ "--redact=100",
+ "--report-format",
+ "json",
+ "--report-path",
+ str(output),
+ "--no-banner",
+ str(repository),
+ )
+ )
+ result = subprocess.run(
+ command,
+ capture_output=True,
+ check=False,
+ text=True,
+ timeout=120,
+ )
+ return result, output
+
+ default_result, default_report = scan(custom=False)
+ custom_result, custom_report = scan(custom=True)
+ self.assertEqual(default_result.returncode, 1)
+ self.assertEqual(custom_result.returncode, 0)
+ combined = root / "combined.json"
+ _write_json(
+ combined,
+ [
+ *json.loads(default_report.read_text(encoding="utf-8")),
+ *json.loads(custom_report.read_text(encoding="utf-8")),
+ ],
+ )
+ sanitized = root / "sanitized.json"
+ gate.sanitize_gitleaks(combined, sanitized)
+ findings = json.loads(sanitized.read_text(encoding="utf-8"))["findings"]
+ actual = {(item["rule_id"], item["fingerprint_sha256"]) for item in findings}
+ registered = json.loads(GITLEAKS_FALSE_POSITIVES.read_text(encoding="utf-8"))
+ expected = {
+ (item["rule_id"], item["fingerprint_sha256"]) for item in registered
+ }
+ self.assertEqual(actual, expected)
+
+ def test_risk_registry_is_exact_bounded_and_tracked(self) -> None:
+ registry = json.loads(ACCEPTED_RISKS.read_text(encoding="utf-8"))
+ self.assertEqual(registry["schema_version"], 1)
+ risks = registry["accepted_risks"]
+ self.assertEqual(risks, [])
+ identities = {
+ (
+ entry["scanner"],
+ entry["target"],
+ entry["advisory_id"],
+ entry["package"],
+ entry["installed_version"],
+ )
+ for entry in risks
+ }
+ self.assertEqual(len(identities), len(risks))
+ for entry in risks:
+ self.assertEqual(entry["scanner"], "trivy")
+ self.assertEqual(entry["target"], "node-agent")
+ self.assertEqual(entry["classification"], "ACCEPTED_TEMPORARY")
+ self.assertEqual(entry["owner"], "ModelForge security owner")
+ self.assertEqual(entry["accepted_on"], "2026-09-02")
+ self.assertEqual(entry["expires_on"], "2026-10-02")
+ self.assertEqual(
+ entry["tracking"],
+ "https://git.example.com/example/modelforge/issues/24",
+ )
+ self.assertNotIn("*", "|".join(str(value) for value in entry.values()))
+
+ def test_false_positives_use_only_exact_hashes(self) -> None:
+ false_positives = json.loads(GITLEAKS_FALSE_POSITIVES.read_text(encoding="utf-8"))
+ self.assertEqual(len(false_positives), 2)
+ for entry in false_positives:
+ self.assertEqual(entry["classification"], "FALSE_POSITIVE")
+ self.assertRegex(entry["fingerprint_sha256"], r"^[0-9a-f]{64}$")
+ self.assertNotIn("path", entry)
+ self.assertNotIn("commit", entry)
+ self.assertNotIn("secret", entry)
+
+ def test_precheckout_fail_evidence_survives_workspace_cleanup(self) -> None:
+ script = _run_script(_named_step(self.workflow, "Prepare fail-closed security evidence"))
+ for fragment in (
+ "GITHUB_RUN_ID",
+ "GITHUB_RUN_ATTEMPT",
+ "GITHUB_SHA",
+ "mktemp -d",
+ "XXXXXXXX",
+ "umask 077",
+ "stat -c '%u'",
+ "stat -c '%a'",
+ ):
+ self.assertIn(fragment, script)
+ self.assertNotIn("mkdir -p", script)
+ sha = "a" * 40
+ execution_script = "\n".join(
+ (
+ "set -euo pipefail",
+ "test_base=$(mktemp -d /tmp/modelforge-root-policy.XXXXXXXX)",
+ "cleanup_test() { code=$?; trap - EXIT; rm -r -- \"${test_base}\"; exit \"${code}\"; }",
+ "trap cleanup_test EXIT",
+ "export RUNNER_TEMP=${test_base}",
+ "export GITHUB_ENV=${test_base}/github-env",
+ f"export GITHUB_SHA={sha}",
+ "export GITHUB_REF=refs/pull/1/head",
+ "export GITHUB_RUN_ID=synthetic-run",
+ "export GITHUB_RUN_ATTEMPT=7",
+ "export GITHUB_SERVER_URL=http://192.0.2.10:3000",
+ "export GITHUB_REPOSITORY=Jens/ITWorx-ModelForge",
+ "export SECURITY_PROFILE=full",
+ "poison_target=${test_base}/poison-target",
+ "mkdir \"${poison_target}\"",
+ "ln -s \"${poison_target}\" \"${test_base}/modelforge-security-evidence-synthetic-run-7-"
+ + sha
+ + "\"",
+ script,
+ "[[ ${SOURCE_REPOSITORY} == "
+ "https://git.example.com/example/modelforge ]]",
+ "first_reports=${REPORTS}",
+ "first_tools=${SECURITY_TOOLS}",
+ "[[ ${first_reports} == ${test_base}/modelforge-security-evidence-synthetic-run-7-"
+ + sha
+ + ".* ]]",
+ "[[ -d ${first_reports} && ! -L ${first_reports} ]]",
+ "[[ $(stat -c '%a' \"${first_reports}\") == 700 ]]",
+ "printf '99\\n' > \"${first_reports}/status/stale.exit\"",
+ "workspace=${test_base}/workspace",
+ "mkdir \"${workspace}\"",
+ "printf 'ephemeral\\n' > \"${workspace}/checkout-marker\"",
+ "rm \"${workspace}/checkout-marker\"",
+ "python3 - \"${first_reports}/security-gate-result.json\" <<'PY'",
+ "import json, sys",
+ "from pathlib import Path",
+ "value = json.loads(Path(sys.argv[1]).read_text(encoding='utf-8'))",
+ "assert value['decision'] == 'FAIL' and value['profile'] == 'full'",
+ "PY",
+ script,
+ "second_reports=${REPORTS}",
+ "second_tools=${SECURITY_TOOLS}",
+ "[[ ${second_reports} != ${first_reports} && ${second_tools} != ${first_tools} ]]",
+ "[[ ! -e ${second_reports}/status/stale.exit ]]",
+ "[[ -f ${first_reports}/security-gate-result.json ]]",
+ )
+ )
+ result = subprocess.run(
+ ["bash"],
+ input=execution_script.encode(),
+ cwd=REPOSITORY_ROOT,
+ capture_output=True,
+ check=False,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace"))
+
+ def test_precheckout_rejects_unknown_insecure_source_server(self) -> None:
+ script = _run_script(_named_step(self.workflow, "Prepare fail-closed security evidence"))
+ execution_script = "\n".join(
+ (
+ "set -euo pipefail",
+ "test_base=$(mktemp -d /tmp/modelforge-source-policy.XXXXXXXX)",
+ "cleanup_test() { code=$?; trap - EXIT; rm -r -- \"${test_base}\"; "
+ "exit \"${code}\"; }",
+ "trap cleanup_test EXIT",
+ "export RUNNER_TEMP=${test_base}",
+ "export GITHUB_ENV=${test_base}/github-env",
+ f"export GITHUB_SHA={'a' * 40}",
+ "export GITHUB_REF=refs/pull/1/head",
+ "export GITHUB_RUN_ID=synthetic-run",
+ "export GITHUB_RUN_ATTEMPT=1",
+ "export GITHUB_SERVER_URL=http://untrusted.invalid",
+ "export GITHUB_REPOSITORY=Jens/ITWorx-ModelForge",
+ "export SECURITY_PROFILE=full",
+ script,
+ )
+ )
+ result = subprocess.run(
+ ["bash"],
+ input=execution_script.encode(),
+ cwd=REPOSITORY_ROOT,
+ capture_output=True,
+ check=False,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Untrusted non-HTTPS Gitea server URL", result.stderr.decode())
+
+ def test_source_inventory_reads_every_optional_group_and_first_party_package(self) -> None:
+ inventory = gate.build_source_inventory(REPOSITORY_ROOT)
+ components = {component["target"]: component for component in inventory["components"]}
+ self.assertEqual(set(components), {"backend", "node-agent", "runtime-worker"})
+ self.assertEqual(components["backend"]["optional_groups"], ["dev"])
+ self.assertEqual(components["node-agent"]["optional_groups"], ["dev"])
+ self.assertEqual(components["runtime-worker"]["optional_groups"], ["dev", "inference"])
+ own = {
+ item["package"]
+ for item in inventory["first_party_coverage"]
+ if item["consumer_target"] == item["provider_target"]
+ }
+ self.assertEqual(
+ own, {"modelforge-api", "modelforge-node-agent", "modelforge-runtime-worker"}
+ )
+
+ def test_pep517_requirements_and_build_toolchain_are_exact_and_hash_locked(self) -> None:
+ inventory = gate.build_source_inventory(REPOSITORY_ROOT)
+ components = {component["target"]: component for component in inventory["components"]}
+ for target in gate.SOURCE_COMPONENTS:
+ by_name = {
+ requirement["name"]: requirement
+ for requirement in components[target]["requirements"]
+ }
+ with self.subTest(target=target):
+ backend_requirement = next(iter(gate.PEP517_BUILD_SYSTEMS[target][1]))
+ self.assertIn("build-system", by_name[backend_requirement]["groups"])
+ for name, version in gate.BUILD_TOOLCHAIN_PINS.items():
+ self.assertEqual(by_name[name]["version"], version)
+ self.assertIn("build-toolchain", by_name[name]["groups"])
+
+ lock_lines = [
+ line.strip()
+ for line in (REPOSITORY_ROOT / gate.BUILD_TOOLCHAIN_LOCK)
+ .read_text(encoding="utf-8")
+ .splitlines()
+ if line.strip() and not line.lstrip().startswith("#")
+ ]
+ self.assertEqual(len(lock_lines), len(gate.BUILD_TOOLCHAIN_PINS))
+ self.assertTrue(all("--hash=sha256:" in line for line in lock_lines))
+ msgpack_line = next(line for line in lock_lines if line.startswith("msgpack=="))
+ self.assertEqual(msgpack_line.count("--hash=sha256:"), 14)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ repository = Path(temporary)
+ _copy_source_gate_inputs(repository)
+ manifest = repository / "backend/pyproject.toml"
+ manifest.write_text(
+ manifest.read_text(encoding="utf-8").replace(
+ ' "hatchling==1.32.4",', ' "hatchling",'
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "not pinned"):
+ gate.build_source_inventory(repository)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ repository = Path(temporary)
+ _copy_source_gate_inputs(repository)
+ dockerfile = repository / "node-agent/Dockerfile"
+ dockerfile.write_text(
+ dockerfile.read_text(encoding="utf-8").replace(
+ "--require-hashes", "", 1
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "hash-locked bootstrap"):
+ gate.build_source_inventory(repository)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ repository = Path(temporary)
+ _copy_source_gate_inputs(repository)
+ dockerfile = repository / "backend/Dockerfile"
+ dockerfile.write_text(
+ dockerfile.read_text(encoding="utf-8").replace(
+ "--no-build-isolation", "", 1
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "disable build isolation"):
+ gate.build_source_inventory(repository)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ repository = Path(temporary)
+ _copy_source_gate_inputs(repository)
+ dockerfile = repository / "node-agent/Dockerfile"
+ dockerfile.write_text(
+ dockerfile.read_text(encoding="utf-8").replace(
+ "--ignore-installed", "", 1
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "runtime dependencies"):
+ gate.build_source_inventory(repository)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ repository = Path(temporary)
+ _copy_source_gate_inputs(repository)
+ dockerfile = repository / "frontend/Dockerfile"
+ dockerfile.write_text(
+ dockerfile.read_text(encoding="utf-8").replace(
+ gate.CANONICAL_SOURCE_REPOSITORY,
+ "https://git.example.com/example/modelforge.git",
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "source label bound"):
+ gate.build_source_inventory(repository)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ repository = Path(temporary)
+ _copy_source_gate_inputs(repository)
+ manifest_files = []
+ for relative_path in gate.IMAGE_DOCKERFILES.values():
+ dockerfile = repository / relative_path
+ rendered = dockerfile.read_text(encoding="utf-8").replace(
+ gate.CANONICAL_SOURCE_REPOSITORY,
+ gate.PUBLIC_SOURCE_LABEL_PLACEHOLDER,
+ )
+ dockerfile.write_text(rendered, encoding="utf-8")
+ rendered_bytes = rendered.encode("utf-8")
+ manifest_files.append(
+ {
+ "path": relative_path,
+ "bytes": len(rendered_bytes),
+ "sha256": hashlib.sha256(rendered_bytes).hexdigest(),
+ }
+ )
+ _write_json(
+ repository / gate.PUBLIC_SOURCE_MANIFEST,
+ {
+ "schemaVersion": 1,
+ "sourceRevision": "a" * 40,
+ "sourceRepository": gate.PUBLIC_SOURCE_LABEL_PLACEHOLDER,
+ "files": manifest_files,
+ },
+ )
+ exporter = (REPOSITORY_ROOT / "scripts" / "export-public-source.mjs").read_text(
+ encoding="utf-8"
+ )
+ self.assertIn(f'"{gate.PUBLIC_SOURCE_LABEL_PLACEHOLDER}"', exporter)
+ gate.build_source_inventory(repository)
+
+ manifest_path = repository / gate.PUBLIC_SOURCE_MANIFEST
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ manifest["sourceRepository"] = "https://attacker.invalid/repository"
+ _write_json(manifest_path, manifest)
+ with self.assertRaisesRegex(gate.GateInputError, "approved sanitized identity"):
+ gate.build_source_inventory(repository)
+
+ manifest["sourceRepository"] = gate.PUBLIC_SOURCE_LABEL_PLACEHOLDER
+ _write_json(manifest_path, manifest)
+
+ dockerfile = repository / "frontend/Dockerfile"
+ dockerfile.write_text(
+ dockerfile.read_text(encoding="utf-8").replace(
+ gate.PUBLIC_SOURCE_LABEL_PLACEHOLDER, "https://attacker.invalid/repository"
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "does not bind the current"):
+ gate.build_source_inventory(repository)
+
+ def test_evaluator_rejects_a_non_https_repository_identity(self) -> None:
+ request = gate.EvaluationRequest(
+ reports_dir=Path("reports"),
+ accepted_risks=Path("accepted-risks.json"),
+ gitleaks_false_positives=Path("false-positives.json"),
+ repository=REPOSITORY_ROOT,
+ commit="a" * 40,
+ ref="refs/heads/master",
+ run_id="test",
+ source_repository="http://git.example.com/Jens/ITWorx-ModelForge",
+ profile="full",
+ output=Path("reports/result.json"),
+ as_of=date(2026, 8, 30),
+ evaluation_time=datetime(2026, 8, 30, 0, 10, tzinfo=UTC),
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "canonical HTTPS"):
+ gate.Evaluator(request)
+
+ def test_source_inventory_emits_exact_pep508_extras(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ gate.collect_source_inventory(
+ REPOSITORY_ROOT,
+ root / "source-inventory.json",
+ root / "source-requirements",
+ )
+ backend = (root / "source-requirements" / "backend.txt").read_text(
+ encoding="utf-8"
+ )
+ self.assertIn("uvicorn[standard]==0.52.4\n", backend)
+ self.assertIn("psycopg[binary]==3.3.4\n", backend)
+ self.assertIn("hatchling==1.32.4\n", backend)
+ self.assertIn("editables==0.5\n", backend)
+ self.assertIn("msgpack==1.2.2\n", backend)
+ self.assertIn("pip==26.2.1\n", backend)
+ self.assertNotIn("\nuvicorn==0.52.4\n", f"\n{backend}")
+ self.assertNotIn("\npsycopg==3.3.4\n", f"\n{backend}")
+
+ def test_source_inventory_rejects_identical_extra_declarations_before_dedup(self) -> None:
+ declarations = ("uvicorn[standard]==0.52.4", "psycopg[binary]==3.3.4")
+ with tempfile.TemporaryDirectory() as temporary:
+ temporary_root = Path(temporary)
+ for attack_index, declaration in enumerate(declarations):
+ with self.subTest(declaration=declaration):
+ repository = temporary_root / f"attack-{attack_index}"
+ _copy_source_gate_inputs(repository)
+ manifest = repository / "backend" / "pyproject.toml"
+ text = manifest.read_text(encoding="utf-8")
+ declaration_line = f' "{declaration}",'
+ first_line = text.splitlines().index(declaration_line) + 1
+ manifest.write_text(
+ text.replace(
+ declaration_line,
+ f"{declaration_line}\n{declaration_line}",
+ 1,
+ ),
+ encoding="utf-8",
+ )
+ with self.assertRaises(gate.GateInputError) as raised:
+ gate.build_source_inventory(repository)
+ message = str(raised.exception)
+ self.assertIn("identical manifest declaration", message)
+ self.assertIn(f"backend/pyproject.toml:{first_line}", message)
+ self.assertIn(f"backend/pyproject.toml:{first_line + 1}", message)
+ self.assertIn("project.dependencies", message)
+
+ def test_all_security_shell_blocks_parse(self) -> None:
+ names = (
+ "Prepare fail-closed security evidence",
+ "Prove complete non-shallow history",
+ "Install checksum-pinned security scanners",
+ "Collect source security reports",
+ "Build and scan exact shipping images",
+ "Evaluate the complete security gate",
+ )
+ for name in names:
+ result = subprocess.run(
+ ["bash", "-n"],
+ input=_run_script(_named_step(self.workflow, name)).encode(),
+ capture_output=True,
+ check=False,
+ )
+ with self.subTest(name=name):
+ self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace"))
+
+
+class GateFixture:
+ commit = "a" * 40
+ as_of = date(2026, 8, 30)
+ evaluation_time = datetime(2026, 8, 30, 0, 10, tzinfo=UTC)
+ source_repository = gate.CANONICAL_SOURCE_REPOSITORY
+
+ def __init__(self, root: Path, *, profile: str = "full") -> None:
+ self.root = root
+ self.profile = profile
+ self.reports = root / "reports"
+ self.risks = root / "accepted-risks.json"
+ self.false_positives = root / "gitleaks-false-positives.json"
+ self.output = self.reports / "security-gate-result.json"
+ (self.reports / "status").mkdir(parents=True)
+ self.reports.chmod(0o700)
+ (self.reports / "versions").mkdir()
+ for status in gate.expected_status_codes(profile):
+ (self.reports / "status" / f"{status}.exit").write_text("0\n", encoding="ascii")
+
+ _write_json(
+ self.reports / "git-history.json",
+ {
+ "schema_version": 1,
+ "target": "full-history",
+ "head": self.commit,
+ "expected_commit": self.commit,
+ "is_shallow": False,
+ "head_commit_count": 151,
+ "all_commit_count": 153,
+ "root_commits": ["b" * 40],
+ "remote_name": "origin",
+ "remote_url": self.source_repository,
+ "advertised_remote_heads": [
+ {"ref": "refs/heads/master", "object": "c" * 40}
+ ],
+ "local_remote_heads": [
+ {"ref": "refs/heads/master", "object": "c" * 40}
+ ],
+ "advertised_tags": [],
+ "local_tags": [],
+ "all_ref_count": 1,
+ "all_objects_reachable": True,
+ },
+ )
+ gate.collect_source_inventory(
+ REPOSITORY_ROOT,
+ self.reports / "source-inventory.json",
+ self.reports / "source-requirements",
+ )
+ source_inventory = json.loads(
+ (self.reports / "source-inventory.json").read_text(encoding="utf-8")
+ )
+ for component in source_inventory["components"]:
+ packages = {
+ item["name"]: item["version"]
+ for item in component["requirements"]
+ if not item["first_party"]
+ }
+ _write_json(
+ self.reports / f"pip-source-{component['target']}.json",
+ self.pip_report(packages),
+ )
+
+ _write_json(
+ self.reports / "gitleaks.json",
+ {
+ "schema_version": 1,
+ "scanner": "gitleaks",
+ "target": "full-history",
+ "redaction_percent": 100,
+ "findings": [],
+ },
+ )
+ clean_npm = {
+ "auditReportVersion": 2,
+ "vulnerabilities": {},
+ "metadata": {
+ "vulnerabilities": {
+ "info": 0,
+ "low": 0,
+ "moderate": 0,
+ "high": 0,
+ "critical": 0,
+ "total": 0,
+ }
+ },
+ }
+ _write_json(self.reports / "npm-audit-all.json", clean_npm)
+ _write_json(self.reports / "npm-audit-production.json", clean_npm)
+ _write_json(
+ self.reports / "trivy-fs.json",
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": str(REPOSITORY_ROOT.resolve()),
+ "ArtifactType": "repository",
+ "Results": [],
+ },
+ )
+
+ image_packages = {
+ "api": {"modelforge-api": PRODUCT_VERSION, "httpx": "0.28.1"},
+ "node-agent": {
+ "modelforge-api": PRODUCT_VERSION,
+ "modelforge-node-agent": PRODUCT_VERSION,
+ "httpx": "0.28.1",
+ },
+ "runtime-worker": {
+ "modelforge-runtime-worker": PRODUCT_VERSION,
+ **gate.IMAGE_NON_PYPI_AUDIT_EXCEPTIONS["runtime-worker"],
+ "httpx": "0.28.1",
+ },
+ }
+ if profile == "security":
+ identity_characters = {
+ "api": "1",
+ "web": "2",
+ "node-agent": "3",
+ "runtime-worker": "4",
+ }
+ for target in gate.IMAGE_TARGETS:
+ image_id = "sha256:" + identity_characters[target] * 64
+ tag = f"modelforge-security-{target}:{self.commit}"
+ identity = {
+ "schema_version": 1,
+ "target": target,
+ "tag": tag,
+ "image_id": image_id,
+ "repo_tags": [tag],
+ "repo_digests": [],
+ "revision": self.commit,
+ "source": self.source_repository,
+ "version": PRODUCT_VERSION,
+ "bundle_path": gate.WEB_BUNDLE_PATH if target == "web" else None,
+ }
+ _write_json(self.reports / f"image-identity-{target}.json", identity)
+ _write_json(
+ self.reports / f"trivy-image-{target}.json",
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": image_id,
+ "ArtifactID": "sha256:" + "a" * 64,
+ "ArtifactType": "container_image",
+ "ScannedImageID": image_id,
+ "Metadata": {
+ "ImageID": image_id,
+ "Reference": tag,
+ "RepoTags": [tag],
+ "RepoDigests": [],
+ },
+ "Results": [],
+ },
+ )
+ _write_json(
+ self.reports / "web-bundle-origin.json",
+ {
+ "schema_version": 1,
+ "target": "web",
+ "image_id": "sha256:" + "2" * 64,
+ "bundle_path": gate.WEB_BUNDLE_PATH,
+ "bundle_sha256": "f" * 64,
+ "bundle_file_count": 3,
+ "provenance_path": gate.WEB_PROVENANCE_PATH,
+ "provenance_sha256": "e" * 64,
+ "served_provenance_sha256": "e" * 64,
+ "provenance_binding_sha256": gate._web_provenance_binding(
+ gate.WEB_SECURITY_ORIGIN
+ ),
+ "runtime_chunk_path": "assets/index.js",
+ "runtime_chunk_sha256": "d" * 64,
+ "api_origins": [gate.WEB_SECURITY_ORIGIN],
+ "non_api_origins": list(gate.WEB_EXPECTED_NON_API_ORIGINS),
+ },
+ )
+ for target, packages in image_packages.items():
+ _write_json(
+ self.reports / f"image-inventory-{target}.json",
+ {
+ "schema_version": 1,
+ "target": target,
+ "python_version": "3.12.0",
+ "packages": [
+ {"name": name, "version": version}
+ for name, version in packages.items()
+ ],
+ },
+ )
+ _write_json(
+ self.reports / f"pip-image-{target}.json",
+ self.pip_report(
+ {
+ name: version
+ for name, version in packages.items()
+ if not name.startswith("modelforge-")
+ and name
+ not in gate.IMAGE_NON_PYPI_AUDIT_EXCEPTIONS.get(target, {})
+ }
+ ),
+ )
+
+ (self.reports / "versions" / "pip-audit.txt").write_text(
+ "pip-audit 2.10.1\n", encoding="utf-8"
+ )
+ (self.reports / "versions" / "gitleaks.txt").write_text(
+ "8.30.1\n", encoding="utf-8"
+ )
+ (self.reports / "versions" / "npm.txt").write_text("11.0.0\n", encoding="utf-8")
+ _write_json(
+ self.reports / "versions" / "trivy.json",
+ {
+ "Version": "0.74.0",
+ "VulnerabilityDB": {
+ "Version": 2,
+ "UpdatedAt": "2026-08-30T00:00:00Z",
+ "DownloadedAt": "2026-08-30T00:05:00Z",
+ },
+ },
+ )
+ self.write_risks([])
+ _write_json(self.false_positives, [])
+
+ @staticmethod
+ def pip_report(packages: dict[str, str]) -> dict[str, Any]:
+ return {
+ "dependencies": [
+ {"name": name, "version": version, "vulns": []}
+ for name, version in sorted(packages.items())
+ ],
+ "fixes": [],
+ }
+
+ def evaluate(self) -> dict[str, Any]:
+ return gate.evaluate_security_gate(
+ gate.EvaluationRequest(
+ reports_dir=self.reports,
+ accepted_risks=self.risks,
+ gitleaks_false_positives=self.false_positives,
+ repository=REPOSITORY_ROOT,
+ commit=self.commit,
+ ref="refs/pull/1/head",
+ run_id="synthetic-1",
+ source_repository=self.source_repository,
+ profile=self.profile,
+ output=self.output,
+ as_of=self.as_of,
+ evaluation_time=self.evaluation_time,
+ )
+ )
+
+ def write_risks(self, entries: list[dict[str, str]]) -> None:
+ _write_json(
+ self.risks,
+ {"schema_version": gate.SCHEMA_VERSION, "accepted_risks": entries},
+ )
+
+ def add_pip_vulnerability(
+ self, *, fix_versions: list[str] | None = None, duplicate: bool = False
+ ) -> dict[str, str]:
+ path = self.reports / "pip-source-backend.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ dependency = report["dependencies"][0]
+ vulnerability = {"id": "PYSEC-TEST-1", "fix_versions": fix_versions or []}
+ dependency["vulns"] = [vulnerability, copy.deepcopy(vulnerability)] if duplicate else [vulnerability]
+ _write_json(path, report)
+ (self.reports / "status" / "pip-source-backend.exit").write_text("1\n")
+ return {
+ "scanner": "pip-audit",
+ "target": "backend",
+ "advisory_id": "PYSEC-TEST-1",
+ "package": dependency["name"],
+ "installed_version": dependency["version"],
+ }
+
+ def risk(self, identity: dict[str, str]) -> dict[str, str]:
+ return {
+ **identity,
+ "classification": "ACCEPTED_TEMPORARY",
+ "owner": "security-owner",
+ "accepted_on": "2026-08-30",
+ "expires_on": "2026-09-29",
+ "tracking": "https://tracker.invalid/RC-DEL-02",
+ "reason": "No upstream fix exists; bounded review is required before release.",
+ }
+
+
+class SecurityGateEvaluatorTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.temporary = tempfile.TemporaryDirectory()
+ self.fixture = GateFixture(Path(self.temporary.name))
+
+ def tearDown(self) -> None:
+ self.temporary.cleanup()
+
+ def assert_failed_with(self, result: dict[str, Any], advisory_id: str) -> None:
+ self.assertEqual(result["decision"], "FAIL")
+ self.assertIn(advisory_id, {item["advisory_id"] for item in result["blocked_findings"]})
+
+ def test_clean_complete_evidence_passes(self) -> None:
+ result = self.fixture.evaluate()
+ self.assertEqual(result["decision"], "PASS")
+ self.assertEqual(result["blocked_findings"], [])
+ self.assertEqual(result["expected_targets"], result["actual_targets"])
+ self.assertEqual(result["scanner_versions"]["trivy"], "0.74.0")
+ self.assertEqual(
+ result["vulnerability_database_times"]["trivy"]["updated_at"],
+ "2026-08-30T00:00:00Z",
+ )
+ self.assertEqual(result["profile"], "full")
+ self.assertNotIn("image_scan", result["expected_targets"])
+
+ def test_security_profile_requires_and_accepts_source_plus_exact_images(self) -> None:
+ fixture = GateFixture(Path(self.temporary.name) / "security", profile="security")
+ result = fixture.evaluate()
+ self.assertEqual(result["decision"], "PASS")
+ self.assertEqual(result["expected_targets"], result["actual_targets"])
+ self.assertEqual(set(result["image_identities"]), set(gate.IMAGE_TARGETS))
+ (fixture.reports / "source-inventory.json").unlink()
+ self.assert_failed_with(fixture.evaluate(), "GATE-SOURCE-INVENTORY")
+
+ def test_runtime_worker_inventory_rejects_installers(self) -> None:
+ fixture = GateFixture(Path(self.temporary.name) / "installers", profile="security")
+ inventory = fixture.reports / "image-inventory-runtime-worker.json"
+ report = json.loads(inventory.read_text(encoding="utf-8"))
+ report["packages"].append({"name": "pip", "version": "26.0"})
+ _write_json(inventory, report)
+ self.assert_failed_with(fixture.evaluate(), "GATE-IMAGE-FORBIDDEN-PACKAGE")
+
+ def test_source_only_profile_defers_but_validates_image_risks(self) -> None:
+ identity = {
+ "scanner": "trivy",
+ "target": "node-agent",
+ "advisory_id": "CVE-2099-0001",
+ "package": "base-package",
+ "installed_version": "1.0",
+ }
+ risk = self.fixture.risk(identity)
+ self.fixture.write_risks([risk])
+ result = self.fixture.evaluate()
+ self.assertEqual(result["decision"], "PASS")
+ self.assertEqual(result["applied_risks"], [])
+ self.assertEqual(result["unused_or_invalid_risks"], [])
+
+ risk["expires_on"] = "2026-08-30"
+ self.fixture.write_risks([risk])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-RISK-INVALID")
+
+ security = GateFixture(Path(self.temporary.name) / "unused-image", profile="security")
+ security.write_risks([security.risk(identity)])
+ self.assert_failed_with(security.evaluate(), "GATE-RISK-UNUSED")
+
+ def test_scanner_timeout_fails(self) -> None:
+ (self.fixture.reports / "status" / "trivy-fs.exit").write_text("124\n")
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-STATUS-FAILED")
+
+ def test_missing_empty_and_malformed_reports_fail(self) -> None:
+ cases = (None, "", "{")
+ path = self.fixture.reports / "npm-audit-all.json"
+ original = path.read_text(encoding="utf-8")
+ for replacement in cases:
+ with self.subTest(replacement=replacement):
+ if replacement is None:
+ path.unlink()
+ else:
+ path.write_text(replacement, encoding="utf-8")
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-NPM-REPORT")
+ path.write_text(original, encoding="utf-8")
+
+ def test_component_gap_fails(self) -> None:
+ path = self.fixture.reports / "pip-source-node-agent.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["dependencies"].pop(0)
+ _write_json(path, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-PIP-COMPONENT-GAP")
+
+ def test_source_direct_packages_are_exact_and_resolved_transitives_are_recorded(self) -> None:
+ path = self.fixture.reports / "pip-source-backend.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["dependencies"].extend(
+ [
+ {"name": "starlette", "version": "0.47.3", "vulns": []},
+ {"name": "uvloop", "version": "0.21.0", "vulns": []},
+ {"name": "psycopg-binary", "version": "3.3.4", "vulns": []},
+ ]
+ )
+ _write_json(path, report)
+ result = self.fixture.evaluate()
+ self.assertEqual(result["decision"], "PASS")
+ for transitive in (
+ {"name": "starlette", "version": "0.47.3"},
+ {"name": "uvloop", "version": "0.21.0"},
+ {"name": "psycopg-binary", "version": "3.3.4"},
+ ):
+ self.assertIn(transitive, result["resolved_transitive_packages"]["backend"])
+
+ report["dependencies"] = [
+ dependency
+ for dependency in report["dependencies"]
+ if gate._normalize_name(dependency["name"]) != "fastapi"
+ ]
+ _write_json(path, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-PIP-COMPONENT-GAP")
+
+ def test_only_pip_audit_packaging_tool_omissions_are_accepted(self) -> None:
+ source = self.fixture.reports / "pip-source-runtime-worker.json"
+ report = json.loads(source.read_text(encoding="utf-8"))
+ report["dependencies"] = [
+ dependency
+ for dependency in report["dependencies"]
+ if gate._normalize_name(dependency["name"])
+ not in gate.PIP_AUDIT_SOURCE_PACKAGING_TOOL_OMISSIONS
+ ]
+ _write_json(source, report)
+ self.assertEqual(self.fixture.evaluate()["decision"], "PASS")
+
+ report["dependencies"] = [
+ dependency
+ for dependency in report["dependencies"]
+ if gate._normalize_name(dependency["name"]) != "sentencepiece"
+ ]
+ _write_json(source, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-PIP-COMPONENT-GAP")
+
+ image_fixture = GateFixture(
+ Path(self.temporary.name) / "packaging-tool-image", profile="security"
+ )
+ image = image_fixture.reports / "pip-image-runtime-worker.json"
+ image_report = json.loads(image.read_text(encoding="utf-8"))
+ image_report["dependencies"] = [
+ dependency
+ for dependency in image_report["dependencies"]
+ if gate._normalize_name(dependency["name"])
+ not in gate.PIP_AUDIT_PACKAGING_TOOL_OMISSIONS
+ ]
+ _write_json(image, image_report)
+ self.assertEqual(image_fixture.evaluate()["decision"], "PASS")
+
+ def test_source_pep508_extra_omission_attack_fails(self) -> None:
+ requirements = self.fixture.reports / "source-requirements" / "backend.txt"
+ retained = requirements.read_text(encoding="utf-8")
+ self.assertIn("uvicorn[standard]==0.52.4", retained)
+ self.assertIn("psycopg[binary]==3.3.4", retained)
+ poisoned = retained.replace("uvicorn[standard]", "uvicorn").replace(
+ "psycopg[binary]", "psycopg"
+ )
+ requirements.write_text(poisoned, encoding="utf-8")
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-SOURCE-REQUIREMENTS")
+
+ def test_identical_duplicate_pip_row_is_malformed(self) -> None:
+ path = self.fixture.reports / "pip-source-backend.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["dependencies"].append(copy.deepcopy(report["dependencies"][0]))
+ _write_json(path, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-PIP-DUPLICATE")
+
+ def test_evidence_symlink_redirection_fails_before_hashing(self) -> None:
+ outside = self.fixture.root / "outside-evidence"
+ outside.mkdir()
+ redirect = self.fixture.reports / "redirected-evidence"
+ if os.name == "nt":
+ subprocess.run(
+ ["cmd.exe", "/d", "/c", "mklink", "/J", str(redirect), str(outside)],
+ capture_output=True,
+ check=True,
+ text=True,
+ timeout=30,
+ )
+ else:
+ redirect.symlink_to(outside, target_is_directory=True)
+ result = self.fixture.evaluate()
+ self.assert_failed_with(result, "GATE-EVIDENCE-PATH")
+ self.assertEqual(result["report_hashes"], {})
+
+ def test_evidence_metadata_rejects_foreign_owner_and_writable_modes(self) -> None:
+ def metadata(mode: int, uid: int) -> Any:
+ return SimpleNamespace(
+ st_dev=1,
+ st_ino=2,
+ st_mode=mode,
+ st_uid=uid,
+ st_size=1,
+ st_mtime_ns=1,
+ st_ctime_ns=1,
+ st_file_attributes=0,
+ )
+
+ with mock.patch.object(os, "geteuid", return_value=1000, create=True):
+ with self.assertRaisesRegex(gate.GateInputError, "not owned"):
+ gate._validate_evidence_metadata(
+ metadata(stat.S_IFREG | 0o600, 65534),
+ path="foreign.json",
+ expected_kind="file",
+ enforce_posix_permissions=True,
+ )
+ for mode, kind, root in (
+ (stat.S_IFDIR | 0o777, "directory", True),
+ (stat.S_IFREG | 0o666, "file", False),
+ ):
+ with self.subTest(mode=oct(mode)):
+ with self.assertRaisesRegex(gate.GateInputError, "permissions|writable"):
+ gate._validate_evidence_metadata(
+ metadata(mode, 1000),
+ path="unsafe",
+ expected_kind=kind,
+ root=root,
+ enforce_posix_permissions=True,
+ )
+
+ def test_evidence_lstat_open_race_fails_before_replacement_is_read(self) -> None:
+ root = self.fixture.root / "race-reports"
+ root.mkdir(mode=0o700)
+ root.chmod(0o700)
+ victim = root / "report.json"
+ victim.write_bytes(b"inside-safe")
+ outside = self.fixture.root / "outside-secret.json"
+ outside.write_bytes(b"outside-must-not-be-consumed")
+ replaced = False
+
+ def replace_after_lstat(event: str, path: Path) -> None:
+ nonlocal replaced
+ if event == "after_lstat" and path == victim and not replaced:
+ replaced = True
+ os.replace(outside, victim)
+
+ with mock.patch.object(
+ gate.SecureEvidenceSnapshot,
+ "_read_fd",
+ wraps=gate.SecureEvidenceSnapshot._read_fd,
+ ) as reader:
+ with self.assertRaisesRegex(gate.GateInputError, "changed before open"):
+ gate.SecureEvidenceSnapshot(
+ root, root / "result.json", race_hook=replace_after_lstat
+ )
+ self.assertTrue(replaced)
+ reader.assert_not_called()
+
+ @unittest.skipUnless(os.name == "posix", "open-handle replacement semantics are Linux-only")
+ def test_evidence_after_open_swap_reads_only_original_descriptor_then_fails(self) -> None:
+ root = self.fixture.root / "open-race-reports"
+ root.mkdir(mode=0o700)
+ root.chmod(0o700)
+ victim = root / "report.json"
+ victim.write_bytes(b"inside-safe")
+ outside = self.fixture.root / "outside-open-race.json"
+ outside.write_bytes(b"outside-must-not-be-consumed")
+ consumed: list[bytes] = []
+ original_reader = gate.SecureEvidenceSnapshot._read_fd
+
+ def swap_after_open(event: str, path: Path) -> None:
+ if event == "after_open" and path == victim:
+ os.replace(outside, victim)
+
+ def read_descriptor(file_descriptor: int) -> bytes:
+ value = original_reader(file_descriptor)
+ consumed.append(value)
+ return value
+
+ with mock.patch.object(
+ gate.SecureEvidenceSnapshot, "_read_fd", side_effect=read_descriptor
+ ):
+ with self.assertRaisesRegex(gate.GateInputError, "changed during read"):
+ gate.SecureEvidenceSnapshot(
+ root, root / "result.json", race_hook=swap_after_open
+ )
+ self.assertEqual(consumed, [b"inside-safe"])
+
+ def test_evidence_directory_substitution_after_snapshot_is_rejected(self) -> None:
+ root = self.fixture.root / "snapshot-reports"
+ nested = root / "nested"
+ nested.mkdir(parents=True)
+ root.chmod(0o700)
+ (nested / "report.json").write_bytes(b"inside-safe")
+ outside = self.fixture.root / "outside-directory"
+ outside.mkdir()
+ (outside / "report.json").write_bytes(b"outside-must-not-be-consumed")
+ snapshot = gate.SecureEvidenceSnapshot(root, root / "result.json")
+ (nested / "report.json").unlink()
+ nested.rmdir()
+ if os.name == "nt":
+ subprocess.run(
+ ["cmd.exe", "/d", "/c", "mklink", "/J", str(nested), str(outside)],
+ capture_output=True,
+ check=True,
+ text=True,
+ timeout=30,
+ )
+ else:
+ nested.symlink_to(outside, target_is_directory=True)
+ with self.assertRaisesRegex(
+ gate.GateInputError, "identity changed|not a real|changed after secure capture"
+ ):
+ snapshot.read_bytes("nested/report.json")
+
+ def test_evidence_in_place_mutation_after_snapshot_is_rejected(self) -> None:
+ root = self.fixture.root / "mutation-reports"
+ root.mkdir(mode=0o700)
+ root.chmod(0o700)
+ report = root / "report.json"
+ report.write_bytes(b"inside-safe")
+ snapshot = gate.SecureEvidenceSnapshot(root, root / "result.json")
+ report.write_bytes(b"outside-poison-is-longer")
+ with self.assertRaisesRegex(gate.GateInputError, "changed after secure capture"):
+ snapshot.read_bytes("report.json")
+
+ def test_record_history_rejects_nonshallow_single_branch_clone(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ origin = root / "origin.git"
+ seed = root / "seed"
+ partial = root / "partial"
+ complete = root / "complete"
+ origin.mkdir()
+ seed.mkdir()
+ _git(origin, "init", "--bare", "--initial-branch=main")
+ _git(seed, "init", "--initial-branch=main")
+ _git(seed, "config", "user.name", "security-gate")
+ _git(seed, "config", "user.email", "security-gate@invalid.example")
+ (seed / "tracked.txt").write_text("history proof\n", encoding="utf-8")
+ _git(seed, "add", "tracked.txt")
+ _git(seed, "commit", "-m", "initial")
+ _git(seed, "branch", "retained-remote-branch")
+ _git(seed, "tag", "retained-tag")
+ _git(seed, "remote", "add", "origin", str(origin))
+ _git(seed, "push", "origin", "--all")
+ _git(seed, "push", "origin", "--tags")
+
+ subprocess.run(
+ [
+ "git",
+ "clone",
+ "--single-branch",
+ "--branch",
+ "main",
+ "--no-tags",
+ str(origin),
+ str(partial),
+ ],
+ capture_output=True,
+ check=True,
+ text=True,
+ timeout=30,
+ )
+ head = _git(partial, "rev-parse", "HEAD").stdout.strip()
+ with self.assertRaisesRegex(gate.GateInputError, "advertised remote heads"):
+ gate.record_history(partial, head, root / "partial-history.json")
+
+ subprocess.run(
+ ["git", "clone", str(origin), str(complete)],
+ capture_output=True,
+ check=True,
+ text=True,
+ timeout=30,
+ )
+ complete_head = _git(complete, "rev-parse", "HEAD").stdout.strip()
+ output = root / "complete-history.json"
+ gate.record_history(complete, complete_head, output)
+ report = json.loads(output.read_text(encoding="utf-8"))
+ self.assertEqual(len(report["advertised_remote_heads"]), 2)
+ self.assertEqual(report["advertised_tags"][0]["ref"], "refs/tags/retained-tag")
+
+ def test_shallow_history_fails(self) -> None:
+ path = self.fixture.reports / "git-history.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["is_shallow"] = True
+ _write_json(path, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-HISTORY-INCOMPLETE")
+
+ def test_history_origin_and_ref_namespace_are_exact(self) -> None:
+ attacks: tuple[tuple[str, Any], ...] = (
+ ("remote_url", "https://attacker.invalid/repository"),
+ (
+ "advertised_remote_heads",
+ [{"ref": "refs/pull/1/head", "object": "c" * 40}],
+ ),
+ ("advertised_remote_heads", []),
+ )
+ for index, (field, value) in enumerate(attacks):
+ with self.subTest(field=field, value=value):
+ fixture = GateFixture(self.fixture.root / f"history-attack-{index}")
+ path = fixture.reports / "git-history.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report[field] = value
+ if field == "advertised_remote_heads" and value:
+ report["local_remote_heads"] = copy.deepcopy(value)
+ _write_json(path, report)
+ self.assert_failed_with(fixture.evaluate(), "GATE-HISTORY-INCOMPLETE")
+
+ def test_unclassified_gitleaks_finding_fails_without_secret_material(self) -> None:
+ fingerprint = "c" * 64
+ _write_json(
+ self.fixture.reports / "gitleaks.json",
+ {
+ "schema_version": 1,
+ "scanner": "gitleaks",
+ "target": "full-history",
+ "redaction_percent": 100,
+ "findings": [
+ {"rule_id": "generic-api-key", "fingerprint_sha256": fingerprint}
+ ],
+ },
+ )
+ (self.fixture.reports / "status" / "gitleaks.exit").write_text("1\n")
+ result = self.fixture.evaluate()
+ self.assert_failed_with(result, "generic-api-key")
+ serialized = json.dumps(result)
+ for forbidden in ('"Secret"', '"Match"', '"Line"', '"author"', '"email"'):
+ self.assertNotIn(forbidden, serialized)
+
+ def test_gitleaks_sanitizer_and_evaluator_reject_metadata_fields(self) -> None:
+ raw = self.fixture.root / "raw-gitleaks.json"
+ sanitized = self.fixture.root / "sanitized-gitleaks.json"
+ _write_json(
+ raw,
+ [
+ {
+ "RuleID": "synthetic-rule",
+ "Fingerprint": "synthetic-fingerprint",
+ "Secret": "must-not-survive",
+ "Match": "must-not-survive",
+ "File": "private/path.txt",
+ "Commit": "b" * 40,
+ "Author": "private-author",
+ }
+ ],
+ )
+ gate.sanitize_gitleaks(raw, sanitized)
+ report = json.loads(sanitized.read_text(encoding="utf-8"))
+ self.assertEqual(set(report), gate.GITLEAKS_REPORT_FIELDS)
+ self.assertEqual(set(report["findings"][0]), gate.GITLEAKS_FINDING_FIELDS)
+ serialized = json.dumps(report)
+ for forbidden in (
+ "must-not-survive",
+ "private/path.txt",
+ "private-author",
+ "synthetic-fingerprint",
+ ):
+ self.assertNotIn(forbidden, serialized)
+
+ unsafe = json.loads(raw.read_text(encoding="utf-8"))
+ unsafe[0]["RuleID"] = "../private/path"
+ _write_json(raw, unsafe)
+ with self.assertRaisesRegex(gate.GateInputError, "safe rule metadata"):
+ gate.sanitize_gitleaks(raw, sanitized)
+
+ for field, value in (
+ ("path", "private/path.txt"),
+ ("commit", "b" * 40),
+ ("secret", "must-not-survive"),
+ ("unknown", "unexpected"),
+ ):
+ with self.subTest(field=field):
+ fixture = GateFixture(self.fixture.root / f"gitleaks-{field}")
+ path = fixture.reports / "gitleaks.json"
+ poisoned = json.loads(path.read_text(encoding="utf-8"))
+ poisoned["findings"] = [
+ {
+ "rule_id": "synthetic-rule",
+ "fingerprint_sha256": "c" * 64,
+ field: value,
+ }
+ ]
+ _write_json(path, poisoned)
+ fixture_status = fixture.reports / "status" / "gitleaks.exit"
+ fixture_status.write_text("1\n", encoding="ascii")
+ self.assert_failed_with(fixture.evaluate(), "GATE-GITLEAKS-SCHEMA")
+
+ def test_every_pip_and_npm_vulnerability_fails(self) -> None:
+ self.fixture.add_pip_vulnerability()
+ self.assert_failed_with(self.fixture.evaluate(), "PYSEC-TEST-1")
+
+ self.fixture = GateFixture(Path(self.temporary.name) / "npm")
+ npm = {
+ "auditReportVersion": 2,
+ "vulnerabilities": {
+ "react": {
+ "name": "react",
+ "severity": "high",
+ "via": [
+ {"source": 1, "url": "https://github.com/advisories/GHSA-test-0001"}
+ ],
+ "fixAvailable": False,
+ }
+ },
+ "metadata": {"vulnerabilities": {"high": 1, "total": 1}},
+ }
+ _write_json(self.fixture.reports / "npm-audit-all.json", npm)
+ (self.fixture.reports / "status" / "npm-audit-all.exit").write_text("1\n")
+ self.assert_failed_with(self.fixture.evaluate(), "GHSA-test-0001")
+
+ def test_trivy_unknown_high_and_critical_fail_but_medium_does_not(self) -> None:
+ path = self.fixture.reports / "trivy-fs.json"
+ for severity, expected in (
+ ("UNKNOWN", "CVE-UNKNOWN"),
+ ("HIGH", "CVE-HIGH"),
+ ("CRITICAL", "CVE-CRITICAL"),
+ ):
+ with self.subTest(severity=severity):
+ _write_json(
+ path,
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": str(REPOSITORY_ROOT.resolve()),
+ "ArtifactType": "repository",
+ "Results": [
+ {
+ "Target": ".",
+ "Vulnerabilities": [
+ {
+ "VulnerabilityID": expected,
+ "PkgName": "package",
+ "InstalledVersion": "1.0",
+ "FixedVersion": "",
+ "Severity": severity,
+ }
+ ],
+ }
+ ],
+ },
+ )
+ self.assert_failed_with(self.fixture.evaluate(), expected)
+ _write_json(
+ path,
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": str(REPOSITORY_ROOT.resolve()),
+ "ArtifactType": "repository",
+ "Results": [
+ {
+ "Target": ".",
+ "Vulnerabilities": [
+ {
+ "VulnerabilityID": "CVE-MEDIUM",
+ "PkgName": "package",
+ "InstalledVersion": "1.0",
+ "Severity": "MEDIUM",
+ }
+ ]
+ }
+ ],
+ },
+ )
+ self.assertEqual(self.fixture.evaluate()["decision"], "PASS")
+
+ def test_trivy_filesystem_and_image_reports_are_bound_to_exact_targets(self) -> None:
+ filesystem = self.fixture.reports / "trivy-fs.json"
+ report = json.loads(filesystem.read_text(encoding="utf-8"))
+ report["ArtifactType"] = "filesystem"
+ _write_json(filesystem, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ report["ArtifactType"] = "repository"
+ report["ArtifactName"] = "swapped-target"
+ _write_json(filesystem, report)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ fixture = GateFixture(Path(self.temporary.name) / "image-target", profile="security")
+ api_report = fixture.reports / "trivy-image-api.json"
+ swapped = json.loads(
+ (fixture.reports / "trivy-image-web.json").read_text(encoding="utf-8")
+ )
+ _write_json(api_report, swapped)
+ self.assert_failed_with(fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ fixture = GateFixture(Path(self.temporary.name) / "nested-target", profile="security")
+ api_report = fixture.reports / "trivy-image-api.json"
+ nested = json.loads(api_report.read_text(encoding="utf-8"))
+ nested["Results"] = [
+ {
+ "Target": "sha256:" + "9" * 64,
+ "ComponentTarget": "Python",
+ "Vulnerabilities": [],
+ }
+ ]
+ _write_json(api_report, nested)
+ self.assert_failed_with(fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ fixture = GateFixture(
+ Path(self.temporary.name) / "duplicate-trivy-alias", profile="security"
+ )
+ api_report = fixture.reports / "trivy-image-api.json"
+ duplicate_alias = json.loads(api_report.read_text(encoding="utf-8"))
+ duplicate_alias["Metadata"]["RepoTags"] *= 2
+ _write_json(api_report, duplicate_alias)
+ self.assert_failed_with(fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ fixture = GateFixture(
+ Path(self.temporary.name) / "missing-scan-binding", profile="security"
+ )
+ api_report = fixture.reports / "trivy-image-api.json"
+ missing_binding = json.loads(api_report.read_text(encoding="utf-8"))
+ del missing_binding["ScannedImageID"]
+ _write_json(api_report, missing_binding)
+ self.assert_failed_with(fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ fixture = GateFixture(
+ Path(self.temporary.name) / "malformed-artifact-id", profile="security"
+ )
+ api_report = fixture.reports / "trivy-image-api.json"
+ malformed_artifact = json.loads(api_report.read_text(encoding="utf-8"))
+ malformed_artifact["ArtifactID"] = "mutable-artifact"
+ _write_json(api_report, malformed_artifact)
+ self.assert_failed_with(fixture.evaluate(), "GATE-TRIVY-SCHEMA")
+
+ fixture = GateFixture(
+ Path(self.temporary.name) / "missing-component-target", profile="security"
+ )
+ api_report = fixture.reports / "trivy-image-api.json"
+ nested = json.loads(api_report.read_text(encoding="utf-8"))
+ nested["Results"] = [
+ {
+ "Target": "sha256:" + "1" * 64,
+ "Vulnerabilities": [],
+ }
+ ]
+ _write_json(api_report, nested)
+ self.assert_failed_with(fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ def test_trivy_binding_preserves_component_and_rejects_swapped_artifact(self) -> None:
+ image_id = "sha256:" + "1" * 64
+ expected_tag = "modelforge-security-api:" + "a" * 40
+ raw = self.fixture.root / "raw-trivy-image.json"
+ bound = self.fixture.root / "bound-trivy-image.json"
+ report = {
+ "SchemaVersion": 2,
+ "ArtifactName": image_id,
+ "ArtifactID": "sha256:" + "2" * 64,
+ "ArtifactType": "container_image",
+ "Metadata": {"ImageID": image_id, "Reference": image_id},
+ "Results": [{"Target": "Python", "Vulnerabilities": []}],
+ }
+ _write_json(raw, report)
+ gate.bind_trivy_image_report(raw, bound, image_id, expected_tag)
+ normalized = json.loads(bound.read_text(encoding="utf-8"))
+ self.assertEqual(normalized["Results"][0]["Target"], image_id)
+ self.assertEqual(normalized["Results"][0]["ComponentTarget"], "Python")
+ self.assertEqual(normalized["ScannedImageID"], image_id)
+
+ report["Metadata"]["Reference"] = expected_tag
+ _write_json(raw, report)
+ gate.bind_trivy_image_report(raw, bound, image_id, expected_tag)
+
+ report["Metadata"]["Reference"] = "attacker:latest"
+ _write_json(raw, report)
+ with self.assertRaisesRegex(gate.GateInputError, "metadata"):
+ gate.bind_trivy_image_report(raw, bound, image_id, expected_tag)
+
+ report["Metadata"]["Reference"] = image_id
+ report["ArtifactName"] = "sha256:" + "9" * 64
+ _write_json(raw, report)
+ with self.assertRaisesRegex(gate.GateInputError, "exact scanned image"):
+ gate.bind_trivy_image_report(raw, bound, image_id, expected_tag)
+
+ report["ArtifactName"] = image_id
+ report["ArtifactID"] = "mutable-artifact"
+ _write_json(raw, report)
+ with self.assertRaisesRegex(gate.GateInputError, "ArtifactID"):
+ gate.bind_trivy_image_report(raw, bound, image_id, expected_tag)
+
+ def test_runtime_non_pypi_exception_is_exact_and_still_inventory_bound(self) -> None:
+ fixture = GateFixture(
+ Path(self.temporary.name) / "non-pypi-runtime", profile="security"
+ )
+ self.assertEqual(fixture.evaluate()["decision"], "PASS")
+ inventory_path = fixture.reports / "image-inventory-runtime-worker.json"
+ inventory = json.loads(inventory_path.read_text(encoding="utf-8"))
+ next(item for item in inventory["packages"] if item["name"] == "conda-build")[
+ "version"
+ ] = "25.5.1"
+ _write_json(inventory_path, inventory)
+ self.assert_failed_with(fixture.evaluate(), "GATE-IMAGE-NON-PYPI-VERSION")
+
+ def test_trivy_malformed_identity_fields_and_result_target_cannot_be_waived(self) -> None:
+ path = self.fixture.reports / "trivy-fs.json"
+ base: dict[str, Any] = {
+ "SchemaVersion": 2,
+ "ArtifactName": str(REPOSITORY_ROOT.resolve()),
+ "ArtifactType": "repository",
+ "Results": [
+ {
+ "Target": ".",
+ "Vulnerabilities": [
+ {
+ "VulnerabilityID": "CVE-SYNTHETIC",
+ "PkgName": "package",
+ "InstalledVersion": "1.0",
+ "Severity": "HIGH",
+ "FixedVersion": "",
+ }
+ ],
+ }
+ ],
+ }
+ for missing in (
+ "VulnerabilityID",
+ "PkgName",
+ "InstalledVersion",
+ "Severity",
+ ):
+ with self.subTest(missing=missing):
+ malformed = copy.deepcopy(base)
+ del malformed["Results"][0]["Vulnerabilities"][0][missing]
+ _write_json(path, malformed)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-SCHEMA")
+ malformed_target = copy.deepcopy(base)
+ del malformed_target["Results"][0]["Target"]
+ _write_json(path, malformed_target)
+ self.fixture.write_risks(
+ [
+ self.fixture.risk(
+ {
+ "scanner": "trivy",
+ "target": "repository",
+ "advisory_id": "TRIVY-UNKNOWN",
+ "package": "unknown",
+ "installed_version": "unknown",
+ }
+ )
+ ]
+ )
+ result = self.fixture.evaluate()
+ self.assert_failed_with(result, "GATE-TRIVY-SCHEMA")
+ self.assertEqual(result["applied_risks"], [])
+
+ def test_trivy_filesystem_result_must_remain_within_repository(self) -> None:
+ path = self.fixture.reports / "trivy-fs.json"
+ _write_json(
+ path,
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": str(REPOSITORY_ROOT.resolve()),
+ "ArtifactType": "repository",
+ "Results": [{"Target": "../outside", "Vulnerabilities": []}],
+ },
+ )
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-TARGET")
+
+ def test_trivy_database_timestamps_are_required_utc_current_and_nonfuture(self) -> None:
+ path = self.fixture.reports / "versions" / "trivy.json"
+ original = json.loads(path.read_text(encoding="utf-8"))
+ cases: list[tuple[str, Any]] = [
+ ("missing", None),
+ ("malformed", "not-a-time"),
+ ("non-utc", "2026-08-30T01:00:00+01:00"),
+ ("future", "2026-08-30T00:11:00Z"),
+ ("stale", "2026-08-27T00:00:00Z"),
+ ]
+ for name, updated in cases:
+ with self.subTest(name=name):
+ document = copy.deepcopy(original)
+ if updated is None:
+ document["VulnerabilityDB"].pop("UpdatedAt")
+ else:
+ document["VulnerabilityDB"]["UpdatedAt"] = updated
+ _write_json(path, document)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-DB-TIME")
+ document = copy.deepcopy(original)
+ document["VulnerabilityDB"]["DownloadedAt"] = "2026-08-29T00:00:00Z"
+ _write_json(path, document)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-DB-TIME")
+
+ def test_exact_active_no_fix_risk_can_be_applied(self) -> None:
+ identity = self.fixture.add_pip_vulnerability()
+ self.fixture.write_risks([self.fixture.risk(identity)])
+ result = self.fixture.evaluate()
+ self.assertEqual(result["decision"], "PASS")
+ self.assertEqual(len(result["applied_risks"]), 1)
+
+ def test_risk_expiration_is_exclusive_after_exactly_thirty_calendar_dates(self) -> None:
+ identity = self.fixture.add_pip_vulnerability()
+ risk = self.fixture.risk(identity)
+ risk["accepted_on"] = "2026-08-01"
+ risk["expires_on"] = "2026-08-31"
+ self.fixture.write_risks([risk])
+ self.fixture.as_of = date(2026, 8, 30)
+ self.assertEqual(self.fixture.evaluate()["decision"], "PASS")
+ self.fixture.as_of = date(2026, 8, 31)
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-RISK-INVALID")
+
+ def test_risk_with_fix_available_fails(self) -> None:
+ identity = self.fixture.add_pip_vulnerability(fix_versions=["9.9.9"])
+ self.fixture.write_risks([self.fixture.risk(identity)])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-RISK-FIX-AVAILABLE")
+
+ def test_risk_registry_version_and_classification_are_exact(self) -> None:
+ _write_json(self.fixture.risks, [])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-RISK-SCHEMA")
+ identity = self.fixture.add_pip_vulnerability()
+ risk = self.fixture.risk(identity)
+ risk["classification"] = "ACCEPTED"
+ self.fixture.write_risks([risk])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-RISK-INVALID")
+
+ def test_image_inventory_requires_exact_first_party_versions_and_web_origin(self) -> None:
+ fixture = GateFixture(Path(self.temporary.name) / "image-policy", profile="security")
+ inventory_path = fixture.reports / "image-inventory-api.json"
+ inventory = json.loads(inventory_path.read_text(encoding="utf-8"))
+ next(
+ package
+ for package in inventory["packages"]
+ if package["name"] == "modelforge-api"
+ )["version"] = "0.0.0"
+ _write_json(inventory_path, inventory)
+ self.assert_failed_with(fixture.evaluate(), "GATE-IMAGE-FIRST-PARTY-VERSION")
+
+ fixture = GateFixture(Path(self.temporary.name) / "web-policy", profile="security")
+ evidence_path = fixture.reports / "web-bundle-origin.json"
+ evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
+ evidence["api_origins"] = [gate.WEB_SECURITY_ORIGIN, "https://attacker.invalid"]
+ _write_json(evidence_path, evidence)
+ self.assert_failed_with(fixture.evaluate(), "GATE-WEB-BUNDLE-SCHEMA")
+
+ def test_web_bundle_evidence_schema_binding_and_origin_controls(self) -> None:
+ origins = gate._extract_http_origins(
+ b'const api="https:\\/\\/security-scan.invalid/v1";'
+ )
+ self.assertEqual(origins, {gate.WEB_SECURITY_ORIGIN})
+ poisoned = gate._extract_http_origins(
+ b"https://security-scan.invalid/v1 https://attacker.invalid/v1"
+ )
+ self.assertEqual(poisoned, {gate.WEB_SECURITY_ORIGIN, "https://attacker.invalid"})
+ split_poison = gate._extract_http_origins(
+ b'const origin = "https" + "://attacker.invalid";'
+ )
+ self.assertEqual(split_poison, {"https://attacker.invalid"})
+ for wrong_context in (
+ b'fetch("https://react.dev/collect")',
+ b'fetch("http://www.w3.org/attacker")',
+ ):
+ with self.subTest(wrong_context=wrong_context):
+ with self.assertRaisesRegex(gate.GateInputError, "pinned context"):
+ gate._validate_non_api_origin_contexts(wrong_context)
+ for origin in (
+ b"http://127.42.0.1:8080",
+ b"http://[::1]:8080",
+ b"http://LOCALHOST.:8080",
+ b"https://api.localhost/v1",
+ ):
+ with self.subTest(origin=origin):
+ with self.assertRaisesRegex(gate.GateInputError, "loopback"):
+ gate._extract_http_origins(origin)
+
+ attacks: tuple[tuple[str, Any], ...] = (
+ ("image_id", "sha256:" + "9" * 64),
+ ("bundle_path", "/tmp/dead-marker"),
+ ("bundle_sha256", "not-a-digest"),
+ ("bundle_file_count", 0),
+ ("provenance_path", "assets/dead-marker.json"),
+ ("served_provenance_sha256", "0" * 64),
+ ("provenance_binding_sha256", "0" * 64),
+ ("runtime_chunk_path", "../outside.js"),
+ ("runtime_chunk_sha256", "not-a-digest"),
+ ("api_origins", [gate.WEB_SECURITY_ORIGIN, "https://attacker.invalid"]),
+ ("non_api_origins", ["https://attacker.invalid"]),
+ ("unexpected", "field"),
+ )
+ for field, value in attacks:
+ with self.subTest(field=field):
+ fixture = GateFixture(self.fixture.root / f"web-{field}", profile="security")
+ path = fixture.reports / "web-bundle-origin.json"
+ evidence = json.loads(path.read_text(encoding="utf-8"))
+ evidence[field] = value
+ _write_json(path, evidence)
+ self.assert_failed_with(fixture.evaluate(), "GATE-WEB-BUNDLE-SCHEMA")
+
+ def test_web_bundle_analyzer_binds_provenance_and_rejects_dead_marker_split_poison(self) -> None:
+ bundle = self.fixture.root / "synthetic-bundle"
+ provenance_path = bundle / gate.WEB_PROVENANCE_PATH
+ runtime_path = bundle / "assets" / "index.js"
+ provenance_path.parent.mkdir(parents=True)
+ runtime_path.parent.mkdir(parents=True)
+ binding = gate._web_provenance_binding(gate.WEB_SECURITY_ORIGIN)
+ provenance = {
+ "schema_version": 1,
+ "target": "modelforge-web",
+ "api_base_url": gate.WEB_SECURITY_ORIGIN,
+ "binding_sha256": binding,
+ }
+ _write_json(provenance_path, provenance)
+ framework_constants = (
+ 'const svg="http://www.w3.org/2000/svg";'
+ 'const docs="https://react.dev/errors/";'
+ 'const runbook="https://git.example.com/example/modelforge/'
+ 'src/branch/master/docs/operations/RUNBOOK_NODE_OFFLINE.md";'
+ )
+ runtime_path.write_text(
+ f'globalThis.__MODELFORGE_BUILD_PROVENANCE__={{api_base_url:'
+ f'"{gate.WEB_SECURITY_ORIGIN}",binding_sha256:"{binding}"}};'
+ f"{framework_constants}\n",
+ encoding="utf-8",
+ )
+ served_provenance = provenance_path.read_bytes()
+ evidence = gate._analyze_web_bundle(
+ bundle,
+ image_id="sha256:" + "2" * 64,
+ bundle_path=gate.WEB_BUNDLE_PATH,
+ expected_origin=gate.WEB_SECURITY_ORIGIN,
+ served_provenance=served_provenance,
+ )
+ self.assertEqual(evidence["api_origins"], [gate.WEB_SECURITY_ORIGIN])
+ self.assertEqual(evidence["runtime_chunk_path"], "assets/index.js")
+ self.assertEqual(evidence["provenance_path"], gate.WEB_PROVENANCE_PATH)
+ self.assertEqual(
+ evidence["non_api_origins"], list(gate.WEB_EXPECTED_NON_API_ORIGINS)
+ )
+
+ with self.assertRaisesRegex(gate.GateInputError, "served Web provenance differs"):
+ gate._analyze_web_bundle(
+ bundle,
+ image_id="sha256:" + "2" * 64,
+ bundle_path=gate.WEB_BUNDLE_PATH,
+ expected_origin=gate.WEB_SECURITY_ORIGIN,
+ served_provenance=b"{}\n",
+ )
+
+ runtime_path.write_text(
+ f'const dead="{gate.WEB_SECURITY_ORIGIN}";const binding="{binding}";'
+ f'{framework_constants}const effective="https" + "://attacker.invalid";\n',
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "network origins are not exact"):
+ gate._analyze_web_bundle(
+ bundle,
+ image_id="sha256:" + "2" * 64,
+ bundle_path=gate.WEB_BUNDLE_PATH,
+ expected_origin=gate.WEB_SECURITY_ORIGIN,
+ served_provenance=served_provenance,
+ )
+
+ runtime_path.write_text(
+ f'const api="{gate.WEB_SECURITY_ORIGIN}";const binding="{binding}";'
+ f"{framework_constants}\n",
+ encoding="utf-8",
+ )
+ (bundle / "assets" / "index.js.map").write_text(
+ '{"sourcesContent":["const evil = `wss://${\"attacker.invalid\"}`;"]}\n',
+ encoding="utf-8",
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "network origins are not exact"):
+ gate._analyze_web_bundle(
+ bundle,
+ image_id="sha256:" + "2" * 64,
+ bundle_path=gate.WEB_BUNDLE_PATH,
+ expected_origin=gate.WEB_SECURITY_ORIGIN,
+ served_provenance=served_provenance,
+ )
+
+ def test_oci_identity_requires_exact_source_revision_version_and_bundle_path(self) -> None:
+ attacks: tuple[tuple[str, str, Any], ...] = (
+ ("api", "source", "https://attacker.invalid/repository"),
+ ("api", "revision", "b" * 40),
+ ("api", "version", "0.0.0"),
+ (
+ "api",
+ "repo_tags",
+ [
+ f"modelforge-security-api:{GateFixture.commit}",
+ "attacker:latest",
+ ],
+ ),
+ (
+ "api",
+ "repo_tags",
+ [
+ f"modelforge-security-api:{GateFixture.commit}",
+ f"modelforge-security-api:{GateFixture.commit}",
+ ],
+ ),
+ ("api", "repo_digests", ["attacker.invalid/api@sha256:" + "9" * 64]),
+ ("web", "bundle_path", "/tmp/dead-marker"),
+ ("runtime-worker", "unexpected", "field"),
+ )
+ for attack_index, (target, field, value) in enumerate(attacks):
+ with self.subTest(target=target, field=field):
+ fixture = GateFixture(
+ self.fixture.root / f"identity-{attack_index}-{target}-{field}",
+ profile="security",
+ )
+ path = fixture.reports / f"image-identity-{target}.json"
+ identity = json.loads(path.read_text(encoding="utf-8"))
+ identity[field] = value
+ _write_json(path, identity)
+ self.assert_failed_with(
+ fixture.evaluate(), "GATE-IMAGE-IDENTITY-SCHEMA"
+ )
+
+ fixture = GateFixture(self.fixture.root / "duplicate-image-id", profile="security")
+ web = fixture.reports / "image-identity-web.json"
+ identity = json.loads(web.read_text(encoding="utf-8"))
+ identity["image_id"] = "sha256:" + "1" * 64
+ _write_json(web, identity)
+ self.assert_failed_with(fixture.evaluate(), "GATE-IMAGE-SET")
+
+ fixture = GateFixture(self.fixture.root / "extra-image", profile="security")
+ _write_json(
+ fixture.reports / "image-identity-attacker.json",
+ {"schema_version": 1, "target": "attacker", "tag": "attacker:latest"},
+ )
+ self.assert_failed_with(fixture.evaluate(), "GATE-EVIDENCE-PATH")
+
+ def test_evidence_manifest_rejects_unknown_bytes_before_hashing(self) -> None:
+ sentinel = b"unexpected-evidence-must-not-be-consumed"
+ (self.fixture.reports / "unexpected-secret.txt").write_bytes(sentinel)
+ original_reader = gate.SecureEvidenceSnapshot._read_fd
+ consumed: list[bytes] = []
+
+ def record_reads(file_descriptor: int) -> bytes:
+ value = original_reader(file_descriptor)
+ consumed.append(value)
+ return value
+
+ with mock.patch.object(gate.SecureEvidenceSnapshot, "_read_fd", side_effect=record_reads):
+ result = self.fixture.evaluate()
+ self.assert_failed_with(result, "GATE-EVIDENCE-PATH")
+ self.assertEqual(result["report_hashes"], {})
+ self.assertNotIn(sentinel, consumed)
+
+ def test_evidence_hardlink_is_rejected_before_hashing(self) -> None:
+ root = self.fixture.root / "hardlink-evidence"
+ root.mkdir(mode=0o700)
+ root.chmod(0o700)
+ outside = self.fixture.root / "outside-evidence.json"
+ outside.write_bytes(b"outside-must-not-be-consumed")
+ linked = root / "report.json"
+ os.link(outside, linked)
+ with mock.patch.object(
+ gate.SecureEvidenceSnapshot,
+ "_read_fd",
+ wraps=gate.SecureEvidenceSnapshot._read_fd,
+ ) as reader:
+ with self.assertRaisesRegex(gate.GateInputError, "hard links"):
+ gate.SecureEvidenceSnapshot(root, root / "result.json")
+ reader.assert_not_called()
+
+ def test_evaluator_output_hardlink_is_rejected_and_never_overwritten(self) -> None:
+ outside = self.fixture.root / "outside-result.json"
+ original = b"external-result-must-survive"
+ outside.write_bytes(original)
+ os.link(outside, self.fixture.output)
+ result = self.fixture.evaluate()
+ self.assert_failed_with(result, "GATE-EVIDENCE-PATH")
+ self.assertEqual(outside.read_bytes(), original)
+
+ # Result persistence replaces a supplied directory entry instead of following it.
+ gate._write_json(self.fixture.output, {"replacement": "safe"})
+ self.assertEqual(outside.read_bytes(), original)
+ self.assertEqual(json.loads(self.fixture.output.read_text(encoding="utf-8")), {"replacement": "safe"})
+
+ def test_duplicate_json_keys_and_normalized_inventory_rows_fail_closed(self) -> None:
+ trivy = self.fixture.reports / "trivy-fs.json"
+ trivy.write_text(
+ '{"SchemaVersion":2,"ArtifactName":"' + str(REPOSITORY_ROOT.resolve()) +
+ '","ArtifactType":"repository","Results":[{"Target":"' +
+ str(REPOSITORY_ROOT.resolve()) +
+ '","Vulnerabilities":[{"VulnerabilityID":"CVE-1","PkgName":"x",'
+ '"InstalledVersion":"1","Severity":"CRITICAL"}]}],"Results":[]}',
+ encoding="utf-8",
+ )
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-TRIVY-REPORT")
+
+ raw_inventory = self.fixture.root / "raw-inventory.json"
+ output = self.fixture.root / "inventory.json"
+ requirements = self.fixture.root / "requirements.txt"
+ for packages in (
+ [{"name": "HTTPX", "version": "0.28.1"}, {"name": "httpx", "version": "0.28.1"}],
+ [{"name": "httpx", "version": "0.28.1"}, {"name": "httpx", "version": "0.28.2"}],
+ ):
+ _write_json(raw_inventory, {"python_version": "3.13", "packages": packages})
+ with self.assertRaisesRegex(gate.GateInputError, "conflicting|duplicate"):
+ gate.normalize_image_inventory("api", raw_inventory, output, requirements)
+
+ runtime_packages = [
+ {"name": name, "version": version}
+ for name, version in gate.IMAGE_NON_PYPI_AUDIT_EXCEPTIONS[
+ "runtime-worker"
+ ].items()
+ ] + [
+ {"name": "httpx", "version": "0.28.1"},
+ ]
+ _write_json(
+ raw_inventory,
+ {"python_version": "3.11", "packages": runtime_packages},
+ )
+ gate.normalize_image_inventory(
+ "runtime-worker", raw_inventory, output, requirements
+ )
+ self.assertNotIn("conda", requirements.read_text(encoding="utf-8"))
+ next(item for item in runtime_packages if item["name"] == "conda")[
+ "version"
+ ] = "25.5.1"
+ _write_json(
+ raw_inventory,
+ {"python_version": "3.11", "packages": runtime_packages},
+ )
+ with self.assertRaisesRegex(gate.GateInputError, "non-PyPI"):
+ gate.normalize_image_inventory(
+ "runtime-worker", raw_inventory, output, requirements
+ )
+
+ def test_wildcard_expired_duplicate_unused_and_ambiguous_risks_fail(self) -> None:
+ identity = self.fixture.add_pip_vulnerability()
+ base = self.fixture.risk(identity)
+ cases: list[tuple[str, list[dict[str, str]], str]] = []
+ wildcard = copy.deepcopy(base)
+ wildcard["target"] = "*"
+ cases.append(("wildcard", [wildcard], "GATE-RISK-INVALID"))
+ expired = copy.deepcopy(base)
+ expired["accepted_on"] = "2026-07-01"
+ expired["expires_on"] = "2026-07-31"
+ cases.append(("expired", [expired], "GATE-RISK-INVALID"))
+ cases.append(("duplicate", [base, copy.deepcopy(base)], "GATE-RISK-INVALID"))
+ unused = copy.deepcopy(base)
+ unused["advisory_id"] = "PYSEC-STALE"
+ cases.append(("unused", [unused], "GATE-RISK-UNUSED"))
+ for name, risks, advisory in cases:
+ with self.subTest(name=name):
+ self.fixture.write_risks(risks)
+ self.assert_failed_with(self.fixture.evaluate(), advisory)
+ self.fixture = GateFixture(Path(self.temporary.name) / "ambiguous")
+ identity = self.fixture.add_pip_vulnerability(duplicate=True)
+ self.fixture.write_risks([self.fixture.risk(identity)])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-RISK-AMBIGUOUS")
+
+ def test_exact_gitleaks_false_positive_applies_and_drift_or_duplicate_fails(self) -> None:
+ fingerprint = "d" * 64
+ _write_json(
+ self.fixture.reports / "gitleaks.json",
+ {
+ "schema_version": 1,
+ "scanner": "gitleaks",
+ "target": "full-history",
+ "redaction_percent": 100,
+ "findings": [
+ {"rule_id": "generic-api-key", "fingerprint_sha256": fingerprint}
+ ],
+ },
+ )
+ (self.fixture.reports / "status" / "gitleaks.exit").write_text("1\n")
+ entry = {
+ "scanner": "gitleaks",
+ "rule_id": "generic-api-key",
+ "fingerprint_sha256": fingerprint,
+ "classification": "FALSE_POSITIVE",
+ "owner": "security-owner",
+ "classified_on": "2026-08-30",
+ "review_on": "2026-09-29",
+ "tracking": "RC-DEL-02",
+ "reason": "Verified synthetic test fixture, never credential material.",
+ }
+ _write_json(self.fixture.false_positives, [entry])
+ result = self.fixture.evaluate()
+ self.assertEqual(result["decision"], "PASS")
+ self.assertEqual(len(result["applied_gitleaks_false_positives"]), 1)
+
+ drifted = copy.deepcopy(entry)
+ drifted["fingerprint_sha256"] = "e" * 64
+ _write_json(self.fixture.false_positives, [drifted])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-GITLEAKS-FP-UNUSED")
+ _write_json(self.fixture.false_positives, [entry, copy.deepcopy(entry)])
+ self.assert_failed_with(self.fixture.evaluate(), "GATE-GITLEAKS-FP-INVALID")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.gitea/workflows/managed-validation.yml b/.gitea/workflows/managed-validation.yml
index b9f8b7e..95687d3 100644
--- a/.gitea/workflows/managed-validation.yml
+++ b/.gitea/workflows/managed-validation.yml
@@ -1,14 +1,8 @@
name: Managed validation
-# The gate on protected master. Before v1 this keyed off manifests at the repository root, and this
-# repository has none — pyproject.toml, package.json and the lock files all live in backend/,
-# frontend/, node-agent/ and runtime-worker/. The "full" profile therefore completed in nine seconds
-# having run a whitespace check, a merge-marker scan and py_compile, and no test suite at all, while
-# reporting success to a branch protection rule that required it.
-#
-# It now targets the component roots explicitly, and refuses to report success when a component that
-# should have run tests ran none. A gate that passes because it found nothing to do is worse than no
-# gate: it produces the paperwork of validation without the fact of it.
+# Protected-source validation. The ordinary validation job preserves the existing profile contract.
+# The security job always scans source for pull requests and full runs; only an explicit dispatched
+# `security` profile pays the cost of building and scanning the four exact shipping images.
on:
# Public exports require explicit owner dispatch; fork PRs never reach private runners.
@@ -31,15 +25,27 @@ concurrency:
jobs:
full:
name: full
- runs-on: ubuntu-latest
+ runs-on: linux-validation
timeout-minutes: 45
env:
PROFILE: ${{ inputs.profile || 'full' }}
- # An explicit interpreter path rather than PATH manipulation: how a runner propagates PATH
- # between steps varies, and a validation gate should not depend on that detail.
- VENV: /tmp/modelforge-validation-venv
+ BACKEND_VENV: /tmp/modelforge-validation-backend-venv
+ NODE_AGENT_VENV: /tmp/modelforge-validation-node-agent-venv
+ RUNTIME_WORKER_VENV: /tmp/modelforge-validation-runtime-worker-venv
+ VALIDATION_COMPOSE_PROJECT: modelforge-validation-${{ gitea.run_id }}-${{ gitea.run_attempt }}
+ MODELFORGE_POSTGRES_ADMIN_PASSWORD: managed-validation-admin
+ MODELFORGE_MIGRATION_DB_PASSWORD: managed-validation-owner
+ MODELFORGE_RUNTIME_DB_PASSWORD: managed-validation-runtime
+ # Port zero asks Docker for a collision-free ephemeral loopback port. The boundary step
+ # exports the resolved URLs only to later steps in this job.
+ MODELFORGE_POSTGRES_PORT: "0"
steps:
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ - name: Check out validation source with full history
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ fetch-depth: 0
+ fetch-tags: true
+ persist-credentials: false
- name: Production delivery policy
shell: bash
@@ -61,6 +67,7 @@ jobs:
run: |
set -euo pipefail
git diff --check
+ python3 scripts/rc_readiness.py
if git grep -nE '^(<<<<<<< |=======$|>>>>>>> )' -- . ':!*.lock' ':!*.patch'; then
echo "Unresolved merge markers detected" >&2
exit 1
@@ -71,10 +78,10 @@ jobs:
shell: bash
run: mkdir -p reports
- # Deliberately not using setup-python/setup-node: this runs on a self-hosted runner whose
- # image already carries both, and the previous workflow depended on that too. What changes is
- # that a missing toolchain now stops the run instead of quietly reducing what gets validated.
+ # Each Python component gets its own environment. Sharing one environment made the later
+ # component inherit the earlier component's dependency resolution and invalidated isolation.
- name: Toolchain
+ if: ${{ env.PROFILE != 'security' }}
shell: bash
run: |
set -euo pipefail
@@ -84,70 +91,17 @@ jobs:
echo " python $(python3 --version)"
echo " node $(node --version)"
echo " npm $(npm --version)"
- python3 -m venv "${VENV}"
- "${VENV}/bin/python" -m pip install --disable-pip-version-check --quiet --upgrade pip
- if [[ "${PROFILE}" == security || "${PROFILE}" == full ]]; then
- command -v gitleaks >/dev/null || {
- echo "gitleaks is required for the security profile" >&2
- exit 1
- }
- "${VENV}/bin/python" -m pip install \
- --disable-pip-version-check --quiet 'pip-audit==2.10.0'
- fi
- echo " venv $("${VENV}/bin/python" --version) at ${VENV}"
-
- - name: Backend — install, lint, typecheck, test
- id: backend
- shell: bash
- working-directory: backend
- run: |
- set -euo pipefail
- "${VENV}/bin/python" -m pip install --disable-pip-version-check --quiet -e '.[dev]'
- if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m ruff check src tests
- fi
- if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m mypy src
- fi
- if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m pytest -q --no-header --junitxml=../reports/backend.xml
- fi
-
- - name: Node Agent — install, lint, typecheck, test
- id: node_agent
- shell: bash
- working-directory: node-agent
- run: |
- set -euo pipefail
- "${VENV}/bin/python" -m pip install --disable-pip-version-check --quiet -e '.[dev]'
- if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m ruff check src
- fi
- if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m mypy src
- fi
- if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m pytest -q --no-header --junitxml=../reports/node-agent.xml
- fi
-
- - name: Runtime Worker — install, lint, typecheck, test
- id: runtime_worker
- shell: bash
- working-directory: runtime-worker
- run: |
- set -euo pipefail
- "${VENV}/bin/python" -m pip install --disable-pip-version-check --quiet -e '.[dev]'
- if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m ruff check src
- fi
- if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m mypy src
- fi
- if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
- "${VENV}/bin/python" -m pytest -q --no-header --junitxml=../reports/runtime-worker.xml
- fi
+ for venv in "${BACKEND_VENV}" "${NODE_AGENT_VENV}" "${RUNTIME_WORKER_VENV}"; do
+ python3 -m venv "${venv}"
+ "${venv}/bin/python" -m pip install --disable-pip-version-check --quiet \
+ --require-hashes --only-binary=:all: --index-url https://pypi.org/simple \
+ -r config/security/build-toolchain.lock
+ done
+ # Build first so the backend packaging policy inspects the artifact that would ship instead
+ # of recording a skip because frontend/dist does not exist yet.
- name: Console — install frozen, typecheck, test, build
+ if: ${{ env.PROFILE != 'security' }}
id: console
shell: bash
working-directory: frontend
@@ -156,21 +110,123 @@ jobs:
npm ci
if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then npx tsc --noEmit; fi
if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
- npx vitest run --reporter=junit --outputFile=../reports/frontend.xml
+ npx vitest run --reporter=default --reporter=junit --outputFile=../reports/frontend.xml
+ node --test ../clients/typescript/modelforge-client.test.mjs
fi
if [[ "${PROFILE}" == build || "${PROFILE}" == full ]]; then npm run build; fi
- - name: Security — secrets and vulnerable dependencies
- if: ${{ env.PROFILE == 'full' || env.PROFILE == 'security' }}
+ - name: Managed PostgreSQL boundary
+ if: ${{ env.PROFILE == 'full' }}
shell: bash
run: |
set -euo pipefail
- gitleaks git . --no-banner --redact --report-format json \
- --report-path reports/gitleaks.json
- "${VENV}/bin/python" -m pip_audit --strict --skip-editable --desc=on \
- --format=json --output=reports/python-audit.json
- cd frontend
- npm audit --audit-level=high --omit=dev --json > ../reports/npm-audit.json
+ command -v docker >/dev/null || {
+ echo "The full profile requires Docker for its disposable PostgreSQL boundary." >&2
+ exit 1
+ }
+ # Compose validates required interpolation for every declared service even though this
+ # step starts PostgreSQL only. Keep these container-internal URLs scoped to this shell;
+ # the backend test process receives only the resolved host-side test URLs below.
+ export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:managed-validation-owner@postgres:5432/modelforge
+ export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:managed-validation-runtime@postgres:5432/modelforge
+ docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" \
+ up --detach --wait postgres
+ docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" ps postgres
+ # The Actions job runs in a container, so the Docker host's loopback is not this job's
+ # 127.0.0.1. Join the disposable Compose network and use Docker service DNS instead.
+ validation_network="${VALIDATION_COMPOSE_PROJECT}_default"
+ job_container="$(hostname)"
+ docker inspect "${job_container}" >/dev/null
+ docker network connect "${validation_network}" "${job_container}"
+ # Docker can report PostgreSQL healthy before the newly attached job container
+ # can reach its listener. Bound the network readiness check to 30 seconds.
+ python3 - <<'PY'
+ import socket
+ import time
+
+ for attempt in range(30):
+ try:
+ with socket.create_connection(("postgres", 5432), timeout=2):
+ break
+ except (OSError, TimeoutError):
+ if attempt == 29:
+ raise
+ time.sleep(1)
+ PY
+ {
+ echo "MODELFORGE_TEST_POSTGRES_URL=postgresql+psycopg://postgres:managed-validation-admin@postgres:5432/modelforge"
+ echo "MODELFORGE_TEST_POSTGRES_OWNER_URL=postgresql+psycopg://modelforge:managed-validation-owner@postgres:5432/modelforge"
+ echo "MODELFORGE_TEST_POSTGRES_RUNTIME_URL=postgresql+psycopg://modelforge_runtime:managed-validation-runtime@postgres:5432/modelforge"
+ echo "MODELFORGE_TEST_POSTGRES_CONTAINER=$(docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" ps -q postgres)"
+ echo "MODELFORGE_TEST_POSTGRES_DATABASE_LIFECYCLE=managed-validation-v1"
+ } >> "${GITHUB_ENV}"
+
+ - name: Backend — install, lint, typecheck, test
+ if: ${{ env.PROFILE != 'security' }}
+ id: backend
+ shell: bash
+ working-directory: backend
+ run: |
+ set -euo pipefail
+ "${BACKEND_VENV}/bin/python" -m pip install --disable-pip-version-check --quiet \
+ --no-build-isolation -e '.[dev]'
+ if [[ "${PROFILE}" == full ]]; then
+ MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}" \
+ "${BACKEND_VENV}/bin/python" -m alembic upgrade head
+ MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}" \
+ "${BACKEND_VENV}/bin/python" -m alembic downgrade 20260828_0022
+ MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}" \
+ "${BACKEND_VENV}/bin/python" -m alembic upgrade head
+ fi
+ if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
+ "${BACKEND_VENV}/bin/python" -m ruff check src tests
+ fi
+ if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
+ "${BACKEND_VENV}/bin/python" -m mypy src
+ fi
+ if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
+ "${BACKEND_VENV}/bin/python" -m pytest -q --no-header --junitxml=../reports/backend.xml
+ fi
+
+ - name: Node Agent — install, lint, typecheck, test
+ if: ${{ env.PROFILE != 'security' }}
+ id: node_agent
+ shell: bash
+ working-directory: node-agent
+ run: |
+ set -euo pipefail
+ "${NODE_AGENT_VENV}/bin/python" -m pip install --disable-pip-version-check --quiet \
+ --no-build-isolation -e '../backend' -e '.[dev]'
+ if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
+ "${NODE_AGENT_VENV}/bin/python" -m ruff check src
+ fi
+ if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
+ "${NODE_AGENT_VENV}/bin/python" -m mypy src
+ fi
+ if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
+ "${NODE_AGENT_VENV}/bin/python" -m pytest -q --no-header \
+ --junitxml=../reports/node-agent.xml
+ fi
+
+ - name: Runtime Worker — install, lint, typecheck, test
+ if: ${{ env.PROFILE != 'security' }}
+ id: runtime_worker
+ shell: bash
+ working-directory: runtime-worker
+ run: |
+ set -euo pipefail
+ "${RUNTIME_WORKER_VENV}/bin/python" -m pip install \
+ --disable-pip-version-check --quiet --no-build-isolation -e '.[dev]'
+ if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
+ "${RUNTIME_WORKER_VENV}/bin/python" -m ruff check src
+ fi
+ if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
+ "${RUNTIME_WORKER_VENV}/bin/python" -m mypy src
+ fi
+ if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
+ "${RUNTIME_WORKER_VENV}/bin/python" -m pytest -q --no-header \
+ --junitxml=../reports/runtime-worker.xml
+ fi
- name: Compose projections
if: ${{ env.PROFILE == 'full' || env.PROFILE == 'build' }}
@@ -178,36 +234,26 @@ jobs:
run: |
set -euo pipefail
if ! command -v docker >/dev/null; then
- echo "SKIPPED: no docker CLI on this runner; compose projections are validated by the"
- echo "local release gate instead. This step never reports a pass it did not earn."
+ echo "SKIPPED: no docker CLI on this runner; image security has its own fail-closed job."
exit 0
fi
- # Compose interpolation needs values, not live credentials. These fixed validation-only
- # strings never reach a service and ensure the fail-closed production projection is
- # actually parsed on every full/build run.
- export MODELFORGE_POSTGRES_DB=modelforge_validation
- export MODELFORGE_POSTGRES_ADMIN_PASSWORD=validation-admin-only
- export MODELFORGE_MIGRATION_DB_PASSWORD=validation-owner-only
- export MODELFORGE_RUNTIME_DB_PASSWORD=validation-runtime-only
- export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:validation-owner-only@postgres:5432/modelforge_validation
- export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:validation-runtime-only@postgres:5432/modelforge_validation
- export MODELFORGE_OPERATOR_API_KEY=validation-operator-key-32-characters # gitleaks:allow — synthetic Compose interpolation only
- export MODELFORGE_BACKUP_ENCRYPTION_KEY=dmFsaWRhdGlvbi1vbmx5LWtleS0zMi1ieXRlcw== # gitleaks:allow — base64 of a public validation-only string
- export MODELFORGE_CORS_ORIGINS=https://modelforge.example.test
- export VITE_API_BASE_URL=https://modelforge.example.test
- export MODELFORGE_VERSION=1.2.1
+ export MODELFORGE_POSTGRES_ADMIN_PASSWORD=compose-validation-admin
+ export MODELFORGE_MIGRATION_DB_PASSWORD=compose-validation-owner
+ export MODELFORGE_RUNTIME_DB_PASSWORD=compose-validation-runtime
+ export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:compose-validation-runtime@postgres:5432/modelforge
+ export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:compose-validation-owner@postgres:5432/modelforge
export MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS=127.0.0.1
export MODELFORGE_AGENT_CA_CERT_PATH=./config/ca.crt
failures=0
check() {
- if docker compose "$@" config -q >/dev/null 2>&1; then
+ if docker compose --env-file .env.example "$@" config -q >/dev/null 2>&1; then
echo " OK $*"
else
echo " FAIL $*"; failures=$((failures + 1))
fi
}
check -f docker-compose.yml
- for overlay in backup dr gpu node-agent node-recovery production runtime-worker; do
+ for overlay in backup dr gpu node-agent node-recovery runtime-worker; do
check -f docker-compose.yml -f "docker-compose.${overlay}.yml"
done
check -f docker-compose.yml -f docker-compose.node-agent.yml \
@@ -220,17 +266,14 @@ jobs:
if: ${{ env.PROFILE == 'full' }}
shell: bash
run: |
- "${VENV}/bin/python" scripts/generate_configuration_docs.py --check
+ "${BACKEND_VENV}/bin/python" scripts/generate_configuration_docs.py --check
- # The rule that makes the rest of this meaningful. Every component above declares a manifest,
- # so every component must have reported a test count. Zero tests where tests were expected is
- # a failure, not a pass — that is exactly how the previous workflow reported success.
- name: Refuse a validation that silently ran no tests
if: ${{ env.PROFILE == 'full' || env.PROFILE == 'test' }}
shell: bash
run: |
set -euo pipefail
- "${VENV}/bin/python" - <<'PY'
+ "${BACKEND_VENV}/bin/python" - <<'PY'
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
@@ -271,6 +314,18 @@ jobs:
sys.exit(1)
PY
+ - name: Remove managed PostgreSQL boundary
+ if: ${{ always() && env.PROFILE == 'full' }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:managed-validation-owner@postgres:5432/modelforge
+ export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:managed-validation-runtime@postgres:5432/modelforge
+ docker network disconnect "${VALIDATION_COMPOSE_PROJECT}_default" "$(hostname)" \
+ >/dev/null 2>&1 || true
+ docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" \
+ down --volumes --remove-orphans
+
- name: Validation summary
if: always()
shell: bash
@@ -279,10 +334,855 @@ jobs:
echo "profile: ${PROFILE}"
ls -l reports/ 2>/dev/null || echo "no reports directory"
+ security:
+ name: security
+ if: ${{ !inputs.profile || inputs.profile == 'security' || inputs.profile == 'full' }}
+ runs-on: linux-validation
+ # Explicit worst-case step bounds total 113 minutes for the image profile. The 17-minute job
+ # reserve means cleanup, the always-run evaluator and immutable upload retain time after
+ # earlier bounds.
+ timeout-minutes: 130
+ env:
+ SECURITY_PROFILE: ${{ gitea.event_name == 'workflow_dispatch' && inputs.profile == 'security' && 'security' || 'full' }}
+ PIP_AUDIT_VERSION: 2.10.1
+ PIP_AUDIT_SHA256: 99ef3f600a317c1945f1e89e227ef26e1c2d618429b8bd3fa6f4f7c440c4611a
+ GITLEAKS_VERSION: 8.30.1
+ GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb
+ TRIVY_VERSION: 0.74.0
+ TRIVY_SHA256: 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a
+ steps:
+ - name: Prepare fail-closed security evidence
+ timeout-minutes: 2
+ shell: bash
+ run: |
+ set -euo pipefail
+ umask 077
+ : "${GITHUB_RUN_ID:?missing run id}"
+ : "${GITHUB_RUN_ATTEMPT:?missing run attempt}"
+ : "${GITHUB_SHA:?missing commit SHA}"
+ : "${GITHUB_SERVER_URL:?missing server URL}"
+ : "${GITHUB_REPOSITORY:?missing repository identity}"
+ : "${GITHUB_ENV:?missing runner environment file}"
+ run_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA}"
+ temp_base="${RUNNER_TEMP:-/tmp}"
+ [[ -d "${temp_base}" && ! -L "${temp_base}" ]]
+ create_private_root() {
+ label="$1"
+ root="$(mktemp -d "${temp_base%/}/modelforge-${label}-${run_key}.XXXXXXXX")"
+ [[ -d "${root}" && ! -L "${root}" ]]
+ [[ "$(stat -c '%u' "${root}")" == "$(id -u)" ]]
+ [[ "$(stat -c '%a' "${root}")" == 700 ]]
+ printf '%s\n' "${root}"
+ }
+ REPORTS="$(create_private_root security-evidence)"
+ SECURITY_TOOLS="$(create_private_root security-tools)"
+ SOURCE_REQUIREMENTS="${REPORTS}/source-requirements"
+ IMAGE_REQUIREMENTS="${REPORTS}/image-requirements"
+ case "${GITHUB_SERVER_URL%/}" in
+ https://git.example.com|http://git.example.com|http://192.0.2.10:3000|http://gitea:3000|http://Gitea:3000|http://gitea-runner-internal:3000)
+ SOURCE_SERVER_URL="https://git.example.com" ;;
+ *) echo "Untrusted Gitea server URL (legacy: Untrusted non-HTTPS Gitea server URL)" >&2; exit 1 ;;
+ esac
+ [[ "${GITHUB_REPOSITORY}" == "Jens/ITWorx-ModelForge" ]] || {
+ echo "Unexpected canonical repository identity: ${GITHUB_REPOSITORY}" >&2
+ exit 1
+ }
+ SOURCE_REPOSITORY="${SOURCE_SERVER_URL}/${GITHUB_REPOSITORY}"
+ [[ "${SOURCE_REPOSITORY}" == "https://git.example.com/example/modelforge" ]]
+ install -d -m 700 "${REPORTS}/status" "${REPORTS}/versions"
+ {
+ printf 'REPORTS=%s\n' "${REPORTS}"
+ printf 'SECURITY_TOOLS=%s\n' "${SECURITY_TOOLS}"
+ printf 'SOURCE_REQUIREMENTS=%s\n' "${SOURCE_REQUIREMENTS}"
+ printf 'IMAGE_REQUIREMENTS=%s\n' "${IMAGE_REQUIREMENTS}"
+ printf 'SOURCE_REPOSITORY=%s\n' "${SOURCE_REPOSITORY}"
+ } >> "${GITHUB_ENV}"
+ export REPORTS SECURITY_TOOLS SOURCE_REQUIREMENTS IMAGE_REQUIREMENTS SOURCE_REPOSITORY
+ python3 - <<'PY'
+ import json
+ import os
+ from datetime import UTC, datetime
+ from pathlib import Path
+
+ result = {
+ "schema_version": 1,
+ "commit": os.environ.get("GITHUB_SHA", "unavailable"),
+ "ref": os.environ.get("GITHUB_REF", "unavailable"),
+ "run_id": os.environ.get("GITHUB_RUN_ID", "unavailable"),
+ "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", "unavailable"),
+ "profile": os.environ.get("SECURITY_PROFILE", "unavailable"),
+ "evaluated_at": datetime.now(UTC).isoformat(),
+ "blocked_findings": [{
+ "scanner": "security-gate",
+ "target": "bootstrap",
+ "advisory_id": "GATE-INCOMPLETE",
+ "severity": "ERROR",
+ "reason": "the central evaluator did not complete",
+ }],
+ "applied_risks": [],
+ "unused_or_invalid_risks": [],
+ "decision": "FAIL",
+ }
+ path = Path(os.environ["REPORTS"]) / "security-gate-result.json"
+ path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+ PY
+
+ - name: Check out security source with full history
+ timeout-minutes: 5
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ fetch-depth: 0
+ fetch-tags: true
+ persist-credentials: false
+
+ - name: Prove complete non-shallow history
+ timeout-minutes: 6
+ shell: bash
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ set -euo pipefail
+ : "${GITHUB_TOKEN:?missing read-only repository token}"
+ cleanup_history_auth() {
+ original_code=$?
+ trap - EXIT
+ git config --local --unset-all http.extraheader >/dev/null 2>&1 || true
+ exit "${original_code}"
+ }
+ trap cleanup_history_auth EXIT
+ git remote set-url origin "${SOURCE_REPOSITORY}"
+ auth_header="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 -w 0)"
+ git config --local http.extraheader "AUTHORIZATION: basic ${auth_header}"
+ unset auth_header
+ set +e
+ timeout --signal=TERM --kill-after=30s 3m git fetch --force --prune \
+ --prune-tags --tags origin \
+ '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*'
+ fetch_code=$?
+ printf '%s\n' "${fetch_code}" > "${REPORTS}/status/history-fetch.exit"
+ timeout --signal=TERM --kill-after=30s 2m python3 scripts/security_gate.py history \
+ --repository . --expected-commit "${GITHUB_SHA}" \
+ --output "${REPORTS}/git-history.json"
+ code=$?
+ printf '%s\n' "${code}" > "${REPORTS}/status/history.exit"
+ MODELFORGE_VERSION="$(tr -d '\r\n' < VERSION)"
+ version_code=$?
+ if [[ "${version_code}" == 0 && \
+ ! "${MODELFORGE_VERSION}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-rc\.(0|[1-9][0-9]*))?$ ]]; then
+ version_code=2
+ fi
+ if [[ "${version_code}" == 0 ]]; then
+ printf 'MODELFORGE_VERSION=%s\n' "${MODELFORGE_VERSION}" >> "${GITHUB_ENV}"
+ fi
+ printf '%s\n' "${version_code}" > "${REPORTS}/status/release-version.exit"
+ set -e
+
+ - name: Install checksum-pinned security scanners
+ timeout-minutes: 15
+ shell: bash
+ run: |
+ set -euo pipefail
+ [[ -d "${SECURITY_TOOLS}" && ! -L "${SECURITY_TOOLS}" ]]
+ install -d -m 700 "${SECURITY_TOOLS}/downloads" "${SECURITY_TOOLS}/bin"
+ record_command() {
+ name="$1"
+ shift
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "$@"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/${name}.exit"
+ }
+
+ record_command pip-audit-download python3 -m pip download \
+ --disable-pip-version-check --index-url https://pypi.org/simple \
+ --require-hashes --only-binary=:all: --dest "${SECURITY_TOOLS}/downloads" \
+ --requirement config/security/pip-audit-toolchain.lock
+ printf '%s %s\n' "${PIP_AUDIT_SHA256}" \
+ "${SECURITY_TOOLS}/downloads/pip_audit-${PIP_AUDIT_VERSION}-py3-none-any.whl" \
+ > "${SECURITY_TOOLS}/pip-audit.sha256"
+ record_command pip-audit-hash sha256sum --check --strict \
+ "${SECURITY_TOOLS}/pip-audit.sha256"
+ set +e
+ if [[ "$(<"${REPORTS}/status/pip-audit-download.exit")" == 0 && \
+ "$(<"${REPORTS}/status/pip-audit-hash.exit")" == 0 ]]; then
+ timeout --signal=TERM --kill-after=30s 10m python3 -m venv \
+ "${SECURITY_TOOLS}/pip-audit-venv" && \
+ timeout --signal=TERM --kill-after=30s 10m \
+ "${SECURITY_TOOLS}/pip-audit-venv/bin/pip" install \
+ --disable-pip-version-check --no-index \
+ --find-links "${SECURITY_TOOLS}/downloads" --require-hashes \
+ --requirement config/security/pip-audit-toolchain.lock
+ code=$?
+ else
+ code=125
+ fi
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-audit-install.exit"
+ set +e
+ timeout --signal=TERM --kill-after=30s 1m \
+ "${SECURITY_TOOLS}/pip-audit-venv/bin/pip-audit" --version \
+ > "${REPORTS}/versions/pip-audit.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-audit-version.exit"
+
+ record_command gitleaks-download curl --fail --location --show-error \
+ --retry 3 --retry-all-errors --output "${SECURITY_TOOLS}/downloads/gitleaks.tar.gz" \
+ "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
+ printf '%s %s\n' "${GITLEAKS_SHA256}" \
+ "${SECURITY_TOOLS}/downloads/gitleaks.tar.gz" > "${SECURITY_TOOLS}/gitleaks.sha256"
+ record_command gitleaks-hash sha256sum --check --strict \
+ "${SECURITY_TOOLS}/gitleaks.sha256"
+ if [[ "$(<"${REPORTS}/status/gitleaks-hash.exit")" == 0 ]]; then
+ record_command gitleaks-extract tar --extract --gzip \
+ --file "${SECURITY_TOOLS}/downloads/gitleaks.tar.gz" \
+ --directory "${SECURITY_TOOLS}/bin" gitleaks
+ else
+ printf '125\n' > "${REPORTS}/status/gitleaks-extract.exit"
+ fi
+ set +e
+ timeout --signal=TERM --kill-after=30s 1m \
+ "${SECURITY_TOOLS}/bin/gitleaks" version > "${REPORTS}/versions/gitleaks.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks-version.exit"
+
+ gitleaks_self_test() (
+ set -euo pipefail
+ test_root="$(mktemp -d "${SECURITY_TOOLS}/gitleaks-self-test.XXXXXXXX")"
+ cleanup_self_test() {
+ original_code=$?
+ trap - EXIT
+ set +e
+ rm -r -- "${test_root}"
+ cleanup_code=$?
+ set -e
+ if [[ "${cleanup_code}" != 0 ]]; then
+ exit 125
+ fi
+ exit "${original_code}"
+ }
+ trap cleanup_self_test EXIT
+ default_trigger="${test_root}/default-trigger"
+ control_repository="${test_root}/control"
+ custom_formats=(ghp gho ghu ghs ghr github-pat)
+ install -d -m 700 "${default_trigger}" "${control_repository}"
+ for format in "${custom_formats[@]}"; do
+ install -d -m 700 "${test_root}/custom-${format}"
+ done
+ for repository in "${default_trigger}" "${control_repository}" \
+ "${test_root}"/custom-*; do
+ git -C "${repository}" init --quiet
+ git -C "${repository}" config user.name security-gate
+ git -C "${repository}" config user.email security-gate@invalid.example
+ done
+ python3 - "${test_root}" "${default_trigger}/credential.txt" <<'PY'
+ import sys
+ from pathlib import Path
+
+ root = Path(sys.argv[1])
+ body = "".join(("aBcDeFgHiJkLmNoPqRsTuVwXyZ", "0123456789"))
+ tokens = {
+ "ghp": "".join(("gh", "p_", body)),
+ "gho": "".join(("gh", "o_", body)),
+ "ghu": "".join(("gh", "u_", body)),
+ "ghs": "".join(("gh", "s_", body)),
+ "ghr": "".join(("gh", "r_", body)),
+ "github-pat": "".join(("github", "_pat_", "A" * 82)),
+ }
+ aws_key = "".join(("AK", "IA", "A1B2C3D4E5F6G7H8"))
+ for name, token in tokens.items():
+ (root / f"custom-{name}" / "credential.txt").write_text(
+ f'token = "{token}"\n', encoding="utf-8"
+ )
+ Path(sys.argv[2]).write_text(f'key = "{aws_key}"\n', encoding="utf-8")
+ PY
+ for repository in "${default_trigger}" "${test_root}"/custom-*; do
+ git -C "${repository}" add credential.txt
+ git -C "${repository}" commit --quiet -m synthetic-trigger
+ printf 'credential removed\n' > "${repository}/credential.txt"
+ git -C "${repository}" add credential.txt
+ git -C "${repository}" commit --quiet -m remove-trigger
+ done
+ printf 'token = "example-not-a-credential"\n' > "${control_repository}/control.txt"
+ git -C "${control_repository}" add control.txt
+ git -C "${control_repository}" commit --quiet -m benign-control
+
+ for format in "${custom_formats[@]}"; do
+ set +e
+ timeout --signal=TERM --kill-after=10s 1m \
+ "${SECURITY_TOOLS}/bin/gitleaks" git \
+ --config "${GITHUB_WORKSPACE}/.gitleaks.toml" --log-opts=--all \
+ --redact=100 --report-format json \
+ --report-path "${test_root}/custom-${format}.json" \
+ --no-banner "${test_root}/custom-${format}" >/dev/null 2>&1
+ custom_trigger_code=$?
+ set -e
+ [[ "${custom_trigger_code}" == 1 ]]
+ done
+ set +e
+ timeout --signal=TERM --kill-after=10s 1m \
+ "${SECURITY_TOOLS}/bin/gitleaks" git \
+ --config "${GITHUB_WORKSPACE}/.gitleaks.toml" --log-opts=--all \
+ --redact=100 --report-format json \
+ --report-path "${test_root}/custom-control.json" \
+ --no-banner "${control_repository}" >/dev/null 2>&1
+ custom_control_code=$?
+ timeout --signal=TERM --kill-after=10s 1m \
+ "${SECURITY_TOOLS}/bin/gitleaks" git --log-opts=--all \
+ --redact=100 --report-format json \
+ --report-path "${test_root}/default-trigger.json" \
+ --no-banner "${default_trigger}" >/dev/null 2>&1
+ default_trigger_code=$?
+ timeout --signal=TERM --kill-after=10s 1m \
+ "${SECURITY_TOOLS}/bin/gitleaks" git --log-opts=--all \
+ --redact=100 --report-format json \
+ --report-path "${test_root}/default-control.json" \
+ --no-banner "${control_repository}" >/dev/null 2>&1
+ default_control_code=$?
+ set -e
+ [[ "${custom_control_code}" == 0 && "${default_trigger_code}" == 1 && \
+ "${default_control_code}" == 0 ]]
+ )
+ set +e
+ gitleaks_self_test
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks-self-test.exit"
+
+ record_command trivy-download curl --fail --location --show-error \
+ --retry 3 --retry-all-errors --output "${SECURITY_TOOLS}/downloads/trivy.tar.gz" \
+ "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
+ printf '%s %s\n' "${TRIVY_SHA256}" \
+ "${SECURITY_TOOLS}/downloads/trivy.tar.gz" > "${SECURITY_TOOLS}/trivy.sha256"
+ record_command trivy-hash sha256sum --check --strict \
+ "${SECURITY_TOOLS}/trivy.sha256"
+ if [[ "$(<"${REPORTS}/status/trivy-hash.exit")" == 0 ]]; then
+ record_command trivy-extract tar --extract --gzip \
+ --file "${SECURITY_TOOLS}/downloads/trivy.tar.gz" \
+ --directory "${SECURITY_TOOLS}/bin" trivy
+ else
+ printf '125\n' > "${REPORTS}/status/trivy-extract.exit"
+ fi
+
+ - name: Collect source security reports
+ timeout-minutes: 30
+ shell: bash
+ run: |
+ set -euo pipefail
+ PIP_AUDIT="${SECURITY_TOOLS}/pip-audit-venv/bin/pip-audit"
+ GITLEAKS="${SECURITY_TOOLS}/bin/gitleaks"
+ TRIVY="${SECURITY_TOOLS}/bin/trivy"
+ GITLEAKS_TEMP="$(mktemp -d "${SECURITY_TOOLS}/gitleaks-run.XXXXXXXX")"
+ RAW_GITLEAKS_DEFAULT="${GITLEAKS_TEMP}/default.json"
+ RAW_GITLEAKS_CUSTOM="${GITLEAKS_TEMP}/custom.json"
+ RAW_GITLEAKS="${GITLEAKS_TEMP}/combined.json"
+ DEFAULT_GITLEAKS_REPOSITORY="${GITLEAKS_TEMP}/default-worktree"
+ cleanup_source_scan() {
+ original_code=$?
+ trap - EXIT
+ set +e
+ cleanup_code=0
+ if [[ -e "${DEFAULT_GITLEAKS_REPOSITORY}/.git" ]]; then
+ git worktree remove --force "${DEFAULT_GITLEAKS_REPOSITORY}" \
+ >/dev/null 2>&1
+ cleanup_code=$?
+ fi
+ git worktree prune
+ prune_code=$?
+ rm -r -- "${GITLEAKS_TEMP}"
+ remove_code=$?
+ if [[ "${cleanup_code}" != 0 || "${prune_code}" != 0 || \
+ "${remove_code}" != 0 ]]; then
+ cleanup_code=125
+ fi
+ printf '%s\n' "${cleanup_code}" > "${REPORTS}/status/gitleaks-cleanup.exit"
+ set -e
+ if [[ "${cleanup_code}" != 0 ]]; then
+ exit 125
+ fi
+ exit "${original_code}"
+ }
+ trap cleanup_source_scan EXIT
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 2m python3 scripts/security_gate.py \
+ source-inventory --repository . --output "${REPORTS}/source-inventory.json" \
+ --requirements-directory "${SOURCE_REQUIREMENTS}"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/source-inventory.exit"
+
+ # Gitleaks 8.30.1 drops newly added rules when extend.useDefault is used (#2170).
+ # Run its built-in defaults on a config-free temporary worktree, run the explicit
+ # repository rules separately, then combine only temporary raw reports.
+ set +e
+ git worktree add --detach "${DEFAULT_GITLEAKS_REPOSITORY}" "${GITHUB_SHA}" \
+ >/dev/null 2>&1
+ default_code=$?
+ if [[ "${default_code}" == 0 ]]; then
+ rm -f -- "${DEFAULT_GITLEAKS_REPOSITORY}/.gitleaks.toml"
+ remove_config_code=$?
+ if [[ "${remove_config_code}" == 0 ]]; then
+ timeout --signal=TERM --kill-after=30s 10m "${GITLEAKS}" git \
+ --log-opts=--all --redact=100 --report-format json \
+ --report-path "${RAW_GITLEAKS_DEFAULT}" --no-banner \
+ "${DEFAULT_GITLEAKS_REPOSITORY}"
+ default_code=$?
+ else
+ default_code=125
+ fi
+ fi
+ set -e
+ printf '%s\n' "${default_code}" > "${REPORTS}/status/gitleaks-default.exit"
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${GITLEAKS}" git \
+ --config .gitleaks.toml --log-opts=--all --redact=100 --report-format json \
+ --report-path "${RAW_GITLEAKS_CUSTOM}" --no-banner .
+ custom_code=$?
+ set -e
+ printf '%s\n' "${custom_code}" > "${REPORTS}/status/gitleaks-custom.exit"
+
+ set +e
+ python3 - "${RAW_GITLEAKS_DEFAULT}" "${RAW_GITLEAKS_CUSTOM}" \
+ "${RAW_GITLEAKS}" <<'PY'
+ import json
+ import sys
+ from pathlib import Path
+
+ findings = []
+ for value in sys.argv[1:3]:
+ report = json.loads(Path(value).read_text(encoding="utf-8"))
+ if not isinstance(report, list):
+ raise TypeError(f"Gitleaks report is not a list: {value}")
+ findings.extend(report)
+ Path(sys.argv[3]).write_text(
+ json.dumps(findings, separators=(",", ":")) + "\n", encoding="utf-8"
+ )
+ PY
+ merge_code=$?
+ set -e
+ printf '%s\n' "${merge_code}" > "${REPORTS}/status/gitleaks-merge.exit"
+ if [[ ( "${default_code}" == 0 || "${default_code}" == 1 ) && \
+ ( "${custom_code}" == 0 || "${custom_code}" == 1 ) && \
+ "${merge_code}" == 0 ]]; then
+ if [[ "${default_code}" == 1 || "${custom_code}" == 1 ]]; then
+ code=1
+ else
+ code=0
+ fi
+ else
+ code=125
+ fi
+ printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks.exit"
+ set +e
+ timeout --signal=TERM --kill-after=30s 2m python3 scripts/security_gate.py \
+ sanitize-gitleaks --input "${RAW_GITLEAKS}" --output "${REPORTS}/gitleaks.json"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks-sanitize.exit"
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict \
+ --progress-spinner off --format json \
+ --output "${REPORTS}/pip-source-backend.json" \
+ --requirement "${SOURCE_REQUIREMENTS}/backend.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-source-backend.exit"
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict \
+ --progress-spinner off --format json \
+ --output "${REPORTS}/pip-source-node-agent.json" \
+ --requirement "${SOURCE_REQUIREMENTS}/node-agent.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-source-node-agent.exit"
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict \
+ --progress-spinner off --format json \
+ --output "${REPORTS}/pip-source-runtime-worker.json" \
+ --requirement "${SOURCE_REQUIREMENTS}/runtime-worker.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-source-runtime-worker.exit"
+
+ set +e
+ npm --version > "${REPORTS}/versions/npm.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/npm-version.exit"
+ set +e
+ (cd frontend && timeout --signal=TERM --kill-after=30s 10m npm audit --json \
+ > "${REPORTS}/npm-audit-all.json")
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/npm-audit-all.exit"
+ set +e
+ (cd frontend && timeout --signal=TERM --kill-after=30s 10m npm audit --omit=dev --json \
+ > "${REPORTS}/npm-audit-production.json")
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/npm-audit-production.exit"
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 20m "${TRIVY}" fs \
+ --scanners vuln,misconfig --severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL \
+ --format json --output "${REPORTS}/trivy-fs.json" "${GITHUB_WORKSPACE}"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/trivy-fs.exit"
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 1m "${TRIVY}" version --format json \
+ > "${REPORTS}/versions/trivy.json"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/trivy-version.exit"
+
+ - name: Build and scan exact shipping images
+ if: ${{ gitea.event_name == 'workflow_dispatch' && inputs.profile == 'security' }}
+ timeout-minutes: 40
+ shell: bash
+ run: |
+ set -euo pipefail
+ PIP_AUDIT="${SECURITY_TOOLS}/pip-audit-venv/bin/pip-audit"
+ TRIVY="${SECURITY_TOOLS}/bin/trivy"
+ API_IMAGE="modelforge-security-api:${GITHUB_SHA}"
+ WEB_IMAGE="modelforge-security-web:${GITHUB_SHA}"
+ NODE_AGENT_IMAGE="modelforge-security-node-agent:${GITHUB_SHA}"
+ RUNTIME_WORKER_IMAGE="modelforge-security-runtime-worker:${GITHUB_SHA}"
+ install -d -m 700 "${IMAGE_REQUIREMENTS}"
+
+ record_command() {
+ name="$1"
+ shift
+ set +e
+ timeout --signal=TERM --kill-after=30s 25m "$@"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/${name}.exit"
+ }
+
+ record_command docker docker version
+ record_command image-build-api docker build --pull --no-cache \
+ --build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
+ --build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
+ --label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
+ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+ --label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
+ --tag "${API_IMAGE}" \
+ --file backend/Dockerfile .
+ record_command image-build-web docker build --pull --no-cache \
+ --build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
+ --build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
+ --build-arg "VITE_API_BASE_URL=https://security-scan.invalid" \
+ --label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
+ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+ --label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
+ --label "io.modelforge.security.bundle-path=/usr/share/nginx/html" \
+ --tag "${WEB_IMAGE}" \
+ --file frontend/Dockerfile frontend
+ record_command image-build-node-agent docker build --pull --no-cache \
+ --build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
+ --build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
+ --label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
+ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+ --label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
+ --tag "${NODE_AGENT_IMAGE}" \
+ --file node-agent/Dockerfile .
+ record_command image-build-runtime-worker docker build --pull --no-cache \
+ --build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
+ --build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
+ --label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
+ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+ --label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
+ --tag "${RUNTIME_WORKER_IMAGE}" \
+ --file runtime-worker/Dockerfile .
+
+ for target in api web node-agent runtime-worker; do
+ case "${target}" in
+ api) image="${API_IMAGE}" ;;
+ web) image="${WEB_IMAGE}" ;;
+ node-agent) image="${NODE_AGENT_IMAGE}" ;;
+ runtime-worker) image="${RUNTIME_WORKER_IMAGE}" ;;
+ esac
+ set +e
+ timeout --signal=TERM --kill-after=30s 3m python3 scripts/security_gate.py \
+ image-identity --target "${target}" --tag "${image}" \
+ --expected-revision "${GITHUB_SHA}" \
+ --expected-source "${SOURCE_REPOSITORY}" \
+ --expected-version "${MODELFORGE_VERSION}" \
+ --output "${REPORTS}/image-identity-${target}.json"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/image-identity-${target}.exit"
+ done
+
+ image_field() {
+ python3 - "$1" "$2" <<'PY'
+ import json
+ import sys
+ from pathlib import Path
+
+ print(json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))[sys.argv[2]])
+ PY
+ }
+ set +e
+ API_ID="$(image_field "${REPORTS}/image-identity-api.json" image_id)" \
+ || API_ID=invalid
+ WEB_ID="$(image_field "${REPORTS}/image-identity-web.json" image_id)" \
+ || WEB_ID=invalid
+ WEB_BUNDLE_PATH="$(image_field "${REPORTS}/image-identity-web.json" bundle_path)" \
+ || WEB_BUNDLE_PATH=invalid
+ NODE_AGENT_ID="$(image_field "${REPORTS}/image-identity-node-agent.json" image_id)" \
+ || NODE_AGENT_ID=invalid
+ RUNTIME_WORKER_ID="$(image_field \
+ "${REPORTS}/image-identity-runtime-worker.json" image_id)" \
+ || RUNTIME_WORKER_ID=invalid
+ set -e
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 3m python3 scripts/security_gate.py \
+ web-bundle-evidence --image-id "${WEB_ID}" \
+ --bundle-path "${WEB_BUNDLE_PATH}" \
+ --expected-origin https://security-scan.invalid \
+ --output "${REPORTS}/web-bundle-origin.json"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/web-bundle-origin.exit"
+
+ INVENTORY_PROGRAM='import importlib.metadata as m,json,platform; print(json.dumps({"python_version":platform.python_version(),"packages":[{"name":d.metadata.get("Name",d.name),"version":d.version} for d in m.distributions()]}))'
+ runtime_python() {
+ image="$1"
+ for candidate in python python3; do
+ if docker run --rm --entrypoint "${candidate}" "${image}" \
+ -c 'import sys; print(sys.executable)' >/dev/null 2>&1; then
+ printf '%s\n' "${candidate}"
+ return 0
+ fi
+ done
+ return 1
+ }
+ collect_inventory() {
+ target="$1"
+ image="$2"
+ raw="${SECURITY_TOOLS}/${target}-inventory.json"
+ set +e
+ python_command="$(runtime_python "${image}")"
+ code=$?
+ if [[ "${code}" == 0 ]]; then
+ timeout --signal=TERM --kill-after=30s 5m docker run --rm \
+ --entrypoint "${python_command}" \
+ "${image}" -c "${INVENTORY_PROGRAM}" > "${raw}"
+ code=$?
+ fi
+ if [[ "${code}" == 0 && "${target}" == "runtime-worker" ]]; then
+ timeout --signal=TERM --kill-after=30s 5m docker run --rm \
+ --entrypoint "${python_command}" \
+ "${image}" -c 'import importlib.metadata as m; import modelforge_runtime_worker.main, sentence_transformers, transformers, filelock, h2, idna, lief, pygments, requests; packages={d.metadata.get("Name", d.name).lower() for d in m.distributions()}; assert not ({"pip", "setuptools", "wheel"} & packages), packages'
+ code=$?
+ fi
+ if [[ "${code}" == 0 ]]; then
+ python3 scripts/security_gate.py image-inventory --target "${target}" \
+ --input "${raw}" --output "${REPORTS}/image-inventory-${target}.json" \
+ --requirements-output "${IMAGE_REQUIREMENTS}/${target}.txt"
+ code=$?
+ fi
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/image-inventory-${target}.exit"
+ }
+ collect_inventory api "${API_ID}"
+ collect_inventory node-agent "${NODE_AGENT_ID}"
+ collect_inventory runtime-worker "${RUNTIME_WORKER_ID}"
+
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict --no-deps \
+ --disable-pip --progress-spinner off --format json \
+ --output "${REPORTS}/pip-image-api.json" \
+ --requirement "${IMAGE_REQUIREMENTS}/api.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-image-api.exit"
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict --no-deps \
+ --disable-pip --progress-spinner off --format json \
+ --output "${REPORTS}/pip-image-node-agent.json" \
+ --requirement "${IMAGE_REQUIREMENTS}/node-agent.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-image-node-agent.exit"
+ set +e
+ timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict --no-deps \
+ --disable-pip --progress-spinner off --format json \
+ --output "${REPORTS}/pip-image-runtime-worker.json" \
+ --requirement "${IMAGE_REQUIREMENTS}/runtime-worker.txt"
+ code=$?
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/pip-image-runtime-worker.exit"
+
+ scan_image() {
+ target="$1"
+ image_id="$2"
+ raw_report="${SECURITY_TOOLS}/trivy-image-${target}.raw.json"
+ set +e
+ timeout --signal=TERM --kill-after=30s 25m "${TRIVY}" image --scanners vuln \
+ --severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL --format json \
+ --output "${raw_report}" "${image_id}"
+ code=$?
+ if [[ "${code}" == 0 ]]; then
+ python3 scripts/security_gate.py bind-trivy-image \
+ --input "${raw_report}" \
+ --output "${REPORTS}/trivy-image-${target}.json" \
+ --image-id "${image_id}" \
+ --expected-tag "modelforge-security-${target}:${GITHUB_SHA}"
+ code=$?
+ fi
+ set -e
+ printf '%s\n' "${code}" > "${REPORTS}/status/trivy-image-${target}.exit"
+ }
+ scan_image api "${API_ID}"
+ scan_image web "${WEB_ID}"
+ scan_image node-agent "${NODE_AGENT_ID}"
+ scan_image runtime-worker "${RUNTIME_WORKER_ID}"
+
+ - name: Remove exact security images
+ if: ${{ always() && gitea.event_name == 'workflow_dispatch' && inputs.profile == 'security' }}
+ timeout-minutes: 5
+ shell: bash
+ run: |
+ set -euo pipefail
+ code=0
+ for target in api web node-agent runtime-worker; do
+ image="modelforge-security-${target}:${GITHUB_SHA}"
+ if docker image inspect "${image}" >/dev/null 2>&1; then
+ docker image rm "${image}" >/dev/null || code=1
+ fi
+ if docker image inspect "${image}" >/dev/null 2>&1; then
+ code=1
+ fi
+ done
+ printf '%s\n' "${code}" > "${REPORTS}/status/image-cleanup.exit"
+ exit "${code}"
+
+ - name: Evaluate the complete security gate
+ if: always()
+ timeout-minutes: 5
+ shell: bash
+ run: |
+ python3 scripts/security_gate.py evaluate --reports-dir "${REPORTS}" \
+ --accepted-risks config/security/accepted-risks.json \
+ --gitleaks-false-positives config/security/gitleaks-false-positives.json \
+ --repository . \
+ --commit "${GITHUB_SHA}" --ref "${GITHUB_REF}" --run-id "${GITHUB_RUN_ID}" \
+ --source-repository "${SOURCE_REPOSITORY}" \
+ --profile "${SECURITY_PROFILE}" \
+ --output "${REPORTS}/security-gate-result.json"
+
+ - name: Upload immutable security evidence
+ if: always()
+ timeout-minutes: 5
+ uses: https://gitea.com/actions/gitea-upload-artifact@81f940d004763f986ba3582c007fd842dd5cb0d7
+ with:
+ name: security-gate-${{ gitea.run_id }}-${{ gitea.run_attempt }}-${{ gitea.sha }}
+ # Do not upload the report root wholesale. The evaluator accepts only this same fixed
+ # evidence manifest, so a scanner/tool side-file can neither be hashed nor exported.
+ path: |
+ ${{ env.REPORTS }}/security-gate-result.json
+ ${{ env.REPORTS }}/git-history.json
+ ${{ env.REPORTS }}/source-inventory.json
+ ${{ env.REPORTS }}/gitleaks.json
+ ${{ env.REPORTS }}/npm-audit-all.json
+ ${{ env.REPORTS }}/npm-audit-production.json
+ ${{ env.REPORTS }}/trivy-fs.json
+ ${{ env.REPORTS }}/versions/pip-audit.txt
+ ${{ env.REPORTS }}/versions/gitleaks.txt
+ ${{ env.REPORTS }}/versions/npm.txt
+ ${{ env.REPORTS }}/versions/trivy.json
+ ${{ env.REPORTS }}/source-requirements/backend.txt
+ ${{ env.REPORTS }}/source-requirements/node-agent.txt
+ ${{ env.REPORTS }}/source-requirements/runtime-worker.txt
+ ${{ env.REPORTS }}/pip-source-backend.json
+ ${{ env.REPORTS }}/pip-source-node-agent.json
+ ${{ env.REPORTS }}/pip-source-runtime-worker.json
+ ${{ env.REPORTS }}/image-identity-api.json
+ ${{ env.REPORTS }}/image-identity-web.json
+ ${{ env.REPORTS }}/image-identity-node-agent.json
+ ${{ env.REPORTS }}/image-identity-runtime-worker.json
+ ${{ env.REPORTS }}/web-bundle-origin.json
+ ${{ env.REPORTS }}/image-inventory-api.json
+ ${{ env.REPORTS }}/image-inventory-node-agent.json
+ ${{ env.REPORTS }}/image-inventory-runtime-worker.json
+ ${{ env.REPORTS }}/image-requirements/api.txt
+ ${{ env.REPORTS }}/image-requirements/node-agent.txt
+ ${{ env.REPORTS }}/image-requirements/runtime-worker.txt
+ ${{ env.REPORTS }}/pip-image-api.json
+ ${{ env.REPORTS }}/pip-image-node-agent.json
+ ${{ env.REPORTS }}/pip-image-runtime-worker.json
+ ${{ env.REPORTS }}/trivy-image-api.json
+ ${{ env.REPORTS }}/trivy-image-web.json
+ ${{ env.REPORTS }}/trivy-image-node-agent.json
+ ${{ env.REPORTS }}/trivy-image-runtime-worker.json
+ ${{ env.REPORTS }}/status/history-fetch.exit
+ ${{ env.REPORTS }}/status/history.exit
+ ${{ env.REPORTS }}/status/release-version.exit
+ ${{ env.REPORTS }}/status/pip-audit-download.exit
+ ${{ env.REPORTS }}/status/pip-audit-hash.exit
+ ${{ env.REPORTS }}/status/pip-audit-install.exit
+ ${{ env.REPORTS }}/status/pip-audit-version.exit
+ ${{ env.REPORTS }}/status/gitleaks-download.exit
+ ${{ env.REPORTS }}/status/gitleaks-hash.exit
+ ${{ env.REPORTS }}/status/gitleaks-extract.exit
+ ${{ env.REPORTS }}/status/gitleaks-version.exit
+ ${{ env.REPORTS }}/status/gitleaks-self-test.exit
+ ${{ env.REPORTS }}/status/trivy-download.exit
+ ${{ env.REPORTS }}/status/trivy-hash.exit
+ ${{ env.REPORTS }}/status/trivy-extract.exit
+ ${{ env.REPORTS }}/status/trivy-version.exit
+ ${{ env.REPORTS }}/status/source-inventory.exit
+ ${{ env.REPORTS }}/status/gitleaks-default.exit
+ ${{ env.REPORTS }}/status/gitleaks-custom.exit
+ ${{ env.REPORTS }}/status/gitleaks-merge.exit
+ ${{ env.REPORTS }}/status/gitleaks-cleanup.exit
+ ${{ env.REPORTS }}/status/gitleaks.exit
+ ${{ env.REPORTS }}/status/gitleaks-sanitize.exit
+ ${{ env.REPORTS }}/status/pip-source-backend.exit
+ ${{ env.REPORTS }}/status/pip-source-node-agent.exit
+ ${{ env.REPORTS }}/status/pip-source-runtime-worker.exit
+ ${{ env.REPORTS }}/status/npm-version.exit
+ ${{ env.REPORTS }}/status/npm-audit-all.exit
+ ${{ env.REPORTS }}/status/npm-audit-production.exit
+ ${{ env.REPORTS }}/status/trivy-fs.exit
+ ${{ env.REPORTS }}/status/docker.exit
+ ${{ env.REPORTS }}/status/image-build-api.exit
+ ${{ env.REPORTS }}/status/image-build-web.exit
+ ${{ env.REPORTS }}/status/image-build-node-agent.exit
+ ${{ env.REPORTS }}/status/image-build-runtime-worker.exit
+ ${{ env.REPORTS }}/status/image-identity-api.exit
+ ${{ env.REPORTS }}/status/image-identity-web.exit
+ ${{ env.REPORTS }}/status/image-identity-node-agent.exit
+ ${{ env.REPORTS }}/status/image-identity-runtime-worker.exit
+ ${{ env.REPORTS }}/status/web-bundle-origin.exit
+ ${{ env.REPORTS }}/status/image-inventory-api.exit
+ ${{ env.REPORTS }}/status/image-inventory-node-agent.exit
+ ${{ env.REPORTS }}/status/image-inventory-runtime-worker.exit
+ ${{ env.REPORTS }}/status/pip-image-api.exit
+ ${{ env.REPORTS }}/status/pip-image-node-agent.exit
+ ${{ env.REPORTS }}/status/pip-image-runtime-worker.exit
+ ${{ env.REPORTS }}/status/trivy-image-api.exit
+ ${{ env.REPORTS }}/status/trivy-image-web.exit
+ ${{ env.REPORTS }}/status/trivy-image-node-agent.exit
+ ${{ env.REPORTS }}/status/trivy-image-runtime-worker.exit
+ ${{ env.REPORTS }}/status/image-cleanup.exit
+ if-no-files-found: error
+ retention-days: 90
+
# Gitea only lists dispatchable workflow files from the default branch. Reusing the dedicated
- # workflow from the existing managed entry point lets an unmerged branch prove its exact public
- # export on the server. Ordinary PR validation and every profile except an explicit `build`
- # dispatch remain unchanged.
+ # workflow from this managed entry point lets an unmerged branch prove its exact public export on
+ # the server. Ordinary PR validation and every profile except an explicit `build` dispatch remain
+ # unchanged.
public_candidate_acceptance:
name: Public candidate server acceptance
if: ${{ gitea.event_name == 'workflow_dispatch' && inputs.profile == 'build' }}
diff --git a/.gitea/workflows/public-candidate-acceptance.yml b/.gitea/workflows/public-candidate-acceptance.yml
index 4b526b2..8cd4978 100644
--- a/.gitea/workflows/public-candidate-acceptance.yml
+++ b/.gitea/workflows/public-candidate-acceptance.yml
@@ -1,6 +1,6 @@
name: Public candidate server acceptance
-# Builds only an exact, curated public-source commit in a disposable Compose namespace. This job
+# Builds only an exact, curated public-source commit in a disposable validation Compose namespace. This job
# never calls the Unraid deploy controller and cannot select the production deployment action.
on:
workflow_dispatch:
@@ -14,6 +14,11 @@ on:
required: true
default: https://modelforge.example.test
type: string
+ retain_image_bundle:
+ description: Retain the verified four-image Docker bundle for P12-P15 qualification
+ required: false
+ default: false
+ type: boolean
workflow_call:
inputs:
source_commit:
@@ -24,6 +29,11 @@ on:
description: Bare API origin compiled into the candidate Console image
required: true
type: string
+ retain_image_bundle:
+ description: Retain the verified four-image Docker bundle for P12-P15 qualification
+ required: false
+ default: false
+ type: boolean
permissions:
contents: read
@@ -140,14 +150,23 @@ jobs:
shell: bash
env:
PUBLIC_API_ORIGIN: ${{ inputs.public_api_origin }}
+ RETAIN_IMAGE_BUNDLE: ${{ inputs.retain_image_bundle }}
+ SOURCE_COMMIT: ${{ inputs.source_commit }}
run: |
set -euo pipefail
cd "${PUBLIC_SOURCE}"
- python3 scripts/rc_server_acceptance.py \
- --public-api-origin "${PUBLIC_API_ORIGIN}" \
- --trivy "${TRIVY}" \
- --output acceptance-evidence \
+ arguments=(
+ --public-api-origin "${PUBLIC_API_ORIGIN}"
+ --canonical-source-commit "${SOURCE_COMMIT}"
+ --trivy "${TRIVY}"
+ --output acceptance-evidence
--project-suffix "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+ )
+ if [[ "${RETAIN_IMAGE_BUNDLE}" == "true" ]]; then
+ arguments+=(--retain-image-bundle)
+ fi
+ python3 scripts/rc_server_acceptance.py \
+ "${arguments[@]}"
- name: Always remove acceptance Docker resources
if: always()
@@ -157,7 +176,7 @@ jobs:
cleanup_failed=0
suffix="$(printf '%s' "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
| tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]' | tail -c 24)"
- project="modelforge-rc-${suffix}"
+ project="modelforge-validation-${suffix}"
for kind in container volume network; do
while IFS= read -r resource; do
[[ -n "${resource}" ]] || continue
@@ -171,7 +190,7 @@ jobs:
done
if [[ -n "${PUBLIC_SOURCE:-}" && -d "${PUBLIC_SOURCE}/.git" ]]; then
- candidate_commit="$(git -C "${PUBLIC_SOURCE}" rev-parse HEAD 2>/dev/null)"
+ canonical_commit="${{ inputs.source_commit }}"
version="$(tr -d '\r\n' < "${PUBLIC_SOURCE}/VERSION" 2>/dev/null)"
for image in modelforge-api modelforge-web modelforge-node-agent \
modelforge-runtime-worker; do
@@ -179,7 +198,7 @@ jobs:
revision="$(docker inspect --format \
'{{index .Config.Labels "org.opencontainers.image.revision"}}' \
"${tag}" 2>/dev/null)"
- if [[ -n "${candidate_commit}" && "${revision}" == "${candidate_commit}" ]]; then
+ if [[ "${revision}" == "${canonical_commit}" ]]; then
docker image rm --force "${tag}" || cleanup_failed=1
fi
done
@@ -208,6 +227,8 @@ jobs:
${{ env.PUBLIC_SOURCE }}/acceptance-evidence/*.json
${{ env.PUBLIC_SOURCE }}/acceptance-evidence/*.txt
${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*.json
+ ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*.tar
+ ${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*.tar.gz
${{ env.PUBLIC_SOURCE }}/acceptance-evidence/release/*SHA256SUMS
if-no-files-found: warn
retention-days: 90
diff --git a/.gitea/workflows/unraid-deploy.yml b/.gitea/workflows/unraid-deploy.yml
deleted file mode 100644
index ee2a351..0000000
--- a/.gitea/workflows/unraid-deploy.yml
+++ /dev/null
@@ -1,132 +0,0 @@
-name: Unraid stable release deployment
-
-on:
- workflow_dispatch:
- inputs:
- release_tag:
- description: Annotated stable release tag (for example, v1.2.1)
- required: true
- type: string
- release_commit:
- description: Exact lowercase 40-character commit SHA referenced by the tag
- required: true
- type: string
- action:
- description: Verify provenance only, or deploy the verified stable release
- required: true
- default: VERIFY_ONLY
- type: choice
- options:
- - VERIFY_ONLY
- - DEPLOY_STABLE_TO_PRODUCTION
-
-concurrency:
- group: unraid-production-itworx-modelforge
- cancel-in-progress: false
-
-permissions:
- contents: read
-
-jobs:
- deploy:
- name: Verify and optionally deploy a stable release
- runs-on: unraid-deploy
- timeout-minutes: 180
- steps:
- - name: Validate immutable dispatch inputs
- shell: bash
- env:
- RELEASE_TAG: ${{ inputs.release_tag }}
- RELEASE_COMMIT: ${{ inputs.release_commit }}
- DEPLOY_ACTION: ${{ inputs.action }}
- run: |
- set -euo pipefail
-
- if [[ "${GITHUB_REF:-}" != "refs/heads/master" ]]; then
- echo "Stable production deployment must be dispatched from refs/heads/master" >&2
- exit 2
- fi
- if [[ ! "${RELEASE_TAG}" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
- echo "release_tag must be a strict stable SemVer tag such as v1.2.1" >&2
- exit 2
- fi
- if [[ ! "${RELEASE_COMMIT}" =~ ^[0-9a-f]{40}$ ]]; then
- echo "release_commit must be an exact lowercase 40-character SHA" >&2
- exit 2
- fi
- case "${DEPLOY_ACTION}" in
- VERIFY_ONLY|DEPLOY_STABLE_TO_PRODUCTION) ;;
- *) echo "action is not allowlisted: ${DEPLOY_ACTION}" >&2; exit 2 ;;
- esac
-
- - name: Check out the exact release commit with full history
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- with:
- ref: ${{ inputs.release_commit }}
- fetch-depth: 0
- # The following provenance step performs an authenticated exact-ref fetch.
- persist-credentials: true
-
- - name: Verify stable release provenance
- shell: bash
- env:
- RELEASE_TAG: ${{ inputs.release_tag }}
- RELEASE_COMMIT: ${{ inputs.release_commit }}
- run: |
- set -euo pipefail
-
- git fetch --force --no-recurse-submodules origin \
- "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}" \
- "refs/heads/master:refs/remotes/origin/master"
-
- tag_ref="refs/tags/${RELEASE_TAG}"
- if [[ "$(git cat-file -t "${tag_ref}")" != "tag" ]]; then
- echo "${RELEASE_TAG} must be an annotated tag" >&2
- exit 1
- fi
-
- tag_object="$(git rev-parse "${tag_ref}^{tag}")"
- tag_commit="$(git rev-parse "${tag_ref}^{commit}")"
- head_commit="$(git rev-parse HEAD)"
- if [[ "${tag_commit}" != "${RELEASE_COMMIT}" ]]; then
- echo "Tag commit ${tag_commit} does not match release_commit ${RELEASE_COMMIT}" >&2
- exit 1
- fi
- if [[ "${head_commit}" != "${RELEASE_COMMIT}" ]]; then
- echo "Checked-out HEAD ${head_commit} does not match release_commit ${RELEASE_COMMIT}" >&2
- exit 1
- fi
-
- if [[ ! -f VERSION ]]; then
- echo "VERSION is missing at the release commit" >&2
- exit 1
- fi
- version="$(tr -d '\r\n' < VERSION)"
- expected_version="${RELEASE_TAG#v}"
- if [[ "${version}" != "${expected_version}" ]]; then
- echo "VERSION ${version} does not match release tag ${RELEASE_TAG}" >&2
- exit 1
- fi
-
- if ! git merge-base --is-ancestor "${RELEASE_COMMIT}" refs/remotes/origin/master; then
- echo "Release commit ${RELEASE_COMMIT} is not an ancestor of origin/master" >&2
- exit 1
- fi
-
- printf 'DEPLOY_COMMIT=%s\n' "${RELEASE_COMMIT}" >> "${GITHUB_ENV}"
- echo "Verified annotated ${RELEASE_TAG} object ${tag_object} at immutable commit ${RELEASE_COMMIT}"
-
- - name: Verification-only result
- if: ${{ inputs.action == 'VERIFY_ONLY' }}
- shell: bash
- run: echo "Stable release provenance verified; production was not changed."
-
- - name: Deploy verified stable release to production
- if: ${{ inputs.action == 'DEPLOY_STABLE_TO_PRODUCTION' }}
- shell: bash
- run: |
- set -euo pipefail
- test -n "${DEPLOY_COMMIT:-}"
- docker exec gitea-deploy-control \
- /opt/gitea-deploy/deploy.py deploy \
- "${GITHUB_REPOSITORY}" "${DEPLOY_COMMIT}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 354c1c9..ebcc4f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,41 @@ commits are not listed unless they change what an operator sees or does.
## Unreleased
+### Release qualification
+
+- Candidate version `1.3.0-rc.1` reports the `rc` channel rather than stable.
+- The P15A path binds the exact canonical source identity, OCI labels and filesystem layer sets
+ into a retained image bundle before isolated RC qualification.
+- This is not a release announcement: no RC tag, Gitea prerelease or production promotion is
+ authorized until the complete qualification matrix is verified.
+
+### Product workflow
+
+- Added the project-first model-management flow: persisted project bindings, human-readable model
+ and update context, guided onboarding, selected current/candidate comparison, impact and
+ migration/reindex boundaries, and explicit lifecycle verification/rollback. Projects continue
+ to bind to stable capabilities rather than model paths or concrete model names.
+- Selected comparisons now retain immutable embedding-space dimensions and exact gateway-bound
+ current/candidate evidence; approved route replacement and rollback are fenced by generation
+ and CAS checks.
+
+### Fixed
+
+- The v1.2.1 upgrade path now detects when `modelforge` is PostgreSQL's protected bootstrap
+ superuser (OID 10) before mutating any role or privilege. Such installations are routed to a
+ documented fresh-cluster logical-restore path; direct system-catalog edits remain unsupported.
+- Compose now passes the retained-admin variable names required by the bounded existing-cluster
+ provisioning script, while continuing to keep administrator and owner credentials out of the
+ API and migration processes.
+
+### Operations
+
+- Recorded the completed v1.2.2 production rollout, exact deployed image identities, role-split
+ cluster migration, verified recovery point, measured downtime and final production-worker LAB
+ probe.
+
+## v1.2.2 — 2026-09-01
+
### Security
- Runtime Worker packaging now upgrades the digest-pinned Ubuntu base before installation and pins
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 818a8d2..957a78f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -24,7 +24,9 @@ python -m ruff check backend/src backend/tests node-agent/src runtime-worker/src
python -m mypy backend/src
python -m mypy --config-file node-agent/pyproject.toml node-agent/src
python -m mypy --config-file runtime-worker/pyproject.toml runtime-worker/src
-python -m pytest backend/tests node-agent/tests runtime-worker/tests -q
+(cd backend && python -m pytest -q)
+(cd node-agent && python -m pytest -q)
+(cd runtime-worker && python -m pytest -q)
cd frontend && npm test -- --run && npm run build
```
@@ -35,6 +37,8 @@ production evidence, model weights, generated backups or personal data.
- Applications depend on versioned capabilities, never concrete model names.
- Production identities use exact commits, artifact digests and image digests.
+- Document source, revision, license and checksum expectations for models and runtime artifacts.
+- Use synthetic identities and configurable node addresses in tests and examples.
- No production promotion occurs without evidence and explicit approval.
- `trust_remote_code` stays false; untrusted model code is not executed.
- Agents and runtime workers use narrow identities and least privilege.
diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md
index e26201f..25d75ea 100644
--- a/PUBLIC_SOURCE_EXPORT.md
+++ b/PUBLIC_SOURCE_EXPORT.md
@@ -1,5 +1,5 @@
# Curated public source export
-Generated from private canonical revision `b8755394117affad0178a79a627cf5574d3c8eef`.
+Generated from private canonical revision `990401737f96c8d46a9f7b0d84994516cadcb6a0`.
This parentless candidate excludes private operational history and uses synthetic example identifiers.
diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json
index cb2aa18..0ad3f93 100644
--- a/PUBLIC_SOURCE_MANIFEST.json
+++ b/PUBLIC_SOURCE_MANIFEST.json
@@ -1,16 +1,17 @@
{
"schemaVersion": 1,
- "sourceRevision": "b8755394117affad0178a79a627cf5574d3c8eef",
+ "sourceRevision": "990401737f96c8d46a9f7b0d84994516cadcb6a0",
+ "sourceRepository": "https://git.example.com/example/modelforge",
"files": [
{
"path": ".dockerignore",
- "bytes": 183,
- "sha256": "156d4a337261c194ec0a791884d7d1a5a588345d769ebf43b6e4f00f8d32008a"
+ "bytes": 1195,
+ "sha256": "b91cc0a07ff2a7aae4f251c8636aa8ab6d034ff5edef14f301488f7aceab3223"
},
{
"path": ".env.example",
- "bytes": 13008,
- "sha256": "b9d5da83afcdd6946bc7897cd6605c5ace2e1460600b1192684b8aff018bd916"
+ "bytes": 15513,
+ "sha256": "bc2bf470871abf659f24d99b9028ccb60d39cbff54098114f13169c57bd3070d"
},
{
"path": ".gitattributes",
@@ -19,33 +20,48 @@
},
{
"path": ".gitea/tests/test_production_deploy_policy.py",
- "bytes": 9934,
- "sha256": "1f5e5f656bbca3ed2992de3bd60aafc7534623d255ac7175a46e286b3853a4fc"
+ "bytes": 7353,
+ "sha256": "f41a5a2a99263cc2577cd5ce33b9702d357860e8c4e29095c9a103c747c362dd"
},
{
"path": ".gitea/tests/test_public_source_export.py",
- "bytes": 8266,
- "sha256": "826d1b83852b44b1bf6c8fe17b6b3300aaceb2894fda54bcfba8ea3c166d8deb"
+ "bytes": 14522,
+ "sha256": "3787851626421635a647b0a7b7280e6d873e351b9fcad18fcf06f992aac3bd53"
},
{
"path": ".gitea/tests/test_rc_acceptance_policy.py",
- "bytes": 13392,
- "sha256": "905b038d9056d2535f111621e340b0cf622ec08d890ad459355d0cacd3503c68"
+ "bytes": 20587,
+ "sha256": "f429a897dd7c583c954222178533b592d5559647d040ff5c21d97128e9fde1f8"
+ },
+ {
+ "path": ".gitea/tests/test_rc_isolation_preflight.py",
+ "bytes": 11192,
+ "sha256": "5fc4dd0d22ad191495398d6e21a6c66c716b00a621e2a9da1d00079d3f07e0aa"
+ },
+ {
+ "path": ".gitea/tests/test_rc_stack_retention.py",
+ "bytes": 13097,
+ "sha256": "3c3548f01ccffca8ffa1aa624fbbe32bffcd5871f33b9a4992117669286a59d7"
+ },
+ {
+ "path": ".gitea/tests/test_security_origin_policy.py",
+ "bytes": 4002,
+ "sha256": "2636626bae46c6f485ea0cc20e78d6866ffc5e6676f21af73c8848958660dcf0"
+ },
+ {
+ "path": ".gitea/tests/test_security_validation_policy.py",
+ "bytes": 113981,
+ "sha256": "c36b24c614fb976a0bd2f90a061139c10845ecd08981bc8fdf9e728f7de3e91a"
},
{
"path": ".gitea/workflows/managed-validation.yml",
- "bytes": 13398,
- "sha256": "cf43e95af3b58f663eca0b22330f5be2d01ba7d0807f45ec87029f2626a7bce7"
+ "bytes": 57275,
+ "sha256": "dd5c18b2c3d41c022d3cc8e22f2aa0171f76419fdd9d4b2c751700c8ba75355f"
},
{
"path": ".gitea/workflows/public-candidate-acceptance.yml",
- "bytes": 9581,
- "sha256": "1702ad62e4f41d0b564927f5667fdce2cbccb5344b4306b8bc2a95247f107dc8"
- },
- {
- "path": ".gitea/workflows/unraid-deploy.yml",
- "bytes": 4842,
- "sha256": "c0a08b4c7e1189d5631fb8d2ad7e7ecb0c8c2dec91a523e32a9a3f4bf3980a94"
+ "bytes": 10418,
+ "sha256": "83e34aa2780d666160b5c5da3bf68c3dcdf26c54414dd16650df898b12b07cbb"
},
{
"path": ".gitignore",
@@ -74,8 +90,8 @@
},
{
"path": "backend/alembic/versions/20260824_0001_m0_foundation.py",
- "bytes": 7318,
- "sha256": "1a9eedbf61eafdadb5237e735e5cee05bf0badfc19696d4494efa4c0d5742ab7"
+ "bytes": 11101,
+ "sha256": "28eb97036a00d8210b5d96c6c9271a6974122c8fe748d74c512acb6cc5d98715"
},
{
"path": "backend/alembic/versions/20260824_0002_m1_hardware_plane.py",
@@ -189,18 +205,93 @@
},
{
"path": "backend/alembic/versions/20260830_0024_audit_chain_checkpoint.py",
- "bytes": 30108,
- "sha256": "62eecea09c21e1ad437fc192510643d85ce94eb4f63e46b3a22b246f8595221d"
+ "bytes": 30107,
+ "sha256": "fab526a4f939beb88f6bc2eac508f02ce2ac75776d7f9caa943433986854bb5a"
+ },
+ {
+ "path": "backend/alembic/versions/20260907_0025_configuration.py",
+ "bytes": 1789,
+ "sha256": "13a4ca879cb46cfb08f5bcbb510658714b8856ac1e8f4ff8faa1f1b4b744a4c2"
+ },
+ {
+ "path": "backend/alembic/versions/20260908_0026_project_onboarding.py",
+ "bytes": 2255,
+ "sha256": "c690da538a1516dd2143da18519468d533d8553b274795e7e1703e2a8c96ee9e"
+ },
+ {
+ "path": "backend/alembic/versions/20260908_0027_managed_operator_identities.py",
+ "bytes": 3690,
+ "sha256": "f66bf1e22728d4006e39b03305eee0b065f3f052294ada5f31e50f60a19e7dd2"
+ },
+ {
+ "path": "backend/alembic/versions/20260908_0028_platform_secret_slots.py",
+ "bytes": 3336,
+ "sha256": "069eaf43a417d5334e71fa39432991be741139400d03da131700e330a715d998"
+ },
+ {
+ "path": "backend/alembic/versions/20260909_0029_external_creative_coordination.py",
+ "bytes": 2891,
+ "sha256": "ac41923b2bd7a8729abeaae5c872afa3186d4b188fe002015d77c9bc23b9c5db"
+ },
+ {
+ "path": "backend/alembic/versions/20260909_0030_model_update_inbox.py",
+ "bytes": 2463,
+ "sha256": "265379bd5e1c2cee78a9c4a4e6004ec802a6dbbc57e5f30c422e8eb91fea99cf"
+ },
+ {
+ "path": "backend/alembic/versions/20260909_0031_external_creative_governance.py",
+ "bytes": 4476,
+ "sha256": "397d4b288bb7564eb50da5950e1478db237f2a91d801de3d3786112b638f2c55"
+ },
+ {
+ "path": "backend/alembic/versions/20260909_0032_audio_creative_workloads.py",
+ "bytes": 1121,
+ "sha256": "ede826fa5ada798e5730ef99b0a8380d1831d3ed11a1baccda9ed5bc0e60d362"
+ },
+ {
+ "path": "backend/alembic/versions/20260909_0033_deployed_hub_convergence.py",
+ "bytes": 6497,
+ "sha256": "373e9ce63db0a22d1491d8ab0523a639cecd776d656e8a54eb107d1cb454dc6a"
+ },
+ {
+ "path": "backend/alembic/versions/20260910_0034_approved_serving_transition.py",
+ "bytes": 1098,
+ "sha256": "6fc188d8f929d587036d7be968f9dd818342a887584a75ffd577fc6278d66ba7"
+ },
+ {
+ "path": "backend/alembic/versions/20260911_0035_user_model_change_cases.py",
+ "bytes": 4936,
+ "sha256": "55b050f9b20c608cc6e545ebb58ecfcaa90c7b64e5bb53bd07fd8c5db9ec869b"
+ },
+ {
+ "path": "backend/alembic/versions/20260923_0036_portfolio_repository_reconciliation.py",
+ "bytes": 6508,
+ "sha256": "c54efb00c3c26eb6c09b3edc606584f2054036c0b60b3b05f19f1f1d28eb0aba"
+ },
+ {
+ "path": "backend/alembic/versions/20260923_0037_alert_notification_outbox.py",
+ "bytes": 4497,
+ "sha256": "d6146257a67db5d70697875fc675057832413006b2dd7d59a9620d2e73555b56"
+ },
+ {
+ "path": "backend/alembic/versions/20260924_0038_lab_execution_runs.py",
+ "bytes": 5677,
+ "sha256": "42b383d2eda6a9bf0f77f1eb557931f2ed32eeabf0339a521387092e9e7d41ee"
+ },
+ {
+ "path": "backend/alembic/versions/20260924_0039_operator_benchmark_runs.py",
+ "bytes": 7556,
+ "sha256": "1487edd78480629b18f4067bf2e796aedfe3b264c69d8cb036286c1f4a37e26d"
},
{
"path": "backend/Dockerfile",
- "bytes": 2533,
- "sha256": "df0916fcdf1fd28f363b06872d9a04295eabb6b57405047e712c5e07fbb30857"
+ "bytes": 2815,
+ "sha256": "62b599253effcd23b357d0efd5ca41350a594748d270f36937ff07e5ef88d327"
},
{
"path": "backend/pyproject.toml",
- "bytes": 1675,
- "sha256": "2c2442e0a1993ab0262a0454bdf41da73d39f4d5d309f0aceccb9341e1a112f7"
+ "bytes": 1713,
+ "sha256": "77d5e11282a03d6ca619107a9bcabe34e11021607459f15304a1c81b28f63553"
},
{
"path": "backend/src/modelforge_api/__init__.py",
@@ -214,13 +305,13 @@
},
{
"path": "backend/src/modelforge_api/api/authorization.py",
- "bytes": 8211,
- "sha256": "ce778ce2de3fb8a3ecca1c1521cf7e19fcd6fa7427831376e8b4c8d1043c37b7"
+ "bytes": 12209,
+ "sha256": "fb90393fcb685e556ceb5f8600dc906ca673c1b78eed5368fe3135991ef35bb5"
},
{
"path": "backend/src/modelforge_api/api/request_limits.py",
- "bytes": 8361,
- "sha256": "b9efd47418abc2bae8f625a4f3d6b838aa753b0098db4ce1d1ca5dc701173cd9"
+ "bytes": 8578,
+ "sha256": "09fc7b445a35648a960083430b02befbcfd30da024ff53c801bfab6f8b238f42"
},
{
"path": "backend/src/modelforge_api/api/routes/__init__.py",
@@ -229,24 +320,39 @@
},
{
"path": "backend/src/modelforge_api/api/routes/acquisition.py",
- "bytes": 4383,
- "sha256": "2f082f9798a867e4a436bc61dcbd8fb39fa3fb1a399a5a9a629b69167254731e"
+ "bytes": 4332,
+ "sha256": "c0402cb57b93a770c8f52f29a0574e0e061e6b144757601ae06c9fc065364ade"
},
{
"path": "backend/src/modelforge_api/api/routes/agent.py",
- "bytes": 11250,
- "sha256": "1901e355fe4aad7a1a1a6be31d009dc3df8e24e2da7193ac2a5bfcd7f2b97d80"
+ "bytes": 12190,
+ "sha256": "e59978cce4721a0eb93d05f519ede39dd26609a9fc54b6f4cb76a65ee9acac89"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/audit_timeline.py",
+ "bytes": 1152,
+ "sha256": "225175ade1e0cae2a871c82cfab4ff404433aed52814ceacd528026a0228fbf2"
},
{
"path": "backend/src/modelforge_api/api/routes/catalog.py",
- "bytes": 11449,
- "sha256": "7984703991e5e459269f7c37836aec22e8f64d2a6e86a26e07c7443e3502434a"
+ "bytes": 21047,
+ "sha256": "fd44a946b9db6d351ddd951ed4e46806ef41626c04682fa88b7ea69e8328b542"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/configuration.py",
+ "bytes": 5220,
+ "sha256": "001b07f15222f86f4155ff617ce2584af9e907315d3b266a2a7158689e458e7b"
},
{
"path": "backend/src/modelforge_api/api/routes/evaluation.py",
"bytes": 12620,
"sha256": "8535fd9b5ab27f9d06005e93e5d3d80c54be6d64f196e310948a531ae22166a7"
},
+ {
+ "path": "backend/src/modelforge_api/api/routes/external_creative.py",
+ "bytes": 5944,
+ "sha256": "efa7b4dd589414cac8a943c05c12681e70996f0261c29de5d48efbe4955c17e6"
+ },
{
"path": "backend/src/modelforge_api/api/routes/hardware.py",
"bytes": 2945,
@@ -257,25 +363,55 @@
"bytes": 3728,
"sha256": "6d268e22b460796e30fe2801f57d103681605b4c15ec803492b03b20dd03e923"
},
+ {
+ "path": "backend/src/modelforge_api/api/routes/identity.py",
+ "bytes": 3017,
+ "sha256": "44e91c8f0995838c8d875de1d643d634f9ae43cea6ecfcb94ad5dcd695850b0c"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/lab.py",
+ "bytes": 5445,
+ "sha256": "e627263d60b406535adcd4c59548af4f34f6d0ac06f4e0fbc6090efbf6edcc71"
+ },
{
"path": "backend/src/modelforge_api/api/routes/lifecycle.py",
- "bytes": 8374,
- "sha256": "57ebee2c1c7aceffc046b38a2c6ef92bd1286fe6422535f2c084e2390f90c558"
+ "bytes": 8615,
+ "sha256": "7c1652ef465d211e3a009df5bc576fb2dc688b349800fdfa15705e819061e67d"
},
{
"path": "backend/src/modelforge_api/api/routes/migrations.py",
- "bytes": 6546,
- "sha256": "de16a2170f4434d9119cff111152ec73909cd12b76d08efd04c84e4f1ff8476c"
+ "bytes": 6622,
+ "sha256": "197278baf24b12601fca370ca93ea79bf401c7f32025901036c129010c1e8843"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/model_updates.py",
+ "bytes": 3343,
+ "sha256": "0ffd52a3bfc08038310ed9085794fc36a1bf5356545f6109900b188e31a738db"
},
{
"path": "backend/src/modelforge_api/api/routes/observability.py",
- "bytes": 7653,
- "sha256": "2be728006566bc0b7b43c73cd1a1ef2fe073d00d6ebe7b00d86db3b617d252d4"
+ "bytes": 10535,
+ "sha256": "d492cb41bc61d403825c92616419f3b2b55372eccbebd0a29798ddfbc23205a9"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/operator_login.py",
+ "bytes": 3141,
+ "sha256": "242d627a28636ad6338fc345dc71142345ed192f70c1a29a4dc79f9019cb4edf"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/portfolio.py",
+ "bytes": 1795,
+ "sha256": "b51b81636939d225582875edc94023c09f04793c66c0c5dc2c3fc8d39473961a"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/projects.py",
+ "bytes": 5707,
+ "sha256": "59e93c3608c284f177d450a92a2959ddd1bba3258820da29198a8ff0af636fd6"
},
{
"path": "backend/src/modelforge_api/api/routes/recovery.py",
- "bytes": 6935,
- "sha256": "7b78ba2e263e939947766b690ef46799c172ba31ef55d852ecd126e5b44ab9e5"
+ "bytes": 7165,
+ "sha256": "bfc212869415bcc79c30e7a337aee3505676163ab7aced089555d2e3539f32f2"
},
{
"path": "backend/src/modelforge_api/api/routes/registry.py",
@@ -287,10 +423,25 @@
"bytes": 6235,
"sha256": "69e33d9ccb9d571a4f76d1cf3ec22493b5c6c1fb5a0128b901b6ccca1ed7a7d9"
},
+ {
+ "path": "backend/src/modelforge_api/api/routes/selected_model_comparison.py",
+ "bytes": 4493,
+ "sha256": "33166b1a82ccd1c1a70a56e1df9eec9d0722a16a27aba31e62dea0650f31898e"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/selected_replacement.py",
+ "bytes": 900,
+ "sha256": "c84c6ebe573c64a15a85a6174ef81ff32d6bd6b7082abad58952949f11d974e0"
+ },
{
"path": "backend/src/modelforge_api/api/routes/serving.py",
- "bytes": 18102,
- "sha256": "4d6cf87cc542a72646e0b2c2df4b0494736ab721a5768da700a737aae0e04959"
+ "bytes": 23066,
+ "sha256": "11d9e3f5755b155146e0d192fbba7bc19ba0a9147c1d1abec8b855e85dfc8ba2"
+ },
+ {
+ "path": "backend/src/modelforge_api/api/routes/update_preflight.py",
+ "bytes": 1218,
+ "sha256": "b3f48d4a457bc2e06a4b20e93022173e3398c857ff2b338c3a406cd1d24e5cfb"
},
{
"path": "backend/src/modelforge_api/cli/__init__.py",
@@ -319,28 +470,58 @@
},
{
"path": "backend/src/modelforge_api/domain/agent_protocol.py",
- "bytes": 3844,
- "sha256": "96036412dee4280f324b03fa44f147c9d8618d73441060ae9695d99f5e33bfee"
+ "bytes": 3884,
+ "sha256": "dbab36a05e7483db502d8ff7a5bdc30da83e4f76bc0220624d1fa953c8859cb1"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/alert_notifications.py",
+ "bytes": 1218,
+ "sha256": "23167a8cb05d987f038bfeb291939393d966bda5a2d92d35f70fa0f2ef08997d"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/audit_timeline.py",
+ "bytes": 2668,
+ "sha256": "e1edebde61f7adc29d5199ecdb7da9c16c10b6b722d665b2c6190a138c5daa82"
},
{
"path": "backend/src/modelforge_api/domain/audit.py",
"bytes": 5045,
"sha256": "d5a6fc6e89538b12b8744f601e0d8f0e3285198203031d339c4431ec703fca80"
},
+ {
+ "path": "backend/src/modelforge_api/domain/benchmarking.py",
+ "bytes": 1704,
+ "sha256": "beebc0ddeea2b5045c5da703a46e803a52b36e6471eee36d4a586eb28e257154"
+ },
{
"path": "backend/src/modelforge_api/domain/capability_evaluation.py",
"bytes": 6253,
"sha256": "38b3a710b5572a4e15ff851c2d7c4b02fac1b9115561e9269262572d2c87fddd"
},
+ {
+ "path": "backend/src/modelforge_api/domain/configuration_policy.py",
+ "bytes": 441,
+ "sha256": "fa8363d3dad971f1abe026609acc8f60986435b48036e43a12b0d98b05bb7d94"
+ },
{
"path": "backend/src/modelforge_api/domain/configuration_reference.py",
- "bytes": 17213,
- "sha256": "e134350e4fc03c94a01c2d175abca2ba52b3eff31f6d7b50a9069a48724d5b7a"
+ "bytes": 19845,
+ "sha256": "994b589ff6ac6e8f99f59f203269d84a5b4c7d865e4db801aa4d61b31de1607d"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/configuration.py",
+ "bytes": 3922,
+ "sha256": "9bf268b6fabdc068423d5bbdb11eac5d964551238535e6c77d401ae1fa1c64c8"
},
{
"path": "backend/src/modelforge_api/domain/contracts.py",
- "bytes": 14817,
- "sha256": "ca8a1b18871ee015faafd71eca27dcd6ca5dc85be34771f57fc3bf51bd09cbd9"
+ "bytes": 14831,
+ "sha256": "0ef79fbac27034beb4f015a9575a78415ef54703a8a6e4fb1390d641e4397075"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/dispatch_context.py",
+ "bytes": 2804,
+ "sha256": "e3978a559063d814fe7ab03a9cee9b48ad4151fbf39f97a352f5b64646b6c3e8"
},
{
"path": "backend/src/modelforge_api/domain/enums.py",
@@ -352,40 +533,85 @@
"bytes": 20073,
"sha256": "f15f0302e645934bb2722fcd8983439adaca755f057e1a7decc753f912237e21"
},
+ {
+ "path": "backend/src/modelforge_api/domain/external_creative.py",
+ "bytes": 4734,
+ "sha256": "1adbd925635b4e7255ce7bec6da16d2a0c536c4528e0edc36650e994bd27ff22"
+ },
{
"path": "backend/src/modelforge_api/domain/hardware.py",
"bytes": 8180,
"sha256": "6893c58f144c613eff3d26917c1d748e8b0b362f2e20ce032c1879672f86d9ea"
},
+ {
+ "path": "backend/src/modelforge_api/domain/lab.py",
+ "bytes": 3970,
+ "sha256": "c63283418b35cba1bdadfc3b8e57dcecbf9717d02de13559376d0d377d02d8d1"
+ },
{
"path": "backend/src/modelforge_api/domain/lifecycle_contracts.py",
- "bytes": 12901,
- "sha256": "d37068fc10193f6b6c8ec63a679b73c70e73f7f85757869fcae0e24440a7d749"
+ "bytes": 13810,
+ "sha256": "34a334fbc61a708dcae1ae8222a8f6b5f51f640d67852de7efd275bdccda7425"
},
{
"path": "backend/src/modelforge_api/domain/lifecycle.py",
"bytes": 3172,
"sha256": "53d144a9b9ff314bdc5134cb14aaa681d804522e9acd7be490953565f411a533"
},
+ {
+ "path": "backend/src/modelforge_api/domain/managed_identity.py",
+ "bytes": 2479,
+ "sha256": "d7ab0b7f6ab60dfeba84a6539ae16ac13f7ad1737ca9d389afb8c32e2f9f61a0"
+ },
{
"path": "backend/src/modelforge_api/domain/migration_contracts.py",
"bytes": 19575,
"sha256": "59d0a0127824665dce28feff7de8b1ef4147357ba44506c8965c48897b697a81"
},
+ {
+ "path": "backend/src/modelforge_api/domain/model_changes.py",
+ "bytes": 4257,
+ "sha256": "1904c4e917573931007b0aa1f8f979a023b6fc2386293a4ff3db194b1e428230"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/model_updates.py",
+ "bytes": 1290,
+ "sha256": "4b75688c589125e90f8e0778c2eceb8ce8e97fa115bc3f25075989119bf52050"
+ },
{
"path": "backend/src/modelforge_api/domain/node_decommission.py",
"bytes": 1742,
"sha256": "6bc976b02bb87e149b1131c1ea11dff0874ba0b98cd8c74e2de1b38ac1a0b848"
},
+ {
+ "path": "backend/src/modelforge_api/domain/node_lifecycle.py",
+ "bytes": 1567,
+ "sha256": "16cda111f9e0a681873c03be272e636240df2a5e3e9d63e6628a1a62ef46af4f"
+ },
{
"path": "backend/src/modelforge_api/domain/observability.py",
- "bytes": 13944,
- "sha256": "fb06302edf24b9096188a62c00197a60c07a35a698e2a9091ff9d0ce0d298a9b"
+ "bytes": 14816,
+ "sha256": "07c6aafee9ff07656135ae96a03d14c81f8328fbe7d2023df690fe1dc6609004"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/operator_identity.py",
+ "bytes": 2406,
+ "sha256": "8560c8e2e608ec205f5e45108fc9784d56f5afa2ad73e4d2a62c6dfe6bb2a8a2"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/portfolio.py",
+ "bytes": 7383,
+ "sha256": "7de222dd98375aa957c483e37720bb9124197a77fc2cff0cce2a6ecea37154ab"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/projects.py",
+ "bytes": 2598,
+ "sha256": "ff765e4395a7e570e3506efe16f16b7451be240d92c8094e300b60dde66b0634"
},
{
"path": "backend/src/modelforge_api/domain/recovery.py",
- "bytes": 16203,
- "sha256": "b675882609c0eb458858714bce5cb1dd7f4736567006eb5c8b072ce5d30d9749"
+ "bytes": 16892,
+ "sha256": "544bca7853c23a01bba4922c1fa3a991459b584b65838277e33ae9d68da75f7a"
},
{
"path": "backend/src/modelforge_api/domain/registry.py",
@@ -394,8 +620,8 @@
},
{
"path": "backend/src/modelforge_api/domain/release.py",
- "bytes": 8211,
- "sha256": "50e77a5c4f08b4cb539225909c06733d7b895f878d959c191e16ce74927ccd86"
+ "bytes": 10297,
+ "sha256": "4ebe294198eb6b123a6bd471c24f3f10b19a9c3595d31241677dfc75942b2070"
},
{
"path": "backend/src/modelforge_api/domain/runtime.py",
@@ -407,10 +633,35 @@
"bytes": 4421,
"sha256": "fef7fcdac138c8f87c1ef5025b0e08e9a82ccf82bb469986d52e6c08816d9741"
},
+ {
+ "path": "backend/src/modelforge_api/domain/secret_slots.py",
+ "bytes": 2679,
+ "sha256": "aaebeb1e6809ea4601dd80da5f32dc7e2ea3a32470b3cb83eeb68b0a437d99dc"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/selected_model_comparison.py",
+ "bytes": 3334,
+ "sha256": "e1a1328b1d0a4ecc7f393c0f8b27f76185a9b8e7762ee464648338d0206cde92"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/selected_replacement.py",
+ "bytes": 1363,
+ "sha256": "964b7fb309c85b6eefa7d5ec316f920188c5fbceaebe05a8dcdc0e05d505475b"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/self_check.py",
+ "bytes": 1911,
+ "sha256": "d819d16a92a44ad09b0d6027ec42e4a82551ae617dfa38a52d1e726c905425ae"
+ },
{
"path": "backend/src/modelforge_api/domain/serving.py",
- "bytes": 25381,
- "sha256": "45de8a4878d1a9a8ea7a1b3f505ecb9047a8a1d19add56a59b021263c0788ac9"
+ "bytes": 26767,
+ "sha256": "4b647b7ef57170b22c1b6788219595ec631c75fac5840a77bbbe410c394f9b77"
+ },
+ {
+ "path": "backend/src/modelforge_api/domain/update_preflight.py",
+ "bytes": 3600,
+ "sha256": "6b3ca9f1de4835d42dd06be9376e5f59522543ab31684dd1c7279883935c18ec"
},
{
"path": "backend/src/modelforge_api/hardware/__init__.py",
@@ -424,13 +675,13 @@
},
{
"path": "backend/src/modelforge_api/main.py",
- "bytes": 23921,
- "sha256": "8c4363d98a28c600884d56580ecd80da96e68883568c2956afb59cec52dd1352"
+ "bytes": 30628,
+ "sha256": "d51d38b6f734baed634d1374d682283f5d162899cd15012b81a28a8da67c0d78"
},
{
"path": "backend/src/modelforge_api/persistence/__init__.py",
- "bytes": 45,
- "sha256": "97a111f6a491e3d07b223a54e2b2630b4d6ac2dd32b9c01fb6bf0263f16debf5"
+ "bytes": 106,
+ "sha256": "b9e0339ade309d716dff6164103b6a06bd844052858338dfeca8d92a34012611"
},
{
"path": "backend/src/modelforge_api/persistence/acquisition_repository.py",
@@ -444,18 +695,23 @@
},
{
"path": "backend/src/modelforge_api/persistence/hardware_repository.py",
- "bytes": 4554,
- "sha256": "5c7a0f8d26e6869da81fe87f472bdbce1d10e47a66af0fc9b9d5669c42e3281b"
+ "bytes": 4635,
+ "sha256": "1d7235fcf8107d86572aa9584877c3cabe5511bccf6cab178f5f9e00b2562f4e"
+ },
+ {
+ "path": "backend/src/modelforge_api/persistence/model_change.py",
+ "bytes": 3867,
+ "sha256": "e3c3740d247ef98fe045b8ce20e40e38ab91c07c75e174698bc0fb64488eaa83"
},
{
"path": "backend/src/modelforge_api/persistence/models.py",
- "bytes": 198975,
- "sha256": "2ebdcf4ce91a100857c9fc55d6d265810774a431c4620f683a49c07b05555df2"
+ "bytes": 228422,
+ "sha256": "c89a60f4015bec556bd137515f8994caba2bad740217e93a4d90fe55885bfb46"
},
{
"path": "backend/src/modelforge_api/persistence/registry_repository.py",
- "bytes": 10529,
- "sha256": "12ff6d8e1c91a2c9954a2b8b3247534fd840d0376156e12b2b7897bfd8bfe236"
+ "bytes": 11430,
+ "sha256": "66a7fbe8220a5b457afae62e54c63c4d7593934a306d86d8eadfb4a8865daa8e"
},
{
"path": "backend/src/modelforge_api/persistence/repositories.py",
@@ -464,8 +720,8 @@
},
{
"path": "backend/src/modelforge_api/persistence/runtime_engine.py",
- "bytes": 671,
- "sha256": "251ad0f5a9599e147be4462790881de2386937f55033e877be057ccbeb5515f4"
+ "bytes": 670,
+ "sha256": "8efe68c0530189544c940546a010c617fad0add4f06f560472a8031fa728ea54"
},
{
"path": "backend/src/modelforge_api/persistence/runtime_repository.py",
@@ -474,8 +730,8 @@
},
{
"path": "backend/src/modelforge_api/persistence/serving_repository.py",
- "bytes": 15734,
- "sha256": "a03a317c8c333a4ef2b567df8ce0eb1a5204ff8963c7c56e62a5101ed6a21dc1"
+ "bytes": 16281,
+ "sha256": "9b147dbe213b7100a98e770d0b6487fbaa5f6381beb2cf401fa1bd03e523b752"
},
{
"path": "backend/src/modelforge_api/providers/__init__.py",
@@ -484,8 +740,8 @@
},
{
"path": "backend/src/modelforge_api/providers/huggingface.py",
- "bytes": 8555,
- "sha256": "dadf052e979dbe34815dae9a063de99dd2035c762d25d798a570d7ac699e527e"
+ "bytes": 8820,
+ "sha256": "aaefd4dc26b6ffdab8201c316845344bac9d36f27e6018929318d7bbd050db9e"
},
{
"path": "backend/src/modelforge_api/py.typed",
@@ -499,23 +755,53 @@
},
{
"path": "backend/src/modelforge_api/services/acquisition.py",
- "bytes": 39577,
- "sha256": "c68445dcd0f2566f86d0af323349f9c15bdf1c0c0adb1136c868ef5979efc053"
+ "bytes": 39969,
+ "sha256": "1b88dbe95fa7b2246acd39357f7f925979d46594531effd8325eb363bd01b599"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/alert_notifications.py",
+ "bytes": 11375,
+ "sha256": "788f4e18e666e878aa2a0c1b3828a334d49d75c4a663e71e4ce7e02aeb0261a7"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/audit_timeline.py",
+ "bytes": 7685,
+ "sha256": "f7c6ae797701d9a5dbaa77aa032ef4ebf00fc17333183cbe3a68a097f66ea51a"
},
{
"path": "backend/src/modelforge_api/services/audit.py",
- "bytes": 22510,
- "sha256": "df7b12f6694f0a363c9613bd40841230143b794a2ffbfa39508ee0550401f340"
+ "bytes": 22624,
+ "sha256": "58b24032fa92183316600730453e347f6bb15b469b905ec77925bebf8f8c7753"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/benchmark_runs.py",
+ "bytes": 50865,
+ "sha256": "cd8f4399d76db34d0dd055af6e6a03067ff0b582c7d5913e8535d6adf90a40fb"
},
{
"path": "backend/src/modelforge_api/services/capability_evaluation.py",
"bytes": 12595,
"sha256": "53ce2cb1ffc3e92b6cd73fff4fde53a184303a6a71c708c7f4c0f7e4ef61892d"
},
+ {
+ "path": "backend/src/modelforge_api/services/configuration_changes.py",
+ "bytes": 13695,
+ "sha256": "69eb071e4587d79c7e74385c83660439a39a93acac74c51247c8f4e82dd9736f"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/configuration.py",
+ "bytes": 5574,
+ "sha256": "2ea4d11f3aa5a216f86076a72f1aebd7e7ec746a854cb10738606e12c5790055"
+ },
{
"path": "backend/src/modelforge_api/services/evaluation.py",
- "bytes": 69120,
- "sha256": "f99c47d1fcc084c1fb61fbebbce5f842dba01d41c952063e1f17a09668532a92"
+ "bytes": 70665,
+ "sha256": "d8e1c24339b376932b06121781fe0464c1bc74e94ad8721e8036f2a87e7e053b"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/external_creative.py",
+ "bytes": 30631,
+ "sha256": "c921cade1bcff6e3fec3c2e89ac6039d3e50eb87266b90f8e2e1d8fcdfaab0ab"
},
{
"path": "backend/src/modelforge_api/services/gpu_scheduler.py",
@@ -542,15 +828,30 @@
"bytes": 29696,
"sha256": "6581dd6c0a92e93556651c5d28e31038380f60e8ee31f19016d181f6bc3239ed"
},
+ {
+ "path": "backend/src/modelforge_api/services/lab_runs.py",
+ "bytes": 32884,
+ "sha256": "60d4ead6ebef33c0263c30b38f10b0caf73316708995633ed876e09afb4b1f64"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/lab.py",
+ "bytes": 9634,
+ "sha256": "ba5d0c4b062797ae2e156b22dd36ad262e27f6eb57017980fe4f1a8a9987d362"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/lifecycle_serving.py",
+ "bytes": 22476,
+ "sha256": "caad43564696fe96c464b383ee3773a8dd32e51ef7953bfd806da66932a56902"
+ },
{
"path": "backend/src/modelforge_api/services/lifecycle.py",
- "bytes": 82097,
- "sha256": "5f8efef25c1136a83bada4789d8c87b7469b9b70443be367fd3aab8f55445120"
+ "bytes": 94034,
+ "sha256": "ae8e35b1620006b733d5feb0986e46ed9469617b87e78e17f0ae72a9c04b5ab9"
},
{
"path": "backend/src/modelforge_api/services/manifest_registry.py",
- "bytes": 5102,
- "sha256": "0b9071e217d383ab02a617c8281b76b4cb207c5fbec9b1dc3a640d702f576d46"
+ "bytes": 5287,
+ "sha256": "1a3d61da752391c7f80b1f4a68febe3adf58b5ecd4b01ee475887db0623fcd26"
},
{
"path": "backend/src/modelforge_api/services/migration_adapters.py",
@@ -559,18 +860,33 @@
},
{
"path": "backend/src/modelforge_api/services/migration_engine.py",
- "bytes": 67305,
- "sha256": "09f0cf152a8cc6b22e1a2b7ac337670c85ea1e98d403aa01d90a464c929265d4"
+ "bytes": 67395,
+ "sha256": "a3d53fa2189fe3394e456447125300d2495c5838e2cae880c5927f0bc66c314d"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/model_changes.py",
+ "bytes": 36063,
+ "sha256": "c133d3e40838484adc3db0380833ad6ed71da12e3d52bc6f6facbb4667ab74ab"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/model_update_polling.py",
+ "bytes": 3356,
+ "sha256": "e68b69766fd15d104b837b28573d07dcc68c51bcbe61008df1674512a2b3ca85"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/model_updates.py",
+ "bytes": 10847,
+ "sha256": "883b4bd26e9ccf5d3ca6f3c1ccec5357e5a89073de51e650fcb53553fa2190f1"
},
{
"path": "backend/src/modelforge_api/services/node_agent.py",
- "bytes": 31646,
- "sha256": "8c3ad2b07827fe310804805de7ae0030e433f0c616182ef27162cc3a210449a0"
+ "bytes": 32491,
+ "sha256": "224411fb6489f0a46a2a9a4575017e099e6d4be48718749d1ee86f303772caee"
},
{
"path": "backend/src/modelforge_api/services/node_decommission.py",
- "bytes": 39759,
- "sha256": "cfbf38baf9411847121af2a12056fc75f371dff34e6869a9ba65ab846b865096"
+ "bytes": 48319,
+ "sha256": "fd35eb02601738c1a7237be9577791ddb67d3053570951316544a1b991f4e55f"
},
{
"path": "backend/src/modelforge_api/services/node_liveness.py",
@@ -579,28 +895,48 @@
},
{
"path": "backend/src/modelforge_api/services/observability_polling.py",
- "bytes": 1799,
- "sha256": "de4885bf874cb890ced5a2d1c2e9663af6dfb7fed3c06201ecff964c232ab2e0"
+ "bytes": 2341,
+ "sha256": "82f937067e2f65a12817074838cb4d70002135f88078a86440e4014307783955"
},
{
"path": "backend/src/modelforge_api/services/observability.py",
- "bytes": 101067,
- "sha256": "1d4f8fb80ca03c2f83d39048f38a741a5c9dfc968cf66633b0a0a44812756068"
+ "bytes": 106617,
+ "sha256": "8df316a3bbf1decc7ba0b1ecb157c43a5f662e6b852c5f454460c1e6497ab5dd"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/operator_identities.py",
+ "bytes": 14290,
+ "sha256": "e24cf34c31737af8a43587d413838cf7aab0e47ff76cf725a12e599acdab424f"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/operator_password.py",
+ "bytes": 3486,
+ "sha256": "1700f0d645c45f2f01d2dd4e5a6d4f618c0a92e85d90c31f287f39037b284292"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/portfolio_registry.py",
+ "bytes": 25502,
+ "sha256": "37b4fd58e7678254afbe1b0a59591e27a38b93b1c4075eb5098b3d54eb916ad1"
},
{
"path": "backend/src/modelforge_api/services/project_registry.py",
- "bytes": 4069,
- "sha256": "ff23c84ddc668c3ac0c0ff2cf184ff7abab6807c86e520c5b404bb550c7595a9"
+ "bytes": 5479,
+ "sha256": "1bfb2671aa280601c19996137df6145202dc308d473970bd9acf7acc7b554d95"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/projects.py",
+ "bytes": 12438,
+ "sha256": "9520ae38e957467a2801f879c6897ca6500e8eaf022b43b3783f1be24517e514"
},
{
"path": "backend/src/modelforge_api/services/recovery_crypto.py",
- "bytes": 5458,
- "sha256": "c2d1125edf389ea1ee99b7ba48d30c53860099ab9f272e1ba12371e30e47e855"
+ "bytes": 5548,
+ "sha256": "bfc1184f229b5550a9c484fbba66a395158a2ecd37001aad7bd99f9e896feed4"
},
{
"path": "backend/src/modelforge_api/services/recovery_fingerprint.py",
- "bytes": 25605,
- "sha256": "2bb80278269a7d7cf22eb1ce79e32bb70c6947395c24a9b30b77b16fde3fc02e"
+ "bytes": 111786,
+ "sha256": "1ab75c04d0a3a79db578560f986839464b189787031ea2bb591e2ce6b9e0e15a"
},
{
"path": "backend/src/modelforge_api/services/recovery_paths.py",
@@ -614,8 +950,8 @@
},
{
"path": "backend/src/modelforge_api/services/recovery.py",
- "bytes": 135885,
- "sha256": "7f4b9d92176ee93f8cb47c7f91795935747fce2c649faddee55df7e1f54a4aa1"
+ "bytes": 150070,
+ "sha256": "1ab21146188dab6ccfe08cb8f2b64a1b7383652ba636087b98cd66ab67dc61ac"
},
{
"path": "backend/src/modelforge_api/services/registry.py",
@@ -632,20 +968,40 @@
"bytes": 46557,
"sha256": "6a2828d6a9ea4245931b46043837b3954debf87039fc2fe5866fc16432bf3254"
},
+ {
+ "path": "backend/src/modelforge_api/services/secret_slots.py",
+ "bytes": 17099,
+ "sha256": "7459bdf184d5f1b122fef5e93b8de586fafb3911dc388ad152b125e8e5526d26"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/selected_model_comparison.py",
+ "bytes": 31406,
+ "sha256": "89d414fd5a0488e3b5ed434ca52112a5783db997d72770757989416db28c591d"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/selected_replacement.py",
+ "bytes": 9557,
+ "sha256": "e18605b93b3d28fa3e483eddb8b0ec7409fc14eea693968dd09182b2e25dfa7e"
+ },
+ {
+ "path": "backend/src/modelforge_api/services/self_check.py",
+ "bytes": 32087,
+ "sha256": "3734df9a4dbd03c091f39d45dfbc74958cf0ddb60a68f9b4b00d4154ffd95a7c"
+ },
{
"path": "backend/src/modelforge_api/services/serving_reconciliation.py",
- "bytes": 1618,
- "sha256": "402ac4c3fac4688f49f2fa6704bcd05683502107d61784e58341c6719f3ab271"
+ "bytes": 2048,
+ "sha256": "37c88f45bcc89082b9024e19e116f5df52a78aeea1378cf541be359b3a1a406f"
},
{
"path": "backend/src/modelforge_api/services/serving.py",
- "bytes": 172209,
- "sha256": "759f7b436bf47e2c6d57c06163a43315f21c8263f4d231068f21e2f6b9a7ca79"
+ "bytes": 213572,
+ "sha256": "ab4e00530c3a9828a42460379515320f79c0dd8948ef439a40ec07ff444a7a14"
},
{
"path": "backend/src/modelforge_api/services/startup_validation.py",
- "bytes": 22792,
- "sha256": "9c2f424a6907ffdb1cfee1d975462ea6d197a4f3958014ed36236f224e140eab"
+ "bytes": 24548,
+ "sha256": "e46365fa7b493821001dc71536abd3ff60c0dcaf33d822c4fc5896c3695d063e"
},
{
"path": "backend/src/modelforge_api/services/transient_payloads.py",
@@ -654,19 +1010,34 @@
},
{
"path": "backend/src/modelforge_api/settings.py",
- "bytes": 5495,
- "sha256": "ea69ba053fc7f47b8de42ed132f2a81c4796c70d70691f6df3bbfe5064de2618"
+ "bytes": 13902,
+ "sha256": "37e48e58494630cdf2a68052cf1fb319310529058ceff39213f954f4d612a3d1"
},
{
"path": "backend/tests/hardware_fakes.py",
"bytes": 6379,
"sha256": "2dd22a5b69221c29266a42b3edd4ca2f338aae2c4839a04592507cd5c912153b"
},
+ {
+ "path": "backend/tests/http_contract_fakes.py",
+ "bytes": 1382,
+ "sha256": "6d917f09b24d736dc66383b28daaaa0377c15514be5b7945bae4f624f5641a4f"
+ },
+ {
+ "path": "backend/tests/recovery_fixtures.py",
+ "bytes": 1332,
+ "sha256": "086c2640d8f708c15f16cf793d743251c41a43e99a32a8709ac0a98af06d3e4d"
+ },
{
"path": "backend/tests/test_acquisition.py",
"bytes": 17693,
"sha256": "468212ae8778de2ec2bd9f0029d86cc2298127b57a34c72fa1b2e46a3ecd9e83"
},
+ {
+ "path": "backend/tests/test_alert_notifications.py",
+ "bytes": 30761,
+ "sha256": "055b951d518ef19f5e5b5573cb217cb63a777fb4d1e418ea106b1bc1975c9a52"
+ },
{
"path": "backend/tests/test_api_security_m16.py",
"bytes": 16717,
@@ -674,28 +1045,83 @@
},
{
"path": "backend/tests/test_api.py",
- "bytes": 5032,
- "sha256": "0364432872f0cf1a545bc413c76e6735c4c72a21cd72936ad8319643cc46ed5e"
+ "bytes": 9226,
+ "sha256": "68e28b19dc7797e0eb3a4441635c795cd1eae1f23881022d8df7e2eacd03e661"
},
{
"path": "backend/tests/test_audit_integrity_rc.py",
- "bytes": 31604,
- "sha256": "2b5d183f6e4c8c77e62d4bb3a69fde6a85d4a3d0de6a0cc748f6a665bcaca0c7"
+ "bytes": 31716,
+ "sha256": "20cc9376ff7bac7a8b7647bc8816f397e0e10996e66873ca0f0eb48abc87d5ed"
},
{
"path": "backend/tests/test_audit_migration_rc.py",
"bytes": 8662,
- "sha256": "ffe01f291541a8e569f853c0607556e6958d99d0b6a48984373dbc4563866841"
+ "sha256": "0b7d012d9189ff006ec6a146b37499902c95676a93c2f224454f49108e0dad29"
},
{
"path": "backend/tests/test_audit_postgres_boundary_rc.py",
- "bytes": 16134,
- "sha256": "306c4cc83dbde2bace32cb95a88b622bb8893408313826d76d292ad0217bd6ec"
+ "bytes": 21131,
+ "sha256": "55f5b3da542c2c8db6ad60e3e065f0091347974190aee535d75550e88fa8cab9"
+ },
+ {
+ "path": "backend/tests/test_audit_timeline_api_rc.py",
+ "bytes": 8239,
+ "sha256": "ce73060c5e412de4a87f096dac1a8b648ac5ee7f01f15a5782f030dbf10718ac"
+ },
+ {
+ "path": "backend/tests/test_benchmark_run_recovery_compatibility.py",
+ "bytes": 9341,
+ "sha256": "1992bd43aa373ec222391ba4ca38aab40d5edf8083d25f87b68cb4aea396997d"
+ },
+ {
+ "path": "backend/tests/test_benchmark_runs_api.py",
+ "bytes": 6738,
+ "sha256": "28ffc958adae5183baf4ee14b1afbc1e1210d19e679b170176e051319d7e6a75"
+ },
+ {
+ "path": "backend/tests/test_benchmark_runs_migration.py",
+ "bytes": 4059,
+ "sha256": "e90da0b69ae7a04c805332258cf67fd4df28d22199cb12e38735cbfc70729fd8"
+ },
+ {
+ "path": "backend/tests/test_benchmark_runs_postgres.py",
+ "bytes": 18898,
+ "sha256": "6e8238c7419c8d2e7209d21c2f4768ed253da94c2a62d6819b44e44adde3ba3d"
+ },
+ {
+ "path": "backend/tests/test_benchmark_runs.py",
+ "bytes": 16302,
+ "sha256": "67b54406f766daa116f20e39fe7b39bffdb3aa3edabcb6f8f6baa14eaeb89387"
+ },
+ {
+ "path": "backend/tests/test_catalog_query_performance.py",
+ "bytes": 19703,
+ "sha256": "672ea64337efd4f801965ece100a93350339cb5dc308c9dff949d84c3475c26a"
},
{
"path": "backend/tests/test_concurrency_m16.py",
- "bytes": 22626,
- "sha256": "a1be3e9048242af57f150af88d193a30193b91ae83ec4dfaab1d6faa2ce24eef"
+ "bytes": 23466,
+ "sha256": "9c6dfca3c218fa663d3f1f9bacc7a0195634bc71d65b6ed46cc247ea19816ce5"
+ },
+ {
+ "path": "backend/tests/test_configuration_catalog.py",
+ "bytes": 8593,
+ "sha256": "970662b64b19164ecbbcf6edc5df8cc6cba835c43a85024af5c10d207d2bf0e4"
+ },
+ {
+ "path": "backend/tests/test_configuration_changes.py",
+ "bytes": 11529,
+ "sha256": "1a7cb2636fdceeff203888f11f9ffb7382ef31bfe0e5ff5065f7939d2339d855"
+ },
+ {
+ "path": "backend/tests/test_configuration_postgres.py",
+ "bytes": 11205,
+ "sha256": "983801c3e205ef332bf77057d5b2e9918b2b479636337325058158e36ca7592d"
+ },
+ {
+ "path": "backend/tests/test_configuration_recovery.py",
+ "bytes": 3754,
+ "sha256": "6d90f1b841e98b888296779002af4503cbeddd6aebaaeb2ff169cd0d2f812951"
},
{
"path": "backend/tests/test_contracts.py",
@@ -707,11 +1133,46 @@
"bytes": 14769,
"sha256": "8053b435bc0b65f0e2eb3715bd7b32ecd13407bed19dcd782dd6d9b163a57160"
},
+ {
+ "path": "backend/tests/test_deployed_recovery_convergence.py",
+ "bytes": 11082,
+ "sha256": "8b99fa5ce8e2234a3d51ff29658288351e5fb5591817f3bb357ec58c93e68627"
+ },
+ {
+ "path": "backend/tests/test_deployed_schema_convergence.py",
+ "bytes": 10977,
+ "sha256": "b93eaeabf359c99049d8c743a94707e033d4b2f43dbb1cced28763a46b070af2"
+ },
+ {
+ "path": "backend/tests/test_dispatch_context.py",
+ "bytes": 23533,
+ "sha256": "cde876ed023c340d1d7d4851fbb8dd7b522d7884305699fb6db2e3f0c87b0e3e"
+ },
+ {
+ "path": "backend/tests/test_evaluation_project_boundary.py",
+ "bytes": 6259,
+ "sha256": "2bf8486f3912cccc9ff15731d54f0bd83c6f656791c917667946702faabdc79c"
+ },
{
"path": "backend/tests/test_evaluation.py",
"bytes": 25077,
"sha256": "2c91cfcb1ba8281159189d73b44627eaa6248b80f0dc4d4ecd01408da61dcdae"
},
+ {
+ "path": "backend/tests/test_external_creative_admission.py",
+ "bytes": 18263,
+ "sha256": "5ab37397582c4a99720c8817f4182ce54a4b9db863acb73270ad56801cb822b6"
+ },
+ {
+ "path": "backend/tests/test_external_creative_governance.py",
+ "bytes": 5000,
+ "sha256": "12ffbfb3e2da079f3240c977cc0015c897fd5630564a6a27f71f99e71430c0a9"
+ },
+ {
+ "path": "backend/tests/test_external_creative_postgres.py",
+ "bytes": 12088,
+ "sha256": "09f8421a05619fbdb52138de93985490cc851ccb923d2d7a38d3c8a318fe7209"
+ },
{
"path": "backend/tests/test_gpu_scheduler.py",
"bytes": 7013,
@@ -719,8 +1180,8 @@
},
{
"path": "backend/tests/test_hardware_api.py",
- "bytes": 4693,
- "sha256": "bb9c9c03f7467bfc177f1c617219cbcd01ea516ffaa7a2b548e7e2651ac3466a"
+ "bytes": 5485,
+ "sha256": "08f5cfc1a3984465ef2280559ae6452307ff6b80783842f3908a230ec739ec3d"
},
{
"path": "backend/tests/test_hardware_collectors.py",
@@ -739,18 +1200,68 @@
},
{
"path": "backend/tests/test_huggingface_provider.py",
- "bytes": 4783,
- "sha256": "041de3c29a3c4c9c68bd9d141288a11205c41ad4bb7b6e5a350404cbbfea163c"
+ "bytes": 5479,
+ "sha256": "5e51f0ad0b0f35d5d6fd10ecf445fc3ac712562325ed7fdbc27c486689532f0c"
},
{
"path": "backend/tests/test_invariants_m16.py",
"bytes": 24602,
"sha256": "6695f1dbd20cfb78721e814cbb78d37e8952af8f6a093b03b2d80c51a69999d1"
},
+ {
+ "path": "backend/tests/test_lab_execution_api.py",
+ "bytes": 3824,
+ "sha256": "f9d4ca32582ca18bb2995e8ee6de3246fa4b17d964207765c0ef4e0cf7fb6d55"
+ },
+ {
+ "path": "backend/tests/test_lab_execution_postgres.py",
+ "bytes": 7274,
+ "sha256": "c765d0bcb305ab72112a781d101aeaa2ec1a18c3e89e256ba885062518e3a6db"
+ },
+ {
+ "path": "backend/tests/test_lab_execution_races_postgres.py",
+ "bytes": 8671,
+ "sha256": "74e731be302e43557d99b22551cfb7c055889c9f41545ebaf90017e585190e9c"
+ },
+ {
+ "path": "backend/tests/test_lab_execution_recovery_compatibility.py",
+ "bytes": 7369,
+ "sha256": "5a48510fbc0ec846b139b46ea081195104375311eaabb9fa84a18b2d77eb5083"
+ },
+ {
+ "path": "backend/tests/test_lab_execution_runs.py",
+ "bytes": 22219,
+ "sha256": "082ec1c386b0ced6503b50a286a1d1f353e5e85084fa0f07bf4e324f91442535"
+ },
+ {
+ "path": "backend/tests/test_lab_preflight_api.py",
+ "bytes": 3220,
+ "sha256": "cc6e0c7fe952c79971f20e8770afcf9120e2dbee8d44d9f4defad93cf47cc3bf"
+ },
+ {
+ "path": "backend/tests/test_lab_preflight.py",
+ "bytes": 11415,
+ "sha256": "1e671aa52f836411e11d3a3322ecf91b9387300a2970da4e3d83082e2779fb94"
+ },
{
"path": "backend/tests/test_lifecycle_m12.py",
- "bytes": 39865,
- "sha256": "5361d437e9e33e8a58f89b7846f2aebd03c2e6709f1a42778690ad8add23bcb9"
+ "bytes": 47887,
+ "sha256": "5f9b18ce9db232a6686c00630afa3ed11c59210c7ab9aa194b646b2ac3516c77"
+ },
+ {
+ "path": "backend/tests/test_lifecycle_promotion_races_postgres.py",
+ "bytes": 5511,
+ "sha256": "8ba43ea3d4d2da7b90abe739991a40697ac56ec260f28d0dea30f2c6396b7197"
+ },
+ {
+ "path": "backend/tests/test_lifecycle_serving_postgres.py",
+ "bytes": 5131,
+ "sha256": "6713738ef015f1fd83e6a73c64af5233c010df23ff25298014f988dbf9dcfee2"
+ },
+ {
+ "path": "backend/tests/test_lifecycle_serving.py",
+ "bytes": 23580,
+ "sha256": "7f8d46a8d3dd1931e5164d804e572c32481f3dce3cb9d460126db0dca6128f51"
},
{
"path": "backend/tests/test_lifecycle.py",
@@ -762,20 +1273,65 @@
"bytes": 2735,
"sha256": "5237fa504e80fba4daa9839cd0102abe164392b4550507e6b2dc816283b3d365"
},
+ {
+ "path": "backend/tests/test_managed_identities_postgres.py",
+ "bytes": 18427,
+ "sha256": "2f8639c4f10b24f554fd0f7ad79b0d653f6c8589f0b605a836dca9ea0e2d7cbe"
+ },
+ {
+ "path": "backend/tests/test_managed_identities.py",
+ "bytes": 20009,
+ "sha256": "48542dad2b257aa52447d9458805de6a3d423b80ec11568e632b01d2e8504742"
+ },
{
"path": "backend/tests/test_manifests.py",
- "bytes": 1170,
- "sha256": "59b10f85cfeddd4b452f58212d19b00457ed31526601d46061f3969fcaf0aa2d"
+ "bytes": 2358,
+ "sha256": "d6882a7469aa0d0ab2795383d7a5d93aeacc83161be29f82d1676a1fa0881894"
},
{
"path": "backend/tests/test_migration_engine_m13.py",
- "bytes": 21030,
- "sha256": "9f5a065b6f033b918cd5d3b0b5c0d4ec426be80b508b48e39175d451af893c7e"
+ "bytes": 22002,
+ "sha256": "01bbbf0c60c9225f918c0415b29631c559a0ab21531732a0077d7ca67a1922d0"
+ },
+ {
+ "path": "backend/tests/test_migration_plan_index_api.py",
+ "bytes": 1235,
+ "sha256": "2d09813652eba9cfdb8d105f8e930bed59286e493293545e5fc4039bfb910afb"
+ },
+ {
+ "path": "backend/tests/test_model_change_recovery.py",
+ "bytes": 4681,
+ "sha256": "f38b272d8993b13f590c67c7e3b2e54065bc6452e9b0bf5e9f010ce60fe7eb42"
+ },
+ {
+ "path": "backend/tests/test_model_changes.py",
+ "bytes": 7758,
+ "sha256": "366b7be89fb61b0b8b0531f7a983a678c6a1c08a3f53dbb513adf5ccdcf859a3"
+ },
+ {
+ "path": "backend/tests/test_model_update_polling.py",
+ "bytes": 3322,
+ "sha256": "98cfbed44539e4b5963f93cfa29e129140ac519ed61a1c76c806d3e6928cf205"
+ },
+ {
+ "path": "backend/tests/test_model_update_recovery.py",
+ "bytes": 5363,
+ "sha256": "b760175cb9496dd77f1b6f136f597325748cc620cd321446e40d01ad74f31dd6"
+ },
+ {
+ "path": "backend/tests/test_model_updates_postgres.py",
+ "bytes": 7857,
+ "sha256": "231afda3e8fbf8f969684918735da83ee66a2d9e25858eff77dec07d20a5a307"
+ },
+ {
+ "path": "backend/tests/test_model_updates.py",
+ "bytes": 16349,
+ "sha256": "3ce0077037a2ae6e7d19731c50a8963110307376d901501d293476527e560f12"
},
{
"path": "backend/tests/test_node_agent_api.py",
- "bytes": 8885,
- "sha256": "dac38351f2ca7d396e016c2a246704e90658cd31b779668c691aa7adb5229442"
+ "bytes": 9751,
+ "sha256": "7b7ce950283b95f56d0fa9ba66a6f7cca838163ec499a86b9f4aa69e9516d3c0"
},
{
"path": "backend/tests/test_node_agent_control_plane.py",
@@ -784,18 +1340,28 @@
},
{
"path": "backend/tests/test_node_auth_scope_migration.py",
- "bytes": 5654,
- "sha256": "4603da2147fb34fd1a7cc23f485c34ae4d2e496935b24fe81f6ba16f458c0f2a"
+ "bytes": 6727,
+ "sha256": "5450d5f3a63e61fcab219290f9c1494721936f6fdf021fd4a1888f718af45669"
},
{
"path": "backend/tests/test_node_decommission.py",
"bytes": 19196,
"sha256": "5cc619ee2e29e2b0775edc41a5b2bed01e2dca83bc20ae2051d2d19e806ca1a0"
},
+ {
+ "path": "backend/tests/test_node_lifecycle_postgres.py",
+ "bytes": 12705,
+ "sha256": "817433801ca63309281112a814ffc3ed633cac194130589308630a4773b960f2"
+ },
+ {
+ "path": "backend/tests/test_node_lifecycle.py",
+ "bytes": 15849,
+ "sha256": "6e246d8ee3ffe7b271784384326d809dce503a682e9ad25197eea0096cd1cb37"
+ },
{
"path": "backend/tests/test_observability_api_m14.py",
- "bytes": 3730,
- "sha256": "e95d1d4784e1a7c643b12c3c86c612590d466deb581848f423bbccc72e875516"
+ "bytes": 15784,
+ "sha256": "02e8ab30c91cfb800030a16547ddd9c0050fae5af1b2949c81337be7c757b71b"
},
{
"path": "backend/tests/test_observability_m14.py",
@@ -804,38 +1370,108 @@
},
{
"path": "backend/tests/test_operator_authorization.py",
- "bytes": 45731,
- "sha256": "e343cf7feea7a0017958979f37b9e751f7fbb6c3d3e1d03d38834f7d1cfbac32"
+ "bytes": 47992,
+ "sha256": "318da084e73801f5feb069e592052fe4bf60895f34f413141aaa18fd57350296"
+ },
+ {
+ "path": "backend/tests/test_operator_identities.py",
+ "bytes": 7789,
+ "sha256": "89a9c1d22665eba89c2f10fd3468d8cba4fc66945ec0216768bebdd178eba3f5"
+ },
+ {
+ "path": "backend/tests/test_operator_password.py",
+ "bytes": 12341,
+ "sha256": "c8874a9448a83979c3a65608d7f42d00ec8f682f61090b7516ea63ee31eb8d8f"
},
{
"path": "backend/tests/test_packaging_m17.py",
- "bytes": 28340,
- "sha256": "45fc2a02d1ca7ee0208e946b3d9ff4dd9931ab5e77af7c0bec2f930578091dbd"
+ "bytes": 28911,
+ "sha256": "690142eaa73dbc0368906f80de4e398dd8ebbf6cfd0f921f054c619b697d8a25"
},
{
"path": "backend/tests/test_persistence.py",
"bytes": 3666,
"sha256": "34bd0f3ba11e5c13ab52c83e8d20a25327c9e3c9a279940ec57ed90803fb791a"
},
+ {
+ "path": "backend/tests/test_portfolio_reconciliation.py",
+ "bytes": 22853,
+ "sha256": "0d60602a331e9221b999269a7d8a0d7d610b4cb0f3b4830477af7168a15143be"
+ },
+ {
+ "path": "backend/tests/test_portfolio_review_artifact.py",
+ "bytes": 5463,
+ "sha256": "56bb4196088fa5646a2ec528cc58128f2041924094dd3b11a3c868f4030c6f51"
+ },
+ {
+ "path": "backend/tests/test_project_model_management.py",
+ "bytes": 2507,
+ "sha256": "c4f090075daa0f45284e8c448f2b92f82007c84ae8eba5486f3dc9338d7214f1"
+ },
+ {
+ "path": "backend/tests/test_project_onboarding_postgres.py",
+ "bytes": 12761,
+ "sha256": "39e748644bff3c733e3fd925127abb363998b65a6bf08b8c1a6773928b20e1f8"
+ },
+ {
+ "path": "backend/tests/test_project_onboarding.py",
+ "bytes": 16416,
+ "sha256": "41b682beac07032c33ed1f1f11fa0e8a89929f32176f3d3e773084219222b9ac"
+ },
+ {
+ "path": "backend/tests/test_project_python_client.py",
+ "bytes": 4968,
+ "sha256": "05051cc834481ebd1f299f939763c3543894c0eeb255b8863babcea704fde5ce"
+ },
{
"path": "backend/tests/test_project_registry.py",
- "bytes": 1486,
- "sha256": "523b448e7aad80140e98db245838e5b21fee5aff089ac3e13b002a8ff5fba29a"
+ "bytes": 5336,
+ "sha256": "084354692604899bddf7bc404eae540d1bc1b7c6de5fdbada2ba22516d75cd5e"
+ },
+ {
+ "path": "backend/tests/test_rc_candidate_freeze.py",
+ "bytes": 8069,
+ "sha256": "e3924da173b98524ef36e307db5f12ab6c8a4caacc0c1446014e2eca86b87b42"
+ },
+ {
+ "path": "backend/tests/test_rc_failure_node.py",
+ "bytes": 4660,
+ "sha256": "d205046f561010f7ae6e8ff14a876ce7edf426261099956baa4ca6867f0a8fe8"
+ },
+ {
+ "path": "backend/tests/test_rc_failure_storage_modalities.py",
+ "bytes": 4493,
+ "sha256": "f785d5b82ddf4589b281f9e29915ead212e05badcfbece4960904a4d46f3d851"
+ },
+ {
+ "path": "backend/tests/test_rc_failure_storage.py",
+ "bytes": 3192,
+ "sha256": "f73c9f97c83ff81ee78c65185be8adf483a31ead4c3c6a291005f87f0eebd0ef"
+ },
+ {
+ "path": "backend/tests/test_rc_readiness_program.py",
+ "bytes": 6136,
+ "sha256": "556ab9b9a890bd860b0d03986e2bdc0dac79b7a5f1d969ec720ac862988d33ee"
},
{
"path": "backend/tests/test_recovery_api_m15.py",
- "bytes": 12376,
- "sha256": "cd2d85d97d23b34b274ed84a34f4edaacd73fb944d9aa88ab7eeae55c4037fd5"
+ "bytes": 13645,
+ "sha256": "5e8e20febc2ffbf1ee6050b1dd4d0a69bc301f78b3c6b711a2a714ea69071dad"
},
{
"path": "backend/tests/test_recovery_audit_integrity_rc.py",
- "bytes": 28396,
- "sha256": "fd8f2f13e033db37ca4162202d45553660339cfb0f442920f2f9c9e1734fc552"
+ "bytes": 30878,
+ "sha256": "2aaefad8bf90e433c6db842202ca89cbf28ef30bbe04e068125eab3d304b278e"
+ },
+ {
+ "path": "backend/tests/test_recovery_audit_order_postgres.py",
+ "bytes": 2389,
+ "sha256": "0eea13d849418fe97e357ae12d9983ca8a1ce6445e23170e7b8ac35d728a66db"
},
{
"path": "backend/tests/test_recovery_m15.py",
- "bytes": 47808,
- "sha256": "1c615f20cf3ce4cc883faee3fe0e5aec69a7b15d89fed7ac8873fe50eab8a29e"
+ "bytes": 56644,
+ "sha256": "2a790145c066f6350047f3ffc5407356993e2770e59cc624297234f10714efe9"
},
{
"path": "backend/tests/test_recovery_security_m15.py",
@@ -844,18 +1480,18 @@
},
{
"path": "backend/tests/test_registry.py",
- "bytes": 13156,
- "sha256": "0d9fbc3a9007bead038e2133304c851fb0b3971f25721c2948dd5fa942930d8e"
+ "bytes": 13313,
+ "sha256": "e314cec8f6e3bf45ff746000784d796e48d294ee8c2f0b747a2f12ba4086efa6"
},
{
"path": "backend/tests/test_release_contract_m17.py",
- "bytes": 13793,
- "sha256": "3ac7a44bb7af5d836e5d8776684243c2b89a1a284e5d8e30c9f222474fd16729"
+ "bytes": 14596,
+ "sha256": "8c75c6c403e17d5a48cce88925b7e933a955c91b270e5575ee1266a94225e8aa"
},
{
"path": "backend/tests/test_release_provenance_v121.py",
- "bytes": 13504,
- "sha256": "9bc98f1c37abca1981eaaf8f79eaf4d7285f7f588dd09a9716dfb2acf39a0c98"
+ "bytes": 13752,
+ "sha256": "946108f6ef04385c34431eede5ef18e2eb65fe7b01906f7b279003b75bd949ab"
},
{
"path": "backend/tests/test_request_limits.py",
@@ -867,10 +1503,80 @@
"bytes": 32402,
"sha256": "cd3e298cbcff1bf8c5fe4b4d8ff59973422cb754cffa2956188da533d48ec725"
},
+ {
+ "path": "backend/tests/test_secret_slots_persistence.py",
+ "bytes": 4902,
+ "sha256": "1d9edb12ff0bab8f499fed1becb930bca058395469585062c0ba59f9e06aed70"
+ },
+ {
+ "path": "backend/tests/test_secret_slots_postgres.py",
+ "bytes": 9976,
+ "sha256": "64ec7fc485b52bf045c9f25cfdf10da4954408d837d1e833165b876052bb1e22"
+ },
+ {
+ "path": "backend/tests/test_secret_slots.py",
+ "bytes": 13500,
+ "sha256": "6b91f859af017dda9a3e16fc6f45d2ada991b79a19f7ea6d881f311f7936f9e3"
+ },
+ {
+ "path": "backend/tests/test_security_gate_rc_metadata.py",
+ "bytes": 1048,
+ "sha256": "2c50fdbf9753db16d2d3b4fb0cb6a4c36d9b02938b5609f54aa6222b20af9202"
+ },
+ {
+ "path": "backend/tests/test_selected_comparison_api.py",
+ "bytes": 6397,
+ "sha256": "b39fc76a392508b317670824af0bdd39fb7c72a36b0714de899314121ee33b00"
+ },
+ {
+ "path": "backend/tests/test_selected_comparison_postgres.py",
+ "bytes": 13272,
+ "sha256": "56a800a164bd094982ce3645bb9fcfa9bec2895b65cfb41f16eb06b70aa649c6"
+ },
+ {
+ "path": "backend/tests/test_selected_comparison_recovery.py",
+ "bytes": 8724,
+ "sha256": "bd83539ac0c2a567161c32fa589a3b76a82aa83a84eea04c00612a1b652473b1"
+ },
+ {
+ "path": "backend/tests/test_selected_model_comparison.py",
+ "bytes": 28618,
+ "sha256": "535c8c3045acb8374831f679394f0c223e3e6b08596231dc5f884fe2b78c42d0"
+ },
+ {
+ "path": "backend/tests/test_selected_replacement.py",
+ "bytes": 6717,
+ "sha256": "e0010fcdf22d2b4689091163c5135f3a0bfaf60b637dea0ad81c875207d41b29"
+ },
+ {
+ "path": "backend/tests/test_self_check.py",
+ "bytes": 20637,
+ "sha256": "7ffc6c08e4a85991ecb2a6cecc734a3c360c0fc0f1dc6c41dfd31406bfb77f34"
+ },
+ {
+ "path": "backend/tests/test_serving_external_budget.py",
+ "bytes": 3910,
+ "sha256": "4ac8bf52de181d035f9a35fdae930a81fb66cd6aefb8fa8c036c7bf52df8b66b"
+ },
+ {
+ "path": "backend/tests/test_serving_generation_recovery.py",
+ "bytes": 7936,
+ "sha256": "27d581e0657c1429cec2f4ecc4948317903e1de9888bac54744b85db52b692c7"
+ },
+ {
+ "path": "backend/tests/test_serving_generation.py",
+ "bytes": 1455,
+ "sha256": "d27f167d96efcdc584ae5ea01f6823e6fb0b37ebe87156cb8c8841a0cfa952e9"
+ },
+ {
+ "path": "backend/tests/test_serving_reconciliation_loop.py",
+ "bytes": 2074,
+ "sha256": "ecd51f19511c27032ab865c6d1e723c7db347bf45e5ade516a6234a800c8e500"
+ },
{
"path": "backend/tests/test_serving.py",
- "bytes": 47117,
- "sha256": "f4cd578bb7c279947a19b9c8b567f7e8fc89fdfa776c90363c69acd936200bf0"
+ "bytes": 54183,
+ "sha256": "b72031b847967b6608d45638851602a6201cdf25279eb19c18a45b2aef7465e1"
},
{
"path": "backend/tests/test_settings.py",
@@ -879,13 +1585,43 @@
},
{
"path": "backend/tests/test_supply_chain_m16.py",
- "bytes": 21024,
- "sha256": "97001a2cac4d2b5a81554183913298a61fee2149ed532ffd09db6933450409f2"
+ "bytes": 23388,
+ "sha256": "730edbfa568832bdf43d6e430475d00d1ad22cebaa970b5e4381882758fa10df"
+ },
+ {
+ "path": "backend/tests/test_update_preflight.py",
+ "bytes": 3030,
+ "sha256": "b987a006751707af2ffbe756d68805630fd5cc489f5aaa282e4290e3aea3b46f"
},
{
"path": "CHANGELOG.md",
- "bytes": 14273,
- "sha256": "0ecfe356d27f686afc0d57e72f1f2b23ef98b4149b004e53dd02d4ac8bbdc5a9"
+ "bytes": 16059,
+ "sha256": "b18e9cd472100411f8ff4bc3ccab7a63c4e56871f3c912de9f96cca9ad239b0e"
+ },
+ {
+ "path": "clients/python/modelforge_client.py",
+ "bytes": 5337,
+ "sha256": "a7dd4386950462d835ee61d96debdb82dc5a900a769d48a7d9f09d26ae184955"
+ },
+ {
+ "path": "clients/README.md",
+ "bytes": 2463,
+ "sha256": "eb366b3e277317283d397e3c597c848a503e3146b82cf1272946b37504179a20"
+ },
+ {
+ "path": "clients/typescript/modelforge-client.d.mts",
+ "bytes": 841,
+ "sha256": "e2db53ba72a1e4375976c6d503e315236c423e77bf4d9505b171c1ea3983ed37"
+ },
+ {
+ "path": "clients/typescript/modelforge-client.mjs",
+ "bytes": 3841,
+ "sha256": "dd8b9fe225bdd790914d11fdaec44070970c42312d80a73fd8eb80158c703c4c"
+ },
+ {
+ "path": "clients/typescript/modelforge-client.test.mjs",
+ "bytes": 3117,
+ "sha256": "7b152b48e034a44ecff1c5fd385b72b3d978453c32bf43c32a5b8c948b3dccb6"
},
{
"path": "config/benchmarks/examplevision-card-retrieval-v1.yaml",
@@ -907,6 +1643,11 @@
"bytes": 1187,
"sha256": "d3feefbab114117e99e341ac3e87b6033cfbd84129a0b4397fbeabbc28bf0046"
},
+ {
+ "path": "config/capabilities/creative.resource.v1.yaml",
+ "bytes": 959,
+ "sha256": "c322a4b78b07b054804ceabe03db9d924578636f98de36f6e2c2483dafc2113a"
+ },
{
"path": "config/capabilities/document.ocr.v1.yaml",
"bytes": 1442,
@@ -959,13 +1700,38 @@
},
{
"path": "config/public-candidate-unfixed-vulnerabilities.json",
- "bytes": 2146,
- "sha256": "a5765d3322544372b8fc261e3338dc151bd27aabbd4cc0770e3a017583b0eee3"
+ "bytes": 149,
+ "sha256": "30ab2f9f3e893908720a7b615b33bd68b10df79246646c902c871f98f410ac24"
+ },
+ {
+ "path": "config/release/rc-program.json",
+ "bytes": 2051,
+ "sha256": "3d66d5f505e4eddd6f0702c88623ce22b8944fcca549e58f0c02dec8dd8f7044"
+ },
+ {
+ "path": "config/security/accepted-risks.json",
+ "bytes": 50,
+ "sha256": "5c097a68386de41d2978c794dfe3f58abf430dda4fe9242c35d3d8b689fbf2c6"
+ },
+ {
+ "path": "config/security/build-toolchain.lock",
+ "bytes": 2286,
+ "sha256": "d6239937e3cdca9d6489e504246ee2dda2bbbd9c1a5aa847016f7b7efac4f6c2"
+ },
+ {
+ "path": "config/security/gitleaks-false-positives.json",
+ "bytes": 1045,
+ "sha256": "9da827205bbf0cf58b71da108c232b72050a12b471879b87f2825d9f8140d9ea"
+ },
+ {
+ "path": "config/security/pip-audit-toolchain.lock",
+ "bytes": 3239,
+ "sha256": "1e0dbd2142ff0bcbc7e36f4e6586e0979f251f6849d92eedcdccc1c07fe15258"
},
{
"path": "CONTRIBUTING.md",
- "bytes": 1957,
- "sha256": "2f14a8540d88bbfcd768ea59d9286ed80a33847e4e1dea88736c0af49cae144d"
+ "bytes": 2182,
+ "sha256": "fd028ae94b8319b69df1464078c80a72ba6c314b193c1f37096764ba315ae95e"
},
{
"path": "deploy/postgres/init/001-modelforge-roles.sql",
@@ -974,18 +1740,23 @@
},
{
"path": "deploy/postgres/provision-existing-1.2.1.sql",
- "bytes": 2882,
- "sha256": "f45a21a6293f52c0ae9f224c73db9a81ea3908c256c4b9b8784abfbc5795a76d"
+ "bytes": 3821,
+ "sha256": "d7c9b6c9704cb9c6fbc899499c1a09eeaac4a99729bc2b3e1bd40732976330b8"
+ },
+ {
+ "path": "deploy/validation/Caddyfile",
+ "bytes": 225,
+ "sha256": "5b84343907057ca50c51d8ace7457e374745cd2d5dc5daacb71e446219ce1059"
},
{
"path": "docker-compose.backup.yml",
- "bytes": 1830,
- "sha256": "c8d2d33c4e8e34ce5fd3e929b39d77ba7cbe1dac708d0e801a3cf690446e2def"
+ "bytes": 1906,
+ "sha256": "0ad787aa5694bbe1bfafc7cd2db27ec9ebbaf0ef76df72d5806daef86bcaaaeb"
},
{
"path": "docker-compose.dr.yml",
- "bytes": 4087,
- "sha256": "7e3f035ec59bcab083323c0b73150b88dec240cef1eebabe37446f8a5c1aacd0"
+ "bytes": 4116,
+ "sha256": "1d94d6a8bb4eb0fb53eec21ca0f70d7599739f161e5b04b70e1f4eac78b2240b"
},
{
"path": "docker-compose.gpu.yml",
@@ -1009,8 +1780,13 @@
},
{
"path": "docker-compose.production.yml",
- "bytes": 3924,
- "sha256": "8762dad33f2f6d238a158dbe2cb298d447ef646e6d4e594fd38eb0d00b54c60c"
+ "bytes": 3974,
+ "sha256": "79c0f42c4d82bee33be7b5f09917d6dce90e3d2ae9beaabb088eddc562577a09"
+ },
+ {
+ "path": "docker-compose.rc-ingress.yml",
+ "bytes": 627,
+ "sha256": "9b97720e1629e386b05234bcbe1642d060370349ef294a3729f2790f63584c14"
},
{
"path": "docker-compose.runtime-worker.private-ca.yml",
@@ -1022,10 +1798,15 @@
"bytes": 1902,
"sha256": "75b0472a6c7115680cd32b458555570e32e4a6fb28bbb9c1bf1a84d21b7f99a1"
},
+ {
+ "path": "docker-compose.validation-tls.yml",
+ "bytes": 1285,
+ "sha256": "34c92d09c72e81e1b1922545dfa038bcb4e13a789019945a054ba312fda55694"
+ },
{
"path": "docker-compose.yml",
- "bytes": 6878,
- "sha256": "4daa43446f5ae49f072b42e62ad445b1dd0b844c64d765c53ab3dadd5713c74e"
+ "bytes": 8742,
+ "sha256": "14ca82e8e9bced15663b6f13dbd2bdd34b52614b9a3d3952e836e3ea9c00af06"
},
{
"path": "docs/architecture/adr/0001-capability-first-routing.md",
@@ -1262,6 +2043,66 @@
"bytes": 1600,
"sha256": "69a888f0d71226b65de16c496d6679a24721da9f4192e40a2f21617405d1db0c"
},
+ {
+ "path": "docs/architecture/adr/0048-transactional-serving-configuration.md",
+ "bytes": 2414,
+ "sha256": "f56b0d6eaeafb95f1f6e3ba59c1c25384192020c581cd88a7d0c8439340394e8"
+ },
+ {
+ "path": "docs/architecture/adr/0049-named-operator-keys.md",
+ "bytes": 2381,
+ "sha256": "c45b609cdcd7447570140e5dd2f1894696d1b7c5bdf525e8c7c62979f59689d4"
+ },
+ {
+ "path": "docs/architecture/adr/0050-project-onboarding-and-credential-overlap.md",
+ "bytes": 2453,
+ "sha256": "be095fb147e077886521835817c6b2fce607972bddf872c175dfc505d2f54476"
+ },
+ {
+ "path": "docs/architecture/adr/0051-manual-canary-observations-are-rehearsal-only.md",
+ "bytes": 2117,
+ "sha256": "29efd25c1d9a3303ef9126d56fea93adbdbf2656b214248c45d0547cb6c1724d"
+ },
+ {
+ "path": "docs/architecture/adr/0052-managed-personal-keys-and-recovery-admission.md",
+ "bytes": 4600,
+ "sha256": "0e0fc1eb8d75b90c660c127b8c379e34647cf6dd953fc3b7794b82782291ec8c"
+ },
+ {
+ "path": "docs/architecture/adr/0053-huggingface-control-plane-secret-slot.md",
+ "bytes": 4104,
+ "sha256": "3fcd564e9d4bd25e8af2ac3af19caef737c37d7e4b55fd83c818027184a44547"
+ },
+ {
+ "path": "docs/architecture/adr/0054-lab-preflight-before-persistent-execution.md",
+ "bytes": 2642,
+ "sha256": "1879a723778915881dc98f1a7abf98629f17958a9fa08c49ba236e5478f8bc35"
+ },
+ {
+ "path": "docs/architecture/adr/0055-selected-model-update-inbox.md",
+ "bytes": 3087,
+ "sha256": "591b2aa5b61b6ec862296089c64557df85a21cebe47f5e48b14022f0a110449d"
+ },
+ {
+ "path": "docs/architecture/adr/0056-selected-model-comparison-intents.md",
+ "bytes": 6031,
+ "sha256": "51ae8926e58b4024f1e1b25fb17e6c9e84581861756ac4f42266952afc54bda4"
+ },
+ {
+ "path": "docs/architecture/adr/0057-deployed-lineage-convergence.md",
+ "bytes": 3691,
+ "sha256": "86eeb800e6c302f2a3f69f2b2b40535541a0e6ddf278b9834b084a505cf3aac5"
+ },
+ {
+ "path": "docs/architecture/adr/0058-approved-serving-transitions.md",
+ "bytes": 6228,
+ "sha256": "34320994aedd8e9a1ed4986d52ad249b827d7dc1548d9a419ce548e1662b1ea2"
+ },
+ {
+ "path": "docs/architecture/adr/0059-isolated-rc-embedding-cutover.md",
+ "bytes": 3915,
+ "sha256": "5ab963ae008f985edc24f50fc9a2b5bce838f468cc2b113861854b6e77b2a827"
+ },
{
"path": "docs/architecture/ADVANCED_GPU_SCHEDULER.md",
"bytes": 3269,
@@ -1284,8 +2125,8 @@
},
{
"path": "docs/architecture/API_BLUEPRINT.md",
- "bytes": 4171,
- "sha256": "8bfaeec866b37097960c7eca977c49a01a0f48b6f03fd457d55542e786937b62"
+ "bytes": 4776,
+ "sha256": "6a6c88cf3dea93a43be663c1b46add8adef0e3c741584782f66d44387703f787"
},
{
"path": "docs/architecture/APPROVAL_POLICIES.md",
@@ -1314,8 +2155,8 @@
},
{
"path": "docs/architecture/BACKUP_AND_RECOVERY.md",
- "bytes": 6786,
- "sha256": "2d14580955dcf39eea96db714d100a9a74d599a6301021c6344a46034d109dfa"
+ "bytes": 7507,
+ "sha256": "6f8deb5239b2ec03bf1ab935885148478c6d4f1853f387de8d263dc04c2186b4"
},
{
"path": "docs/architecture/CANARY_AND_TRAFFIC_SHIFT.md",
@@ -1489,8 +2330,8 @@
},
{
"path": "docs/architecture/NODE_AGENT_PROTOCOL.md",
- "bytes": 6614,
- "sha256": "07a61ec404e1d5b4c7991d0e2262dad558566080564d0c58bffc0aa2ea5f43a3"
+ "bytes": 7634,
+ "sha256": "b54a5eb239051f68a1890035752e8c8ae704fed3bb0a03a8f19af2b8c7f6dcc7"
},
{
"path": "docs/architecture/NODE_DECOMMISSION.md",
@@ -1532,6 +2373,11 @@
"bytes": 5619,
"sha256": "a5a7e532d8d9a0718608bae04a0f917772e03afb5812ac63160366a32b70564a"
},
+ {
+ "path": "docs/architecture/PRIVATE_MUSE_AUDIO_RESOURCE_CONTROL.md",
+ "bytes": 3382,
+ "sha256": "b677e0223ee877da3de6ad886ee7842070614cf30446cb16545b703b6f3bcecb"
+ },
{
"path": "docs/architecture/PROJECT_BINDINGS.md",
"bytes": 1535,
@@ -1539,8 +2385,8 @@
},
{
"path": "docs/architecture/PROJECT_CAPABILITY_INTEGRATIONS.md",
- "bytes": 1259,
- "sha256": "e1c3911e4e78c238630ede056732ad89f70b41273e20eb6225594fb2ff8871ba"
+ "bytes": 1633,
+ "sha256": "a45d99854e996e1c3579940225fa0acb8fe01c8f5b31c322cd354124a9440568"
},
{
"path": "docs/architecture/PROJECT_FIT_EVALUATION.md",
@@ -1549,8 +2395,8 @@
},
{
"path": "docs/architecture/PROJECT_INTEGRATIONS.md",
- "bytes": 1312,
- "sha256": "756d52ae92b60affafc94026b7024f40eb9aff0721a02b1cc9bdb37c7827f516"
+ "bytes": 1664,
+ "sha256": "33196654fca26d7a75c6a7d24102efc612041f433555631818f5ee5a2b3d005d"
},
{
"path": "docs/architecture/PROMOTION_PLANS.md",
@@ -1689,13 +2535,13 @@
},
{
"path": "docs/COMPATIBILITY.md",
- "bytes": 5703,
- "sha256": "80af6c8bfcc3caca63fe719d2785c06f30b4bb7fe26c6602526fc66497787b75"
+ "bytes": 6141,
+ "sha256": "367b00fa022adacac2ac9f0dbec7dca7ed7d861711da591f8370128e865ba5c8"
},
{
"path": "docs/CONFIGURATION.md",
- "bytes": 19082,
- "sha256": "948888c5f2624dccbbf54ca3cf5878e217b43328bee6ee53ff2b3e8592b509a9"
+ "bytes": 22406,
+ "sha256": "d84f1411ceda37d80284accd9c0ae662c74a012b8e3be0a038bef301f3c77e48"
},
{
"path": "docs/FIRST_RUN.md",
@@ -1704,13 +2550,13 @@
},
{
"path": "docs/INSTALLATION.md",
- "bytes": 7713,
- "sha256": "fac59f8017eb7e65c63341f487615e6456fc2c6d05cffc4440f507b8cf9ab5d2"
+ "bytes": 9422,
+ "sha256": "0bfc4e1c85e204fee586a18d121c1a4a0d534158aa64cf3c037d308319163576"
},
{
"path": "docs/NODE_AGENT.md",
- "bytes": 6145,
- "sha256": "25b8f4a85ea0f5d0a128e12427350ef561bd68d0ebede008479c7a4c649b0e71"
+ "bytes": 8568,
+ "sha256": "f65969f37f5adb6f6e2e0bec700f67aa3131c6d88c1a42a870bac67f7423916a"
},
{
"path": "docs/OPERATIONS.md",
@@ -1722,6 +2568,11 @@
"bytes": 6042,
"sha256": "256fede908692cdfb5c96337708f25b18ef42dd82bd1b84c5589da38f25aa12b"
},
+ {
+ "path": "docs/operations/CONFIGURATION_WORKFLOW.md",
+ "bytes": 3726,
+ "sha256": "20e6b946bf75966151afcd5d038fd047291bdbba31af21e4e7f2399b990243d5"
+ },
{
"path": "docs/operations/FAILURE_AND_HEALTH_MODEL.md",
"bytes": 1410,
@@ -1747,10 +2598,30 @@
"bytes": 3600,
"sha256": "950f3310dedb50f00ff9de56294f92ced3dbb413fb00205bcdee73030c9f7514"
},
+ {
+ "path": "docs/operations/MODEL_UPDATES.md",
+ "bytes": 2488,
+ "sha256": "df91f3bc863ea3f344efee532c68603df7ea047e78f1528dd9688868d910bab2"
+ },
+ {
+ "path": "docs/operations/OPERATOR_ACCESS.md",
+ "bytes": 7561,
+ "sha256": "765e8127679f1b060199e817c0cec50021c9d5aa3419f3c3b21e55b29e287e63"
+ },
+ {
+ "path": "docs/operations/PLATFORM_UPDATE_PREFLIGHT.md",
+ "bytes": 1554,
+ "sha256": "f14d48098499e26d5e0835cafa78035a32bf1022b74bed8fe17c757aeced1249"
+ },
+ {
+ "path": "docs/operations/PROJECT_ONBOARDING.md",
+ "bytes": 4552,
+ "sha256": "a477392cba03a402dc9337f578a652f8e25bd9be3a174ccc2bf1e672d61bf855"
+ },
{
"path": "docs/operations/RC_ACCEPTANCE.md",
- "bytes": 2419,
- "sha256": "a74eab1e570d06de61e62aadd4cc2073282553d0de54f64beb2dc7cd165c3422"
+ "bytes": 7981,
+ "sha256": "553e87ccab96759d44e402879e45b1af73ddf000636ada6a141dc6b0d23c706d"
},
{
"path": "docs/operations/RESIDENCY_MANAGER.md",
@@ -1807,10 +2678,15 @@
"bytes": 2325,
"sha256": "389feddf825a404ea7f010d0fabff8ca0dcb87f091f167cdd10528c9aa3c938d"
},
+ {
+ "path": "docs/operations/RUNBOOK_NODE_LIFECYCLE.md",
+ "bytes": 2010,
+ "sha256": "b1c414f5471451156c458df115fe45e6278b3a7b05d09255f23e40f0d7edf156"
+ },
{
"path": "docs/operations/RUNBOOK_NODE_LOSS.md",
- "bytes": 2561,
- "sha256": "bf8e67789b8a5f57773669312d4fa5e5780ce2ded35e89a1c57ddbe76d3ec297"
+ "bytes": 3187,
+ "sha256": "f9cad0b3b1f65087c964a0b951690a3c977044ecf6261d4ec2631835db9ce4de"
},
{
"path": "docs/operations/RUNBOOK_NODE_OFFLINE.md",
@@ -1832,6 +2708,21 @@
"bytes": 2047,
"sha256": "fec1460f2637a49927759f7b906dbbab76ea69a735dbc67f857ee06b22a6b9f6"
},
+ {
+ "path": "docs/operations/SCHEDULED_BACKUPS.md",
+ "bytes": 2280,
+ "sha256": "e498ee3ef956689e904d63b1ed0156a06c75758a7377f4e9a9d5754f0d83725b"
+ },
+ {
+ "path": "docs/operations/SECRET_LIFECYCLE.md",
+ "bytes": 3794,
+ "sha256": "7020e2b6d726d4b54127a917e03e9a7f5b35157295a7fa6e02a932376184dc38"
+ },
+ {
+ "path": "docs/operations/SELECTED_MODEL_COMPARISONS.md",
+ "bytes": 8841,
+ "sha256": "b691668f17bfc959a700cca222a1a1b30e077afaa80ffa0a1dfdfbd97d6b7d91"
+ },
{
"path": "docs/operations/SERVER_AGENT_DEPLOYMENT.md",
"bytes": 6154,
@@ -1842,6 +2733,11 @@
"bytes": 4808,
"sha256": "a8224bbbc43910a2377c3c41614940fcce88c6ed9a998d3e98d4b24ec6f87068"
},
+ {
+ "path": "docs/operations/VALIDATION_TLS_TEST_PORT.md",
+ "bytes": 3617,
+ "sha256": "a66e931139a120cd8176ed92162b5e038a5a4c7e699b9f213150baf7df2309bf"
+ },
{
"path": "docs/product/MODEL_LIFECYCLE.md",
"bytes": 2921,
@@ -1849,13 +2745,13 @@
},
{
"path": "docs/PROJECT_INTEGRATION.md",
- "bytes": 4942,
- "sha256": "e12d1c2beabf70970fb55aaaf1aa02769fd8c0fa8193793a0f3c84c8f99e7609"
+ "bytes": 5133,
+ "sha256": "468ddec1b058fbc1ecc17bb853396c421c1a5e122cb06f0f3fb3898cb9126d7a"
},
{
"path": "docs/PUBLICATION.md",
- "bytes": 2688,
- "sha256": "b64478a3547b2776c45566dddb54595184247f1a584acb0083d277d36d6f0b3d"
+ "bytes": 3523,
+ "sha256": "4f794719587124bf3bef20e9ff4ae88d4afe3970945b9b395930a49a53102cee"
},
{
"path": "docs/RELEASE_NOTES_v1.0.0.md",
@@ -1884,13 +2780,38 @@
},
{
"path": "docs/RELEASE_NOTES_v1.2.2.md",
- "bytes": 2318,
- "sha256": "9fc39f703b4126c295b8cd2b0822166c0577307c44475e0e20ba0a9e6d61ad91"
+ "bytes": 3820,
+ "sha256": "6c56c13ef7f58d292c9d77c10c4fc7453acefb8f69bb1df58ad1218e25e6b73e"
+ },
+ {
+ "path": "docs/RELEASE_NOTES_v1.2.3.md",
+ "bytes": 2359,
+ "sha256": "184582efb996619c84ad019596f993dd78a04a0f7eb812cd504851fb4da1c584"
+ },
+ {
+ "path": "docs/RELEASE_NOTES_v1.3.0-rc.1.md",
+ "bytes": 1798,
+ "sha256": "7ac793ae81907c16b882bfefcd5f39be937931f546ecd59dd1a6db5400756dab"
+ },
+ {
+ "path": "docs/screenshots/console-demo.png",
+ "bytes": 137762,
+ "sha256": "ed0d6a2aed99c64e950aeb43b7fa4b74f29c2a38a6ddc0f54de074a2db13ac05"
+ },
+ {
+ "path": "docs/screenshots/models-demo.png",
+ "bytes": 92426,
+ "sha256": "5b92ca1863390ff27605c2635cd1b9c3e82d8e3eed6467d6255e3a35629be3b3"
+ },
+ {
+ "path": "docs/screenshots/README.md",
+ "bytes": 463,
+ "sha256": "da570e7443c5ed9da60c45daa62aa80e40a1348a4821812741447d1ae5d5f50d"
},
{
"path": "docs/SECURITY.md",
- "bytes": 4225,
- "sha256": "2af6d5e0ac311ef543e55b0ea2d31fd8c4236cbe4619a6873f9acc38976354be"
+ "bytes": 4371,
+ "sha256": "b2ef8ef5ca20863f8fe4a7fe7ae5358cce4eaaddee8deffb359bc4d4ad75a907"
},
{
"path": "docs/security/ADVERSARIAL_API_TESTING.md",
@@ -1929,13 +2850,13 @@
},
{
"path": "docs/security/SERVICE_AUTHENTICATION.md",
- "bytes": 703,
- "sha256": "a1a5f7b5caf803683a57d0bd230be6d382864c33e5d2c428dfbd47d800104872"
+ "bytes": 985,
+ "sha256": "2f70d75067a142ece7c91ca25750606d457a34b8fad5193c9aec0b1e1f82c33d"
},
{
"path": "docs/security/SUPPLY_CHAIN_RELEASE_GATE.md",
- "bytes": 4524,
- "sha256": "26c0e02b466a0246d48363961475e1e9d2714d5fd9bd7f7a2dc8aa16dc0a6f57"
+ "bytes": 20404,
+ "sha256": "9ce3ee08883ef6ec42ed4ca05d042bce0e46f34580d443087a0fb2d0e338e4d3"
},
{
"path": "docs/security/THREAT_MODEL.md",
@@ -1949,13 +2870,13 @@
},
{
"path": "docs/UNRAID_DEPLOYMENT.md",
- "bytes": 3308,
- "sha256": "23a759d75df84d0580db26a3f7076cb4f6b7c2650826f063f7925d90b137013d"
+ "bytes": 3513,
+ "sha256": "ae1a8c9328a0cc2fb4890f53dd74a697cae2025835a3a324fec4bb3b6672257f"
},
{
"path": "docs/UPGRADE.md",
- "bytes": 10085,
- "sha256": "3f90263b13530ba3afd4758912f820e16f2f1a1f5d7a5c74f1659d49cf35d2ee"
+ "bytes": 14137,
+ "sha256": "ee71624ea69e7f53a2fe1ca4aa44fea36b3599ed518eda899bb5090c80e5f6ac"
},
{
"path": "frontend/.dockerignore",
@@ -1964,8 +2885,8 @@
},
{
"path": "frontend/Dockerfile",
- "bytes": 3068,
- "sha256": "1fb254d786ff9e5799b4efd8f1e4a33a1d7505f8329224f768ab73fc506e5337"
+ "bytes": 3273,
+ "sha256": "2d2553f96da53ef45020188ccb7af7f0f42729263e0b237747e3ae8594eb7dbc"
},
{
"path": "frontend/index.html",
@@ -1974,33 +2895,48 @@
},
{
"path": "frontend/nginx.conf",
- "bytes": 1808,
- "sha256": "2628b4fad935e549f503c31be4503cf93fb6a3a39b81d30740214fa6065d2ae2"
+ "bytes": 2213,
+ "sha256": "908161929b932420d26e8ac027ded882a6c5a546b557146fd96ff2630feee931"
},
{
"path": "frontend/package-lock.json",
- "bytes": 83083,
- "sha256": "e659484f6917736a18c98990be47f683992f0302276ff808e94052447866f02a"
+ "bytes": 83093,
+ "sha256": "08eb405ec1703bfbe8946b37326abe40a4de476e2ccec7295b98d6517be78143"
},
{
"path": "frontend/package.json",
- "bytes": 710,
- "sha256": "efded9d3f0458a78a9da5afd971bd63fe2ff11452a346b25eda8892f67601db1"
+ "bytes": 919,
+ "sha256": "130446e32ba0e5403cb6e611e895592ccfe3133553611b895d66c5dcefdd3544"
+ },
+ {
+ "path": "frontend/scripts/accessibility-contrast.mjs",
+ "bytes": 2325,
+ "sha256": "510c75ba7ff493fc3f5deca10ec8984d6c03d9a92e47023a71a20824e8513925"
},
{
"path": "frontend/security-headers.inc.template",
- "bytes": 1167,
- "sha256": "f2e120277ce59303aa94faca53130600448390d269fb2fac4f2c0254688382f5"
+ "bytes": 1160,
+ "sha256": "41f793a6709ac98b02dbfb787c4409d5a3432212aaa8ab32427a6e7eabd32634"
},
{
"path": "frontend/src/App.test.tsx",
- "bytes": 28514,
- "sha256": "48e654d1850178778acbe38ec54aec1729d7848925ef4ae73f09c402ada8bcbd"
+ "bytes": 24459,
+ "sha256": "53266eadeeda25093ea747ad976f5a83c087d134676bd6a06311d0163eab2e42"
},
{
"path": "frontend/src/App.tsx",
- "bytes": 47369,
- "sha256": "92491fe037b1ceaedd01cc6d7724ba952352f2bd6169be0aaf1ac38992abc5f5"
+ "bytes": 28548,
+ "sha256": "7362e56b605949a193b1c8675a3d5ce02b34aaf460e4136c8ed99a87848802b2"
+ },
+ {
+ "path": "frontend/src/benchmark-workspace.css",
+ "bytes": 15004,
+ "sha256": "f56881ee37ef0d29ecf1e63e29539248a484d874168220be6a28d878d1af12f2"
+ },
+ {
+ "path": "frontend/src/components/AdvancedHub.tsx",
+ "bytes": 2565,
+ "sha256": "46aa2ce1b87701a9e2563b7d1144f98331c7d4d3e1205feeb9629476e22c27e1"
},
{
"path": "frontend/src/components/AdvisorWorkspace.test.tsx",
@@ -2012,35 +2948,120 @@
"bytes": 3388,
"sha256": "068ba7f7f17b81cf927929369257940d674ed495225fedbc485759f62c3b1d94"
},
+ {
+ "path": "frontend/src/components/AuditTimelinePanel.test.tsx",
+ "bytes": 7099,
+ "sha256": "780d80f44ca341b7398b8bf67aadf23fa55aae35d6ea125dbf7775c5300d5aa1"
+ },
+ {
+ "path": "frontend/src/components/AuditTimelinePanel.tsx",
+ "bytes": 8480,
+ "sha256": "8675198e9800ceced61537e8fcda3072a266dfb77e0d4bec3d4c567e8a41144d"
+ },
+ {
+ "path": "frontend/src/components/BenchmarkWorkspace.test.tsx",
+ "bytes": 16923,
+ "sha256": "1c728f960672e19489f06c3fd23c15e77d1992b863c9e4d9a166cfcf2a25e9f7"
+ },
+ {
+ "path": "frontend/src/components/BenchmarkWorkspace.tsx",
+ "bytes": 36326,
+ "sha256": "9d350f9fdfc2fdf2c28caf3a8725290e2038ec5150a305218e8577991ddf8409"
+ },
+ {
+ "path": "frontend/src/components/BuildIdentitySummary.test.tsx",
+ "bytes": 3087,
+ "sha256": "52e4a278188dd0c365c87e7ea5a93875391c725b5c29f877069dbf33f5a2512b"
+ },
+ {
+ "path": "frontend/src/components/BuildIdentitySummary.tsx",
+ "bytes": 2532,
+ "sha256": "84961845c87c33cacd4b7b6dac3941e46729a445d948822d1dd01e59f41f8e88"
+ },
+ {
+ "path": "frontend/src/components/CapabilityDetails.model-identity.test.tsx",
+ "bytes": 2605,
+ "sha256": "1da75bedb660849a0d6a59d0c042d0553213f7bbb9fa30e87f50bcaa433a56cc"
+ },
+ {
+ "path": "frontend/src/components/CapabilityDetails.test.tsx",
+ "bytes": 11698,
+ "sha256": "35a684ecb62053944a52a6b330de44e5cdcc48bb7f0959711ac079dd88c454e5"
+ },
+ {
+ "path": "frontend/src/components/CapabilityDetails.tsx",
+ "bytes": 12206,
+ "sha256": "e60df025b4777e6c0420e401531380f3e0f5c307d2565f72dacad9e0681df5d9"
+ },
{
"path": "frontend/src/components/CapabilityWorkspace.test.tsx",
- "bytes": 13522,
- "sha256": "9568afeea15762f2872bcec1880fbfc0e76131d323e35c5bed8551073952d6a5"
+ "bytes": 23703,
+ "sha256": "755276d604fe1cb3d3e82bfaef5eac1bb4dc844e101b034897789c0f1d5d10a6"
},
{
"path": "frontend/src/components/CapabilityWorkspace.tsx",
- "bytes": 32801,
- "sha256": "901a2d2f0a3b3830e09891f951209e366f206e6b4ac2723523aac3054db8d578"
+ "bytes": 28871,
+ "sha256": "66c1ffebc63f17aa40aab04a8d6e4cbbd8db1515b1c88b46f8f3c2d666bd0c15"
},
{
"path": "frontend/src/components/CommandCenter.tsx",
- "bytes": 18557,
- "sha256": "31f028cc0f3aa9c4ce238751f448a4c9e6215e23cc34006c03470480fd28444e"
+ "bytes": 14752,
+ "sha256": "2871c0bd8c0c414a96a373096fc53d8d4bb80eeb9e7e508657878c8dc8c30881"
+ },
+ {
+ "path": "frontend/src/components/CommandPalette.test.tsx",
+ "bytes": 13314,
+ "sha256": "a3348855d73bc5c1a089dce3e2cfc61497245f8203f9551476c3df3bd6bae7a9"
},
{
"path": "frontend/src/components/CommandPalette.tsx",
- "bytes": 4775,
- "sha256": "b0f8fddf900af56c9a133691daae33f31f37b754d277458aa2942c50653b2a55"
+ "bytes": 15601,
+ "sha256": "ffdc4aecf52ac9a3ee39f38f456c634f0fb6142d3427c61796fbfb91328e3d68"
+ },
+ {
+ "path": "frontend/src/components/ConfigurationEditor.test.tsx",
+ "bytes": 8718,
+ "sha256": "613e79751130dffa65bde55bab80115cd18c0cb88b0b556220483fec7f0ec797"
+ },
+ {
+ "path": "frontend/src/components/ConfigurationEditor.tsx",
+ "bytes": 9433,
+ "sha256": "94a92ee6ecbfcae6f93e7c0a865f054eb1f7ab51ead9a28b615c756c8f4116e6"
+ },
+ {
+ "path": "frontend/src/components/configurationValidation.ts",
+ "bytes": 1652,
+ "sha256": "41dc487b0f2c6360da1937f8155df5659e5744f675636229215527b0f136fb85"
+ },
+ {
+ "path": "frontend/src/components/ConfigurationWorkspace.test.tsx",
+ "bytes": 4654,
+ "sha256": "885169dada3e962893c166e26241fb8399173df65135209ad6bd44975a231095"
+ },
+ {
+ "path": "frontend/src/components/ConfigurationWorkspace.tsx",
+ "bytes": 8379,
+ "sha256": "b792ea7a6757f65c8fa2a6654655fb7594596d24506934615c6b0487f3f57654"
+ },
+ {
+ "path": "frontend/src/components/CreativeResourcePanel.test.tsx",
+ "bytes": 9411,
+ "sha256": "9fb62f769792fa9c4f7bd08695ac21e5e7a09095c7777b9669b900006e485661"
+ },
+ {
+ "path": "frontend/src/components/CreativeResourcePanel.tsx",
+ "bytes": 12486,
+ "sha256": "5ef9cb12df112489514d1bc0308cfd646a868fdd74209ed7622a8009608639c1"
},
{
"path": "frontend/src/components/DiscoverWorkspace.test.tsx",
- "bytes": 7992,
- "sha256": "a45d171db2fa5a6d3a636391d23e30f5ba15cf95b4e276af95a3bc0d079e70f9"
+ "bytes": 26737,
+ "sha256": "1d034f202600655ca9184f523b90bb42cefc91557d77dabb74e07bab8bf61e9e"
},
{
"path": "frontend/src/components/DiscoverWorkspace.tsx",
- "bytes": 14277,
- "sha256": "86f24515aa7ea6d5a213ab6083c46268f37416c2eaa81cefe7ed1a1e5822a8b5"
+ "bytes": 26318,
+ "sha256": "70731b43ed6e7f33c1a4226ef6656d5c829f3ee66ddf0bf415ae88bc0499a62a"
},
{
"path": "frontend/src/components/EvaluationWorkspace.test.tsx",
@@ -2052,15 +3073,70 @@
"bytes": 17225,
"sha256": "385b268d888054dfbaa95da80bb8486d447fd58bf623a903be66a1ee9fba47b8"
},
+ {
+ "path": "frontend/src/components/GpuResidencyDrawer.test.tsx",
+ "bytes": 3719,
+ "sha256": "92ac43b61862050e98cfc82aeb950ed34e0842003a1dccdbec0755018cf7c950"
+ },
+ {
+ "path": "frontend/src/components/GpuResidencyDrawer.tsx",
+ "bytes": 13841,
+ "sha256": "9ddb903dd36c399e65af21e0eef3774076f2fb42f6105bc645a12498ebde6a9f"
+ },
+ {
+ "path": "frontend/src/components/HardwareWorkspace.test.tsx",
+ "bytes": 43720,
+ "sha256": "6e909bd4c058972d35b9466569b48aa5c99cb8374fb223cd1cbfb07955fc541d"
+ },
+ {
+ "path": "frontend/src/components/HardwareWorkspace.tsx",
+ "bytes": 30017,
+ "sha256": "15265c79f7c1248c9bd88246367b73cd6c472c4c201f979159d29da6cdbfc5b4"
+ },
+ {
+ "path": "frontend/src/components/LabExecutionWorkspace.test.tsx",
+ "bytes": 18044,
+ "sha256": "00479b310f6681e093694388434c673e8ce073c17601a4e90bfb0f3e44c581f4"
+ },
+ {
+ "path": "frontend/src/components/LabExecutionWorkspace.tsx",
+ "bytes": 43038,
+ "sha256": "6475c696c79b4a52c87e30275afbfaa46be5cd80aa5b0ea62dcaa69be54cb4c4"
+ },
+ {
+ "path": "frontend/src/components/LabPreflightWorkspace.test.tsx",
+ "bytes": 16872,
+ "sha256": "3098ed99e14e5766ebdf9bf6567bab2cdb5930b18f3f49ad5d25abe9b3152c96"
+ },
+ {
+ "path": "frontend/src/components/LabPreflightWorkspace.tsx",
+ "bytes": 19068,
+ "sha256": "bca70158b1ec39f260441136a89322d0f91a47e6cef13aef086bf953c5cc991f"
+ },
+ {
+ "path": "frontend/src/components/LabWorkspace.tsx",
+ "bytes": 1356,
+ "sha256": "50c3f88940d4675a2f9919e67fac84757e832ce52ea4386c92db70d4ca8be595"
+ },
{
"path": "frontend/src/components/LifecycleWorkspace.test.tsx",
- "bytes": 6253,
- "sha256": "199d1f5bed4a488f89fdcd0df5f2ad7b7494bf5834fc57ce7ebf3ddef4ee3215"
+ "bytes": 7482,
+ "sha256": "04d8392e3ddec590bce005a529baa9a5ca30fe4e2548262abcf641ee0fc91ea4"
},
{
"path": "frontend/src/components/LifecycleWorkspace.tsx",
- "bytes": 23613,
- "sha256": "95b9e37d5fe658b8edcb7b85dddd5917e3a3dd83862cdb40a3a967eef5205920"
+ "bytes": 23212,
+ "sha256": "70ead98c0f4a6717d8c2a201fbe67ca80f50bd21c95527ce7d590f8876058cb8"
+ },
+ {
+ "path": "frontend/src/components/ManagedIdentitiesPanel.test.tsx",
+ "bytes": 13707,
+ "sha256": "ea006d654e332476b608ff0bc1c9f5d9f0ef768e04aceaf0bf60fdfdd6567fa7"
+ },
+ {
+ "path": "frontend/src/components/ManagedIdentitiesPanel.tsx",
+ "bytes": 12981,
+ "sha256": "340ad93452d539ecc8e913f176d094c699130796bf29afd23be57bd95a892640"
},
{
"path": "frontend/src/components/MetricCard.tsx",
@@ -2069,54 +3145,194 @@
},
{
"path": "frontend/src/components/MigrationWorkspace.test.tsx",
- "bytes": 4676,
- "sha256": "bad5355c9862a72081d7d48064bcc7bae7717bbf1ac5b82742bfa674519e2c66"
+ "bytes": 12960,
+ "sha256": "d487217efb8aa2c6c496dc77e6915c54dbe91a74143acb7fbb7031acd9d41cca"
},
{
"path": "frontend/src/components/MigrationWorkspace.tsx",
- "bytes": 11098,
- "sha256": "880960af19c090f458a7e68a43e46a992fdfebf8c212dfd94c2b2ee718d117ee"
+ "bytes": 19656,
+ "sha256": "6638fc382ebe65f8e12bc797758a026e051c6956090ba2976f67d979da191236"
+ },
+ {
+ "path": "frontend/src/components/ModelChangeCasePanel.test.tsx",
+ "bytes": 1212,
+ "sha256": "cf42fdbcdfd57be57e0e5f3f2a3494504eff960b8cc94fea30185eeef86fb63e"
+ },
+ {
+ "path": "frontend/src/components/ModelChangeCasePanel.tsx",
+ "bytes": 13685,
+ "sha256": "b3aae29968901a04d22b1f55971e86c9c1aae4a329534218c08df1c11972d61e"
+ },
+ {
+ "path": "frontend/src/components/ModelsRegistry.project-management.test.tsx",
+ "bytes": 4137,
+ "sha256": "8b89df88d93871214e1df96ebc97a410c07eefc4190226728ca0199f60845c56"
},
{
"path": "frontend/src/components/ModelsRegistry.tsx",
- "bytes": 28410,
- "sha256": "1078c8a8391521319ae9218442942a402c261f16961eef02f79e46295b19e795"
+ "bytes": 20343,
+ "sha256": "a2db52b6957c351dd399990078bb0b4c6acab75ee3748c28582e85dfb56d46bf"
+ },
+ {
+ "path": "frontend/src/components/ModelUpdateAttention.test.tsx",
+ "bytes": 1811,
+ "sha256": "766ecd745cc0a9d75d1f5cbed599965101b8ae9573ba2619e3d1c3c6b3ffc236"
+ },
+ {
+ "path": "frontend/src/components/ModelUpdateAttention.tsx",
+ "bytes": 2242,
+ "sha256": "2ca74d93cb920383ef954cc50dcc953aebce42fa75938057609a2206a3196a40"
+ },
+ {
+ "path": "frontend/src/components/ModelUpdatesPanel.test.tsx",
+ "bytes": 12649,
+ "sha256": "61aa87717fdbc929ef31e06f6c91ac4f952abd65048fdcffe42dd17733905be1"
+ },
+ {
+ "path": "frontend/src/components/ModelUpdatesPanel.tsx",
+ "bytes": 17380,
+ "sha256": "8c029209808f80d9ec1672b25c62b118946923724155f7c89332a1d73bf2deef"
+ },
+ {
+ "path": "frontend/src/components/NodeEnrollmentWizard.tsx",
+ "bytes": 38499,
+ "sha256": "b5466469b57f7e8c2b581c8358b9d304d5c9a977e6388455c179ae15ccd1af3f"
+ },
+ {
+ "path": "frontend/src/components/OperationsCockpit.test.tsx",
+ "bytes": 8189,
+ "sha256": "8a66c5934fec2ddd2de51f34bf1ad21719ca9c52077b85ce741d0aa116f63493"
+ },
+ {
+ "path": "frontend/src/components/OperationsCockpit.tsx",
+ "bytes": 11480,
+ "sha256": "34cc3db7630e258391dd8f70abf24490e9df74b9ad0b87ad6bdae4c1b251220a"
},
{
"path": "frontend/src/components/OperationsWorkspace.test.tsx",
- "bytes": 5663,
- "sha256": "0164c5fdcda15d3ee309e3d5245da678271e138088b4e2edf7fabf53d0dfc775"
+ "bytes": 19072,
+ "sha256": "34469ce7460bee8f5cde92c00a757229de1a8db61cc07dddaf8290e37249ca6c"
},
{
"path": "frontend/src/components/OperationsWorkspace.tsx",
- "bytes": 16110,
- "sha256": "9a2f756f49c670386449bf7052825dadb7640764dca3b6d974385319668fe6ce"
+ "bytes": 20772,
+ "sha256": "ec9c66e9e404a8a6e29571f36b5a9d84e50aead3d06899109a5024e9b1b2fc35"
+ },
+ {
+ "path": "frontend/src/components/PortfolioWorkspace.test.tsx",
+ "bytes": 5975,
+ "sha256": "ae539c2c5e53766bf52558b94a6ce0c88847b99fa6685f7479a793278a26ae56"
+ },
+ {
+ "path": "frontend/src/components/PortfolioWorkspace.tsx",
+ "bytes": 9360,
+ "sha256": "4377088bc8716476c63eaf95d888c9731b7b09d0444987e101dc5916f0f081fb"
+ },
+ {
+ "path": "frontend/src/components/ProjectFirstWorkspace.tsx",
+ "bytes": 17048,
+ "sha256": "4472c09eb33c94209dd42d271598da046a8bdffe1706fd89a59cf78304702390"
+ },
+ {
+ "path": "frontend/src/components/ProjectOnboardingWizard.tsx",
+ "bytes": 11011,
+ "sha256": "745109ed6319b962e3e1bd3187c24ed686703850cd94af3961f9a92258b025ec"
+ },
+ {
+ "path": "frontend/src/components/ProjectsWorkspace.model-management.test.tsx",
+ "bytes": 3505,
+ "sha256": "e7c21f76b9e34b19b8fd46cd71a4c385d6a416e637b36ba5c49b068b35561b36"
+ },
+ {
+ "path": "frontend/src/components/ProjectsWorkspace.test.tsx",
+ "bytes": 8823,
+ "sha256": "65a55a6abe0b40a1eefae6f6723c8e2433ee271d445e07c30f38f766c6d5807c"
+ },
+ {
+ "path": "frontend/src/components/ProjectsWorkspace.tsx",
+ "bytes": 23828,
+ "sha256": "0d8de83937450668cb0207f404420de4e1fe8a705b1dbd546c6bfb3f8a845d2f"
+ },
+ {
+ "path": "frontend/src/components/PromotionGate.test.tsx",
+ "bytes": 10210,
+ "sha256": "87c042d83c44f985dc1189a9a494fe49e948dc606222d7b7e94af11ad9239684"
+ },
+ {
+ "path": "frontend/src/components/PromotionGate.tsx",
+ "bytes": 8577,
+ "sha256": "3aa1212dcbab187cd897f78cd12a0d5932ecaaa72286978d2db262790e349924"
},
{
"path": "frontend/src/components/RecoveryWorkspace.test.tsx",
- "bytes": 17790,
- "sha256": "f03c603258204571c1abbf9b1fababe1e48dcf3e6f189ad4e081c4cfce82ce17"
+ "bytes": 22432,
+ "sha256": "6661bd328ec8aab6a550b016372bf570affc9e6f85d917bea4980d3a93615891"
},
{
"path": "frontend/src/components/RecoveryWorkspace.tsx",
- "bytes": 19633,
- "sha256": "8f8073992684882f8c0513d595181ef0e0935a7dd09604f697a4f4cb11b7ad16"
+ "bytes": 23115,
+ "sha256": "9b724361773efbce6ae4141a0c7e665494207112858c1d2cf712884bce44583a"
},
{
"path": "frontend/src/components/RuntimeWorkspace.test.tsx",
- "bytes": 7647,
- "sha256": "1c73ef3a23011362d93b1ce7bc5d823ab82560f18951d008d41cc870c8b3c482"
+ "bytes": 19196,
+ "sha256": "6e9aa5b3526ee733c2980f6fb501d6c6c84b4fd5571b163f1e3f429f2ee2856b"
},
{
"path": "frontend/src/components/RuntimeWorkspace.tsx",
- "bytes": 11977,
- "sha256": "4d7c5a352d200529c1a8124848d24b010f5539e760d89b5ed9e4beda43ff8312"
+ "bytes": 16631,
+ "sha256": "ba39634e337b3b5718cc70f9baa4128af34e7dbb57536c61a36ddf7a42695b48"
+ },
+ {
+ "path": "frontend/src/components/SafetyDialog.test.tsx",
+ "bytes": 1960,
+ "sha256": "66a69967036a496e7a8e7886aa518b01333eea938f40d880a416127fc6755fa5"
},
{
"path": "frontend/src/components/SafetyDialog.tsx",
"bytes": 4599,
"sha256": "6e521026287cb79964b4ea01d60a01a4499ec7c8a25e6f833c420ab63c85fc40"
},
+ {
+ "path": "frontend/src/components/SecretSlotsPanel.test.tsx",
+ "bytes": 7227,
+ "sha256": "8a348668a879c96344a29ca5d14f2b12e95539d9e940ff6f92685630b5e7b289"
+ },
+ {
+ "path": "frontend/src/components/SecretSlotsPanel.tsx",
+ "bytes": 8539,
+ "sha256": "dc00ba573d50cfd2537971b3d358fde663bc69743b2540b73cd139cc671cdd2a"
+ },
+ {
+ "path": "frontend/src/components/SelectedModelComparisonsPanel.test.tsx",
+ "bytes": 7786,
+ "sha256": "41c34ee9badf7d1e83d42027b08f2160fc0e32d321ec237bddd1b9b14dfd0e4b"
+ },
+ {
+ "path": "frontend/src/components/SelectedModelComparisonsPanel.tsx",
+ "bytes": 12980,
+ "sha256": "c5f208fbacaf9c560611f0cab3a8e9a1807a05b11f1b530267a81b7a4b4541e3"
+ },
+ {
+ "path": "frontend/src/components/SelectedModelReplacementPanel.test.tsx",
+ "bytes": 4553,
+ "sha256": "e2d901207f5f2a7502bb19849c07f23d65fcabf566589ebd3336ac50e2c66d3f"
+ },
+ {
+ "path": "frontend/src/components/SelectedModelReplacementPanel.tsx",
+ "bytes": 4484,
+ "sha256": "a8638b99da6a5de220377031d80aebb86f93a7fffda0b7299407c0ddc0e0966e"
+ },
+ {
+ "path": "frontend/src/components/SelfCheckPanel.test.tsx",
+ "bytes": 8618,
+ "sha256": "2911b120cf3a6abf4f7ac1ae4c5c1b22a39a00f0eeb0e441f8bb1facf0703ff1"
+ },
+ {
+ "path": "frontend/src/components/SelfCheckPanel.tsx",
+ "bytes": 5286,
+ "sha256": "10fd0118439b702321be0aed687a4135e80879425aa3c43ab186c8c2869c1736"
+ },
{
"path": "frontend/src/components/TabList.test.tsx",
"bytes": 3382,
@@ -2127,6 +3343,31 @@
"bytes": 2418,
"sha256": "7a8d9de39066d74f737dce5c49c88621b161a86c12871dc223a5acb4178519f6"
},
+ {
+ "path": "frontend/src/components/TroubleshootingPanel.test.tsx",
+ "bytes": 5111,
+ "sha256": "ac6cf899bd0e10342022aba9841dfd7ddeb6ba16d503c0e23e89610433a8ee22"
+ },
+ {
+ "path": "frontend/src/components/TroubleshootingPanel.tsx",
+ "bytes": 7337,
+ "sha256": "cde75d0f716d74e62b71d14ffdc78ad0cdffb2f42eb63aeb8ab00ae6d7d3f4fd"
+ },
+ {
+ "path": "frontend/src/components/UpdatePreflightPanel.tsx",
+ "bytes": 5629,
+ "sha256": "d7daf9ddd57f3e46fd1a4a7ffd39058f2c74b7e1897300b7699cc2b9a91ddef9"
+ },
+ {
+ "path": "frontend/src/components/WorkspaceLoadBoundary.test.tsx",
+ "bytes": 3324,
+ "sha256": "b38ac7de44c4ee216e7c8105f623c78d8962df6637b0a0c74af1c88d11757905"
+ },
+ {
+ "path": "frontend/src/components/WorkspaceLoadBoundary.tsx",
+ "bytes": 1320,
+ "sha256": "b190a7b3a947d8f9ed4e8c5ff30ad94c5cb3e9f5774a40e74fe4a526e1776ae2"
+ },
{
"path": "frontend/src/lib/api.origin.test.ts",
"bytes": 2105,
@@ -2134,28 +3375,98 @@
},
{
"path": "frontend/src/lib/api.test.ts",
- "bytes": 5557,
- "sha256": "4c2afdc96da88608c8d59c885f6fce52a6d01da12eed52b669ed8771663733c2"
+ "bytes": 21299,
+ "sha256": "0eccf8f7386c63decfdd4a6c5f3ec7008e6a5e236c79d093e54d5ff8d3fbdbdd"
},
{
"path": "frontend/src/lib/api.ts",
- "bytes": 27782,
- "sha256": "a3f52e4edb4e11af93376ea6d27436e0febe162f7afea08bfce1c0c045fc9dcc"
+ "bytes": 53206,
+ "sha256": "c39df3e5742e4c266ebf050e4afd538c214ffc3b4053b1c112af7362b4a90d8e"
+ },
+ {
+ "path": "frontend/src/lib/build-identity.test.ts",
+ "bytes": 1372,
+ "sha256": "0be13e961c30add5d045bcf9fc7406761ec684d982bf1c86c0993446bc6dc893"
+ },
+ {
+ "path": "frontend/src/lib/build-identity.ts",
+ "bytes": 1834,
+ "sha256": "0ed8997e10f757664c688c7e9555bdd720703b4ba073e1df934370668bcf2268"
+ },
+ {
+ "path": "frontend/src/lib/model-management.test.ts",
+ "bytes": 4891,
+ "sha256": "431bd194cccddaf0a8df55d1e0d2f024081db63d09942a65d2f274778a5940ae"
+ },
+ {
+ "path": "frontend/src/lib/model-management.ts",
+ "bytes": 1959,
+ "sha256": "b9fc6d1cfba3721fec7d8d1b6d78ef2a852ed22e18358b23c1e86737deeeb92b"
+ },
+ {
+ "path": "frontend/src/lib/project-integration.ts",
+ "bytes": 2184,
+ "sha256": "954b0833c2d1e78d369cad90450600d69c28750cea1f47272a2b992cc995673b"
+ },
+ {
+ "path": "frontend/src/lib/troubleshooting.test.ts",
+ "bytes": 12043,
+ "sha256": "f78e58800b1d1af215b6b0836cc88782f3faedd02a14239b15bf1cba10a7d6d0"
+ },
+ {
+ "path": "frontend/src/lib/troubleshooting.ts",
+ "bytes": 22890,
+ "sha256": "75c38b8fd286042694a37955952c466b589b4d1ded9dc197bc693bdc8b15eb29"
+ },
+ {
+ "path": "frontend/src/lib/user-language.ts",
+ "bytes": 6345,
+ "sha256": "0e3d634f07b2b7d6a6632282d75df3b63e31c007de584e2e7a4434e38dfdaa42"
},
{
"path": "frontend/src/main.tsx",
- "bytes": 239,
- "sha256": "a0d8cf21a990ece4cf836c1a252e0e63e1ffc326ee7f87efce1725b63ef7c8e7"
+ "bytes": 389,
+ "sha256": "0706ec06720e6db8c63f3df7b30416f061270efd13bdab8d51c7f2f54c40ba5b"
+ },
+ {
+ "path": "frontend/src/model-management-types.ts",
+ "bytes": 2876,
+ "sha256": "3f7c5a7f62f07ef9af60bba99b4acc53dcf7a76beac0a252876a8cbfd654c81f"
+ },
+ {
+ "path": "frontend/src/navigation.test.ts",
+ "bytes": 1748,
+ "sha256": "2e2f0a1218c7f5dc83c3230abfbbfaf81f501017ea1827e264e2abe4e1de5499"
},
{
"path": "frontend/src/navigation.ts",
- "bytes": 5470,
- "sha256": "1edefd5cd5b837cc61148e9cba27659a197a4aaf650af789e36a22c6753ac5d8"
+ "bytes": 7420,
+ "sha256": "4e4e4257dd656ad6cbf5a0208f48a7224838912ec37307a1784b339b7abe4d35"
+ },
+ {
+ "path": "frontend/src/product-ux.css",
+ "bytes": 14428,
+ "sha256": "dcc655f2c20505f53126476d8d431ebc9d15a733ea5ecf7181723c90dd10bbd3"
+ },
+ {
+ "path": "frontend/src/project-wizard.css",
+ "bytes": 4646,
+ "sha256": "bcfed2ef2001c6862adca7633b2c1df7107d24c471a2ac54d1e86b77820caad8"
+ },
+ {
+ "path": "frontend/src/simple-workflows.css",
+ "bytes": 23661,
+ "sha256": "b773696b1f7050454a348c5d57dd20ece4371494da4dbe4aabfa27cbc3b0f98c"
+ },
+ {
+ "path": "frontend/src/simple.css",
+ "bytes": 10565,
+ "sha256": "354ebbd7cea46fad093e5367767f4ec021eaea1744ea16ebb8e19a93bc34a46a"
},
{
"path": "frontend/src/styles.css",
- "bytes": 98483,
- "sha256": "8f607ef346b09b7c77d9618a09a6a6ed8d3471cadc27e5f5e5c92ed9f6bb23fa"
+ "bytes": 125852,
+ "sha256": "ca04602b000c36b0148de2d8ceb7ba6415b3ce3dafad788ffb25dbb9dd311c5c"
},
{
"path": "frontend/src/test/setup.ts",
@@ -2164,13 +3475,13 @@
},
{
"path": "frontend/src/types.ts",
- "bytes": 43701,
- "sha256": "efb34f6a6cf5dd9aeee4bd72ef3e9fcd3e3894ea55b8166db82006bd0dfaf873"
+ "bytes": 65025,
+ "sha256": "866011f26fccb4c185dedc8522132ebbc555f7d5056e4bf9a020d56e741a6262"
},
{
"path": "frontend/src/vite-env.d.ts",
- "bytes": 38,
- "sha256": "65996936fbb042915f7b74a200fcdde7e410f32a669b1ab9597cfaa4b0faddb5"
+ "bytes": 387,
+ "sha256": "edf8c4f302aa857945780ec02c2b063c4b5c530a950395b8a9d15d755f541fe8"
},
{
"path": "frontend/tsconfig.json",
@@ -2179,8 +3490,8 @@
},
{
"path": "frontend/vite.config.ts",
- "bytes": 1019,
- "sha256": "39196ecbd0c830b1bced59602aa4ab3ee6c52f38b547b3648ac584591ae03e26"
+ "bytes": 3027,
+ "sha256": "0f85bafc1a3033d574aafe0e4e2832ed1aaa5c31826a356764dc70500f1d03f3"
},
{
"path": "LICENSE",
@@ -2189,18 +3500,18 @@
},
{
"path": "node-agent/Dockerfile",
- "bytes": 2269,
- "sha256": "1357b603034e01caea72d4e758123719abc89a76e3054c4c361326efbef8c3f0"
+ "bytes": 5038,
+ "sha256": "2e94100a8bc3bd5040e9410cfd5a6082a138369cbaa4928bf5bd016afe7eed96"
},
{
"path": "node-agent/pyproject.toml",
- "bytes": 1421,
- "sha256": "c83a2c79a0c5f68edea89a6e9b60760da8453f54a8eaadfe164871533def275a"
+ "bytes": 1440,
+ "sha256": "6e7a73b4b211ae5f933fcfa8432b33f46aff1a9fdcaf142cdfc88446fc52f47a"
},
{
"path": "node-agent/src/modelforge_node_agent/__init__.py",
- "bytes": 22,
- "sha256": "bae7f854db53340f777cc86802eafd61a7f368a2459dccd8f87d524824ae4554"
+ "bytes": 27,
+ "sha256": "59619b05de8ed2b0cac4b2976b43a676fbac3d4c90f194a691aae3686eab96da"
},
{
"path": "node-agent/src/modelforge_node_agent/acquisition.py",
@@ -2229,8 +3540,8 @@
},
{
"path": "node-agent/src/modelforge_node_agent/state.py",
- "bytes": 1645,
- "sha256": "4091878754fc04ea51489f17a333887a070d7a98fc57cc852818c97d2ce175a4"
+ "bytes": 2106,
+ "sha256": "2931aa6d619b32002e3a3513eb442258cbafa527757538687e2786474c3bac8d"
},
{
"path": "node-agent/src/modelforge_node_agent/transport.py",
@@ -2244,8 +3555,8 @@
},
{
"path": "node-agent/tests/test_agent.py",
- "bytes": 14180,
- "sha256": "b888aa2e95d1a41173fd48e9c6c25db91e21592fabb899eba2338f16f11a932f"
+ "bytes": 15088,
+ "sha256": "9843205bd900b9037bac8d6d9c40913260c544df7d852f4810c16ba6d04fa3ab"
},
{
"path": "node-agent/tests/test_main.py",
@@ -2259,23 +3570,23 @@
},
{
"path": "README.md",
- "bytes": 6078,
- "sha256": "b1004ea4aa619be307b28aa77bfa06b832c8653fa6c690146b79e93a7706122c"
+ "bytes": 7383,
+ "sha256": "4c59c74659ade42f194b16ec8d9db7849e12e42c4cc29729af3b90657c104f94"
},
{
"path": "runtime-worker/Dockerfile",
- "bytes": 1714,
- "sha256": "e521bba10dcce424068eba51f4b8af97bf2ca4a01d403b69b264dddc930bd225"
+ "bytes": 2370,
+ "sha256": "aa3d92ce95d33633dae74327428cf40e414484fa4a2b6a76b2fef99d270e9d74"
},
{
"path": "runtime-worker/pyproject.toml",
- "bytes": 2194,
- "sha256": "821a45d7f2318cb73220556c5ceaf81e5ea479601e7a43ebfe20c8ab06aea3d2"
+ "bytes": 2319,
+ "sha256": "4eb4ff6b1568d46e66b6f4b4dfdd18a7d2aecae2c612689ab777b4d5c931b581"
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/__init__.py",
- "bytes": 22,
- "sha256": "bae7f854db53340f777cc86802eafd61a7f368a2459dccd8f87d524824ae4554"
+ "bytes": 27,
+ "sha256": "59619b05de8ed2b0cac4b2976b43a676fbac3d4c90f194a691aae3686eab96da"
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/adapters.py",
@@ -2284,8 +3595,8 @@
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/contracts.py",
- "bytes": 5716,
- "sha256": "52a1d931e03253e25ad0df310536f6080ba9cda6d067ec3cd3121ce7d7da7484"
+ "bytes": 5756,
+ "sha256": "402517f7a7b93a8c6462a00d74bbb6236b77d7f1960ae9a79f394e00761e2fd9"
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/main.py",
@@ -2299,8 +3610,8 @@
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/serving.py",
- "bytes": 18445,
- "sha256": "17723e0ed89254a26f6f5ef7ce82a7ab05260c419768759415e7e3b02b508708"
+ "bytes": 18911,
+ "sha256": "9eabb0bd251989f7cf9fbebcc313e7e05a50572478bb8ebef4d47d5d869d9c86"
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/settings.py",
@@ -2309,18 +3620,18 @@
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/transport.py",
- "bytes": 2949,
- "sha256": "5c57a2e411c44aee6358870f2fc65c12938cb2f2b9cc8bb9fcb35a0dd3127ba3"
+ "bytes": 3684,
+ "sha256": "52f13a6a6eda85d11cbbb7efadd26f19280c80b23b90102b48c4a6b9166f68ac"
},
{
"path": "runtime-worker/src/modelforge_runtime_worker/worker.py",
- "bytes": 9185,
- "sha256": "bcfd7d63f5ff108fd14d7ea23c6ab42ce6a5e1c6ee44a5d92e258f87787b94a6"
+ "bytes": 12624,
+ "sha256": "15f817df260cf3db094bc9d6a508b5346e2ddbcd78f1a5d0d830351cafa747b8"
},
{
"path": "runtime-worker/tests/test_worker.py",
- "bytes": 21438,
- "sha256": "a3f4e00f03a206a04b1db2d3a25046dbcd1100a0dc0d62d4aea5c7f13e5e2f89"
+ "bytes": 24596,
+ "sha256": "fa67d8c73d132598a55e7becdf9a388505e8bf375991b4a733aa4364b87014d1"
},
{
"path": "scripts/bootstrap.py",
@@ -2329,23 +3640,23 @@
},
{
"path": "scripts/check.ps1",
- "bytes": 1690,
- "sha256": "ca6671659a7349a8caa2bba65e85facbfc8a716028b54198338f4cdbf8286e6f"
+ "bytes": 3561,
+ "sha256": "17f952b7f7f197155e3af194ebcc10d3b974b5f643715c5852694a5ee9113749"
},
{
"path": "scripts/check.sh",
- "bytes": 2206,
- "sha256": "118ec71a31bb919b71958a477c0b48e0c251b73c12c18eac9fe0be0d1c59b60a"
+ "bytes": 3380,
+ "sha256": "b5576ac74e1b4902a51c38da88c4811ac94c582e5588631c79015503ff5fb244"
},
{
"path": "scripts/export-public-source.mjs",
- "bytes": 8885,
- "sha256": "e8c3a3311c1528e4da5fdc2f1396dc95d188e26b5031591fc81da88447a658ac"
+ "bytes": 9540,
+ "sha256": "bde0c53d4ca1d20dcd4d283215fce82a9b374e45cc4d71d1093742d529d55a44"
},
{
"path": "scripts/generate_configuration_docs.py",
- "bytes": 11151,
- "sha256": "0fc3f0a702e68d0721a08b6e9f518ef07b5222f7387b8c5da3b4e1a299526c82"
+ "bytes": 11439,
+ "sha256": "42486cd0af5bef01b11d856c000d6369f9dcc946fea3b36312bc02c2b5667130"
},
{
"path": "scripts/m14_live_rehearsal.py",
@@ -2394,29 +3705,64 @@
},
{
"path": "scripts/modelforge_scheduled_backup.sh",
- "bytes": 2661,
- "sha256": "e7b5f8a9ad5b66a3fe97247731a7648aa164561229f257b92514d5f7578ce830"
+ "bytes": 2758,
+ "sha256": "0f7ca0ac3abfb9e1cdbc58af5fbbe9c4c2ab6bfa3bb5288f8a7013d7b386ce6c"
+ },
+ {
+ "path": "scripts/operator_identity.py",
+ "bytes": 1380,
+ "sha256": "406b008066db0ba0234725efb8b48f3b0120ef795312ac20cd8e6d3f9cf3f6e6"
+ },
+ {
+ "path": "scripts/operator_password.py",
+ "bytes": 677,
+ "sha256": "27a89c02e022f3d71e5b15c065a63a75fa2f1fa13b4185da4368c1c5a303ec77"
},
{
"path": "scripts/preflight.py",
"bytes": 8997,
"sha256": "8e1dedf444f128114c37a097a1e1cf70ed4bd6fb81aa4aa1c24ec322db4a7dac"
},
+ {
+ "path": "scripts/rc_isolation_preflight.py",
+ "bytes": 17813,
+ "sha256": "03f2a0b6534997cd09493d53b0c6c07cde1c3c2c12b1174a12f059c8d161dcd7"
+ },
+ {
+ "path": "scripts/rc_readiness.py",
+ "bytes": 6811,
+ "sha256": "dd8cdced7f50a1c9926a65e0d203e8035946f58e462699d8bac6a1a6e33c801b"
+ },
{
"path": "scripts/rc_server_acceptance.py",
- "bytes": 15359,
- "sha256": "0baef82d94137305acb773e9484e5729ae9ce928c352291ef15a70cca991299b"
+ "bytes": 18350,
+ "sha256": "6f013ec2c4925876a844c5bfa82da47a6d194e5e627620cbb5e8121742692ef9"
+ },
+ {
+ "path": "scripts/rc_stack_retention.py",
+ "bytes": 16137,
+ "sha256": "fefea79831b84e2b989a74f6d566664815a02719fdd6088790b5d94ff7662677"
},
{
"path": "scripts/release_build.py",
- "bytes": 19029,
- "sha256": "1b32834ab4fa2270cf8d10905bb6fd74c2f00503f134878003a46e0ff6cbc20d"
+ "bytes": 34085,
+ "sha256": "716680b0dceca744a98b9a94fc529c7ed31fb0ce99861faef032027402987570"
},
{
"path": "scripts/release_image_acceptance.py",
"bytes": 7747,
"sha256": "77b2c28c3bd87386715e6aeb6d9503ed1154ccd6bb32d06abcd8a1d141421ddd"
},
+ {
+ "path": "scripts/security_gate.py",
+ "bytes": 164848,
+ "sha256": "61a865c3985b6b6e8266852fd1ff2d93ec6588afaf830eb6d3eb1ed2bad7c8b9"
+ },
+ {
+ "path": "scripts/test_modelforge_scheduled_backup_auth.sh",
+ "bytes": 3449,
+ "sha256": "77d804f7d3b7512b3bbb4f44a355460f44ee24c115732f1111a0dd224bbfb430"
+ },
{
"path": "scripts/upgrade.py",
"bytes": 12360,
@@ -2429,8 +3775,8 @@
},
{
"path": "scripts/validate-public-source.mjs",
- "bytes": 3702,
- "sha256": "37ec733fcfe126dd595187d08b775007278216ec7b5f8afaf4e3dc62b41bca18"
+ "bytes": 4945,
+ "sha256": "1e936dec9e8648afe65c20b0bc64dd1f468aa9306fd50be8e8f9a0605c4a0ceb"
},
{
"path": "SECURITY.md",
@@ -2439,8 +3785,8 @@
},
{
"path": "VERSION",
- "bytes": 6,
- "sha256": "c62a5cc61ac4525935753f5ec284c0a052a4b1da4b3a3580dce2b7d77a2154ae"
+ "bytes": 11,
+ "sha256": "82d60e10e0d6a08ee84288108bcb9b15e933935160a975ccd55136660bf75ad7"
}
]
}
diff --git a/README.md b/README.md
index fcbbc9c..2c8ebcb 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,26 @@
# ITWorx ModelForge
+One stable capability interface for models running on your own hardware.
+Console tour · Workflow · Capabilities · Quick start · Architecture
+
ModelForge is a self-hosted control plane for running AI models on hardware you control. Applications
ask for a stable capability such as `rag.embedding@1`; ModelForge selects an approved model revision,
places it on an eligible compute node, and retains the evidence needed to explain and roll back that
decision.
-Current release: **1.2.1**
+## Console tour
+
+
+
+*Overview: the operator's starting point for model and infrastructure status.*
+
+
+
+*Model library: inspect registered revisions and their details before planning a deployment.*
+
+Both images show the actual web console with synthetic API responses. No real operator, model, project or infrastructure data appear in them, and they do not imply a validated deployment. The current console uses Dutch interface labels.
+
+The current source line is **1.3.0-rc.1**. A source version does not by itself identify a qualified release or a running deployment; use the exact release evidence and deployment revision when operating an instance. The [publication guide](docs/PUBLICATION.md) explains the release boundary.
## What you get
@@ -69,13 +84,15 @@ The caller does not need to know the model name, artifact path, runtime or GPU n
## Architecture
-```text
-Applications -> Capability Gateway -> Scheduler -> typed runtime workers
- | ^
- v |
- Control Plane <--- outbound node agents
- |
- PostgreSQL + Redis + verified artifacts
+```mermaid
+flowchart LR
+ Apps[Applications] --> Gateway[Capability gateway]
+ Gateway --> Scheduler[Placement scheduler]
+ Scheduler --> Workers[Typed runtime workers]
+ Agents[Outbound node agents] --> Control[Control plane]
+ Control --> Scheduler
+ Control --> DB[(PostgreSQL and Redis)]
+ Control --> Artifacts[Verified model artifacts]
```
Important boundaries:
@@ -103,7 +120,7 @@ detail.
| Operate and recover it | [Operations](docs/OPERATIONS.md) · [Troubleshooting](docs/TROUBLESHOOTING.md) |
| Review security | [Security](docs/SECURITY.md) · [Threat model](docs/security/THREAT_MODEL.md) |
| Understand compatibility | [Compatibility](docs/COMPATIBILITY.md) |
-| See release changes | [Changelog](CHANGELOG.md) · [1.2.1 release notes](docs/RELEASE_NOTES_v1.2.1.md) |
+| See release changes | [Changelog](CHANGELOG.md) · [1.2.2 release notes](docs/RELEASE_NOTES_v1.2.2.md) |
## Repository layout
diff --git a/VERSION b/VERSION
index 23aa839..bf16dde 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.2.2
+1.3.0-rc.1
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 8155821..fcc1efb 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -25,22 +25,24 @@ LABEL org.opencontainers.image.description="Capability-first local AI ModelOps a
LABEL org.opencontainers.image.version="${MODELFORGE_VERSION}"
LABEL org.opencontainers.image.revision="${MODELFORGE_COMMIT}"
LABEL org.opencontainers.image.created="${MODELFORGE_BUILT_AT}"
-LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge.git"
+LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge"
LABEL org.opencontainers.image.vendor="ITWorx"
LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
WORKDIR /app
-COPY pyproject.toml ./
-COPY src ./src
-COPY alembic.ini ./
-COPY alembic ./alembic
-# Upgrade the installer before it resolves anything: the pinned base image ships a pip
-# carrying archive-extraction advisories. pip never runs at runtime, but a release image
-# should not carry a known-vulnerable installer.
-RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" "msgpack>=1.2.1" \
- && pip install --no-cache-dir . \
+COPY config/security/build-toolchain.lock /tmp/modelforge-build-toolchain.lock
+COPY backend/pyproject.toml ./
+COPY backend/src ./src
+COPY backend/alembic.ini ./
+COPY backend/alembic ./alembic
+# Build tooling is preinstalled from a fully resolved hash lock. Disabling PEP 517 isolation
+# makes pip use those exact tools rather than fetching a second, unverified build environment.
+RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: \
+ --index-url https://pypi.org/simple -r /tmp/modelforge-build-toolchain.lock \
+ && python -m pip install --no-cache-dir --no-build-isolation . \
&& pip check \
- && python -m pip uninstall --yes pip setuptools
+ && python -m pip uninstall --yes pip setuptools wheel editables hatchling pathspec pluggy \
+ tomlkit trove-classifiers
RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
&& mkdir -p /data/state /data/hf-cache /data/artifacts /data/quarantine \
@@ -49,4 +51,4 @@ RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
USER modelforge
EXPOSE 8000
-CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-proxy-headers"]
diff --git a/backend/alembic/versions/20260824_0001_m0_foundation.py b/backend/alembic/versions/20260824_0001_m0_foundation.py
index 13ddf2a..345f8e6 100644
--- a/backend/alembic/versions/20260824_0001_m0_foundation.py
+++ b/backend/alembic/versions/20260824_0001_m0_foundation.py
@@ -19,6 +19,49 @@ def upgrade() -> None:
# The declarative metadata is the reviewed M0 schema contract. create_all is
# used only from this baseline migration; later changes require explicit ops.
Base.metadata.create_all(bind=op.get_bind(), checkfirst=False)
+ # BenchmarkRun was part of M0. The evolving metadata also contains RC-LAB-02's later
+ # capability-target, actor/idempotency and environment fields, so rebuild its original
+ # shape here; migration 0039 adds those fields at the correct historical boundary.
+ op.drop_table("benchmark_runs")
+ op.create_table(
+ "benchmark_runs",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("deployment_id", sa.Uuid(), nullable=False),
+ sa.Column("suite_id", sa.Uuid(), nullable=False),
+ sa.Column("model_revision_sha", sa.String(length=64), nullable=False),
+ sa.Column("artifact_sha256", sa.String(length=64), nullable=False),
+ sa.Column("runtime_type", sa.String(length=32), nullable=False),
+ sa.Column("runtime_version", sa.String(length=128), nullable=False),
+ sa.Column("runtime_image_digest", sa.String(length=71), nullable=True),
+ sa.Column("launch_arguments", sa.JSON(), nullable=False),
+ sa.Column("suite_revision", sa.String(length=128), nullable=False),
+ sa.Column("dataset_revision", sa.String(length=255), nullable=False),
+ sa.Column("environment_fingerprint", sa.JSON(), nullable=False),
+ sa.Column("environment_digest", sa.String(length=64), nullable=False),
+ sa.Column("seed", sa.Integer(), nullable=True),
+ sa.Column("results", sa.JSON(), nullable=False),
+ sa.Column("status", sa.String(length=32), nullable=False),
+ sa.Column("comparable", sa.Boolean(), nullable=False),
+ sa.Column("stale_reason", sa.Text(), nullable=True),
+ sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.ForeignKeyConstraint(["deployment_id"], ["deployments.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["suite_id"], ["benchmark_suites.id"], ondelete="RESTRICT"),
+ sa.PrimaryKeyConstraint("id"),
+ )
+ op.create_index("ix_benchmark_runs_deployment_id", "benchmark_runs", ["deployment_id"])
+ op.create_index("ix_benchmark_runs_suite_id", "benchmark_runs", ["suite_id"])
+ op.create_index("ix_benchmark_runs_artifact_sha256", "benchmark_runs", ["artifact_sha256"])
+ op.create_index(
+ "ix_benchmark_runs_environment_digest", "benchmark_runs", ["environment_digest"]
+ )
+ op.create_index(
+ "ix_benchmark_runs_deployment_completed",
+ "benchmark_runs",
+ ["deployment_id", "completed_at"],
+ )
+ op.drop_constraint("ck_capability_serving_generation", "capability_contracts", type_="check")
+ op.drop_column("capability_contracts", "serving_generation")
# Freeze the original M0 shape even though the declarative metadata evolves.
# Remove newer foreign-key columns before dropping the tables they target.
for column in (
@@ -36,6 +79,27 @@ def upgrade() -> None:
# Later milestone tables are present in the evolving declarative metadata
# used to build this baseline. Drop dependants before their M5/M0 targets
# so a fresh upgrade still reconstructs each historical revision exactly.
+ "model_change_cases",
+ "selected_model_comparisons",
+ "external_creative_action_attestations",
+ "external_creative_profile_observations",
+ "external_creative_resources",
+ "external_creative_allocations",
+ "model_update_notices",
+ "model_update_watches",
+ "portfolio_repository_observations",
+ "portfolio_reconciliation_runs",
+ "platform_secret_activations",
+ "platform_secret_tests",
+ "platform_secret_versions",
+ "platform_secret_state",
+ "managed_operator_credential_activations",
+ "managed_operator_credentials",
+ "managed_operator_identities",
+ "managed_operator_state",
+ "configuration_revisions",
+ "project_onboarding",
+ "configuration_state",
"audit_chain_heads",
"node_decommission_operations",
# M15 recovery tables come first: artifact_recovery_operations references
@@ -50,6 +114,8 @@ def upgrade() -> None:
"recovery_asset_records",
"recovery_policy_revisions",
"incident_timeline_events",
+ # The M37 outbox restricts deletion of its alert history parent.
+ "alert_notification_deliveries",
"alert_history_events",
"operational_alerts",
"operational_incidents",
@@ -107,6 +173,9 @@ def upgrade() -> None:
"project_evaluation_bindings",
"serving_gpu_leases",
"serving_jobs",
+ # M38 durable LAB history owns a restrictive reference to gateway_requests;
+ # drop it before the M0 gateway table while reconstructing the historical baseline.
+ "lab_execution_runs",
"gateway_requests",
"residency_allocations",
"service_credentials",
diff --git a/backend/alembic/versions/20260830_0024_audit_chain_checkpoint.py b/backend/alembic/versions/20260830_0024_audit_chain_checkpoint.py
index bc300f8..ec88418 100644
--- a/backend/alembic/versions/20260830_0024_audit_chain_checkpoint.py
+++ b/backend/alembic/versions/20260830_0024_audit_chain_checkpoint.py
@@ -581,7 +581,7 @@ def _remove_postgres_audit_boundary(connection: Connection) -> None:
"drop trigger if exists trg_modelforge_audit_head_owner on public.audit_chain_heads; "
"drop trigger if exists trg_modelforge_audit_head_truncate_owner "
"on public.audit_chain_heads; "
- "drop function if exists modelforge_audit.append_event_v2("
+ "drop function if exists modelforge_audit.append_event_v2("
"uuid, timestamptz, text, text, text, text, text, text, text, jsonb, "
"bigint, bigint, text, text, bigint, bigint, text); "
"drop function if exists modelforge_audit.enforce_owner_mutation(); "
diff --git a/backend/alembic/versions/20260907_0025_configuration.py b/backend/alembic/versions/20260907_0025_configuration.py
new file mode 100644
index 0000000..e585b45
--- /dev/null
+++ b/backend/alembic/versions/20260907_0025_configuration.py
@@ -0,0 +1,46 @@
+"""Durable operator configuration and revision history.
+
+Revision ID: 20260907_0025
+Revises: 20260830_0024
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260907_0025"
+down_revision = "20260830_0024"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "configuration_state",
+ sa.Column("id", sa.Integer(), primary_key=True),
+ sa.Column("revision", sa.Integer(), nullable=False),
+ sa.Column("overrides", sa.JSON(), nullable=False),
+ sa.CheckConstraint("id = 1"),
+ sa.CheckConstraint("revision >= 0"),
+ )
+ op.create_table(
+ "configuration_revisions",
+ sa.Column("revision", sa.Integer(), primary_key=True),
+ sa.Column("previous_revision", sa.Integer(), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("actor", sa.String(255), nullable=False),
+ sa.Column("correlation_id", sa.String(255), nullable=False),
+ sa.Column("overrides", sa.JSON(), nullable=False),
+ sa.Column("preview_digest", sa.String(64), nullable=False),
+ )
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute("GRANT SELECT, INSERT, UPDATE ON configuration_state TO modelforge_runtime")
+ op.execute("REVOKE UPDATE, DELETE, TRUNCATE ON configuration_revisions FROM modelforge_runtime")
+ op.execute("GRANT SELECT, INSERT ON configuration_revisions TO modelforge_runtime")
+
+
+def downgrade() -> None:
+ if op.get_bind().execute(sa.text("SELECT count(*) FROM configuration_revisions")).scalar_one():
+ raise RuntimeError("Configuration history exists; recover the pre-migration backup instead.")
+ op.drop_table("configuration_revisions")
+ op.drop_table("configuration_state")
diff --git a/backend/alembic/versions/20260908_0026_project_onboarding.py b/backend/alembic/versions/20260908_0026_project_onboarding.py
new file mode 100644
index 0000000..0b669ca
--- /dev/null
+++ b/backend/alembic/versions/20260908_0026_project_onboarding.py
@@ -0,0 +1,51 @@
+"""Project onboarding and bounded credential overlap.
+
+Revision ID: 20260908_0026
+Revises: 20260907_0025
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260908_0026"
+down_revision = "20260907_0025"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "project_onboarding",
+ sa.Column("project_id", sa.Uuid(), sa.ForeignKey("projects.id", ondelete="RESTRICT"), primary_key=True),
+ sa.Column("owner", sa.String(128), nullable=False),
+ sa.Column("environment", sa.String(32), nullable=False),
+ sa.Column("labels", sa.JSON(), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ )
+ op.add_column("service_credentials", sa.Column("superseded_at", sa.DateTime(timezone=True)))
+ op.drop_index("uq_active_service_client_credential", table_name="service_credentials")
+ op.create_index(
+ "uq_active_service_client_credential", "service_credentials", ["service_client_id"],
+ unique=True,
+ postgresql_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
+ sqlite_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
+ )
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute("GRANT SELECT, INSERT ON project_onboarding TO modelforge_runtime")
+ op.execute("REVOKE UPDATE, DELETE, TRUNCATE ON project_onboarding FROM modelforge_runtime")
+
+
+def downgrade() -> None:
+ connection = op.get_bind()
+ if connection.execute(sa.text("SELECT count(*) FROM project_onboarding")).scalar_one() or connection.execute(
+ sa.text("SELECT count(*) FROM service_credentials WHERE superseded_at IS NOT NULL")
+ ).scalar_one():
+ raise RuntimeError("Onboarding or rotation history exists; recover the pre-migration backup instead.")
+ op.drop_index("uq_active_service_client_credential", table_name="service_credentials")
+ op.create_index(
+ "uq_active_service_client_credential", "service_credentials", ["service_client_id"],
+ unique=True, postgresql_where=sa.text("revoked_at IS NULL"), sqlite_where=sa.text("revoked_at IS NULL"),
+ )
+ op.drop_column("service_credentials", "superseded_at")
+ op.drop_table("project_onboarding")
diff --git a/backend/alembic/versions/20260908_0027_managed_operator_identities.py b/backend/alembic/versions/20260908_0027_managed_operator_identities.py
new file mode 100644
index 0000000..d5e19e5
--- /dev/null
+++ b/backend/alembic/versions/20260908_0027_managed_operator_identities.py
@@ -0,0 +1,60 @@
+"""Managed personal identities with recovery-cleared credential admission.
+
+Revision ID: 20260908_0027
+Revises: 20260908_0026
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260908_0027"
+down_revision = "20260908_0026"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table("managed_operator_state",
+ sa.Column("singleton_id", sa.Integer(), primary_key=True),
+ sa.Column("revision", sa.Integer(), nullable=False),
+ sa.CheckConstraint("singleton_id = 1", name="ck_managed_operator_singleton"),
+ sa.CheckConstraint("revision >= 0", name="ck_managed_operator_revision"))
+ op.execute("INSERT INTO managed_operator_state (singleton_id, revision) VALUES (1, 0)")
+ op.create_table("managed_operator_identities",
+ sa.Column("slug", sa.String(63), primary_key=True),
+ sa.Column("role", sa.String(16), nullable=False),
+ sa.Column("enabled", sa.Boolean(), nullable=False),
+ sa.Column("version", sa.Integer(), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.CheckConstraint("role IN ('viewer','operator','admin')", name="ck_managed_operator_role"),
+ sa.CheckConstraint("version >= 1", name="ck_managed_operator_version"))
+ op.create_table("managed_operator_credentials",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column("identity_slug", sa.String(63), sa.ForeignKey("managed_operator_identities.slug", ondelete="RESTRICT"), nullable=False),
+ sa.Column("token_sha256", sa.String(64), nullable=False, unique=True),
+ sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("revoked_at", sa.DateTime(timezone=True)),
+ sa.Column("superseded_at", sa.DateTime(timezone=True)),
+ sa.Column("last_used_at", sa.DateTime(timezone=True)),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False))
+ op.create_index("ix_managed_operator_credentials_identity_slug", "managed_operator_credentials", ["identity_slug"])
+ op.create_index("uq_managed_operator_current", "managed_operator_credentials", ["identity_slug"], unique=True,
+ postgresql_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
+ sqlite_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"))
+ op.create_table("managed_operator_credential_activations",
+ sa.Column("credential_id", sa.Uuid(), sa.ForeignKey("managed_operator_credentials.id", ondelete="RESTRICT"), primary_key=True),
+ sa.Column("activated_at", sa.DateTime(timezone=True), nullable=False))
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute("GRANT SELECT, INSERT, UPDATE ON managed_operator_state, managed_operator_identities, managed_operator_credentials TO modelforge_runtime")
+ op.execute("GRANT SELECT, INSERT, DELETE ON managed_operator_credential_activations TO modelforge_runtime")
+ op.execute("REVOKE DELETE, TRUNCATE ON managed_operator_state, managed_operator_identities, managed_operator_credentials FROM modelforge_runtime")
+
+
+def downgrade() -> None:
+ connection = op.get_bind()
+ if connection.execute(sa.text("SELECT revision FROM managed_operator_state WHERE singleton_id = 1")).scalar_one() != 0 or connection.execute(sa.text("SELECT count(*) FROM managed_operator_identities")).scalar_one():
+ raise RuntimeError("Managed identity history exists; recover the pre-migration backup instead.")
+ op.drop_table("managed_operator_credential_activations")
+ op.drop_table("managed_operator_credentials")
+ op.drop_table("managed_operator_identities")
+ op.drop_table("managed_operator_state")
diff --git a/backend/alembic/versions/20260908_0028_platform_secret_slots.py b/backend/alembic/versions/20260908_0028_platform_secret_slots.py
new file mode 100644
index 0000000..091fd67
--- /dev/null
+++ b/backend/alembic/versions/20260908_0028_platform_secret_slots.py
@@ -0,0 +1,56 @@
+"""Encrypted HF control-plane secret history and recovery-cleared admission.
+
+Revision ID: 20260908_0028
+Revises: 20260908_0027
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260908_0028"
+down_revision = "20260908_0027"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table("platform_secret_state",
+ sa.Column("singleton_id", sa.Integer(), primary_key=True),
+ sa.Column("revision", sa.Integer(), nullable=False),
+ sa.Column("initialized", sa.Boolean(), nullable=False),
+ sa.CheckConstraint("singleton_id = 1", name="ck_secret_state_singleton"),
+ sa.CheckConstraint("revision >= 0", name="ck_secret_state_revision"))
+ op.execute("INSERT INTO platform_secret_state (singleton_id, revision, initialized) VALUES (1, 0, false)")
+ op.create_table("platform_secret_versions",
+ sa.Column("version", sa.Integer(), primary_key=True, autoincrement=False),
+ sa.Column("key_id", sa.String(64), nullable=False),
+ sa.Column("secret_ciphertext", sa.Text(), nullable=False),
+ sa.Column("secret_nonce", sa.String(32), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("revoked_at", sa.DateTime(timezone=True)),
+ sa.CheckConstraint("version > 0", name="ck_secret_version_positive"))
+ op.create_table("platform_secret_tests",
+ sa.Column("version", sa.Integer(), sa.ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), primary_key=True),
+ sa.Column("passed", sa.Boolean(), nullable=False),
+ sa.Column("tested_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("code", sa.String(64), nullable=False))
+ op.create_table("platform_secret_activations",
+ sa.Column("singleton_id", sa.Integer(), primary_key=True),
+ sa.Column("version", sa.Integer(), sa.ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), nullable=False),
+ sa.Column("activated_at", sa.DateTime(timezone=True), nullable=False),
+ sa.CheckConstraint("singleton_id = 1", name="ck_secret_activation_singleton"))
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute("REVOKE ALL ON platform_secret_state, platform_secret_versions, platform_secret_tests, platform_secret_activations FROM modelforge_runtime")
+ op.execute("GRANT SELECT, UPDATE ON platform_secret_state TO modelforge_runtime")
+ op.execute("GRANT SELECT, INSERT ON platform_secret_versions TO modelforge_runtime")
+ op.execute("GRANT UPDATE (revoked_at) ON platform_secret_versions TO modelforge_runtime")
+ op.execute("GRANT SELECT, INSERT, UPDATE, DELETE ON platform_secret_tests, platform_secret_activations TO modelforge_runtime")
+
+
+def downgrade() -> None:
+ connection = op.get_bind()
+ state = connection.execute(sa.text("SELECT revision, initialized FROM platform_secret_state WHERE singleton_id=1")).one()
+ if state.revision != 0 or state.initialized or connection.scalar(sa.text("SELECT count(*) FROM platform_secret_versions")):
+ raise RuntimeError("Secret slot history exists; recover the pre-migration backup instead.")
+ for table in ("platform_secret_activations", "platform_secret_tests", "platform_secret_versions", "platform_secret_state"):
+ op.drop_table(table)
diff --git a/backend/alembic/versions/20260909_0029_external_creative_coordination.py b/backend/alembic/versions/20260909_0029_external_creative_coordination.py
new file mode 100644
index 0000000..ba77edc
--- /dev/null
+++ b/backend/alembic/versions/20260909_0029_external_creative_coordination.py
@@ -0,0 +1,57 @@
+"""External creative GPU coordination.
+
+Revision ID: 20260909_0029
+Revises: 20260908_0028
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0029"
+down_revision = "20260908_0028"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "external_creative_allocations",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("service_client_id", sa.Uuid(), nullable=False),
+ sa.Column("project_id", sa.Uuid(), nullable=False),
+ sa.Column("accelerator_id", sa.Uuid()),
+ sa.Column("request_id", sa.String(128), nullable=False),
+ sa.Column("idempotency_key", sa.String(128), nullable=False),
+ sa.Column("request_sha256", sa.String(64), nullable=False),
+ sa.Column("workload", sa.String(16), nullable=False),
+ sa.Column("resource_ref", sa.String(255), nullable=False),
+ sa.Column("provider_resource_id", sa.String(128), nullable=False),
+ sa.Column("required_vram_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("priority", sa.String(32), nullable=False),
+ sa.Column("ttl_seconds", sa.Integer(), nullable=False),
+ sa.Column("state", sa.String(32), nullable=False),
+ sa.Column("decision_code", sa.String(64), nullable=False),
+ sa.Column("decision_evidence", sa.JSON(), nullable=False),
+ sa.Column("required_actions", sa.JSON(), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.Column("acquired_at", sa.DateTime(timezone=True)),
+ sa.Column("heartbeat_at", sa.DateTime(timezone=True)),
+ sa.Column("expires_at", sa.DateTime(timezone=True)),
+ sa.Column("uncertain_at", sa.DateTime(timezone=True)),
+ sa.Column("released_at", sa.DateTime(timezone=True)),
+ sa.CheckConstraint("required_vram_bytes > 0", name="ck_external_creative_vram_positive"),
+ sa.CheckConstraint("ttl_seconds >= 30 AND ttl_seconds <= 3600", name="ck_external_creative_ttl"),
+ sa.ForeignKeyConstraint(["service_client_id"], ["service_clients.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"], ondelete="RESTRICT"),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint("service_client_id", "idempotency_key", name="uq_external_creative_idempotency"),
+ )
+ for column in ("service_client_id", "project_id", "accelerator_id", "request_id", "state", "expires_at"):
+ op.create_index(f"ix_external_creative_allocations_{column}", "external_creative_allocations", [column])
+
+
+def downgrade() -> None:
+ op.drop_table("external_creative_allocations")
diff --git a/backend/alembic/versions/20260909_0030_model_update_inbox.py b/backend/alembic/versions/20260909_0030_model_update_inbox.py
new file mode 100644
index 0000000..7e7f784
--- /dev/null
+++ b/backend/alembic/versions/20260909_0030_model_update_inbox.py
@@ -0,0 +1,43 @@
+"""Explicit selected-model upstream watches and deduplicated revision notices."""
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0030"
+down_revision = "20260909_0029"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table("model_update_watches",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column("model_id", sa.Uuid(), sa.ForeignKey("models.id", ondelete="RESTRICT"), nullable=False),
+ sa.Column("baseline_revision_id", sa.Uuid(), sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"), nullable=False),
+ sa.Column("baseline_commit_sha", sa.String(64), nullable=False),
+ sa.Column("upstream_ref", sa.String(255), nullable=False),
+ sa.Column("last_checked_at", sa.DateTime(timezone=True)),
+ sa.Column("last_error_code", sa.String(64)),
+ sa.Column("check_until", sa.DateTime(timezone=True)),
+ sa.Column("check_token", sa.Uuid()),
+ sa.UniqueConstraint("model_id", "upstream_ref", name="uq_model_watch_ref"))
+ op.create_table("model_update_notices",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column("watch_id", sa.Uuid(), sa.ForeignKey("model_update_watches.id", ondelete="RESTRICT"), nullable=False),
+ sa.Column("model_id", sa.Uuid(), sa.ForeignKey("models.id", ondelete="RESTRICT"), nullable=False),
+ sa.Column("baseline_commit_sha", sa.String(64), nullable=False),
+ sa.Column("candidate_revision_id", sa.Uuid(), sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"), nullable=False),
+ sa.Column("candidate_commit_sha", sa.String(64), nullable=False),
+ sa.Column("status", sa.String(16), nullable=False),
+ sa.Column("discovered_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.UniqueConstraint("watch_id", "candidate_commit_sha", name="uq_model_notice_commit"),
+ sa.CheckConstraint("status IN ('new', 'dismissed')", name="ck_model_notice_status"))
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute("GRANT SELECT, INSERT, UPDATE ON model_update_watches, model_update_notices TO modelforge_runtime")
+
+
+def downgrade() -> None:
+ if op.get_bind().scalar(sa.text("SELECT count(*) FROM model_update_watches")):
+ raise RuntimeError("Model update history exists; restore the pre-migration backup instead.")
+ op.drop_table("model_update_notices")
+ op.drop_table("model_update_watches")
diff --git a/backend/alembic/versions/20260909_0031_external_creative_governance.py b/backend/alembic/versions/20260909_0031_external_creative_governance.py
new file mode 100644
index 0000000..ca523d2
--- /dev/null
+++ b/backend/alembic/versions/20260909_0031_external_creative_governance.py
@@ -0,0 +1,114 @@
+"""Govern external creative resources, measurements, and action attestations.
+
+Revision ID: 20260909_0031
+Revises: 20260909_0030
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0031"
+down_revision = "20260909_0030"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "external_creative_resources",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column(
+ "project_id",
+ sa.Uuid(),
+ sa.ForeignKey("projects.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column("resource_ref", sa.String(255), nullable=False),
+ sa.Column("workload", sa.String(16), nullable=False),
+ sa.Column("provider_resource_id", sa.String(128), nullable=False),
+ sa.Column("approved", sa.Boolean(), nullable=False, server_default=sa.false()),
+ sa.Column("configured_vram_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("measured_peak_vram_bytes", sa.BigInteger()),
+ sa.Column("measurement_count", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column(
+ "updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+ ),
+ sa.UniqueConstraint(
+ "project_id",
+ "resource_ref",
+ "workload",
+ "provider_resource_id",
+ name="uq_external_creative_resource_binding",
+ ),
+ sa.CheckConstraint("configured_vram_bytes > 0", name="ck_external_resource_vram_positive"),
+ )
+ op.create_index(
+ "ix_external_creative_resources_project",
+ "external_creative_resources",
+ ["project_id", "approved"],
+ )
+ op.create_table(
+ "external_creative_action_attestations",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column(
+ "allocation_id",
+ sa.Uuid(),
+ sa.ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column("action", sa.String(64), nullable=False),
+ sa.Column("before_vram_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("after_vram_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("validated", sa.Boolean(), nullable=False),
+ sa.Column(
+ "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+ ),
+ sa.UniqueConstraint(
+ "allocation_id", "action", name="uq_external_creative_action_attestation"
+ ),
+ sa.CheckConstraint(
+ "before_vram_bytes >= 0 AND after_vram_bytes >= 0",
+ name="ck_external_action_vram_nonnegative",
+ ),
+ )
+ op.create_table(
+ "external_creative_profile_observations",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column(
+ "allocation_id",
+ sa.Uuid(),
+ sa.ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column("resource_ref", sa.String(255), nullable=False),
+ sa.Column("provider_resource_id", sa.String(128), nullable=False),
+ sa.Column("workload", sa.String(16), nullable=False),
+ sa.Column("observed_peak_vram_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("reserved_vram_bytes", sa.BigInteger(), nullable=False),
+ sa.Column("duration_ms", sa.BigInteger(), nullable=False),
+ sa.Column("outcome", sa.String(16), nullable=False),
+ sa.Column(
+ "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
+ ),
+ sa.UniqueConstraint(
+ "allocation_id", name="uq_external_creative_profile_observation_allocation"
+ ),
+ sa.CheckConstraint(
+ "observed_peak_vram_bytes > 0 AND reserved_vram_bytes > 0",
+ name="ck_external_profile_vram_positive",
+ ),
+ sa.CheckConstraint(
+ "duration_ms >= 0 AND duration_ms <= 604800000", name="ck_external_profile_duration"
+ ),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("external_creative_profile_observations")
+ op.drop_table("external_creative_action_attestations")
+ op.drop_index(
+ "ix_external_creative_resources_project", table_name="external_creative_resources"
+ )
+ op.drop_table("external_creative_resources")
diff --git a/backend/alembic/versions/20260909_0032_audio_creative_workloads.py b/backend/alembic/versions/20260909_0032_audio_creative_workloads.py
new file mode 100644
index 0000000..5d4f9d7
--- /dev/null
+++ b/backend/alembic/versions/20260909_0032_audio_creative_workloads.py
@@ -0,0 +1,44 @@
+"""Allow explicit audio and film creative workload names.
+
+Revision ID: 20260909_0032
+Revises: 20260909_0031
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0032"
+down_revision = "20260909_0031"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ for table in (
+ "external_creative_allocations",
+ "external_creative_resources",
+ "external_creative_profile_observations",
+ ):
+ with op.batch_alter_table(table) as batch:
+ batch.alter_column(
+ "workload",
+ existing_type=sa.String(16),
+ type_=sa.String(32),
+ existing_nullable=False,
+ )
+
+
+def downgrade() -> None:
+ for table in (
+ "external_creative_profile_observations",
+ "external_creative_resources",
+ "external_creative_allocations",
+ ):
+ with op.batch_alter_table(table) as batch:
+ batch.alter_column(
+ "workload",
+ existing_type=sa.String(32),
+ type_=sa.String(16),
+ existing_nullable=False,
+ )
diff --git a/backend/alembic/versions/20260909_0033_deployed_hub_convergence.py b/backend/alembic/versions/20260909_0033_deployed_hub_convergence.py
new file mode 100644
index 0000000..1626721
--- /dev/null
+++ b/backend/alembic/versions/20260909_0033_deployed_hub_convergence.py
@@ -0,0 +1,154 @@
+"""Persist one explicitly delegated current/candidate comparison per project binding."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260909_0033"
+down_revision = "20260909_0032"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "model_update_watches",
+ sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
+ )
+ op.add_column(
+ "model_update_watches",
+ sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+ )
+ op.create_table(
+ "selected_model_comparisons",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column(
+ "project_binding_id",
+ sa.Uuid(),
+ sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column(
+ "consumer_client_id",
+ sa.Uuid(),
+ sa.ForeignKey("service_clients.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column(
+ "suite_revision_id",
+ sa.Uuid(),
+ sa.ForeignKey("evaluation_suite_revisions.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column(
+ "current_deployment_id",
+ sa.Uuid(),
+ sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column(
+ "candidate_deployment_id",
+ sa.Uuid(),
+ sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column(
+ "candidate_route_id",
+ sa.Uuid(),
+ sa.ForeignKey("capability_experiment_routes.id", ondelete="RESTRICT"),
+ nullable=False,
+ ),
+ sa.Column("idempotency_key", sa.String(128), nullable=False),
+ sa.Column("request_digest", sa.String(64), nullable=False),
+ sa.Column("actor_id", sa.String(128), nullable=False),
+ sa.Column("pins", sa.JSON(), nullable=False),
+ sa.Column("status", sa.String(32), nullable=False),
+ sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+ sa.Column("current_completed_cases", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("candidate_completed_cases", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("expected_cases", sa.Integer(), nullable=False),
+ sa.Column("claim_token_hash", sa.String(64)),
+ sa.Column("claim_expires_at", sa.DateTime(timezone=True)),
+ sa.Column("deadline_at", sa.DateTime(timezone=True)),
+ sa.Column("claimed_at", sa.DateTime(timezone=True)),
+ sa.Column("cancel_requested_at", sa.DateTime(timezone=True)),
+ sa.Column("finished_at", sa.DateTime(timezone=True)),
+ sa.Column("error_code", sa.String(64)),
+ sa.Column("result_digest", sa.String(64)),
+ sa.Column(
+ "baseline_run_id", sa.Uuid(), sa.ForeignKey("evaluation_runs.id", ondelete="RESTRICT")
+ ),
+ sa.Column(
+ "candidate_run_id", sa.Uuid(), sa.ForeignKey("evaluation_runs.id", ondelete="RESTRICT")
+ ),
+ sa.Column(
+ "evaluation_comparison_id",
+ sa.Uuid(),
+ sa.ForeignKey("evaluation_comparisons.id", ondelete="RESTRICT"),
+ ),
+ sa.Column(
+ "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+ ),
+ sa.Column(
+ "updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+ ),
+ sa.UniqueConstraint(
+ "project_binding_id", "idempotency_key", name="uq_selected_comparison_key"
+ ),
+ sa.CheckConstraint(
+ "status IN ('pending', 'running', 'cancel_requested', 'completed', 'failed', 'cancelled', 'interrupted')",
+ name="ck_selected_comparison_status",
+ ),
+ sa.CheckConstraint("expected_cases BETWEEN 1 AND 100", name="ck_selected_comparison_cases"),
+ sa.CheckConstraint(
+ "current_completed_cases BETWEEN 0 AND expected_cases AND candidate_completed_cases BETWEEN 0 AND expected_cases",
+ name="ck_selected_comparison_progress",
+ ),
+ sa.CheckConstraint("version > 0", name="ck_selected_comparison_version"),
+ sa.CheckConstraint(
+ "current_deployment_id <> candidate_deployment_id",
+ name="ck_selected_comparison_distinct_targets",
+ ),
+ )
+ op.create_index(
+ "uq_selected_comparison_active_binding",
+ "selected_model_comparisons",
+ ["project_binding_id"],
+ unique=True,
+ postgresql_where=sa.text("status IN ('pending', 'running', 'cancel_requested')"),
+ sqlite_where=sa.text("status IN ('pending', 'running', 'cancel_requested')"),
+ )
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute(
+ "REVOKE DELETE, TRUNCATE ON model_update_watches, model_update_notices FROM modelforge_runtime"
+ )
+ op.execute(
+ "GRANT SELECT, INSERT, UPDATE ON model_update_watches, model_update_notices TO modelforge_runtime"
+ )
+ op.execute("REVOKE DELETE, TRUNCATE ON selected_model_comparisons FROM modelforge_runtime")
+ op.execute(
+ "GRANT SELECT, INSERT, UPDATE ON selected_model_comparisons TO modelforge_runtime"
+ )
+
+
+def downgrade() -> None:
+ # Earlier deployed migrations have destructive downgrades. Never permit this
+ # release to cross back into that graph while authoritative history exists.
+ for table in (
+ "selected_model_comparisons",
+ "model_update_watches",
+ "model_update_notices",
+ "external_creative_allocations",
+ "external_creative_resources",
+ "external_creative_action_attestations",
+ "external_creative_profile_observations",
+ ):
+ if op.get_bind().scalar(sa.select(sa.func.count()).select_from(sa.table(table))):
+ raise RuntimeError(
+ "Control-plane history exists; restore the pre-migration backup instead."
+ )
+ op.drop_index("uq_selected_comparison_active_binding", table_name="selected_model_comparisons")
+ op.drop_table("selected_model_comparisons")
+ op.drop_column("model_update_watches", "version")
+ op.drop_column("model_update_watches", "enabled")
+ # Deliberately retain the safer runtime privilege restrictions on empty old tables.
diff --git a/backend/alembic/versions/20260910_0034_approved_serving_transition.py b/backend/alembic/versions/20260910_0034_approved_serving_transition.py
new file mode 100644
index 0000000..6bcddd5
--- /dev/null
+++ b/backend/alembic/versions/20260910_0034_approved_serving_transition.py
@@ -0,0 +1,28 @@
+"""Fence approved stable serving transitions and rollback against ABA changes."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260910_0034"
+down_revision = "20260909_0033"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.add_column("capability_contracts", sa.Column(
+ "serving_generation", sa.BigInteger(), nullable=False, server_default="0"))
+ with op.batch_alter_table("capability_contracts") as batch:
+ batch.create_check_constraint("ck_capability_serving_generation", "serving_generation >= 0")
+
+
+def downgrade() -> None:
+ contracts = sa.table("capability_contracts", sa.column("serving_generation"))
+ if op.get_bind().scalar(sa.select(sa.func.count()).select_from(contracts).where(
+ contracts.c.serving_generation != 0
+ )):
+ raise RuntimeError("Serving route history exists; restore the pre-migration backup instead.")
+ with op.batch_alter_table("capability_contracts") as batch:
+ batch.drop_constraint("ck_capability_serving_generation", type_="check")
+ batch.drop_column("serving_generation")
diff --git a/backend/alembic/versions/20260911_0035_user_model_change_cases.py b/backend/alembic/versions/20260911_0035_user_model_change_cases.py
new file mode 100644
index 0000000..168967c
--- /dev/null
+++ b/backend/alembic/versions/20260911_0035_user_model_change_cases.py
@@ -0,0 +1,95 @@
+"""Add the user-facing model replacement case without duplicating lifecycle truth."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260911_0035"
+down_revision = "20260910_0034"
+branch_labels = None
+depends_on = None
+
+
+STATUSES = (
+ "searching",
+ "found",
+ "source_checked",
+ "downloaded",
+ "ready_to_test",
+ "tested",
+ "comparison_available",
+ "migration_required",
+ "ready_to_activate",
+ "active",
+ "checked",
+ "rolled_back",
+ "rejected",
+)
+
+
+def upgrade() -> None:
+ op.create_table(
+ "model_change_cases",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("project_binding_id", sa.Uuid(), nullable=False),
+ sa.Column("current_deployment_id", sa.Uuid(), nullable=False),
+ sa.Column("candidate_model_id", sa.Uuid(), nullable=True),
+ sa.Column("candidate_revision_id", sa.Uuid(), nullable=True),
+ sa.Column("candidate_artifact_set_id", sa.Uuid(), nullable=True),
+ sa.Column("candidate_deployment_id", sa.Uuid(), nullable=True),
+ sa.Column("discovery_assessment_id", sa.Uuid(), nullable=True),
+ sa.Column("download_plan_id", sa.Uuid(), nullable=True),
+ sa.Column("artifact_job_id", sa.Uuid(), nullable=True),
+ sa.Column("selected_comparison_id", sa.Uuid(), nullable=True),
+ sa.Column("migration_plan_id", sa.Uuid(), nullable=True),
+ sa.Column("lifecycle_plan_id", sa.Uuid(), nullable=True),
+ sa.Column("status", sa.String(length=32), nullable=False, server_default="searching"),
+ sa.Column("status_reason", sa.Text(), nullable=False, server_default=""),
+ sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
+ sa.Column("created_by", sa.String(length=255), nullable=False),
+ sa.Column("updated_by", sa.String(length=255), nullable=False),
+ sa.Column("context", sa.JSON(), nullable=False),
+ sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("checked_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("closed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.ForeignKeyConstraint(["project_binding_id"], ["project_bindings.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["current_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["candidate_model_id"], ["models.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["candidate_revision_id"], ["model_revisions.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["candidate_artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["candidate_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["discovery_assessment_id"], ["discovery_candidate_assessments.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["download_plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["artifact_job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["selected_comparison_id"], ["selected_model_comparisons.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["migration_plan_id"], ["migration_plans.id"], ondelete="RESTRICT"),
+ sa.ForeignKeyConstraint(["lifecycle_plan_id"], ["lifecycle_promotion_plans.id"], ondelete="RESTRICT"),
+ sa.PrimaryKeyConstraint("id"),
+ sa.CheckConstraint("version >= 1", name="ck_model_change_case_version"),
+ sa.CheckConstraint(
+ "status IN (" + ",".join(repr(value) for value in STATUSES) + ")",
+ name="ck_model_change_case_status",
+ ),
+ )
+ op.create_index("ix_model_change_cases_binding", "model_change_cases", ["project_binding_id"])
+ op.create_index("ix_model_change_cases_status", "model_change_cases", ["status"])
+ op.create_index(
+ "uq_model_change_cases_open_binding",
+ "model_change_cases",
+ ["project_binding_id"],
+ unique=True,
+ postgresql_where=sa.text("closed_at IS NULL"),
+ sqlite_where=sa.text("closed_at IS NULL"),
+ )
+
+
+def downgrade() -> None:
+ changes = sa.table("model_change_cases", sa.column("id"))
+ if op.get_bind().scalar(sa.select(sa.func.count()).select_from(changes)):
+ raise RuntimeError("Model change cases exist; restore the pre-migration backup instead.")
+ op.drop_index("uq_model_change_cases_open_binding", table_name="model_change_cases")
+ op.drop_index("ix_model_change_cases_status", table_name="model_change_cases")
+ op.drop_index("ix_model_change_cases_binding", table_name="model_change_cases")
+ op.drop_table("model_change_cases")
diff --git a/backend/alembic/versions/20260923_0036_portfolio_repository_reconciliation.py b/backend/alembic/versions/20260923_0036_portfolio_repository_reconciliation.py
new file mode 100644
index 0000000..247177f
--- /dev/null
+++ b/backend/alembic/versions/20260923_0036_portfolio_repository_reconciliation.py
@@ -0,0 +1,122 @@
+"""Persist secret-free repository observations and snapshot idempotency."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260923_0036"
+down_revision = "20260911_0035"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "portfolio_reconciliation_runs",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("source_system", sa.String(length=16), nullable=False),
+ sa.Column("snapshot_id", sa.String(length=128), nullable=False),
+ sa.Column("payload_digest", sa.String(length=64), nullable=False),
+ sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("complete", sa.Boolean(), nullable=False),
+ sa.Column("stale_snapshot", sa.Boolean(), nullable=False),
+ sa.Column("projects_created", sa.Integer(), nullable=False),
+ sa.Column("repositories_created", sa.Integer(), nullable=False),
+ sa.Column("repositories_updated", sa.Integer(), nullable=False),
+ sa.Column("stale_observations", sa.Integer(), nullable=False),
+ sa.Column("repositories_marked_stale", sa.Integer(), nullable=False),
+ sa.Column("runtime_observations_updated", sa.Integer(), nullable=False),
+ sa.Column("head_changes_review_required", sa.Integer(), nullable=False),
+ sa.Column("head_changes_accepted", sa.Integer(), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint("source_system", "snapshot_id", name="uq_portfolio_snapshot_identity"),
+ sa.CheckConstraint("length(payload_digest) = 64", name="ck_portfolio_snapshot_digest_length"),
+ sa.CheckConstraint(
+ "projects_created >= 0 AND repositories_created >= 0 "
+ "AND repositories_updated >= 0 AND stale_observations >= 0 "
+ "AND repositories_marked_stale >= 0 AND runtime_observations_updated >= 0 "
+ "AND head_changes_review_required >= 0 AND head_changes_accepted >= 0",
+ name="ck_portfolio_snapshot_counts_nonnegative",
+ ),
+ )
+ op.create_index(
+ "ix_portfolio_snapshot_source_observed",
+ "portfolio_reconciliation_runs",
+ ["source_system", "complete", "observed_at"],
+ )
+ op.create_table(
+ "portfolio_repository_observations",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("project_id", sa.Uuid(), nullable=True),
+ sa.Column("project_key", sa.String(length=128), nullable=True),
+ sa.Column("source_system", sa.String(length=16), nullable=False),
+ sa.Column("external_repository_id", sa.String(length=128), nullable=False),
+ sa.Column("slug", sa.String(length=255), nullable=False),
+ sa.Column("default_branch", sa.String(length=255), nullable=False),
+ sa.Column("head_sha", sa.String(length=40), nullable=False),
+ sa.Column("classification", sa.String(length=32), nullable=False),
+ sa.Column("code_evidence", sa.JSON(), nullable=False),
+ sa.Column("runtime_evidence", sa.JSON(), nullable=False),
+ sa.Column("source_observed_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("source_snapshot_id", sa.String(length=128), nullable=False),
+ sa.Column("source_snapshot_digest", sa.String(length=64), nullable=False),
+ sa.Column("pending_head_sha", sa.String(length=40), nullable=True),
+ sa.Column("pending_default_branch", sa.String(length=255), nullable=True),
+ sa.Column("pending_project_key", sa.String(length=128), nullable=True),
+ sa.Column("pending_classification", sa.String(length=32), nullable=True),
+ sa.Column("pending_code_evidence", sa.JSON(), nullable=True),
+ sa.Column("pending_observed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("pending_snapshot_id", sa.String(length=128), nullable=True),
+ sa.Column("pending_snapshot_digest", sa.String(length=64), nullable=True),
+ sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("stale_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint(
+ "source_system", "external_repository_id", name="uq_portfolio_repo_external_id"
+ ),
+ sa.UniqueConstraint(
+ "source_system", "project_key", name="uq_portfolio_source_project_key"
+ ),
+ sa.UniqueConstraint("project_id", name="uq_portfolio_repository_project_id"),
+ sa.CheckConstraint("length(head_sha) = 40", name="ck_portfolio_repo_head_sha_length"),
+ sa.CheckConstraint(
+ "pending_head_sha IS NULL OR length(pending_head_sha) = 40",
+ name="ck_portfolio_repo_pending_sha_length",
+ ),
+ sa.CheckConstraint(
+ "classification IN ('local_model', 'hosted_model', 'gateway', 'mixed', "
+ "'model_management', 'runtime_observer', 'non_ai', 'unknown')",
+ name="ck_portfolio_repo_classification",
+ ),
+ )
+ op.create_index(
+ "ix_portfolio_repository_observations_project_id",
+ "portfolio_repository_observations",
+ ["project_id"],
+ )
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ runs = sa.table("portfolio_reconciliation_runs", sa.column("id"))
+ repositories = sa.table("portfolio_repository_observations", sa.column("id"))
+ if bind.scalar(sa.select(sa.func.count()).select_from(runs)) or bind.scalar(
+ sa.select(sa.func.count()).select_from(repositories)
+ ):
+ raise RuntimeError(
+ "portfolio reconciliation history exists; restore the pre-migration backup instead."
+ )
+ op.drop_index(
+ "ix_portfolio_repository_observations_project_id",
+ table_name="portfolio_repository_observations",
+ )
+ op.drop_table("portfolio_repository_observations")
+ op.drop_index(
+ "ix_portfolio_snapshot_source_observed", table_name="portfolio_reconciliation_runs"
+ )
+ op.drop_table("portfolio_reconciliation_runs")
diff --git a/backend/alembic/versions/20260923_0037_alert_notification_outbox.py b/backend/alembic/versions/20260923_0037_alert_notification_outbox.py
new file mode 100644
index 0000000..ee5e6e9
--- /dev/null
+++ b/backend/alembic/versions/20260923_0037_alert_notification_outbox.py
@@ -0,0 +1,99 @@
+"""Add durable, single-route operational alert notification outbox."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260923_0037"
+down_revision = "20260923_0036"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "alert_notification_deliveries",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("alert_history_event_id", sa.Uuid(), nullable=True),
+ sa.Column("route_key", sa.String(length=64), nullable=False),
+ sa.Column("idempotency_key", sa.String(length=160), nullable=False),
+ sa.Column("event_kind", sa.String(length=16), nullable=False),
+ sa.Column("severity", sa.String(length=16), nullable=True),
+ sa.Column("payload", sa.JSON(), nullable=False),
+ sa.Column("status", sa.String(length=24), nullable=False),
+ sa.Column("attempt_count", sa.Integer(), nullable=False),
+ sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("delivered_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("claim_token_hash", sa.String(length=64), nullable=True),
+ sa.Column("last_http_status", sa.Integer(), nullable=True),
+ sa.Column("last_error_code", sa.String(length=64), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.CheckConstraint(
+ "event_kind IN ('FIRING', 'RESOLVED', 'TEST')",
+ name="ck_alert_notification_event_kind",
+ ),
+ sa.CheckConstraint(
+ "status IN ('PENDING', 'DISPATCHING', 'RETRY_WAIT', 'DELIVERED', 'FAILED', 'HELD_FOR_REVIEW')",
+ name="ck_alert_notification_status",
+ ),
+ sa.CheckConstraint("attempt_count >= 0", name="ck_alert_notification_attempt_count"),
+ sa.ForeignKeyConstraint(
+ ["alert_history_event_id"], ["alert_history_events.id"], ondelete="RESTRICT"
+ ),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint("idempotency_key", name="uq_alert_notification_idempotency"),
+ )
+ op.create_index(
+ "ix_alert_notification_deliveries_alert_history_event_id",
+ "alert_notification_deliveries",
+ ["alert_history_event_id"],
+ )
+ op.create_index(
+ "ix_alert_notification_deliveries_event_kind",
+ "alert_notification_deliveries",
+ ["event_kind"],
+ )
+ op.create_index(
+ "ix_alert_notification_deliveries_severity",
+ "alert_notification_deliveries",
+ ["severity"],
+ )
+ op.create_index(
+ "ix_alert_notification_deliveries_status",
+ "alert_notification_deliveries",
+ ["status"],
+ )
+ op.create_index(
+ "ix_alert_notification_deliveries_next_attempt_at",
+ "alert_notification_deliveries",
+ ["next_attempt_at"],
+ )
+ op.create_index(
+ "ix_alert_notification_deliveries_lease_expires_at",
+ "alert_notification_deliveries",
+ ["lease_expires_at"],
+ )
+ op.create_index(
+ "ix_alert_notification_dispatch_due",
+ "alert_notification_deliveries",
+ ["status", "next_attempt_at", "created_at"],
+ )
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ deliveries = sa.table("alert_notification_deliveries", sa.column("id"))
+ if bind.scalar(sa.select(sa.func.count()).select_from(deliveries)):
+ raise RuntimeError(
+ "alert notification history exists; restore the pre-migration backup instead."
+ )
+ op.drop_index("ix_alert_notification_dispatch_due", table_name="alert_notification_deliveries")
+ op.drop_index("ix_alert_notification_deliveries_lease_expires_at", table_name="alert_notification_deliveries")
+ op.drop_index("ix_alert_notification_deliveries_next_attempt_at", table_name="alert_notification_deliveries")
+ op.drop_index("ix_alert_notification_deliveries_status", table_name="alert_notification_deliveries")
+ op.drop_index("ix_alert_notification_deliveries_severity", table_name="alert_notification_deliveries")
+ op.drop_index("ix_alert_notification_deliveries_event_kind", table_name="alert_notification_deliveries")
+ op.drop_index("ix_alert_notification_deliveries_alert_history_event_id", table_name="alert_notification_deliveries")
+ op.drop_table("alert_notification_deliveries")
diff --git a/backend/alembic/versions/20260924_0038_lab_execution_runs.py b/backend/alembic/versions/20260924_0038_lab_execution_runs.py
new file mode 100644
index 0000000..17cb4c3
--- /dev/null
+++ b/backend/alembic/versions/20260924_0038_lab_execution_runs.py
@@ -0,0 +1,104 @@
+"""Persist actor-scoped Model Lab execution identities."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260924_0038"
+down_revision = "20260923_0037"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "lab_execution_runs",
+ sa.Column("id", sa.Uuid(), nullable=False),
+ sa.Column("actor_subject", sa.String(length=255), nullable=False),
+ sa.Column("idempotency_key", sa.String(length=128), nullable=False),
+ sa.Column("request_fingerprint", sa.String(length=64), nullable=False),
+ sa.Column("status", sa.String(length=24), server_default="accepted", nullable=False),
+ sa.Column("capability", sa.String(length=128), server_default="rag.embedding@1", nullable=False),
+ sa.Column("target_snapshot", sa.JSON(), nullable=False),
+ sa.Column("input_sha256", sa.String(length=64), nullable=False),
+ sa.Column("input_count", sa.Integer(), nullable=False),
+ sa.Column("input_encoded_bytes", sa.Integer(), nullable=False),
+ sa.Column("payload_reference", sa.String(length=128), nullable=True),
+ sa.Column("input_payload_expires_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("gateway_request_id", sa.Uuid(), nullable=False),
+ sa.Column("timeout_seconds", sa.Integer(), server_default="60", nullable=False),
+ sa.Column("queue_timeout_seconds", sa.Integer(), server_default="10", nullable=False),
+ sa.Column("retention_seconds", sa.Integer(), server_default="300", nullable=False),
+ sa.Column("output_metadata", sa.JSON(), nullable=True),
+ sa.Column("worker_observations", sa.JSON(), nullable=True),
+ sa.Column("failure_code", sa.String(length=64), nullable=True),
+ sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("dispatch_started_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("cancellation_requested_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("cancellation_acknowledged_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+ sa.CheckConstraint(
+ "status IN ('accepted', 'preparing', 'queued', 'running', 'cancel_requested', "
+ "'completed', 'failed', 'cancelled', 'interrupted')",
+ name="ck_lab_execution_run_status",
+ ),
+ sa.CheckConstraint("input_count BETWEEN 1 AND 8", name="ck_lab_execution_run_input_count"),
+ sa.CheckConstraint("timeout_seconds BETWEEN 5 AND 300", name="ck_lab_execution_run_timeout"),
+ sa.CheckConstraint("queue_timeout_seconds BETWEEN 1 AND 60", name="ck_lab_execution_run_queue_timeout"),
+ sa.CheckConstraint("retention_seconds BETWEEN 60 AND 3600", name="ck_lab_execution_run_retention"),
+ sa.ForeignKeyConstraint(
+ ["gateway_request_id"], ["gateway_requests.id"], ondelete="RESTRICT"
+ ),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint("actor_subject", "idempotency_key", name="uq_lab_run_actor_idempotency"),
+ sa.UniqueConstraint("gateway_request_id", name="uq_lab_execution_run_gateway_request"),
+ )
+ op.create_index("ix_lab_execution_runs_actor_subject", "lab_execution_runs", ["actor_subject"])
+ op.create_index("ix_lab_execution_runs_status", "lab_execution_runs", ["status"])
+ op.create_index("ix_lab_execution_runs_gateway_request_id", "lab_execution_runs", ["gateway_request_id"])
+ op.create_index("ix_lab_execution_runs_actor_status_created", "lab_execution_runs", ["actor_subject", "status", "created_at"])
+ op.add_column(
+ "serving_jobs",
+ sa.Column("lab_execution_run_id", sa.Uuid(), nullable=True),
+ )
+ op.create_foreign_key(
+ "fk_serving_jobs_lab_execution_run_id_lab_execution_runs",
+ "serving_jobs",
+ "lab_execution_runs",
+ ["lab_execution_run_id"],
+ ["id"],
+ ondelete="SET NULL",
+ )
+ op.create_index(
+ "ix_serving_jobs_lab_execution_run_id", "serving_jobs", ["lab_execution_run_id"]
+ )
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute(
+ "REVOKE DELETE, TRUNCATE ON lab_execution_runs FROM modelforge_runtime"
+ )
+ op.execute(
+ "GRANT SELECT, INSERT, UPDATE ON lab_execution_runs TO modelforge_runtime"
+ )
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ if bind.scalar(sa.select(sa.func.count()).select_from(sa.table("lab_execution_runs", sa.column("id")))):
+ raise RuntimeError(
+ "LAB run history exists; restore the pre-migration backup instead of deleting it."
+ )
+ op.drop_index("ix_serving_jobs_lab_execution_run_id", table_name="serving_jobs")
+ op.drop_constraint(
+ "fk_serving_jobs_lab_execution_run_id_lab_execution_runs",
+ "serving_jobs",
+ type_="foreignkey",
+ )
+ op.drop_column("serving_jobs", "lab_execution_run_id")
+ op.drop_index("ix_lab_execution_runs_actor_status_created", table_name="lab_execution_runs")
+ op.drop_index("ix_lab_execution_runs_gateway_request_id", table_name="lab_execution_runs")
+ op.drop_index("ix_lab_execution_runs_status", table_name="lab_execution_runs")
+ op.drop_index("ix_lab_execution_runs_actor_subject", table_name="lab_execution_runs")
+ op.drop_table("lab_execution_runs")
diff --git a/backend/alembic/versions/20260924_0039_operator_benchmark_runs.py b/backend/alembic/versions/20260924_0039_operator_benchmark_runs.py
new file mode 100644
index 0000000..769c676
--- /dev/null
+++ b/backend/alembic/versions/20260924_0039_operator_benchmark_runs.py
@@ -0,0 +1,164 @@
+"""Bind persisted benchmark evidence to authenticated capability runs."""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision = "20260924_0039"
+down_revision = "20260924_0038"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ if op.get_bind().dialect.name == "sqlite":
+ # SQLite cannot ALTER TABLE to add the FK/check/unique constraints. Alembic's batch
+ # recreation preserves historical benchmark rows and their indexes.
+ with op.batch_alter_table("benchmark_runs", recreate="always") as batch:
+ batch.alter_column("deployment_id", existing_type=sa.Uuid(), nullable=True)
+ batch.add_column(sa.Column("capability_deployment_id", sa.Uuid(), nullable=True))
+ batch.add_column(sa.Column("target_snapshot", sa.JSON(), nullable=True))
+ batch.add_column(sa.Column("actor_subject", sa.String(255), nullable=True))
+ batch.add_column(sa.Column("idempotency_key", sa.String(128), nullable=True))
+ batch.add_column(sa.Column("request_fingerprint", sa.String(64), nullable=True))
+ batch.add_column(sa.Column("failure_code", sa.String(64), nullable=True))
+ batch.add_column(sa.Column("cancellation_requested_at", sa.DateTime(timezone=True)))
+ batch.add_column(
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ )
+ )
+ batch.add_column(
+ sa.Column("generation", sa.Integer(), server_default="1", nullable=False)
+ )
+ batch.create_foreign_key(
+ "fk_benchmark_runs_capability_deployment",
+ "capability_deployments",
+ ["capability_deployment_id"],
+ ["id"],
+ ondelete="RESTRICT",
+ )
+ batch.create_check_constraint(
+ "ck_benchmark_runs_operator_identity",
+ "(actor_subject IS NULL AND idempotency_key IS NULL AND "
+ "request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
+ "(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
+ "request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
+ )
+ batch.create_unique_constraint(
+ "uq_benchmark_runs_actor_idempotency", ["actor_subject", "idempotency_key"]
+ )
+ else:
+ op.alter_column("benchmark_runs", "deployment_id", existing_type=sa.Uuid(), nullable=True)
+ op.add_column(
+ "benchmark_runs", sa.Column("capability_deployment_id", sa.Uuid(), nullable=True)
+ )
+ op.add_column("benchmark_runs", sa.Column("target_snapshot", sa.JSON(), nullable=True))
+ op.add_column("benchmark_runs", sa.Column("actor_subject", sa.String(255), nullable=True))
+ op.add_column("benchmark_runs", sa.Column("idempotency_key", sa.String(128), nullable=True))
+ op.add_column(
+ "benchmark_runs", sa.Column("request_fingerprint", sa.String(64), nullable=True)
+ )
+ op.add_column("benchmark_runs", sa.Column("failure_code", sa.String(64), nullable=True))
+ op.add_column(
+ "benchmark_runs", sa.Column("cancellation_requested_at", sa.DateTime(timezone=True))
+ )
+ op.add_column(
+ "benchmark_runs",
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ )
+ op.add_column(
+ "benchmark_runs",
+ sa.Column("generation", sa.Integer(), server_default="1", nullable=False),
+ )
+ op.create_foreign_key(
+ "fk_benchmark_runs_capability_deployment",
+ "benchmark_runs",
+ "capability_deployments",
+ ["capability_deployment_id"],
+ ["id"],
+ ondelete="RESTRICT",
+ )
+ op.create_check_constraint(
+ "ck_benchmark_runs_operator_identity",
+ "benchmark_runs",
+ "(actor_subject IS NULL AND idempotency_key IS NULL AND "
+ "request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
+ "(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
+ "request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
+ )
+ op.create_unique_constraint(
+ "uq_benchmark_runs_actor_idempotency",
+ "benchmark_runs",
+ ["actor_subject", "idempotency_key"],
+ )
+ op.create_index("ix_benchmark_runs_actor_subject", "benchmark_runs", ["actor_subject"])
+ op.create_index(
+ "ix_benchmark_runs_capability_deployment_completed",
+ "benchmark_runs",
+ ["capability_deployment_id", "completed_at"],
+ )
+ if op.get_bind().dialect.name == "postgresql":
+ op.execute("REVOKE DELETE, TRUNCATE ON benchmark_runs FROM modelforge_runtime")
+ op.execute("GRANT SELECT, INSERT, UPDATE ON benchmark_runs TO modelforge_runtime")
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ benchmark_runs = sa.table("benchmark_runs", sa.column("actor_subject", sa.String()))
+ operator_history = bind.scalar(
+ sa.select(sa.func.count())
+ .select_from(benchmark_runs)
+ .where(benchmark_runs.c.actor_subject.is_not(None))
+ )
+ if operator_history:
+ raise RuntimeError(
+ "Operator benchmark history exists; restore the pre-migration backup instead of deleting it."
+ )
+ op.drop_index("ix_benchmark_runs_capability_deployment_completed", table_name="benchmark_runs")
+ op.drop_index("ix_benchmark_runs_actor_subject", table_name="benchmark_runs")
+ if op.get_bind().dialect.name == "sqlite":
+ with op.batch_alter_table("benchmark_runs", recreate="always") as batch:
+ batch.drop_constraint("uq_benchmark_runs_actor_idempotency", type_="unique")
+ batch.drop_constraint("ck_benchmark_runs_operator_identity", type_="check")
+ batch.drop_constraint("fk_benchmark_runs_capability_deployment", type_="foreignkey")
+ for column in (
+ "generation",
+ "created_at",
+ "cancellation_requested_at",
+ "request_fingerprint",
+ "failure_code",
+ "idempotency_key",
+ "actor_subject",
+ "target_snapshot",
+ "capability_deployment_id",
+ ):
+ batch.drop_column(column)
+ batch.alter_column("deployment_id", existing_type=sa.Uuid(), nullable=False)
+ else:
+ op.drop_constraint("uq_benchmark_runs_actor_idempotency", "benchmark_runs", type_="unique")
+ op.drop_constraint("ck_benchmark_runs_operator_identity", "benchmark_runs", type_="check")
+ op.drop_constraint(
+ "fk_benchmark_runs_capability_deployment", "benchmark_runs", type_="foreignkey"
+ )
+ for column in (
+ "generation",
+ "created_at",
+ "cancellation_requested_at",
+ "request_fingerprint",
+ "failure_code",
+ "idempotency_key",
+ "actor_subject",
+ "target_snapshot",
+ "capability_deployment_id",
+ ):
+ op.drop_column("benchmark_runs", column)
+ op.alter_column("benchmark_runs", "deployment_id", existing_type=sa.Uuid(), nullable=False)
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index e9d0b5f..1062970 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -1,10 +1,12 @@
[build-system]
-requires = ["hatchling"]
+requires = [
+ "hatchling==1.32.4",
+]
build-backend = "hatchling.build"
[project]
name = "modelforge-api"
-version = "1.2.2"
+version = "1.3.0-rc.1"
description = "ITWorx ModelForge control-plane API"
license = "AGPL-3.0-or-later"
requires-python = ">=3.12"
@@ -19,6 +21,7 @@ dependencies = [
"cryptography==50.0.1",
"redis==6.4.0",
"httpx==0.28.1",
+ "msgpack==1.2.2",
"huggingface-hub==1.29.0",
"nvidia-ml-py==13.610.43",
"psutil==7.2.2",
@@ -28,7 +31,7 @@ dependencies = [
[project.optional-dependencies]
dev = [
- "pytest==8.4.2",
+ "pytest==9.0.3",
"pytest-asyncio==1.4.0",
"ruff==0.16.5",
"mypy==1.20.2",
diff --git a/backend/src/modelforge_api/api/authorization.py b/backend/src/modelforge_api/api/authorization.py
index 9b253f7..8bef1e9 100644
--- a/backend/src/modelforge_api/api/authorization.py
+++ b/backend/src/modelforge_api/api/authorization.py
@@ -1,13 +1,16 @@
from __future__ import annotations
+import hashlib
import hmac
import re
from dataclasses import dataclass
+from datetime import UTC, datetime
from enum import StrEnum
from typing import Annotated
from fastapi import Depends, Header, HTTPException, Request
+from modelforge_api.domain.operator_identity import parse_operator_identities
from modelforge_api.settings import Settings, get_settings
@@ -35,6 +38,7 @@ class AccessBoundary(StrEnum):
_PUBLIC_REQUESTS = {
+ ("POST", "/api/v1/operator/login"),
("GET", "/"),
("GET", "/api/v1/health/live"),
("GET", "/api/v1/health/ready"),
@@ -56,14 +60,18 @@ _NODE_REQUESTS = {
("GET", "/api/v1/agent/serving-jobs/next"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
+ ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancellation"),
+ ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancelled"),
("POST", "/api/v1/agent/serving-state"),
}
_NODE_PARAMETERIZED_REQUESTS = (
("POST", re.compile(r"^/api/v1/agent/artifact-jobs/[^/]+/(?:progress|complete|fail)$")),
("POST", re.compile(r"^/api/v1/agent/runtime-probes/[^/]+/(?:progress|complete|fail)$")),
- ("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail)$")),
+ ("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail|cancellation|cancelled)$")),
)
_CAPABILITY_REQUIREMENTS = {
+ ("POST", "/api/v1/capabilities/creative.resource@1/allocations"): "creative.resource@1",
+ ("POST", "/api/v1/capabilities/creative.resource@1/profiles/observations"): "creative.resource@1",
("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): "rag.embedding@1",
("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): "rag.reranking@1",
("POST", "/api/v1/capabilities/document.ocr@1/invoke"): "document.ocr@1",
@@ -72,8 +80,19 @@ _CAPABILITY_REQUIREMENTS = {
("POST", "/api/v1/capability-experiments/{route_key}/invoke"): "rag.embedding@1",
("POST", "/v1/embeddings"): "rag.embedding@1",
}
+for _capability in (
+ "rag.embedding@1", "rag.reranking@1", "document.ocr@1",
+ "vision.embedding@1", "speech.transcription@1",
+):
+ _CAPABILITY_REQUIREMENTS[("GET", f"/api/v1/capabilities/{_capability}/check")] = _capability
_CAPABILITY_CLIENT_REQUESTS = set(_CAPABILITY_REQUIREMENTS)
+_CAPABILITY_REQUIREMENTS[("GET", "/api/v1/selected-model-comparisons/assigned")] = "rag.embedding@1"
+_CAPABILITY_CLIENT_REQUESTS.add(("GET", "/api/v1/selected-model-comparisons/assigned"))
_CAPABILITY_EXPERIMENT_REQUEST = re.compile(r"^/api/v1/capability-experiments/[^/]+/invoke$")
+_CREATIVE_ALLOCATION_REQUEST = re.compile(r"^/api/v1/capabilities/creative\.resource@1/allocations/[^/]+(?:/(?:attest|heartbeat|release|uncertain))?$")
+_COMPARISON_CONSUMER_REQUEST = re.compile(
+ r"^/api/v1/selected-model-comparisons/[^/]+/(claim|progress|result|finish)$"
+)
def access_boundary_for_request(
@@ -98,7 +117,12 @@ def access_boundary_for_request(
):
return AccessBoundary.NODE
if (normalized_method, path) in _CAPABILITY_CLIENT_REQUESTS or (
- normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+ normalized_method in {"GET", "POST"} and _CREATIVE_ALLOCATION_REQUEST.fullmatch(path)
+ ) or (
+ normalized_method == "POST" and (
+ _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+ or _COMPARISON_CONSUMER_REQUEST.fullmatch(path)
+ )
):
return AccessBoundary.CAPABILITY_CLIENT
return AccessBoundary.CONTROL_PLANE
@@ -111,7 +135,12 @@ def required_capability_for_request(method: str, path: str) -> str | None:
requirement = _CAPABILITY_REQUIREMENTS.get((normalized_method, path))
if requirement is not None:
return requirement
- if normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path):
+ if normalized_method in {"GET", "POST"} and _CREATIVE_ALLOCATION_REQUEST.fullmatch(path):
+ return "creative.resource@1"
+ if normalized_method == "POST" and (
+ _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+ or _COMPARISON_CONSUMER_REQUEST.fullmatch(path)
+ ):
return "rag.embedding@1"
return None
@@ -145,28 +174,70 @@ class Principal:
subject: str
role: PrincipalRole
authentication_method: str
+ credential_id: str | None = None
+ expires_at: datetime | None = None
def permits(self, required: PrincipalRole) -> bool:
return _ROLE_RANK[self.role] >= _ROLE_RANK[required]
def authenticate_operator_token(settings: Settings, token: str | None) -> Principal:
- """Authenticate the backward-compatible operator key as an admin principal.
-
- The legacy key is the sole human credential in this release. OIDC/session authentication can
- add another principal producer later without changing route authorization policy.
- """
+ """Authenticate a named expiring human key or the legacy bootstrap administrator."""
configured = settings.operator_api_key
- if configured is None or not configured.get_secret_value():
+ document = settings.operator_identities
+ identities = parse_operator_identities(document.get_secret_value()) if (
+ document is not None and document.get_secret_value()
+ ) else ()
+ if not identities and (configured is None or not configured.get_secret_value()):
raise HTTPException(status_code=503, detail="operator API authentication is not configured")
- if token is None or not hmac.compare_digest(token, configured.get_secret_value()):
+ if token is None or len(token) > 512 or token.startswith(("mfopm_", "mfsvc_", "mfnode_", "mfenroll_")):
raise HTTPException(status_code=401, detail="invalid operator credential")
- return Principal(
- subject="legacy-operator",
- role=PrincipalRole.ADMIN,
- authentication_method="legacy_admin_token",
- )
+ if token.startswith("mfop_"):
+ if not re.fullmatch(r"mfop_[A-Za-z0-9_-]{43}", token):
+ raise HTTPException(status_code=401, detail="invalid operator credential")
+ digest = hashlib.sha256(token.encode()).hexdigest()
+ for identity in identities:
+ if hmac.compare_digest(digest, identity.token_sha256):
+ if identity.expires_at <= datetime.now(UTC):
+ raise HTTPException(status_code=401, detail="invalid operator credential")
+ return Principal(
+ subject=identity.subject,
+ role=PrincipalRole(identity.role),
+ authentication_method="named_operator_key",
+ credential_id=identity.credential_id,
+ expires_at=identity.expires_at,
+ )
+ # The named namespace never falls back to the legacy administrator.
+ raise HTTPException(status_code=401, detail="invalid operator credential")
+ if configured is not None and hmac.compare_digest(
+ token.encode("utf-8"), configured.get_secret_value().encode("utf-8")
+ ):
+ return Principal(
+ subject="legacy-operator",
+ role=PrincipalRole.ADMIN,
+ authentication_method="legacy_admin_token",
+ )
+ if configured is not None and token.startswith("mfpw_"):
+ from modelforge_api.services.operator_password import verify_session
+
+ password_hash = settings.operator_password_hash
+ if password_hash is not None and verify_session(
+ token, configured.get_secret_value(), password_hash.get_secret_value()
+ ):
+ return Principal(
+ "password-operator", PrincipalRole.ADMIN, "password_session",
+ expires_at=datetime.fromtimestamp(int(token[5:].split(".")[1]), UTC),
+ )
+ raise HTTPException(status_code=401, detail="invalid operator credential")
+
+
+def operator_token_from_headers(admin_token: str | None, authorization: str | None) -> str | None:
+ if admin_token is not None:
+ return admin_token
+ if authorization is not None and authorization.startswith("Bearer mfpw_"):
+ return authorization.removeprefix("Bearer ")
+ return None
def authenticate_operator_principal(
@@ -179,7 +250,9 @@ def authenticate_operator_principal(
principal = getattr(request.state, "principal", None)
if isinstance(principal, Principal):
return principal
- return authenticate_operator_token(settings, token)
+ return authenticate_operator_token(
+ settings, operator_token_from_headers(token, request.headers.get("authorization"))
+ )
AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_principal)]
@@ -187,7 +260,10 @@ AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_prin
def _require_role(principal: Principal, required: PrincipalRole) -> Principal:
if not principal.permits(required):
- raise HTTPException(status_code=403, detail="operator role is not authorized")
+ raise HTTPException(
+ status_code=403,
+ detail=f"This action requires the {required.value} role; your role is {principal.role.value}.",
+ )
return principal
diff --git a/backend/src/modelforge_api/api/request_limits.py b/backend/src/modelforge_api/api/request_limits.py
index 9308859..31e874a 100644
--- a/backend/src/modelforge_api/api/request_limits.py
+++ b/backend/src/modelforge_api/api/request_limits.py
@@ -79,7 +79,14 @@ class RequestBodyLimitMiddleware:
"details": details or {},
}
},
- headers={"X-Correlation-ID": str(correlation_id)},
+ headers={
+ "X-Correlation-ID": str(correlation_id),
+ **(
+ {"Cache-Control": "no-store"}
+ if scope.get("path") == "/api/v1/operator/login"
+ else {}
+ ),
+ },
)
await response(scope, receive, send)
diff --git a/backend/src/modelforge_api/api/routes/acquisition.py b/backend/src/modelforge_api/api/routes/acquisition.py
index e8444ed..c90f9fc 100644
--- a/backend/src/modelforge_api/api/routes/acquisition.py
+++ b/backend/src/modelforge_api/api/routes/acquisition.py
@@ -18,8 +18,8 @@ from modelforge_api.domain.acquisition import (
UpstreamRefreshRequest,
UpstreamSnapshotResponse,
)
-from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
from modelforge_api.settings import Settings, get_settings
router = APIRouter(
@@ -33,9 +33,8 @@ def get_acquisition_service(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
) -> AcquisitionService:
- token = settings.hf_token.get_secret_value() if settings.hf_token else None
- provider = OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds)
- return AcquisitionService(session, settings, provider)
+ provider = SecretAwareHuggingFaceProvider(session, settings)
+ return AcquisitionService(session, settings, provider, credential_configured=provider.credential_configured)
Service = Annotated[AcquisitionService, Depends(get_acquisition_service)]
diff --git a/backend/src/modelforge_api/api/routes/agent.py b/backend/src/modelforge_api/api/routes/agent.py
index d3fbe26..71d205d 100644
--- a/backend/src/modelforge_api/api/routes/agent.py
+++ b/backend/src/modelforge_api/api/routes/agent.py
@@ -37,6 +37,12 @@ from modelforge_api.domain.node_decommission import (
NodeDecommissionPreview,
NodeDecommissionResult,
)
+from modelforge_api.domain.node_lifecycle import (
+ NodeLifecycleChange,
+ NodeLifecyclePreview,
+ NodeLifecyclePreviewRequest,
+ NodeLifecycleResult,
+)
from modelforge_api.domain.runtime import (
AgentRuntimeProbeComplete,
AgentRuntimeProbeControl,
@@ -46,11 +52,11 @@ from modelforge_api.domain.runtime import (
RuntimeProbeResponse,
)
from modelforge_api.persistence.models import ComputeNode, NodeCredential
-from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
from modelforge_api.services.acquisition import AcquisitionService
from modelforge_api.services.node_agent import NodeAgentService, NodeAuthenticationEvidence
from modelforge_api.services.node_decommission import NodeDecommissionService
from modelforge_api.services.runtime import RuntimeService
+from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
from modelforge_api.settings import Settings, get_settings
router = APIRouter(tags=["node-agent"])
@@ -119,6 +125,7 @@ def list_enrollments(service: Service, _admin: Admin) -> list[EnrollmentTokenSum
expires_at=item.expires_at,
used_at=item.used_at,
revoked_at=item.revoked_at,
+ enrolled_node_id=str(item.enrolled_node_id) if item.enrolled_node_id is not None else None,
)
for item in service.repository.enrollments()
]
@@ -141,6 +148,32 @@ def update_node(
return ActionResponse()
+@router.post(
+ "/api/v1/admin/hardware/nodes/{node_id}/lifecycle/preview",
+ response_model=NodeLifecyclePreview,
+)
+def preview_node_lifecycle(
+ node_id: uuid.UUID,
+ request: NodeLifecyclePreviewRequest,
+ service: DecommissionService,
+ _admin: Admin,
+) -> NodeLifecyclePreview:
+ return service.preview_enabled_change(node_id, request)
+
+
+@router.post(
+ "/api/v1/admin/hardware/nodes/{node_id}/lifecycle",
+ response_model=NodeLifecycleResult,
+)
+def change_node_lifecycle(
+ node_id: uuid.UUID,
+ request: NodeLifecycleChange,
+ service: DecommissionService,
+ _admin: Admin,
+) -> NodeLifecycleResult:
+ return service.apply_enabled_change(node_id, request)
+
+
@router.post(
"/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
response_model=NodeDecommissionPreview,
@@ -213,13 +246,14 @@ def get_agent_acquisition_service(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
) -> AcquisitionService:
- token = settings.hf_token.get_secret_value() if settings.hf_token else None
+ provider = SecretAwareHuggingFaceProvider(session, settings)
return AcquisitionService(
session,
settings,
- OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds),
+ provider,
actor_type="node_agent",
actor_id="authenticated-node",
+ credential_configured=provider.credential_configured,
)
diff --git a/backend/src/modelforge_api/api/routes/audit_timeline.py b/backend/src/modelforge_api/api/routes/audit_timeline.py
new file mode 100644
index 0000000..1e2d70f
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/audit_timeline.py
@@ -0,0 +1,37 @@
+"""Admin-only, read-only audit timeline API."""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin
+from modelforge_api.db import get_session
+from modelforge_api.domain.audit_timeline import AuditTimelineRequest, AuditTimelineResponse
+from modelforge_api.services.audit_timeline import (
+ AuditTimelineCursorError,
+ AuditTimelineService,
+)
+
+router = APIRouter(prefix="/api/v1/admin/audit", tags=["audit"])
+
+
+def get_service(session: Annotated[Session, Depends(get_session)]) -> AuditTimelineService:
+ return AuditTimelineService(session)
+
+
+Service = Annotated[AuditTimelineService, Depends(get_service)]
+
+
+@router.get("/events", response_model=AuditTimelineResponse)
+def audit_timeline(
+ query: Annotated[AuditTimelineRequest, Query()],
+ service: Service,
+ _admin: Admin,
+) -> AuditTimelineResponse:
+ try:
+ return service.list_events(query)
+ except AuditTimelineCursorError as error:
+ raise HTTPException(status_code=422, detail=str(error)) from None
diff --git a/backend/src/modelforge_api/api/routes/catalog.py b/backend/src/modelforge_api/api/routes/catalog.py
index 39fb63f..24ed9e5 100644
--- a/backend/src/modelforge_api/api/routes/catalog.py
+++ b/backend/src/modelforge_api/api/routes/catalog.py
@@ -1,9 +1,13 @@
+from collections import defaultdict
+from uuid import UUID
+
from fastapi import APIRouter, Depends
-from sqlalchemy import select
-from sqlalchemy.orm import Session
+from sqlalchemy import select, tuple_
+from sqlalchemy.orm import Session, aliased, selectinload
from modelforge_api.api.authorization import require_viewer
from modelforge_api.db import get_session
+from modelforge_api.domain.contracts import ProjectBindingManifest
from modelforge_api.domain.schemas import (
CapabilityContractResponse,
CapabilityEstateResponse,
@@ -40,22 +44,31 @@ def list_capabilities(
registry: ManifestRegistry = Depends(get_manifest_registry),
session: Session = Depends(get_session),
) -> list[CapabilityContractResponse]:
- responses = []
- for contract in registry.capabilities():
- stable = session.execute(
- select(CapabilityDeployment)
+ manifests = registry.capabilities()
+ identities = {(item.capability, item.version) for item in manifests}
+ stable_by_contract: dict[tuple[str, int], CapabilityDeployment] = {}
+ if identities:
+ stable_rows = session.execute(
+ select(Capability.key, CapabilityContract.version, CapabilityDeployment)
.join(
CapabilityContract,
CapabilityContract.id == CapabilityDeployment.capability_contract_id,
)
.join(Capability, Capability.id == CapabilityContract.capability_id)
.where(
- Capability.key == contract.capability,
- CapabilityContract.version == contract.version,
+ tuple_(Capability.key, CapabilityContract.version).in_(identities),
CapabilityDeployment.status == "stable",
CapabilityDeployment.production.is_(True),
)
- ).scalar_one_or_none()
+ )
+ for key, version, deployment in stable_rows:
+ identity = (key, version)
+ if identity in stable_by_contract:
+ raise ValueError(f"multiple stable deployments for {key}@{version}")
+ stable_by_contract[identity] = deployment
+ responses = []
+ for contract in manifests:
+ stable = stable_by_contract.get((contract.capability, contract.version))
responses.append(
CapabilityContractResponse(
key=contract.capability,
@@ -82,45 +95,121 @@ def capability_estate(
registry: ManifestRegistry = Depends(get_manifest_registry),
session: Session = Depends(get_session),
) -> list[CapabilityEstateResponse]:
- responses: list[CapabilityEstateResponse] = []
- for manifest in registry.capabilities():
- contract = session.scalar(
- select(CapabilityContract)
+ manifests = registry.capabilities()
+ contracts_by_identity: dict[tuple[str, int], CapabilityContract] = {}
+ identities = [(item.capability, item.version) for item in manifests]
+ if identities:
+ for contract, capability_key in session.execute(
+ select(CapabilityContract, Capability.key)
.join(Capability, Capability.id == CapabilityContract.capability_id)
- .where(Capability.key == manifest.capability, CapabilityContract.version == manifest.version)
+ .where(tuple_(Capability.key, CapabilityContract.version).in_(identities))
+ ):
+ contracts_by_identity[capability_key, contract.version] = contract
+
+ contract_ids = [contract.id for contract in contracts_by_identity.values()]
+ deployments_by_contract: dict[UUID, CapabilityDeployment] = {}
+ if contract_ids:
+ deployment_candidate = aliased(CapabilityDeployment)
+ selected_deployment_id = (
+ select(deployment_candidate.id)
+ .where(
+ deployment_candidate.capability_contract_id == CapabilityContract.id
+ )
+ .order_by(
+ deployment_candidate.production.desc(),
+ deployment_candidate.created_at.desc(),
+ )
+ .limit(1)
+ .correlate(CapabilityContract)
+ .scalar_subquery()
)
- deployment = None
- model = None
- revision = None
- profile = None
- envelope = None
- latest_evaluation = None
- if contract:
- deployment = session.scalar(
- select(CapabilityDeployment)
- .where(CapabilityDeployment.capability_contract_id == contract.id)
- .order_by(
- CapabilityDeployment.production.desc(),
- CapabilityDeployment.created_at.desc(),
- )
- .limit(1)
+ for selected_deployment in session.scalars(
+ select(CapabilityDeployment)
+ .join(
+ CapabilityContract,
+ CapabilityContract.id == CapabilityDeployment.capability_contract_id,
)
- if deployment:
- artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
- revision = session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
- model = session.get(Model, revision.model_id) if revision else None
- profile = session.get(RuntimeProfile, deployment.runtime_profile_id)
- envelope = session.scalar(
- select(CapabilityResourceEnvelope).where(
- CapabilityResourceEnvelope.capability_deployment_id == deployment.id
- )
+ .where(
+ CapabilityContract.id.in_(contract_ids),
+ CapabilityDeployment.id == selected_deployment_id,
)
- latest_evaluation = session.scalar(
- select(CapabilityEvaluationRun)
- .where(CapabilityEvaluationRun.capability_deployment_id == deployment.id)
- .order_by(CapabilityEvaluationRun.created_at.desc())
- .limit(1)
+ ):
+ deployments_by_contract[selected_deployment.capability_contract_id] = selected_deployment
+
+ deployments = list(deployments_by_contract.values())
+ deployment_ids = [item.id for item in deployments]
+ artifact_set_ids = list({item.artifact_set_id for item in deployments})
+ artifact_sets = (
+ session.scalars(select(ArtifactSet).where(ArtifactSet.id.in_(artifact_set_ids))).all()
+ if artifact_set_ids
+ else []
+ )
+ artifact_sets_by_id = {item.id: item for item in artifact_sets}
+ revision_ids = list({item.revision_id for item in artifact_sets})
+ revisions = (
+ session.scalars(select(ModelRevision).where(ModelRevision.id.in_(revision_ids))).all()
+ if revision_ids
+ else []
+ )
+ revisions_by_id = {item.id: item for item in revisions}
+ model_ids = list({item.model_id for item in revisions})
+ models = (
+ session.scalars(select(Model).where(Model.id.in_(model_ids))).all() if model_ids else []
+ )
+ models_by_id = {item.id: item for item in models}
+ profile_ids = list({item.runtime_profile_id for item in deployments})
+ profiles = (
+ session.scalars(select(RuntimeProfile).where(RuntimeProfile.id.in_(profile_ids))).all()
+ if profile_ids
+ else []
+ )
+ profiles_by_id = {item.id: item for item in profiles}
+ envelopes = (
+ session.scalars(
+ select(CapabilityResourceEnvelope).where(
+ CapabilityResourceEnvelope.capability_deployment_id.in_(deployment_ids)
)
+ ).all()
+ if deployment_ids
+ else []
+ )
+ envelopes_by_deployment = {item.capability_deployment_id: item for item in envelopes}
+ latest_evaluations: dict[UUID, CapabilityEvaluationRun] = {}
+ if deployment_ids:
+ evaluation_candidate = aliased(CapabilityEvaluationRun)
+ latest_evaluation_id = (
+ select(evaluation_candidate.id)
+ .where(
+ evaluation_candidate.capability_deployment_id == CapabilityDeployment.id
+ )
+ .order_by(evaluation_candidate.created_at.desc())
+ .limit(1)
+ .correlate(CapabilityDeployment)
+ .scalar_subquery()
+ )
+ for evaluation in session.scalars(
+ select(CapabilityEvaluationRun)
+ .join(
+ CapabilityDeployment,
+ CapabilityDeployment.id == CapabilityEvaluationRun.capability_deployment_id,
+ )
+ .where(
+ CapabilityDeployment.id.in_(deployment_ids),
+ CapabilityEvaluationRun.id == latest_evaluation_id,
+ )
+ ):
+ latest_evaluations[evaluation.capability_deployment_id] = evaluation
+
+ responses: list[CapabilityEstateResponse] = []
+ for manifest in manifests:
+ contract = contracts_by_identity.get((manifest.capability, manifest.version))
+ deployment = deployments_by_contract.get(contract.id) if contract else None
+ artifact_set = artifact_sets_by_id.get(deployment.artifact_set_id) if deployment else None
+ revision = revisions_by_id.get(artifact_set.revision_id) if artifact_set else None
+ model = models_by_id.get(revision.model_id) if revision else None
+ profile = profiles_by_id.get(deployment.runtime_profile_id) if deployment else None
+ envelope = envelopes_by_deployment.get(deployment.id) if deployment else None
+ latest_evaluation = latest_evaluations.get(deployment.id) if deployment else None
operational_state = (
deployment.status
if deployment
@@ -157,67 +246,142 @@ def capability_estate(
def model_installation_rationale(
session: Session = Depends(get_session),
) -> list[ModelInstallationRationaleResponse]:
- responses: list[ModelInstallationRationaleResponse] = []
- for model in session.scalars(select(Model).order_by(Model.display_name)).all():
- revisions = session.scalars(
- select(ModelRevision).where(ModelRevision.model_id == model.id)
- ).all()
- revision_ids = [item.id for item in revisions]
- sets = (
- session.scalars(select(ArtifactSet).where(ArtifactSet.revision_id.in_(revision_ids))).all()
- if revision_ids
- else []
+ models = session.scalars(select(Model).order_by(Model.display_name)).all()
+ installed_sets_by_model: dict[UUID, list[ArtifactSet]] = defaultdict(list)
+ if models:
+ for artifact_set, model_id in session.execute(
+ select(ArtifactSet, ModelRevision.model_id)
+ .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
+ .join(Model, Model.id == ModelRevision.model_id)
+ .where(ArtifactSet.availability == "local")
+ ):
+ installed_sets_by_model[model_id].append(artifact_set)
+
+ installed_sets = [
+ artifact_set
+ for model in models
+ for artifact_set in installed_sets_by_model.get(model.id, [])
+ ]
+ deployments_by_artifact_set: dict[UUID, list[CapabilityDeployment]] = defaultdict(list)
+ if installed_sets:
+ for deployment in session.scalars(
+ select(CapabilityDeployment)
+ .join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
+ .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
+ .join(Model, Model.id == ModelRevision.model_id)
+ .where(ArtifactSet.availability == "local")
+ ):
+ deployments_by_artifact_set[deployment.artifact_set_id].append(deployment)
+
+ contract_and_capability_by_deployment: dict[
+ UUID, tuple[CapabilityContract, Capability]
+ ] = {}
+ if installed_sets:
+ for deployment, contract, capability in session.execute(
+ select(CapabilityDeployment, CapabilityContract, Capability)
+ .join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
+ .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
+ .join(Model, Model.id == ModelRevision.model_id)
+ .join(
+ CapabilityContract,
+ CapabilityContract.id == CapabilityDeployment.capability_contract_id,
+ )
+ .join(Capability, Capability.id == CapabilityContract.capability_id)
+ .where(ArtifactSet.availability == "local")
+ ):
+ contract_and_capability_by_deployment[deployment.id] = (contract, capability)
+
+ eligible_deployments = (
+ select(
+ CapabilityDeployment.id.label("deployment_id"),
+ CapabilityDeployment.capability_contract_id.label("contract_id"),
)
- installed_sets = [item for item in sets if item.availability == "local"]
+ .join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
+ .join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
+ .join(Model, Model.id == ModelRevision.model_id)
+ .join(
+ CapabilityContract,
+ CapabilityContract.id == CapabilityDeployment.capability_contract_id,
+ )
+ .join(Capability, Capability.id == CapabilityContract.capability_id)
+ .where(ArtifactSet.availability == "local")
+ .subquery()
+ )
+ eligible_contract_ids = select(eligible_deployments.c.contract_id).distinct()
+ eligible_deployment_ids = select(eligible_deployments.c.deployment_id)
+
+ project_consumers: dict[UUID, list[str]] = defaultdict(list)
+ if installed_sets:
+ for contract_id, project_key in session.execute(
+ select(ProjectBinding.capability_contract_id, Project.key)
+ .join(Project, Project.id == ProjectBinding.project_id)
+ .where(
+ ProjectBinding.capability_contract_id.in_(eligible_contract_ids),
+ ProjectBinding.deprecated_at.is_(None),
+ )
+ .distinct()
+ ):
+ project_consumers[contract_id].append(project_key)
+
+ active_project_consumers: dict[UUID, list[str]] = defaultdict(list)
+ if installed_sets:
+ for contract_id, project_key in session.execute(
+ select(ProjectBinding.capability_contract_id, Project.key)
+ .join(Project, Project.id == ProjectBinding.project_id)
+ .join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
+ .where(
+ ProjectBinding.capability_contract_id.in_(eligible_contract_ids),
+ ServiceClient.status == "active",
+ )
+ .distinct()
+ ):
+ active_project_consumers[contract_id].append(project_key)
+
+ fit_evidence_ids: dict[UUID, list[UUID]] = defaultdict(list)
+ if installed_sets:
+ for contract_id, evidence_id in session.execute(
+ select(ProjectBinding.capability_contract_id, ProjectFitEvidence.id)
+ .join(ProjectBinding, ProjectBinding.id == ProjectFitEvidence.project_binding_id)
+ .where(ProjectBinding.capability_contract_id.in_(eligible_contract_ids))
+ ):
+ fit_evidence_ids[contract_id].append(evidence_id)
+
+ evaluation_ids: dict[UUID, list[UUID]] = defaultdict(list)
+ if installed_sets:
+ for deployment_id, evaluation_id in session.execute(
+ select(
+ CapabilityEvaluationRun.capability_deployment_id,
+ CapabilityEvaluationRun.id,
+ ).where(
+ CapabilityEvaluationRun.capability_deployment_id.in_(eligible_deployment_ids)
+ )
+ ):
+ evaluation_ids[deployment_id].append(evaluation_id)
+
+ residency_by_deployment = (
+ {
+ item.capability_deployment_id: item
+ for item in session.scalars(
+ select(ResidencyAllocation).where(
+ ResidencyAllocation.capability_deployment_id.in_(eligible_deployment_ids)
+ )
+ )
+ }
+ if installed_sets
+ else {}
+ )
+
+ responses: list[ModelInstallationRationaleResponse] = []
+ for model in models:
+ model_installed_sets = installed_sets_by_model.get(model.id, [])
dependencies: list[InstallationDependencyResponse] = []
- for artifact_set in installed_sets:
- deployments = session.scalars(
- select(CapabilityDeployment).where(
- CapabilityDeployment.artifact_set_id == artifact_set.id
- )
- ).all()
- for deployment in deployments:
- contract = session.get(CapabilityContract, deployment.capability_contract_id)
- capability = session.get(Capability, contract.capability_id) if contract else None
- if not contract or not capability:
+ for artifact_set in model_installed_sets:
+ for deployment in deployments_by_artifact_set.get(artifact_set.id, []):
+ relation = contract_and_capability_by_deployment.get(deployment.id)
+ if not relation:
continue
- projects = session.scalars(
- select(Project)
- .join(ProjectBinding, ProjectBinding.project_id == Project.id)
- .where(
- ProjectBinding.capability_contract_id == contract.id,
- ProjectBinding.deprecated_at.is_(None),
- )
- .distinct()
- ).all()
- active_projects = session.scalars(
- select(Project)
- .join(ProjectBinding, ProjectBinding.project_id == Project.id)
- .join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
- .where(
- ProjectBinding.capability_contract_id == contract.id,
- ServiceClient.status == "active",
- )
- .distinct()
- ).all()
- project_fit_ids = session.scalars(
- select(ProjectFitEvidence.id)
- .join(
- ProjectBinding,
- ProjectBinding.id == ProjectFitEvidence.project_binding_id,
- )
- .where(ProjectBinding.capability_contract_id == contract.id)
- ).all()
- evaluations = session.scalars(
- select(CapabilityEvaluationRun).where(
- CapabilityEvaluationRun.capability_deployment_id == deployment.id
- )
- ).all()
- residency = session.scalar(
- select(ResidencyAllocation).where(
- ResidencyAllocation.capability_deployment_id == deployment.id
- )
- )
+ contract, capability = relation
+ residency = residency_by_deployment.get(deployment.id)
dependencies.append(
InstallationDependencyResponse(
capability=capability.key,
@@ -225,10 +389,10 @@ def model_installation_rationale(
deployment_id=deployment.id,
channel=deployment.channel,
production=deployment.production,
- project_consumers=[item.key for item in projects],
- active_project_consumers=[item.key for item in active_projects],
- project_fit_evidence_ids=list(project_fit_ids),
- evaluation_run_ids=[item.id for item in evaluations],
+ project_consumers=project_consumers.get(contract.id, []),
+ active_project_consumers=active_project_consumers.get(contract.id, []),
+ project_fit_evidence_ids=fit_evidence_ids.get(contract.id, []),
+ evaluation_run_ids=evaluation_ids.get(deployment.id, []),
last_used_at=residency.last_used_at if residency else None,
)
)
@@ -242,10 +406,10 @@ def model_installation_rationale(
model_id=model.id,
display_name=model.display_name,
upstream_source=model.upstream_source,
- installed=bool(installed_sets),
- installed_bytes=sum(item.total_size_bytes for item in installed_sets),
+ installed=bool(model_installed_sets),
+ installed_bytes=sum(item.total_size_bytes for item in model_installed_sets),
dependencies=dependencies,
- can_delete=bool(installed_sets) and not blockers,
+ can_delete=bool(model_installed_sets) and not blockers,
deletion_blockers=blockers,
)
)
@@ -255,8 +419,9 @@ def model_installation_rationale(
@router.get("/projects", response_model=list[ProjectResponse])
def list_projects(
registry: ManifestRegistry = Depends(get_manifest_registry),
+ session: Session = Depends(get_session),
) -> list[ProjectResponse]:
- return [
+ declared = [
ProjectResponse(
**project.project.model_dump(),
notes=project.notes,
@@ -271,3 +436,57 @@ def list_projects(
)
for project in registry.projects()
]
+ responses = {project.id: project for project in declared}
+ projects = session.scalars(
+ select(Project).options(selectinload(Project.bindings)).order_by(Project.key)
+ ).all()
+ contract_ids = {
+ binding.capability_contract_id
+ for project in projects
+ for binding in project.bindings
+ if binding.deprecated_at is None
+ }
+ contracts = (
+ {
+ item.id: item
+ for item in session.scalars(
+ select(CapabilityContract).where(CapabilityContract.id.in_(contract_ids))
+ )
+ }
+ if contract_ids
+ else {}
+ )
+ capability_ids = {item.capability_id for item in contracts.values()}
+ capabilities = (
+ {
+ item.id: item
+ for item in session.scalars(
+ select(Capability).where(Capability.id.in_(capability_ids))
+ )
+ }
+ if capability_ids
+ else {}
+ )
+ for item in projects:
+ bindings = []
+ for binding in item.bindings:
+ if binding.deprecated_at:
+ continue
+ contract = contracts.get(binding.capability_contract_id)
+ capability = capabilities.get(contract.capability_id) if contract else None
+ if not contract or not capability:
+ continue
+ bindings.append(ProjectBindingResponse(
+ capability=capability.key, contract_version=contract.version,
+ binding=ProjectBindingManifest.model_validate({
+ "contract_version": contract.version, "channel": binding.channel,
+ "priority": binding.priority, "optional": binding.optional,
+ "fallback": binding.fallback_policy, "migration_support": binding.migration_support,
+ "slo": binding.slo, "benchmark_suites": binding.benchmark_requirements,
+ }),
+ ))
+ responses[item.key] = ProjectResponse(
+ id=item.key, name=item.name, description=item.description, bindings=bindings,
+ notes=responses[item.key].notes if item.key in responses else [],
+ )
+ return sorted(responses.values(), key=lambda item: item.id)
diff --git a/backend/src/modelforge_api/api/routes/configuration.py b/backend/src/modelforge_api/api/routes/configuration.py
new file mode 100644
index 0000000..0f06dbd
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/configuration.py
@@ -0,0 +1,154 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Path, Query, Request
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, Operator, Viewer, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.configuration import (
+ ConfigurationApplied,
+ ConfigurationApply,
+ ConfigurationCatalogResponse,
+ ConfigurationChange,
+ ConfigurationExport,
+ ConfigurationImport,
+ ConfigurationPreview,
+ ConfigurationProposal,
+ ConfigurationRevisionResponse,
+ ConfigurationRollback,
+)
+from modelforge_api.domain.secret_slots import (
+ SecretSlotCatalog,
+ SecretSlotMetadata,
+ SecretSlotRevoke,
+ SecretSlotSet,
+ SecretSlotVersionRequest,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.configuration import build_configuration_catalog
+from modelforge_api.services.configuration_changes import (
+ ConfigurationService,
+ resolved_configuration,
+)
+from modelforge_api.services.secret_slots import SecretSlotService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+ prefix="/api/v1/configuration",
+ tags=["configuration"],
+ dependencies=[Depends(require_viewer)],
+)
+Database = Annotated[Session, Depends(get_session)]
+RuntimeSettings = Annotated[Settings, Depends(get_settings)]
+
+
+def get_configuration_service(
+ session: Database,
+ settings: RuntimeSettings,
+ principal: Viewer,
+ request: Request,
+) -> ConfigurationService:
+ return ConfigurationService(
+ session,
+ settings,
+ AuditContext(
+ actor_type="operator",
+ actor_id=principal.subject,
+ correlation_id=request.state.correlation_id,
+ ),
+ )
+
+
+Service = Annotated[ConfigurationService, Depends(get_configuration_service)]
+
+
+@router.get("/settings", response_model=ConfigurationCatalogResponse)
+def configuration_settings(
+ session: Database,
+ settings: RuntimeSettings,
+) -> ConfigurationCatalogResponse:
+ revision, overrides, effective = resolved_configuration(session, settings)
+ return build_configuration_catalog(
+ effective,
+ revision=revision,
+ overrides=overrides,
+ )
+
+
+@router.post("/preview", response_model=ConfigurationPreview)
+def configuration_preview(
+ body: ConfigurationChange, service: Service, _principal: Operator
+) -> ConfigurationPreview:
+ return service.preview(body)
+
+
+@router.post("/apply", response_model=ConfigurationApplied)
+def configuration_apply(
+ body: ConfigurationApply, service: Service, _principal: Admin
+) -> ConfigurationApplied:
+ return service.apply(body)
+
+
+@router.get("/history", response_model=list[ConfigurationRevisionResponse])
+def configuration_history(
+ service: Service, limit: Annotated[int, Query(ge=1, le=100)] = 30
+) -> list[ConfigurationRevisionResponse]:
+ return service.history(limit)
+
+
+@router.post("/revisions/{revision}/preview", response_model=ConfigurationProposal)
+def configuration_rollback_preview(
+ revision: Annotated[int, Path(ge=0)],
+ body: ConfigurationRollback,
+ service: Service,
+ _principal: Operator,
+) -> ConfigurationProposal:
+ change = service.rollback_change(revision, body.expected_revision)
+ return ConfigurationProposal(change=change, preview=service.preview(change))
+
+
+@router.get("/export", response_model=ConfigurationExport)
+def configuration_export(service: Service) -> ConfigurationExport:
+ return service.export()
+
+
+@router.post("/import/preview", response_model=ConfigurationProposal)
+def configuration_import_preview(
+ body: ConfigurationImport,
+ service: Service,
+ _principal: Operator,
+) -> ConfigurationProposal:
+ change = service.import_change(body)
+ return ConfigurationProposal(change=change, preview=service.preview(change))
+
+
+def secret_slot_service(session: Database, settings: RuntimeSettings, principal: Viewer, request: Request) -> SecretSlotService:
+ return SecretSlotService(session, settings, principal, request.state.correlation_id)
+
+
+Secrets = Annotated[SecretSlotService, Depends(secret_slot_service)]
+
+
+@router.get("/secret-slots", response_model=SecretSlotCatalog)
+def secret_slots(service: Secrets) -> SecretSlotCatalog:
+ return service.catalog()
+
+
+@router.post("/secret-slots/huggingface_control_plane/set", response_model=SecretSlotMetadata)
+def secret_slot_set(body: SecretSlotSet, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+ return service.set(body)
+
+
+@router.post("/secret-slots/huggingface_control_plane/test", response_model=SecretSlotMetadata)
+def secret_slot_test(body: SecretSlotVersionRequest, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+ return service.test(body)
+
+
+@router.post("/secret-slots/huggingface_control_plane/activate", response_model=SecretSlotMetadata)
+def secret_slot_activate(body: SecretSlotVersionRequest, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+ return service.activate(body)
+
+
+@router.post("/secret-slots/huggingface_control_plane/revoke", response_model=SecretSlotMetadata)
+def secret_slot_revoke(body: SecretSlotRevoke, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
+ return service.revoke(body)
diff --git a/backend/src/modelforge_api/api/routes/external_creative.py b/backend/src/modelforge_api/api/routes/external_creative.py
new file mode 100644
index 0000000..7b41527
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/external_creative.py
@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, Request, status
+
+from modelforge_api.api.authorization import Operator
+from modelforge_api.api.routes.serving import Service, _authenticate_capability_client
+from modelforge_api.domain.external_creative import (
+ ExternalAllocationAction,
+ ExternalAllocationCreate,
+ ExternalAllocationList,
+ ExternalAllocationReconcile,
+ ExternalAllocationResponse,
+ ExternalProfileObservation,
+ ExternalResourceApproval,
+ ExternalResourceList,
+ ExternalResourceResponse,
+)
+from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.external_creative import ExternalCreativeService
+
+router = APIRouter(tags=["creative-resource-coordination"])
+
+
+def creative_client(
+ request: Request,
+ service: Service,
+ authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+ return _authenticate_capability_client(request, service, authorization, "creative.resource@1")
+
+
+Client = Annotated[ServiceClient, Depends(creative_client)]
+
+
+@router.post(
+ "/api/v1/capabilities/creative.resource@1/allocations",
+ response_model=ExternalAllocationResponse,
+ status_code=status.HTTP_201_CREATED,
+)
+def acquire(
+ body: ExternalAllocationCreate, service: Service, client: Client
+) -> ExternalAllocationResponse:
+ return ExternalCreativeService(service.session, service).acquire(body, client)
+
+
+@router.get(
+ "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}",
+ response_model=ExternalAllocationResponse,
+)
+def allocation(
+ allocation_id: uuid.UUID, service: Service, client: Client
+) -> ExternalAllocationResponse:
+ return ExternalCreativeService(service.session, service).get(allocation_id, client)
+
+
+def _action(
+ allocation_id: uuid.UUID,
+ body: ExternalAllocationAction,
+ action: str,
+ service: Service,
+ client: ServiceClient,
+) -> ExternalAllocationResponse:
+ return ExternalCreativeService(service.session, service).action(
+ allocation_id,
+ body.request_id,
+ action,
+ client,
+ body.observed_peak_vram_bytes,
+ body.attestation,
+ )
+
+
+@router.post(
+ "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/heartbeat",
+ response_model=ExternalAllocationResponse,
+)
+def heartbeat(
+ allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+ return _action(allocation_id, body, "heartbeat", service, client)
+
+
+@router.post(
+ "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/attest",
+ response_model=ExternalAllocationResponse,
+)
+def attest(
+ allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+ return _action(allocation_id, body, "attest", service, client)
+
+
+@router.post(
+ "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/release",
+ response_model=ExternalAllocationResponse,
+)
+def release(
+ allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+ return _action(allocation_id, body, "release", service, client)
+
+
+@router.post(
+ "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/uncertain",
+ response_model=ExternalAllocationResponse,
+)
+def uncertain(
+ allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
+) -> ExternalAllocationResponse:
+ return _action(allocation_id, body, "uncertain", service, client)
+
+
+@router.post(
+ "/api/v1/capabilities/creative.resource@1/profiles/observations",
+ response_model=ExternalResourceResponse,
+)
+def observe_profile(
+ body: ExternalProfileObservation, service: Service, client: Client
+) -> ExternalResourceResponse:
+ return ExternalCreativeService(service.session, service).observe(body, client)
+
+
+@router.put(
+ "/api/v1/operator/creative-resource/resources",
+ response_model=ExternalResourceResponse,
+)
+def configure_resource(
+ body: ExternalResourceApproval, service: Service, operator: Operator
+) -> ExternalResourceResponse:
+ return ExternalCreativeService(service.session, service).upsert_resource(body, operator.subject)
+
+
+@router.get(
+ "/api/v1/operator/creative-resource/resources",
+ response_model=ExternalResourceList,
+)
+def operator_resources(
+ service: Service,
+ _operator: Operator,
+ project_id: uuid.UUID | None = None,
+ limit: int = 50,
+ offset: int = 0,
+) -> ExternalResourceList:
+ return ExternalCreativeService(service.session, service).list_resources(
+ project_id,
+ min(max(limit, 1), 200),
+ max(offset, 0),
+ )
+
+
+@router.get(
+ "/api/v1/operator/creative-resource/allocations",
+ response_model=ExternalAllocationList,
+)
+def operator_allocations(
+ service: Service,
+ _operator: Operator,
+ state: str | None = None,
+ limit: int = 50,
+ offset: int = 0,
+) -> ExternalAllocationList:
+ if state not in {None, "granted", "queued", "blocked", "uncertain", "released"}:
+ from fastapi import HTTPException
+
+ raise HTTPException(status_code=422, detail="unsupported allocation state")
+ return ExternalCreativeService(service.session, service).list_allocations(
+ state, min(max(limit, 1), 200), max(offset, 0)
+ )
+
+
+@router.post(
+ "/api/v1/operator/creative-resource/allocations/{allocation_id}/reconcile",
+ response_model=ExternalAllocationResponse,
+)
+def reconcile_allocation(
+ allocation_id: uuid.UUID,
+ body: ExternalAllocationReconcile,
+ service: Service,
+ operator: Operator,
+) -> ExternalAllocationResponse:
+ return ExternalCreativeService(service.session, service).reconcile(
+ allocation_id,
+ body.action,
+ body.reason,
+ operator.subject,
+ )
diff --git a/backend/src/modelforge_api/api/routes/identity.py b/backend/src/modelforge_api/api/routes/identity.py
new file mode 100644
index 0000000..1fbf0d6
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/identity.py
@@ -0,0 +1,79 @@
+from datetime import UTC, datetime
+from typing import Annotated, Literal
+
+from fastapi import APIRouter, Depends, Request
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, PrincipalRole, Viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.managed_identity import (
+ ManagedIdentityChange,
+ ManagedIdentityCreate,
+ ManagedIdentityIssued,
+ ManagedIdentityList,
+ ManagedIdentityResult,
+ ManagedIdentityRevoke,
+ ManagedIdentityRotate,
+)
+from modelforge_api.services.operator_identities import ManagedIdentityService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/operator", tags=["operator-identity"])
+
+
+class CurrentOperator(BaseModel):
+ subject: str
+ role: PrincipalRole
+ authentication_method: str
+ credential_id: str | None
+ credential_expires_at: datetime | None
+ server_time: datetime
+ console_lock_after_seconds: Literal[900] = 900
+
+
+@router.get("/identity", response_model=CurrentOperator)
+def current_operator(principal: Viewer) -> CurrentOperator:
+ return CurrentOperator(
+ subject=principal.subject,
+ role=principal.role,
+ authentication_method=principal.authentication_method,
+ credential_id=principal.credential_id,
+ credential_expires_at=principal.expires_at,
+ server_time=datetime.now(UTC),
+ )
+
+
+def managed_identity_service(
+ session: Annotated[Session, Depends(get_session)], principal: Admin, request: Request,
+) -> ManagedIdentityService:
+ return ManagedIdentityService(session, principal, request.state.correlation_id)
+
+
+Service = Annotated[ManagedIdentityService, Depends(managed_identity_service)]
+
+
+@router.get("/managed-identities", response_model=ManagedIdentityList)
+def managed_identities(service: Service, _admin: Admin,
+ settings: Annotated[Settings, Depends(get_settings)]) -> ManagedIdentityList:
+ return service.list(settings)
+
+
+@router.post("/managed-identities", response_model=ManagedIdentityIssued, status_code=201)
+def create_managed_identity(body: ManagedIdentityCreate, service: Service, _admin: Admin) -> ManagedIdentityIssued:
+ return service.create(body)
+
+
+@router.post("/managed-identities/{slug}/rotate", response_model=ManagedIdentityIssued)
+def rotate_managed_identity(slug: str, body: ManagedIdentityRotate, service: Service, _admin: Admin) -> ManagedIdentityIssued:
+ return service.rotate(slug, body)
+
+
+@router.patch("/managed-identities/{slug}", response_model=ManagedIdentityResult)
+def change_managed_identity(slug: str, body: ManagedIdentityChange, service: Service, _admin: Admin) -> ManagedIdentityResult:
+ return service.change(slug, body)
+
+
+@router.post("/managed-identities/{slug}/revoke", response_model=ManagedIdentityResult)
+def revoke_managed_identity(slug: str, body: ManagedIdentityRevoke, service: Service, _admin: Admin) -> ManagedIdentityResult:
+ return service.revoke(slug, body)
diff --git a/backend/src/modelforge_api/api/routes/lab.py b/backend/src/modelforge_api/api/routes/lab.py
new file mode 100644
index 0000000..12bfdcb
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/lab.py
@@ -0,0 +1,162 @@
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, Response, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Operator, require_operator
+from modelforge_api.api.routes.serving import Service as ServingDependency
+from modelforge_api.db import get_session
+from modelforge_api.domain.benchmarking import (
+ BenchmarkRunResponse,
+ BenchmarkRunStartRequest,
+ BenchmarkSuiteResponse,
+)
+from modelforge_api.domain.lab import (
+ LabExecutionRunResponse,
+ LabExecutionStartRequest,
+ LabPreflightRequest,
+ LabPreflightResponse,
+)
+from modelforge_api.services.benchmark_runs import BenchmarkRunService
+from modelforge_api.services.lab import LabPreflightService
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+ prefix="/api/v1/lab", tags=["model-lab"], dependencies=[Depends(require_operator)]
+)
+
+
+def get_lab_service(
+ session: Annotated[Session, Depends(get_session)],
+ settings: Annotated[Settings, Depends(get_settings)],
+) -> LabPreflightService:
+ return LabPreflightService(session, settings)
+
+
+def get_lab_run_service(serving: ServingDependency) -> LabRunService:
+ return LabRunService(serving)
+
+
+def get_benchmark_run_service(serving: ServingDependency) -> BenchmarkRunService:
+ return BenchmarkRunService(serving)
+
+
+LabRunServiceDependency = Annotated[LabRunService, Depends(get_lab_run_service)]
+BenchmarkRunServiceDependency = Annotated[BenchmarkRunService, Depends(get_benchmark_run_service)]
+
+
+@router.post("/preflight", response_model=LabPreflightResponse)
+def preflight(
+ request: LabPreflightRequest,
+ service: Annotated[LabPreflightService, Depends(get_lab_service)],
+) -> LabPreflightResponse:
+ """Inspect a bounded LAB request without storing input or starting execution."""
+ return service.preflight(request)
+
+
+@router.post("/runs", response_model=LabExecutionRunResponse, status_code=status.HTTP_202_ACCEPTED)
+def start_run(
+ request: LabExecutionStartRequest,
+ response: Response,
+ service: LabRunServiceDependency,
+ operator: Operator,
+ idempotency_key: Annotated[
+ str,
+ Header(
+ alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"
+ ),
+ ],
+) -> LabExecutionRunResponse:
+ response.headers["Cache-Control"] = "no-store"
+ return service.start(request, actor_subject=operator.subject, idempotency_key=idempotency_key)
+
+
+@router.get("/runs/{run_id}", response_model=LabExecutionRunResponse)
+def get_run(
+ run_id: uuid.UUID,
+ response: Response,
+ service: LabRunServiceDependency,
+ operator: Operator,
+) -> LabExecutionRunResponse:
+ response.headers["Cache-Control"] = "no-store"
+ return service.get(run_id, actor_subject=operator.subject)
+
+
+@router.post("/runs/{run_id}/cancel", response_model=LabExecutionRunResponse)
+def cancel_run(
+ run_id: uuid.UUID,
+ response: Response,
+ service: LabRunServiceDependency,
+ operator: Operator,
+) -> LabExecutionRunResponse:
+ response.headers["Cache-Control"] = "no-store"
+ return service.cancel(run_id, actor_subject=operator.subject)
+
+
+@router.get("/benchmark-suites", response_model=list[BenchmarkSuiteResponse])
+def benchmark_suites(
+ response: Response,
+ service: BenchmarkRunServiceDependency,
+) -> list[BenchmarkSuiteResponse]:
+ response.headers["Cache-Control"] = "no-store"
+ return service.suites()
+
+
+@router.get("/benchmark-runs", response_model=list[BenchmarkRunResponse])
+def benchmark_history(
+ response: Response,
+ service: BenchmarkRunServiceDependency,
+ operator: Operator,
+ limit: int = 50,
+ cursor: str | None = None,
+) -> list[BenchmarkRunResponse]:
+ response.headers["Cache-Control"] = "no-store"
+ page = service.list_page(actor_subject=operator.subject, limit=limit, cursor=cursor)
+ response.headers["X-Next-Cursor"] = page.next_cursor or ""
+ response.headers["Access-Control-Expose-Headers"] = "X-Next-Cursor"
+ return page.runs
+
+
+@router.post(
+ "/benchmark-runs",
+ response_model=BenchmarkRunResponse,
+ status_code=status.HTTP_202_ACCEPTED,
+)
+def start_benchmark_run(
+ request: BenchmarkRunStartRequest,
+ response: Response,
+ service: BenchmarkRunServiceDependency,
+ operator: Operator,
+ idempotency_key: Annotated[
+ str,
+ Header(
+ alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"
+ ),
+ ],
+) -> BenchmarkRunResponse:
+ response.headers["Cache-Control"] = "no-store"
+ return service.start(request, actor_subject=operator.subject, idempotency_key=idempotency_key)
+
+
+@router.get("/benchmark-runs/{run_id}", response_model=BenchmarkRunResponse)
+def get_benchmark_run(
+ run_id: uuid.UUID,
+ response: Response,
+ service: BenchmarkRunServiceDependency,
+ operator: Operator,
+) -> BenchmarkRunResponse:
+ response.headers["Cache-Control"] = "no-store"
+ return service.get(run_id, actor_subject=operator.subject)
+
+
+@router.post("/benchmark-runs/{run_id}/cancel", response_model=BenchmarkRunResponse)
+def cancel_benchmark_run(
+ run_id: uuid.UUID,
+ response: Response,
+ service: BenchmarkRunServiceDependency,
+ operator: Operator,
+) -> BenchmarkRunResponse:
+ response.headers["Cache-Control"] = "no-store"
+ return service.cancel(run_id, actor_subject=operator.subject)
diff --git a/backend/src/modelforge_api/api/routes/lifecycle.py b/backend/src/modelforge_api/api/routes/lifecycle.py
index 95be55a..1d98bc6 100644
--- a/backend/src/modelforge_api/api/routes/lifecycle.py
+++ b/backend/src/modelforge_api/api/routes/lifecycle.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import uuid
from typing import Annotated
-from fastapi import APIRouter, Depends, Query, status
+from fastapi import APIRouter, Depends, Query, Request, status
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, require_viewer
@@ -28,6 +28,7 @@ from modelforge_api.domain.lifecycle_contracts import (
RetentionPolicyCreate,
RetentionPolicyResponse,
)
+from modelforge_api.services.audit import AuditContext
from modelforge_api.services.lifecycle import LifecycleService
router = APIRouter(
@@ -171,9 +172,13 @@ def operation(
@router.post("/operations/{operation_id}/canary", response_model=LifecycleOperationResponse)
def observe_canary(
- operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin
+ operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin,
+ http_request: Request,
) -> LifecycleOperationResponse:
- return service.observe_canary(operation_id, request)
+ return service.observe_canary(operation_id, request, audit_context=AuditContext(
+ actor_type="operator", actor_id=_admin.subject,
+ correlation_id=http_request.state.correlation_id,
+ ))
@router.post("/operations/{operation_id}/commit", response_model=LifecycleOperationResponse)
diff --git a/backend/src/modelforge_api/api/routes/migrations.py b/backend/src/modelforge_api/api/routes/migrations.py
index 4abf928..d276499 100644
--- a/backend/src/modelforge_api/api/routes/migrations.py
+++ b/backend/src/modelforge_api/api/routes/migrations.py
@@ -60,8 +60,12 @@ def create_validation_policy(
@router.get("/plans", response_model=list[MigrationPlanResponse])
-def plans(service: Service, _admin: Admin) -> list[MigrationPlanResponse]:
- return service.plans()
+def plans(
+ service: Service,
+ _admin: Admin,
+ limit: Annotated[int, Query(ge=1, le=200)] = 200,
+) -> list[MigrationPlanResponse]:
+ return service.plans(limit=limit)
@router.post("/plans", response_model=MigrationPlanResponse, status_code=status.HTTP_201_CREATED)
diff --git a/backend/src/modelforge_api/api/routes/model_updates.py b/backend/src/modelforge_api/api/routes/model_updates.py
new file mode 100644
index 0000000..82b5240
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/model_updates.py
@@ -0,0 +1,80 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, Query, Request
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, require_operator, require_viewer
+from modelforge_api.api.routes.acquisition import get_acquisition_service
+from modelforge_api.db import get_session
+from modelforge_api.domain.model_updates import (
+ ModelUpdateMonitorResponse,
+ ModelUpdateNoticeResponse,
+ ModelWatchCreate,
+ ModelWatchResponse,
+ ModelWatchUpdate,
+)
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/model-updates", tags=["model-updates"],
+ dependencies=[Depends(require_viewer)])
+
+
+def get_model_update_service(
+ request: Request,
+ session: Annotated[Session, Depends(get_session)],
+ acquisition: Annotated[AcquisitionService, Depends(get_acquisition_service)],
+ principal: Annotated[Principal, Depends(require_viewer)],
+) -> ModelUpdateService:
+ context = AuditContext("operator", principal.subject,
+ getattr(request.state, "correlation_id", None))
+ return ModelUpdateService(session, acquisition, context)
+
+
+Service = Annotated[ModelUpdateService, Depends(get_model_update_service)]
+Limit = Annotated[int, Query(ge=1, le=100)]
+Offset = Annotated[int, Query(ge=0, le=10000)]
+
+
+@router.get("/monitor", response_model=ModelUpdateMonitorResponse)
+def monitor(settings: Annotated[Settings, Depends(get_settings)]) -> ModelUpdateMonitorResponse:
+ return ModelUpdateMonitorResponse(
+ enabled=settings.env == "production" and settings.model_update_monitor_enabled,
+ check_interval_seconds=settings.model_update_check_interval_seconds,
+ )
+
+
+@router.get("/watches", response_model=list[ModelWatchResponse])
+def watches(service: Service, limit: Limit = 50, offset: Offset = 0) -> list[ModelWatchResponse]:
+ return service.watches(limit, offset)
+
+
+@router.post("/watches", response_model=ModelWatchResponse, dependencies=[Depends(require_operator)])
+def create(request: ModelWatchCreate, service: Service) -> ModelWatchResponse:
+ return service.create(request)
+
+
+@router.post("/watches/{watch_id}/check", response_model=ModelWatchResponse,
+ dependencies=[Depends(require_operator)])
+def check(watch_id: UUID, service: Service) -> ModelWatchResponse:
+ return service.check(watch_id)
+
+
+@router.patch("/watches/{watch_id}", response_model=ModelWatchResponse,
+ dependencies=[Depends(require_operator)])
+def change(watch_id: UUID, request: ModelWatchUpdate, service: Service) -> ModelWatchResponse:
+ return service.change(watch_id, request)
+
+
+@router.get("/notices", response_model=list[ModelUpdateNoticeResponse])
+def notices(service: Service, limit: Limit = 50, offset: Offset = 0) -> list[ModelUpdateNoticeResponse]:
+ return service.notices(limit, offset)
+
+
+@router.post("/notices/{notice_id}/dismiss", response_model=ModelUpdateNoticeResponse,
+ dependencies=[Depends(require_operator)])
+def dismiss(notice_id: UUID, service: Service) -> ModelUpdateNoticeResponse:
+ return service.dismiss(notice_id)
diff --git a/backend/src/modelforge_api/api/routes/observability.py b/backend/src/modelforge_api/api/routes/observability.py
index 3d48891..6b30b92 100644
--- a/backend/src/modelforge_api/api/routes/observability.py
+++ b/backend/src/modelforge_api/api/routes/observability.py
@@ -1,16 +1,21 @@
from __future__ import annotations
import uuid
-from typing import Annotated, Any
+from typing import Annotated, Any, Literal
-from fastapi import APIRouter, Depends, Query, Response, status
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status
from sqlalchemy import select
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, require_viewer
from modelforge_api.db import get_session
+from modelforge_api.domain.alert_notifications import (
+ NotificationDeliveryResponse,
+ NotificationTestResponse,
+)
from modelforge_api.domain.observability import (
+ ActiveSchedulerLeaseInventoryResponse,
AlertAction,
AlertHistoryResponse,
AlertResponse,
@@ -28,12 +33,17 @@ from modelforge_api.domain.observability import (
TrendResponse,
metrics,
)
+from modelforge_api.domain.self_check import SelfCheckReport
from modelforge_api.persistence.models import (
AlertHistoryEvent,
IncidentTimelineEvent,
OperationalIncident,
)
+from modelforge_api.services.alert_notifications import AlertNotificationService
+from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.services.self_check import run_self_check
+from modelforge_api.settings import Settings, get_settings
router = APIRouter(tags=["operations"], dependencies=[Depends(require_viewer)])
@@ -63,6 +73,47 @@ def overview(service: Service, _admin: Admin) -> OperationsOverview:
return service.overview()
+@router.get("/api/v1/admin/operations/self-check", response_model=SelfCheckReport)
+def self_check(
+ _admin: Admin,
+ session: Annotated[Session, Depends(get_session)],
+ settings: Annotated[Settings, Depends(get_settings)],
+ registry: Annotated[ManifestRegistry, Depends(get_manifest_registry)],
+) -> SelfCheckReport:
+ """Run passive diagnostics only; no repair, write probe, or external action is performed."""
+
+ return run_self_check(session, settings, registry)
+
+
+@router.get(
+ "/api/v1/admin/operations/notification-deliveries",
+ response_model=list[NotificationDeliveryResponse],
+)
+def notification_deliveries(
+ session: Annotated[Session, Depends(get_session)],
+ settings: Annotated[Settings, Depends(get_settings)],
+ _admin: Admin,
+ limit: int = Query(default=100, ge=1, le=500),
+) -> list[NotificationDeliveryResponse]:
+ return AlertNotificationService(session, settings).deliveries(limit)
+
+
+@router.post(
+ "/api/v1/admin/operations/notifications/test",
+ response_model=NotificationTestResponse,
+ status_code=status.HTTP_202_ACCEPTED,
+)
+def queue_notification_test(
+ session: Annotated[Session, Depends(get_session)],
+ settings: Annotated[Settings, Depends(get_settings)],
+ principal: Admin,
+ request: Request,
+) -> NotificationTestResponse:
+ return AlertNotificationService(session, settings).queue_test(
+ principal, getattr(request.state, "correlation_id", None)
+ )
+
+
@router.get(
"/api/v1/admin/operations/slis", response_model=list[SLIDefinitionResponse]
)
@@ -185,6 +236,20 @@ def capacity(
return service.capacity(limit)
+@router.get(
+ "/api/v1/admin/operations/active-leases",
+ response_model=ActiveSchedulerLeaseInventoryResponse,
+)
+def active_scheduler_leases(
+ service: Service,
+ _admin: Admin,
+ limit: int = Query(default=100, ge=1, le=500),
+ sort_by: Literal["expires_at", "created_at", "priority"] = Query(default="expires_at"),
+ order: Literal["asc", "desc"] = Query(default="asc"),
+) -> ActiveSchedulerLeaseInventoryResponse:
+ return service.active_scheduler_leases(limit=limit, sort_by=sort_by, order=order)
+
+
@router.post(
"/api/v1/admin/operations/capacity/collect", response_model=list[CapacitySnapshotResponse]
)
@@ -208,10 +273,20 @@ def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
@router.get("/api/v1/admin/operations/incidents", response_model=list[IncidentResponse])
-def incidents(service: Service, _admin: Admin) -> list[IncidentResponse]:
- rows = service.session.scalars(
- select(OperationalIncident).order_by(OperationalIncident.last_seen_at.desc())
+def incidents(
+ service: Service,
+ _admin: Admin,
+ limit: int | None = Query(default=None, ge=1, le=500),
+ offset: int = Query(default=0, ge=0),
+) -> list[IncidentResponse]:
+ if limit is None and offset:
+ raise HTTPException(status_code=422, detail="offset requires limit")
+ statement = select(OperationalIncident).order_by(
+ OperationalIncident.last_seen_at.desc(), OperationalIncident.id
)
+ if limit is not None:
+ statement = statement.offset(offset).limit(limit)
+ rows = service.session.scalars(statement)
return [IncidentResponse.model_validate(item) for item in rows]
diff --git a/backend/src/modelforge_api/api/routes/operator_login.py b/backend/src/modelforge_api/api/routes/operator_login.py
new file mode 100644
index 0000000..fd55ca8
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/operator_login.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime
+from typing import Annotated
+
+import redis
+from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+
+from modelforge_api.services.operator_password import (
+ SESSION_SECONDS,
+ issue_session,
+ verify_password,
+)
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/operator", tags=["operator"])
+# One atomic operation across all API processes; keys expire even for failed attempts.
+_RATE_SCRIPT = """
+local global = redis.call('INCR', KEYS[1])
+if global == 1 then redis.call('EXPIRE', KEYS[1], 60) end
+if global > 30 then return 0 end
+local peer = redis.call('INCR', KEYS[2])
+if peer == 1 then redis.call('EXPIRE', KEYS[2], 60) end
+if peer > 5 then return 0 end
+return 1
+"""
+
+
+def limit_login(request: Request, settings: Settings) -> None:
+ # Use only the direct socket peer. Forwarded headers do not identify trusted clients.
+ peer = request.client.host if request.client else "unknown"
+ peer_digest = hashlib.sha256(peer.encode()).hexdigest()
+ try:
+ with redis.Redis.from_url(
+ settings.redis_url, socket_connect_timeout=2, socket_timeout=2
+ ) as client:
+ accepted = client.eval(
+ _RATE_SCRIPT, 2, "modelforge:login:global", f"modelforge:login:peer:{peer_digest}"
+ )
+ except (redis.RedisError, OSError, ValueError):
+ raise HTTPException(503, "Password login is temporarily unavailable") from None
+ if accepted != 1:
+ raise HTTPException(429, "Too many login attempts; try again later")
+
+
+class PasswordLogin(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ password: SecretStr = Field(min_length=1, max_length=1024)
+
+
+class PasswordSession(BaseModel):
+ access_token: str
+ expires_at: datetime
+ expires_in: int = SESSION_SECONDS
+ token_type: str = "bearer" # noqa: S105 - protocol token type, not a secret
+
+
+@router.post("/login", response_model=PasswordSession)
+def login(
+ payload: PasswordLogin,
+ request: Request,
+ response: Response,
+ settings: Annotated[Settings, Depends(get_settings)],
+) -> PasswordSession:
+ # The pre-body middleware applies the shared limiter, including malformed requests.
+ configured = settings.operator_password_hash
+ bootstrap = settings.operator_api_key
+ if configured is None or bootstrap is None or len(bootstrap.get_secret_value()) < 32:
+ raise HTTPException(503, "Password login is not configured")
+ try:
+ valid = verify_password(payload.password.get_secret_value(), configured.get_secret_value())
+ except (ValueError, UnicodeError):
+ raise HTTPException(503, "Password login is not configured") from None
+ if not valid:
+ raise HTTPException(401, "Invalid password")
+ token, expires = issue_session(bootstrap.get_secret_value(), configured.get_secret_value())
+ response.headers["Cache-Control"] = "no-store"
+ return PasswordSession(access_token=token, expires_at=datetime.fromtimestamp(expires, UTC))
diff --git a/backend/src/modelforge_api/api/routes/portfolio.py b/backend/src/modelforge_api/api/routes/portfolio.py
new file mode 100644
index 0000000..36d928d
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/portfolio.py
@@ -0,0 +1,61 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Operator, Viewer, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.portfolio import (
+ PortfolioInventoryResponse,
+ PortfolioReconcileRequest,
+ PortfolioReconcileResponse,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.portfolio_registry import (
+ PortfolioRegistryError,
+ PortfolioRegistryService,
+)
+
+router = APIRouter(
+ prefix="/api/v1/portfolio",
+ tags=["portfolio"],
+ dependencies=[Depends(require_viewer)],
+)
+
+
+def get_portfolio_registry_service(
+ session: Annotated[Session, Depends(get_session)],
+) -> PortfolioRegistryService:
+ return PortfolioRegistryService(session)
+
+
+Service = Annotated[PortfolioRegistryService, Depends(get_portfolio_registry_service)]
+
+
+@router.get("", response_model=PortfolioInventoryResponse)
+def portfolio_inventory(service: Service, _viewer: Viewer) -> PortfolioInventoryResponse:
+ return service.inventory()
+
+
+@router.post(
+ "/reconcile",
+ response_model=PortfolioReconcileResponse,
+)
+def reconcile_portfolio(
+ body: PortfolioReconcileRequest,
+ service: Service,
+ operator: Operator,
+ request: Request,
+) -> PortfolioReconcileResponse:
+ context = AuditContext(
+ actor_type="operator",
+ actor_id=operator.subject,
+ correlation_id=getattr(request.state, "correlation_id", None),
+ )
+ try:
+ return service.reconcile(body, context)
+ except PortfolioRegistryError as exc:
+ raise HTTPException(
+ status_code=exc.status_code,
+ detail={"code": exc.code, "message": exc.message},
+ ) from exc
diff --git a/backend/src/modelforge_api/api/routes/projects.py b/backend/src/modelforge_api/api/routes/projects.py
new file mode 100644
index 0000000..ab618cc
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/projects.py
@@ -0,0 +1,172 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+
+from modelforge_api.api.authorization import Operator, Viewer
+from modelforge_api.api.routes.serving import Service as Serving
+from modelforge_api.domain.model_changes import (
+ ModelChangeCandidateLink,
+ ModelChangeCreate,
+ ModelChangeEvidenceLink,
+ ModelChangeList,
+ ModelChangeReject,
+ ModelChangeResponse,
+)
+from modelforge_api.domain.projects import (
+ ProjectCreate,
+ ProjectRequirementCreate,
+ ProjectWorkspaceResponse,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.model_changes import ModelChangeError, ModelChangeService
+from modelforge_api.services.projects import ProjectService
+
+router = APIRouter(prefix="/api/v1/project-workspace", tags=["project-onboarding"])
+
+
+def get_project_service(serving: Serving, principal: Viewer, request: Request) -> ProjectService:
+ serving.audit = AuditWriter(serving.session, context=AuditContext(
+ "operator", principal.subject, request.state.correlation_id,
+ ))
+ return ProjectService(serving)
+
+
+def get_model_change_service(
+ serving: Serving, principal: Viewer, request: Request
+) -> ModelChangeService:
+ return ModelChangeService(
+ serving.session,
+ AuditContext("operator", principal.subject, request.state.correlation_id),
+ )
+
+
+Service = Annotated[ProjectService, Depends(get_project_service)]
+ChangeService = Annotated[ModelChangeService, Depends(get_model_change_service)]
+
+
+def _change_error(exc: ModelChangeError) -> HTTPException:
+ return HTTPException(status_code=exc.status_code, detail=exc.message)
+
+
+@router.get("", response_model=list[ProjectWorkspaceResponse])
+def projects(
+ service: Service,
+ _viewer: Viewer,
+ limit: int | None = Query(default=None, ge=1, le=200),
+ after_key: str | None = Query(default=None, min_length=1, max_length=128),
+) -> list[ProjectWorkspaceResponse]:
+ if after_key is not None and limit is None:
+ raise HTTPException(status_code=422, detail="after_key requires limit")
+ return service.list(limit=limit, after_key=after_key)
+
+
+@router.post("", response_model=ProjectWorkspaceResponse, status_code=201)
+def create_project(body: ProjectCreate, service: Service, _operator: Operator) -> ProjectWorkspaceResponse:
+ return service.create(body)
+
+
+@router.get("/changes", response_model=ModelChangeList)
+def model_changes(
+ service: ChangeService,
+ _viewer: Viewer,
+ project_key: str | None = None,
+ status: str | None = None,
+ project_binding_id: UUID | None = None,
+ open_only: bool = False,
+ limit: int = Query(default=100, ge=1, le=200),
+ offset: int = Query(default=0, ge=0),
+) -> ModelChangeList:
+ try:
+ return service.list(
+ project_key=project_key,
+ status=status,
+ project_binding_id=project_binding_id,
+ open_only=open_only,
+ limit=limit,
+ offset=offset,
+ )
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.post("/changes", response_model=ModelChangeResponse, status_code=201)
+def create_model_change(
+ body: ModelChangeCreate, service: ChangeService, _operator: Operator
+) -> ModelChangeResponse:
+ try:
+ return service.create(body)
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.get("/changes/{case_id}", response_model=ModelChangeResponse)
+def model_change(case_id: UUID, service: ChangeService, _viewer: Viewer) -> ModelChangeResponse:
+ try:
+ return service.get(case_id)
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.put("/changes/{case_id}/candidate", response_model=ModelChangeResponse)
+def link_model_change_candidate(
+ case_id: UUID,
+ body: ModelChangeCandidateLink,
+ service: ChangeService,
+ _operator: Operator,
+) -> ModelChangeResponse:
+ try:
+ return service.link_candidate(case_id, body)
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.put("/changes/{case_id}/evidence", response_model=ModelChangeResponse)
+def link_model_change_evidence(
+ case_id: UUID,
+ body: ModelChangeEvidenceLink,
+ service: ChangeService,
+ _operator: Operator,
+) -> ModelChangeResponse:
+ try:
+ return service.link_evidence(case_id, body)
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.post("/changes/{case_id}/refresh", response_model=ModelChangeResponse)
+def refresh_model_change(
+ case_id: UUID,
+ service: ChangeService,
+ _operator: Operator,
+ expected_version: int = Query(ge=1),
+) -> ModelChangeResponse:
+ try:
+ return service.refresh(case_id, expected_version)
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.post("/changes/{case_id}/reject", response_model=ModelChangeResponse)
+def reject_model_change(
+ case_id: UUID,
+ body: ModelChangeReject,
+ service: ChangeService,
+ _operator: Operator,
+) -> ModelChangeResponse:
+ try:
+ return service.reject(case_id, body)
+ except ModelChangeError as exc:
+ raise _change_error(exc) from exc
+
+
+@router.get("/{key}", response_model=ProjectWorkspaceResponse)
+def project(key: str, service: Service, _viewer: Viewer) -> ProjectWorkspaceResponse:
+ return service.detail(key)
+
+
+@router.post("/{key}/requirements", response_model=ProjectWorkspaceResponse, status_code=201)
+def bind_project(
+ key: str, body: ProjectRequirementCreate, service: Service, _operator: Operator,
+) -> ProjectWorkspaceResponse:
+ return service.bind(key, body)
diff --git a/backend/src/modelforge_api/api/routes/recovery.py b/backend/src/modelforge_api/api/routes/recovery.py
index a2fc7cd..a9a31f0 100644
--- a/backend/src/modelforge_api/api/routes/recovery.py
+++ b/backend/src/modelforge_api/api/routes/recovery.py
@@ -26,6 +26,7 @@ from modelforge_api.domain.recovery import (
RecoveryDashboard,
RecoveryPolicyCreate,
RecoveryPolicyResponse,
+ RecoveryRetentionPreview,
RestoreAdvanceRequest,
RestoreOperationEventResponse,
RestoreOperationResponse,
@@ -116,6 +117,11 @@ def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
return service.apply_retention()
+@router.get("/retention/preview", response_model=RecoveryRetentionPreview)
+def preview_retention(service: Service, _admin: Admin) -> RecoveryRetentionPreview:
+ return service.preview_retention()
+
+
@router.get("/restore-plans", response_model=list[RestorePlanResponse])
def restore_plans(
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
diff --git a/backend/src/modelforge_api/api/routes/selected_model_comparison.py b/backend/src/modelforge_api/api/routes/selected_model_comparison.py
new file mode 100644
index 0000000..f48f577
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/selected_model_comparison.py
@@ -0,0 +1,106 @@
+"""Separate operator planning from designated project-consumer execution reports."""
+
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, Query, Request, Response, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, require_operator, require_viewer
+from modelforge_api.api.routes.serving import (
+ Service as Serving,
+)
+from modelforge_api.api.routes.serving import (
+ authenticate_rag_embedding_client,
+)
+from modelforge_api.db import get_session
+from modelforge_api.domain.selected_model_comparison import (
+ ComparisonCancel,
+ ComparisonClaim,
+ ComparisonCreate,
+ ComparisonFinish,
+ ComparisonList,
+ ComparisonProgress,
+ ComparisonResponse,
+ ComparisonResult,
+)
+from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
+
+router = APIRouter(prefix="/api/v1/selected-model-comparisons", tags=["selected-model-comparisons"])
+Client = Annotated[ServiceClient, Depends(authenticate_rag_embedding_client)]
+Limit = Annotated[int, Query(ge=1, le=100)]
+Offset = Annotated[int, Query(ge=0, le=10000)]
+
+
+def operator_service(
+ request: Request,
+ session: Annotated[Session, Depends(get_session)],
+ principal: Annotated[Principal, Depends(require_viewer)],
+) -> SelectedModelComparisonService:
+ return SelectedModelComparisonService(session, audit_context=AuditContext(
+ "operator", principal.subject, getattr(request.state, "correlation_id", None),
+ ))
+
+
+def consumer_service(request: Request, response: Response, serving: Serving, client: Client) -> SelectedModelComparisonService:
+ response.headers["Cache-Control"] = "no-store"
+ return SelectedModelComparisonService(serving.session, audit_context=AuditContext(
+ "service_client", str(client.id), getattr(request.state, "correlation_id", None),
+ ))
+
+
+OperatorService = Annotated[SelectedModelComparisonService, Depends(operator_service)]
+ConsumerService = Annotated[SelectedModelComparisonService, Depends(consumer_service)]
+
+
+@router.get("", response_model=ComparisonList, dependencies=[Depends(require_viewer)])
+def history(service: OperatorService, project_binding_id: UUID | None = None,
+ limit: Limit = 50, offset: Offset = 0) -> ComparisonList:
+ return service.list(project_binding_id=project_binding_id, limit=limit, offset=offset)
+
+
+@router.post("", response_model=ComparisonResponse, status_code=status.HTTP_201_CREATED,
+ dependencies=[Depends(require_operator)])
+def create(body: ComparisonCreate, service: OperatorService) -> ComparisonResponse:
+ return service.create(body)
+
+
+@router.get("/assigned", response_model=ComparisonList)
+def assigned(service: ConsumerService, client: Client, limit: Limit = 50) -> ComparisonList:
+ return service.assigned(client, limit=limit)
+
+
+@router.get("/{comparison_id}", response_model=ComparisonResponse, dependencies=[Depends(require_viewer)])
+def detail(comparison_id: UUID, service: OperatorService) -> ComparisonResponse:
+ return service.get(comparison_id)
+
+
+@router.post("/{comparison_id}/cancel", response_model=ComparisonResponse,
+ dependencies=[Depends(require_operator)])
+def cancel(comparison_id: UUID, body: ComparisonCancel, service: OperatorService) -> ComparisonResponse:
+ return service.cancel(comparison_id, body)
+
+
+@router.post("/{comparison_id}/claim", response_model=ComparisonClaim)
+def claim(comparison_id: UUID, service: ConsumerService, client: Client) -> ComparisonClaim:
+ return service.claim(comparison_id, client)
+
+
+@router.post("/{comparison_id}/progress", response_model=ComparisonResponse)
+def progress(comparison_id: UUID, body: ComparisonProgress, service: ConsumerService,
+ client: Client) -> ComparisonResponse:
+ return service.progress(comparison_id, body, client)
+
+
+@router.post("/{comparison_id}/result", response_model=ComparisonResponse)
+def result(comparison_id: UUID, body: ComparisonResult, service: ConsumerService,
+ client: Client) -> ComparisonResponse:
+ return service.result(comparison_id, body, client)
+
+
+@router.post("/{comparison_id}/finish", response_model=ComparisonResponse)
+def finish(comparison_id: UUID, body: ComparisonFinish, service: ConsumerService,
+ client: Client) -> ComparisonResponse:
+ return service.finish(comparison_id, body, client)
diff --git a/backend/src/modelforge_api/api/routes/selected_replacement.py b/backend/src/modelforge_api/api/routes/selected_replacement.py
new file mode 100644
index 0000000..2d08697
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/selected_replacement.py
@@ -0,0 +1,20 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, Response
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.selected_replacement import ReplacementPreview
+from modelforge_api.services.selected_replacement import SelectedReplacementService
+
+router = APIRouter(prefix="/api/v1/selected-model-comparisons", tags=["selected-model-comparisons"])
+
+
+@router.get("/{comparison_id}/replacement-preview", response_model=ReplacementPreview,
+ dependencies=[Depends(require_viewer)])
+def preview(comparison_id: UUID, response: Response,
+ session: Annotated[Session, Depends(get_session)]) -> ReplacementPreview:
+ response.headers["Cache-Control"] = "no-store"
+ return SelectedReplacementService(session).preview(comparison_id)
diff --git a/backend/src/modelforge_api/api/routes/serving.py b/backend/src/modelforge_api/api/routes/serving.py
index 6f2b83f..bc289dd 100644
--- a/backend/src/modelforge_api/api/routes/serving.py
+++ b/backend/src/modelforge_api/api/routes/serving.py
@@ -7,10 +7,14 @@ from typing import Annotated
from fastapi import APIRouter, Depends, Header, Query, Request, Response, status
from sqlalchemy.orm import Session
-from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.api.authorization import Admin, require_viewer, required_capability_for_request
from modelforge_api.api.routes.agent import AgentIdentity
from modelforge_api.db import get_session
+from modelforge_api.domain.projects import ProjectConnectivityResponse
from modelforge_api.domain.serving import (
+ AgentServingJobCancellationAck,
+ AgentServingJobCancellationCheck,
+ AgentServingJobCancellationStatus,
AgentServingJobComplete,
AgentServingJobFailure,
AgentServingJobLease,
@@ -48,6 +52,9 @@ from modelforge_api.domain.serving import (
ServiceClientCreate,
ServiceClientCreated,
ServiceClientResponse,
+ ServiceCredentialResponse,
+ ServiceCredentialRevoke,
+ ServiceCredentialRotate,
SpeechTranscriptionInvokeRequest,
SpeechTranscriptionInvokeResponse,
StableEmbeddingInvokeResponse,
@@ -55,6 +62,7 @@ from modelforge_api.domain.serving import (
VisionEmbeddingInvokeResponse,
)
from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
from modelforge_api.services.serving import (
CapabilityAuthenticationEvidence,
@@ -146,6 +154,60 @@ SpeechTranscriptionClient = Annotated[
]
+def authenticate_check_client(
+ request: Request,
+ service: Service,
+ authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+ capability = required_capability_for_request(request.method, request.url.path)
+ if capability is None:
+ raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability is not registered")
+ return _authenticate_capability_client(request, service, authorization, capability)
+
+
+@router.get(
+ "/api/v1/capabilities/rag.embedding@1/check", response_model=ProjectConnectivityResponse
+)
+@router.get(
+ "/api/v1/capabilities/rag.reranking@1/check", response_model=ProjectConnectivityResponse
+)
+@router.get("/api/v1/capabilities/document.ocr@1/check", response_model=ProjectConnectivityResponse)
+@router.get(
+ "/api/v1/capabilities/vision.embedding@1/check", response_model=ProjectConnectivityResponse
+)
+@router.get(
+ "/api/v1/capabilities/speech.transcription@1/check", response_model=ProjectConnectivityResponse
+)
+def check_project_connectivity(
+ request: Request,
+ response: Response,
+ service: Service,
+ client: Annotated[ServiceClient, Depends(authenticate_check_client)],
+) -> ProjectConnectivityResponse:
+ response.headers["Cache-Control"] = "no-store"
+ capability = required_capability_for_request(request.method, request.url.path)
+ if capability is None:
+ raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability is not registered")
+ service.audit = AuditWriter(
+ service.session,
+ context=AuditContext(
+ "service_client",
+ str(client.id),
+ request.state.correlation_id,
+ ),
+ )
+ service.audit.write(
+ "PROJECT_CONNECTIVITY_CHECKED", "service_client", str(client.id), {"capability": capability}
+ )
+ service.session.commit()
+ return ProjectConnectivityResponse(
+ client_id=client.id,
+ project_key=service.client_response(client).project_key,
+ capability=capability,
+ correlation_id=request.state.correlation_id,
+ )
+
+
@router.post(
"/api/v1/admin/deployment-candidates/{candidate_id}/production-approvals",
response_model=ProductionApprovalResponse,
@@ -254,11 +316,74 @@ def create_service_client(
service: Service,
_admin: Admin,
response: Response,
+ http_request: Request,
) -> ServiceClientCreated:
response.headers["Cache-Control"] = "no-store"
+ service.audit = AuditWriter(
+ service.session,
+ context=AuditContext(
+ "operator",
+ _admin.subject,
+ http_request.state.correlation_id,
+ ),
+ )
return service.create_client(request)
+@router.get(
+ "/api/v1/admin/service-clients/{client_id}/credentials",
+ response_model=list[ServiceCredentialResponse],
+)
+def service_credential_history(
+ client_id: uuid.UUID, service: Service, _admin: Admin
+) -> list[ServiceCredentialResponse]:
+ return service.credential_history(client_id)
+
+
+@router.post(
+ "/api/v1/admin/service-clients/{client_id}/credentials/rotate",
+ response_model=ServiceClientCreated,
+)
+def rotate_project_credential(
+ client_id: uuid.UUID,
+ body: ServiceCredentialRotate,
+ service: Service,
+ principal: Admin,
+ request: Request,
+) -> ServiceClientCreated:
+ service.audit = AuditWriter(
+ service.session,
+ context=AuditContext(
+ "operator",
+ principal.subject,
+ request.state.correlation_id,
+ ),
+ )
+ return service.rotate_client_credential(client_id, body)
+
+
+@router.post(
+ "/api/v1/admin/service-clients/{client_id}/credentials/revoke",
+ response_model=ServiceClientResponse,
+)
+def revoke_project_credential(
+ client_id: uuid.UUID,
+ body: ServiceCredentialRevoke,
+ service: Service,
+ principal: Admin,
+ request: Request,
+) -> ServiceClientResponse:
+ service.audit = AuditWriter(
+ service.session,
+ context=AuditContext(
+ "operator",
+ principal.subject,
+ request.state.correlation_id,
+ ),
+ )
+ return service.revoke_client_credential(client_id, body)
+
+
@router.get("/api/v1/admin/service-clients", response_model=list[ServiceClientResponse])
def service_clients(service: Service, _admin: Admin) -> list[ServiceClientResponse]:
return service.clients()
@@ -311,7 +436,7 @@ def record_project_fit_evidence(
dependencies=[Depends(require_viewer)],
)
def scheduler_overview(service: Service) -> list[SchedulerBudgetResponse]:
- return service.scheduler_overview()
+ return service.scheduler_overview(commit=False, update_pressure=False)
@router.get(
@@ -569,6 +694,32 @@ def fail_serving_job(
return {"status": "accepted"}
+@router.post(
+ "/api/v1/agent/serving-jobs/{job_id}/cancellation",
+ response_model=AgentServingJobCancellationStatus,
+)
+def check_serving_job_cancellation(
+ job_id: uuid.UUID,
+ request: AgentServingJobCancellationCheck,
+ service: Service,
+ identity: AgentIdentity,
+) -> AgentServingJobCancellationStatus:
+ _credential, node = identity
+ return service.check_lab_cancellation(job_id, node, request)
+
+
+@router.post("/api/v1/agent/serving-jobs/{job_id}/cancelled")
+def acknowledge_serving_job_cancellation(
+ job_id: uuid.UUID,
+ request: AgentServingJobCancellationAck,
+ service: Service,
+ identity: AgentIdentity,
+) -> dict[str, str]:
+ _credential, node = identity
+ service.acknowledge_lab_cancellation(job_id, node, request)
+ return {"status": "accepted"}
+
+
@router.post(
"/api/v1/agent/serving-state",
response_model=AgentServingStateAck,
diff --git a/backend/src/modelforge_api/api/routes/update_preflight.py b/backend/src/modelforge_api/api/routes/update_preflight.py
new file mode 100644
index 0000000..9237903
--- /dev/null
+++ b/backend/src/modelforge_api/api/routes/update_preflight.py
@@ -0,0 +1,36 @@
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+
+from modelforge_api.api.authorization import Admin
+from modelforge_api.domain.release import build_identity
+from modelforge_api.domain.update_preflight import UpdatePreflight, update_preflight
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1/admin/updates", tags=["update-preflight"])
+
+
+@router.get(
+ "/preflight",
+ response_model=UpdatePreflight,
+)
+def read_update_preflight(
+ _admin: Admin,
+ settings: Annotated[Settings, Depends(get_settings)],
+) -> UpdatePreflight:
+ """Report local update evidence; schema is the app build target, not the applied DB revision."""
+
+ identity = build_identity(
+ source_commit=settings.build_commit,
+ built_at=settings.build_timestamp,
+ image_digest=settings.build_image_digest,
+ )
+ return update_preflight(
+ version=identity.version,
+ channel=identity.channel,
+ schema_revision=identity.schema_revision,
+ agent_protocol_version=identity.agent_protocol_version,
+ source_commit=identity.source_commit,
+ built_at=identity.built_at,
+ image_digest=identity.image_digest,
+ )
diff --git a/backend/src/modelforge_api/domain/agent_protocol.py b/backend/src/modelforge_api/domain/agent_protocol.py
index bdae529..d469fe9 100644
--- a/backend/src/modelforge_api/domain/agent_protocol.py
+++ b/backend/src/modelforge_api/domain/agent_protocol.py
@@ -123,6 +123,7 @@ class EnrollmentTokenSummary(AgentProtocolModel):
expires_at: datetime
used_at: datetime | None
revoked_at: datetime | None
+ enrolled_node_id: str | None = None
class NodeManagementUpdate(AgentProtocolModel):
diff --git a/backend/src/modelforge_api/domain/alert_notifications.py b/backend/src/modelforge_api/domain/alert_notifications.py
new file mode 100644
index 0000000..437c47b
--- /dev/null
+++ b/backend/src/modelforge_api/domain/alert_notifications.py
@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict
+
+
+class NotificationStatus(StrEnum):
+ PENDING = "PENDING"
+ DISPATCHING = "DISPATCHING"
+ RETRY_WAIT = "RETRY_WAIT"
+ DELIVERED = "DELIVERED"
+ FAILED = "FAILED"
+ HELD_FOR_REVIEW = "HELD_FOR_REVIEW"
+
+
+class NotificationKind(StrEnum):
+ FIRING = "FIRING"
+ RESOLVED = "RESOLVED"
+ TEST = "TEST"
+
+
+class NotificationDeliveryResponse(BaseModel):
+ """Safe operator-facing delivery metadata. Payload and destination are never returned."""
+
+ model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+ id: UUID
+ alert_history_event_id: UUID | None
+ route_key: str
+ event_kind: NotificationKind
+ severity: str | None
+ status: NotificationStatus
+ attempt_count: int
+ next_attempt_at: datetime | None
+ last_attempt_at: datetime | None
+ delivered_at: datetime | None
+ last_http_status: int | None
+ last_error_code: str | None
+ created_at: datetime
+
+
+class NotificationTestResponse(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ delivery: NotificationDeliveryResponse
+ message: str
diff --git a/backend/src/modelforge_api/domain/audit_timeline.py b/backend/src/modelforge_api/domain/audit_timeline.py
new file mode 100644
index 0000000..855aa30
--- /dev/null
+++ b/backend/src/modelforge_api/domain/audit_timeline.py
@@ -0,0 +1,69 @@
+"""Typed, bounded query and response models for the operator audit timeline."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import UUID
+
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, model_validator
+
+AUDIT_TIMELINE_MAX_PAGE_SIZE = 100
+AUDIT_TIMELINE_MAX_TIME_RANGE = timedelta(days=366)
+
+
+class AuditTimelineRequest(BaseModel):
+ """Validated filters for one bounded page of audit events.
+
+ Text filters are case-insensitive substring matches. A time filter must specify both ends,
+ and its window is limited to one year plus one leap day.
+ """
+
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+
+ principal: str | None = Field(default=None, min_length=1, max_length=255)
+ action: str | None = Field(default=None, min_length=1, max_length=128)
+ resource: str | None = Field(default=None, min_length=1, max_length=128)
+ outcome: str | None = Field(default=None, min_length=1, max_length=32)
+ occurred_after: AwareDatetime | None = None
+ occurred_before: AwareDatetime | None = None
+ correlation_id: str | None = Field(default=None, min_length=1, max_length=64)
+ limit: int = Field(default=50, ge=1, le=AUDIT_TIMELINE_MAX_PAGE_SIZE)
+ cursor: str | None = Field(default=None, min_length=1, max_length=256)
+
+ @model_validator(mode="after")
+ def validate_time_window(self) -> AuditTimelineRequest:
+ if (self.occurred_after is None) != (self.occurred_before is None):
+ raise ValueError("occurred_after and occurred_before must be supplied together")
+ if self.occurred_after is not None and self.occurred_before is not None:
+ if self.occurred_after >= self.occurred_before:
+ raise ValueError("occurred_after must be earlier than occurred_before")
+ if self.occurred_before - self.occurred_after > AUDIT_TIMELINE_MAX_TIME_RANGE:
+ raise ValueError("audit time range exceeds the 366-day maximum")
+ return self
+
+
+class AuditTimelineEvent(BaseModel):
+ """Safe event fields returned to the operator console."""
+
+ model_config = ConfigDict(frozen=True)
+
+ id: UUID
+ sequence: int = Field(ge=1)
+ occurred_at: AwareDatetime
+ correlation_id: str
+ actor_type: str
+ actor_id: str
+ action: str
+ resource_type: str
+ resource_id: str | None
+ outcome: str
+
+
+class AuditTimelineResponse(BaseModel):
+ """One page of audit events and an opaque keyset cursor for the next page."""
+
+ model_config = ConfigDict(frozen=True)
+
+ events: list[AuditTimelineEvent]
+ next_cursor: str | None
+ limit: int = Field(ge=1, le=AUDIT_TIMELINE_MAX_PAGE_SIZE)
diff --git a/backend/src/modelforge_api/domain/benchmarking.py b/backend/src/modelforge_api/domain/benchmarking.py
new file mode 100644
index 0000000..0808bbe
--- /dev/null
+++ b/backend/src/modelforge_api/domain/benchmarking.py
@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import Field
+
+from modelforge_api.domain.contracts import BenchmarkRunContract, StrictModel
+from modelforge_api.domain.lab import LabTarget
+
+
+class BenchmarkSuiteResponse(StrictModel):
+ id: uuid.UUID
+ key: str
+ revision: str
+ capability: Literal["rag.embedding@1"]
+ dataset_revision: str
+ definition: dict[str, Any]
+ immutable_at: datetime
+
+
+class BenchmarkRunStartRequest(StrictModel):
+ capability: Literal["rag.embedding@1"]
+ suite_id: uuid.UUID
+ target: LabTarget
+ deployment_id: uuid.UUID
+ experiment_route_id: uuid.UUID
+
+
+BenchmarkRunStatus = Literal[
+ "accepted",
+ "running",
+ "cancel_requested",
+ "completed",
+ "failed",
+ "cancelled",
+ "interrupted",
+]
+
+
+class BenchmarkRunProgress(StrictModel):
+ completed_invocations: int = Field(ge=0)
+ total_invocations: int = Field(ge=1)
+ active_case: str | None = None
+
+
+class BenchmarkRunResponse(StrictModel):
+ id: uuid.UUID
+ capability: Literal["rag.embedding@1"]
+ suite_id: uuid.UUID
+ capability_deployment_id: uuid.UUID
+ status: BenchmarkRunStatus
+ contract: BenchmarkRunContract
+ target_snapshot: dict[str, Any]
+ progress: BenchmarkRunProgress
+ results: dict[str, Any]
+ failure_code: str | None = None
+ cancellation_requested_at: datetime | None = None
+ created_at: datetime
+ completed_at: datetime | None = None
+
+
+class BenchmarkRunHistoryPage(StrictModel):
+ """A bounded page of actor-scoped benchmark history."""
+
+ runs: list[BenchmarkRunResponse]
+ next_cursor: str | None
+ limit: int = Field(ge=1, le=100)
diff --git a/backend/src/modelforge_api/domain/configuration.py b/backend/src/modelforge_api/domain/configuration.py
new file mode 100644
index 0000000..36ad841
--- /dev/null
+++ b/backend/src/modelforge_api/domain/configuration.py
@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from modelforge_api.domain.configuration_reference import Sensitivity
+
+
+class ConfigurationSource(StrEnum):
+ """Source that supplied the effective value in Pydantic's unchanged priority order."""
+
+ PYDANTIC_DEFAULT = "pydantic_default"
+ INITIALIZATION = "initialization"
+ PROCESS_ENVIRONMENT = "process_environment"
+ DOTENV = "dotenv"
+ SECRETS_FILE = "secrets_file"
+ RUNTIME_UPDATE = "runtime_update"
+ OPERATOR_OVERRIDE = "operator_override"
+
+
+class ConfigurationApplicationMode(StrEnum):
+ """How a setting can take effect in the current product contract."""
+
+ DYNAMIC = "dynamic"
+ RESTART_REQUIRED = "restart_required"
+
+
+class ConfigurationScope(StrEnum):
+ CONTROL_PLANE = "control_plane"
+
+
+class ConfigurationSettingResponse(BaseModel):
+ key: str
+ environment_variable: str
+ description: str
+ sensitivity: Sensitivity
+ scope: ConfigurationScope = ConfigurationScope.CONTROL_PLANE
+ required_in_production: bool
+ application_mode: ConfigurationApplicationMode
+ editable: bool = False
+ configured: bool
+ source: ConfigurationSource
+ value: str | int | float | bool | None = None
+ display_name: str = ""
+ category: str = "process"
+ data_type: str = "string"
+ default_value: str | int | float | bool | None = None
+ validation_rules: dict[str, str | int | float] = Field(default_factory=dict)
+ consumer: str | None = None
+ rollback: str = "Restore deployment configuration and restart the component."
+
+
+class ConfigurationCatalogResponse(BaseModel):
+ environment: str
+ read_only: bool = True
+ settings: list[ConfigurationSettingResponse] = Field(default_factory=list)
+ revision: int = 0
+ overrides: dict[str, int | float] = Field(default_factory=dict)
+
+
+class ConfigurationChange(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+ expected_revision: int = Field(ge=0)
+ # Null removes the operator override and reveals the deployment-resolved value.
+ changes: dict[str, int | float | None] = Field(min_length=1, max_length=14)
+
+
+class ConfigurationApply(ConfigurationChange):
+ preview_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+ confirm: Literal[True]
+
+
+class ConfigurationDifference(BaseModel):
+ key: str
+ before: int | float
+ after: int | float
+ source_before: ConfigurationSource
+ source_after: ConfigurationSource
+
+
+class ConfigurationPreview(BaseModel):
+ expected_revision: int
+ preview_digest: str
+ differences: list[ConfigurationDifference]
+ affected_components: list[str]
+ impact: str
+ rollback: str
+ overrides: dict[str, int | float]
+
+
+class ConfigurationApplied(BaseModel):
+ revision: int
+ status: Literal["accepted_for_new_requests"] = "accepted_for_new_requests"
+ acknowledged_by: str
+ preview_digest: str
+
+
+class ConfigurationRevisionResponse(BaseModel):
+ revision: int
+ previous_revision: int
+ created_at: datetime
+ actor: str
+ correlation_id: str
+ overrides: dict[str, int | float]
+ preview_digest: str
+
+
+class ConfigurationExport(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+ format: str
+ product: Literal["ModelForge"] = "ModelForge"
+ source_version: str
+ overrides: dict[str, int | float] = Field(max_length=14)
+ secret_slots: list[str] = Field(max_length=20)
+
+
+class ConfigurationImport(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+ expected_revision: int = Field(ge=0)
+ document: ConfigurationExport
+
+
+class ConfigurationProposal(BaseModel):
+ change: ConfigurationChange
+ preview: ConfigurationPreview
+
+
+class ConfigurationRollback(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+ expected_revision: int = Field(ge=0)
diff --git a/backend/src/modelforge_api/domain/configuration_policy.py b/backend/src/modelforge_api/domain/configuration_policy.py
new file mode 100644
index 0000000..ca337be
--- /dev/null
+++ b/backend/src/modelforge_api/domain/configuration_policy.py
@@ -0,0 +1,14 @@
+"""The closed set of settings with a proven per-request serving consumer."""
+
+SERVING_OVERRIDE_KEYS = frozenset(
+ {
+ "gateway_max_batch_size",
+ "gateway_max_input_characters",
+ "gateway_request_timeout_seconds",
+ "gateway_queue_timeout_seconds",
+ "scheduler_telemetry_stale_seconds",
+ }
+)
+
+CONFIGURATION_FORMAT = "modelforge.serving-configuration.v1"
+CONFIGURATION_CONSUMER = "control_plane.serving"
diff --git a/backend/src/modelforge_api/domain/configuration_reference.py b/backend/src/modelforge_api/domain/configuration_reference.py
index a72a40a..bfcfe24 100644
--- a/backend/src/modelforge_api/domain/configuration_reference.py
+++ b/backend/src/modelforge_api/domain/configuration_reference.py
@@ -73,14 +73,30 @@ SETTING_DOCS: dict[str, SettingDoc] = {
"A wildcard is refused in production because requests are credentialed."
),
# ---------------------------------------------------------------- credentials
+ "operator_identities": _secret(
+ "Optional JSON array of named human identities with role, credential ID, SHA-256 key "
+ "digest and timezone-aware expiry. Provision with scripts/operator_identity.py. "
+ "Replace this deployment setting and recreate the API to rotate or revoke access."
+ ),
+ "operator_password_hash": _secret(
+ "Optional salted scrypt verifier generated by scripts/operator_password.py. Enables "
+ "password login with 15-minute sessions; requires the separate strong operator API key. "
+ "Changing either value invalidates password sessions."
+ ),
"operator_api_key": _secret(
- "Operator API key guarding every admin route. Generate at least 32 random characters; "
- "ModelForge never mints one for you.",
- required=True,
+ "Legacy bootstrap administrator key. Generate at least 32 random characters. "
+ "Production requires this key or an unexpired named administrator in operator_identities."
),
"hf_token": _secret(
- "Optional Hugging Face token, used only for acquiring gated repositories. "
- "It is never passed to a runtime and never leaves the control plane."
+ "Optional deployment Hugging Face token for control-plane metadata. Compose may also "
+ "provision it independently to node-agent downloads; API secret activation does not update nodes."
+ ),
+ "secret_slot_root_key": _secret(
+ "External base64 32-byte AES-256 key for the managed Hugging Face control-plane slot. "
+ "Keep independent of backup and operator keys; retain it externally for historical restore."
+ ),
+ "secret_slot_root_key_id": _p(
+ "External key identifier bound to encrypted HF slot versions. Replacing a key does not rewrap history."
),
"backup_encryption_key": _secret(
"Base64 AES-256 key for backup encryption. Without it no backup can be produced, and "
@@ -160,6 +176,25 @@ SETTING_DOCS: dict[str, SettingDoc] = {
),
"observability_monitor_enabled": _p("Run SLO and alert evaluation in this process."),
"observability_poll_interval_seconds": _p("Interval between observability evaluation passes."),
+ "alert_notifications_enabled": _p("Queue and dispatch the deployment-managed operational alert webhook."),
+ "alert_webhook_url": _secret(
+ "Deployment-managed HTTPS notification destination. Its exact hostname must also appear "
+ "in alert_webhook_allowed_hosts; the URL is never returned by the API."
+ ),
+ "alert_webhook_allowed_hosts": _p(
+ "Comma-separated exact destination host allowlist for the alert webhook. Wildcards and "
+ "redirects are not accepted."
+ ),
+ "alert_webhook_min_severity": _p(
+ "Minimum alert severity sent to the single webhook route: INFO, WARNING or CRITICAL."
+ ),
+ "alert_notification_timeout_seconds": _p("Total timeout for one webhook delivery."),
+ "alert_notification_max_attempts": _p("Maximum webhook attempts before a delivery fails permanently."),
+ "alert_notification_batch_size": _p("Maximum outbox rows claimed by one notification dispatch pass."),
+ "alert_notification_lease_seconds": _p("Lease duration before an interrupted delivery can be reclaimed."),
+ "alert_notification_retry_base_seconds": _p("Base delay for bounded exponential webhook retries."),
+ "model_update_monitor_enabled": _p("In production, check explicitly followed model references for changed revisions. Metadata only; never downloads, tests or promotes a model."),
+ "model_update_check_interval_seconds": _p("Minimum interval between automatic checks of each followed model reference; default one day. Each API process checks at most one due reference per minute."),
"recovery_reconciliation_enabled": _p("Reconcile interrupted backups and restores at startup."),
# ---------------------------------------------------------------- recovery
"backup_pg_dump_path": _p("pg_dump executable. Must match the server major version."),
diff --git a/backend/src/modelforge_api/domain/contracts.py b/backend/src/modelforge_api/domain/contracts.py
index ee6b3af..9a1050e 100644
--- a/backend/src/modelforge_api/domain/contracts.py
+++ b/backend/src/modelforge_api/domain/contracts.py
@@ -341,12 +341,12 @@ class BenchmarkEnvironmentFingerprint(StrictModel):
launch_arguments: dict[str, Any]
cuda_version: str | None
driver_version: str | None
- accelerator_name: str
+ accelerator_name: str | None
accelerator_uuid: str | None
context_length: int | None
concurrency: int = Field(ge=1)
seed: int | None
- operating_system: str
+ operating_system: str | None
python_version: str | None = None
@property
diff --git a/backend/src/modelforge_api/domain/dispatch_context.py b/backend/src/modelforge_api/domain/dispatch_context.py
new file mode 100644
index 0000000..5b27c46
--- /dev/null
+++ b/backend/src/modelforge_api/domain/dispatch_context.py
@@ -0,0 +1,53 @@
+"""Internal trusted scheduler identity; this is not an authorization or public input contract."""
+from __future__ import annotations
+
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+DispatchPriority = Literal["production", "interactive", "background", "benchmark", "lab"]
+
+
+class DispatchContext(BaseModel):
+ """Identifiers only, constructed after gateway authentication or operator admission.
+
+ An operator context does not grant permission to run a candidate. The eventual caller must
+ independently admit the exact target and persist its operation before using this context.
+ """
+
+ model_config = ConfigDict(frozen=True, extra="forbid", strict=True)
+ actor_kind: Literal["service_client", "operator"]
+ actor_id: str = Field(min_length=1, max_length=128)
+ lease_owner: str = Field(min_length=1, max_length=255)
+ workload_priority: DispatchPriority
+ service_client_id: UUID | None = None
+ operation_id: UUID | None = None
+
+ @model_validator(mode="after")
+ def coherent_identity(self) -> DispatchContext:
+ if self.actor_kind == "service_client":
+ if (self.service_client_id is None or self.actor_id != str(self.service_client_id)
+ or self.operation_id is not None or self.workload_priority == "lab"):
+ raise ValueError("service dispatch requires its exact client identity and configured priority")
+ elif (self.service_client_id is not None or self.operation_id is None
+ or self.workload_priority != "lab" or self.lease_owner != f"operator:{self.actor_id}"):
+ raise ValueError("operator dispatch requires a persisted operation and fixed lab priority")
+ return self
+
+ @classmethod
+ def for_service_client(cls, client_id: UUID, name: str, priority: str) -> DispatchContext:
+ """Priority is read from the authenticated persisted client, never the request body."""
+ return cls.model_validate({"actor_kind": "service_client", "actor_id": str(client_id),
+ "lease_owner": name, "service_client_id": client_id, "workload_priority": priority})
+
+ @classmethod
+ def for_operator(cls, subject: str, operation_id: UUID) -> DispatchContext:
+ return cls(actor_kind="operator", actor_id=subject, lease_owner=f"operator:{subject}",
+ workload_priority="lab", operation_id=operation_id)
+
+ def provenance(self) -> dict[str, str | None]:
+ return {"actor_kind": self.actor_kind, "actor_id": self.actor_id,
+ "service_client_id": str(self.service_client_id) if self.service_client_id else None,
+ "operation_id": str(self.operation_id) if self.operation_id else None,
+ "workload_priority": self.workload_priority}
diff --git a/backend/src/modelforge_api/domain/external_creative.py b/backend/src/modelforge_api/domain/external_creative.py
new file mode 100644
index 0000000..d399309
--- /dev/null
+++ b/backend/src/modelforge_api/domain/external_creative.py
@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+CreativeWorkload = Literal[
+ "image",
+ "video",
+ "chat",
+ "speech_generation",
+ "voice_clone",
+ "music_generation",
+ "sound_generation",
+ "foley_generation",
+ "transcription",
+ "audio_enhancement",
+ "lip_sync",
+]
+CreativeAllocationState = Literal["granted", "queued", "blocked", "uncertain", "released"]
+CreativeRequiredAction = Literal["release_idle_comfy", "unload_configured_ollama"]
+
+
+class ExternalAllocationCreate(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+ request_id: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
+ idempotency_key: str = Field(
+ min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+ )
+ installation_id: str = Field(
+ min_length=1, max_length=120, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+ )
+ workload: CreativeWorkload
+ resource_ref: str = Field(
+ min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
+ )
+ provider_resource_id: str = Field(
+ min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+ )
+ required_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+ priority: Literal["production", "interactive", "background", "benchmark"] = "interactive"
+ ttl_seconds: int = Field(default=600, ge=30, le=3600)
+ use_measured_profile: bool = True
+
+
+class ExternalAllocationResponse(BaseModel):
+ allocation_id: uuid.UUID
+ project_id: uuid.UUID
+ request_id: str
+ installation_id: str
+ workload: CreativeWorkload
+ resource_ref: str
+ provider_resource_id: str
+ required_vram_bytes: int
+ priority: str
+ state: CreativeAllocationState
+ decision_code: str
+ required_actions: list[CreativeRequiredAction]
+ accelerator_id: uuid.UUID | None
+ acquired_at: datetime | None
+ heartbeat_at: datetime | None
+ expires_at: datetime | None
+ uncertain_at: datetime | None
+ released_at: datetime | None
+ heartbeat_interval_seconds: int = 30
+ decision_evidence: dict[str, Any]
+ queue_position: int | None = None
+
+
+class ExternalAllocationAction(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ request_id: str = Field(min_length=1, max_length=128)
+ observed_peak_vram_bytes: int | None = Field(default=None, gt=0, le=256 * 1024**3)
+ attestation: ExternalActionAttestation | None = None
+
+
+class ExternalResourceApproval(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+ project_id: uuid.UUID
+ resource_ref: str = Field(
+ min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
+ )
+ workload: CreativeWorkload
+ provider_resource_id: str = Field(
+ min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
+ )
+ approved: bool = True
+ configured_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+
+
+class ExternalResourceResponse(ExternalResourceApproval):
+ resource_id: uuid.UUID
+ measured_peak_vram_bytes: int | None = None
+ measurement_count: int = 0
+ updated_at: datetime
+
+
+class ExternalResourceList(BaseModel):
+ items: list[ExternalResourceResponse]
+ total: int
+
+
+class ExternalActionAttestation(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+ action: Literal["release_idle_comfy", "unload_configured_ollama"]
+ observed_before: int = Field(ge=0, le=256 * 1024**3)
+ observed_after: int = Field(ge=0, le=256 * 1024**3)
+ completed_at: datetime
+
+
+class ExternalActionAttestationResponse(ExternalActionAttestation):
+ attestation_id: uuid.UUID
+ allocation_id: uuid.UUID
+ validated: bool
+
+
+class ExternalAllocationReconcile(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+ action: Literal["confirm_released", "keep_uncertain"]
+ reason: str = Field(min_length=3, max_length=500)
+
+
+class ExternalAllocationList(BaseModel):
+ items: list[ExternalAllocationResponse]
+ total: int
+
+
+class ExternalProfileObservation(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+ allocation_id: uuid.UUID
+ resource_ref: str = Field(min_length=1, max_length=255)
+ provider_resource_id: str = Field(min_length=1, max_length=128)
+ workload: CreativeWorkload
+ observed_peak_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+ reserved_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
+ duration_ms: int = Field(ge=0, le=7 * 24 * 3600 * 1000)
+ outcome: Literal["succeeded", "failed", "cancelled", "uncertain"]
diff --git a/backend/src/modelforge_api/domain/lab.py b/backend/src/modelforge_api/domain/lab.py
new file mode 100644
index 0000000..bcdb93c
--- /dev/null
+++ b/backend/src/modelforge_api/domain/lab.py
@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from modelforge_api.domain.serving import (
+ EmbeddingInvokeRequest,
+ OCRInvokeRequest,
+ RerankingInvokeRequest,
+ SpeechTranscriptionInvokeRequest,
+ VisionEmbeddingInvokeRequest,
+)
+
+
+class LabModel(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class LabTarget(LabModel):
+ deployment_candidate_id: uuid.UUID
+ artifact_set_id: uuid.UUID
+ runtime_profile_id: uuid.UUID
+ compute_node_id: uuid.UUID
+ execution_approval_id: uuid.UUID
+
+
+class LabEmbeddingInput(LabModel):
+ capability: Literal["rag.embedding"]
+ payload: EmbeddingInvokeRequest
+
+
+class LabRerankingInput(LabModel):
+ capability: Literal["rag.reranking"]
+ payload: RerankingInvokeRequest
+
+
+class LabOCRInput(LabModel):
+ capability: Literal["document.ocr"]
+ payload: OCRInvokeRequest
+
+
+class LabVisionInput(LabModel):
+ capability: Literal["vision.embedding"]
+ payload: VisionEmbeddingInvokeRequest
+
+
+class LabSpeechInput(LabModel):
+ capability: Literal["speech.transcription"]
+ payload: SpeechTranscriptionInvokeRequest
+
+
+LabInput = Annotated[
+ LabEmbeddingInput | LabRerankingInput | LabOCRInput | LabVisionInput | LabSpeechInput,
+ Field(discriminator="capability"),
+]
+
+
+class LabLimits(LabModel):
+ timeout_seconds: int = Field(default=60, ge=5, le=300, strict=True)
+ queue_timeout_seconds: int = Field(default=10, ge=1, le=60, strict=True)
+ retention_seconds: int = Field(default=300, ge=60, le=3600, strict=True)
+ max_output_bytes: int = Field(default=65536, ge=1, le=1048576, strict=True)
+ concurrency: Literal[1] = 1
+
+ @model_validator(mode="after")
+ def bounded_queue(self) -> LabLimits:
+ if self.queue_timeout_seconds > self.timeout_seconds:
+ raise ValueError("queue timeout cannot exceed total timeout")
+ return self
+
+
+class LabPreflightRequest(LabModel):
+ target: LabTarget
+ input: LabInput
+ contract_version: Literal[1] = 1
+ limits: LabLimits = Field(default_factory=LabLimits)
+
+
+class LabExecutionStartRequest(LabPreflightRequest):
+ """An explicitly selected embedding experiment; body input is never echoed in status."""
+
+ deployment_id: uuid.UUID
+ experiment_route_id: uuid.UUID
+
+
+class LabInputSummary(LabModel):
+ sha256: str
+ item_count: int
+ encoded_bytes: int
+
+
+class LabBinding(LabTarget):
+ compatibility_assessment_id: uuid.UUID
+ runtime_probe_id: uuid.UUID
+ capability: str
+ contract_version: Literal[1] = 1
+ runtime_profile_fingerprint: str
+ assessment_fingerprint: str
+ approval_evidence_fingerprint: str
+ probe_environment_fingerprint: str
+ accelerator_id: uuid.UUID | None = None
+
+
+class LabPreflightResponse(LabModel):
+ input_summary: LabInputSummary
+ binding: LabBinding | None = None
+ limits: LabLimits
+ blockers: list[str]
+ checks_passed: bool
+ execution_started: Literal[False] = False
+ permission_granted: Literal[False] = False
+
+
+LabExecutionStatus = Literal[
+ "accepted",
+ "preparing",
+ "queued",
+ "running",
+ "cancel_requested",
+ "completed",
+ "failed",
+ "cancelled",
+ "interrupted",
+]
+
+
+class LabExecutionRunResponse(LabModel):
+ id: uuid.UUID
+ capability: Literal["rag.embedding@1"] = "rag.embedding@1"
+ status: LabExecutionStatus
+ target_snapshot: dict[str, object]
+ input_summary: LabInputSummary
+ output_metadata: dict[str, object] | None = None
+ worker_observations: dict[str, object] | None = None
+ failure_code: str | None = None
+ input_payload_expires_at: datetime
+ retention_seconds: int = Field(ge=60, le=3600)
+ created_at: datetime
+ started_at: datetime | None = None
+ cancellation_requested_at: datetime | None = None
+ cancellation_acknowledged_at: datetime | None = None
+ finished_at: datetime | None = None
diff --git a/backend/src/modelforge_api/domain/lifecycle_contracts.py b/backend/src/modelforge_api/domain/lifecycle_contracts.py
index df6a3f9..31aed18 100644
--- a/backend/src/modelforge_api/domain/lifecycle_contracts.py
+++ b/backend/src/modelforge_api/domain/lifecycle_contracts.py
@@ -133,7 +133,31 @@ def assert_deployment_transition(
raise ValueError(f"lifecycle transition {current} -> {target} is not allowed")
+class ServingTransitionSelection(LifecycleModel):
+ """Explicit selection; the server derives the complete immutable serving contract."""
+
+ current_deployment_id: uuid.UUID
+ candidate_deployment_id: uuid.UUID
+ production_approval_id: uuid.UUID
+ expected_route_generation: int = Field(ge=0)
+ comparison_ids: dict[uuid.UUID, uuid.UUID] = Field(min_length=1, max_length=1000)
+
+
+class ServingTransitionContract(LifecycleModel):
+ version: Literal[1] = 1
+ production_validation: Literal["SATISFIED"] = "SATISFIED"
+ selection: ServingTransitionSelection
+ capability_contract_id: uuid.UUID
+ current: dict[str, Any]
+ candidate: dict[str, Any]
+ experiment_routes: dict[str, dict[str, Any]]
+ bindings: dict[str, dict[str, Any]]
+ comparisons: dict[str, dict[str, Any]]
+ production_approval_fingerprint: str
+
+
class LifecycleEvidenceBundle(LifecycleModel):
+ serving_transition: ServingTransitionSelection | None = None
artifact_set_id: uuid.UUID | None = None
model_revision_id: uuid.UUID | None = None
runtime_profile_id: uuid.UUID | None = None
diff --git a/backend/src/modelforge_api/domain/managed_identity.py b/backend/src/modelforge_api/domain/managed_identity.py
new file mode 100644
index 0000000..0dee9e2
--- /dev/null
+++ b/backend/src/modelforge_api/domain/managed_identity.py
@@ -0,0 +1,89 @@
+"""Bounded, secret-safe contracts for the managed personal-key provider."""
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+Role = Literal["viewer", "operator", "admin"]
+
+
+class ManagedIdentityRequest(BaseModel):
+ model_config = ConfigDict(extra="forbid", hide_input_in_errors=True, strict=True)
+ reason: str = Field(min_length=8, max_length=500)
+
+ @field_validator("reason")
+ @classmethod
+ def safe_reason(cls, value: str) -> str:
+ value = value.strip()
+ if len(value) < 8 or any(prefix in value for prefix in ("mfopm_", "mfop_", "mfsvc_", "mfnode_", "mfenroll_")):
+ raise ValueError("Use a non-secret explanation of at least eight characters")
+ return value
+
+
+class ManagedIdentityCreate(ManagedIdentityRequest):
+ slug: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$")
+ role: Role
+ expires_in_days: int = Field(ge=1, le=365)
+ expected_revision: int = Field(ge=0)
+
+
+class ManagedIdentityRotate(ManagedIdentityRequest):
+ expected_version: int = Field(ge=1)
+ expires_in_days: int = Field(ge=1, le=365)
+ overlap_seconds: int = Field(ge=0, le=900)
+
+
+class ManagedIdentityChange(ManagedIdentityRequest):
+ expected_version: int = Field(ge=1)
+ role: Role
+ enabled: bool
+
+
+class ManagedIdentityRevoke(ManagedIdentityRequest):
+ expected_version: int = Field(ge=1)
+ confirm_subject: str = Field(max_length=71)
+
+
+class ManagedCredentialMetadata(BaseModel):
+ id: str
+ expires_at: datetime
+ revoked_at: datetime | None
+ superseded_at: datetime | None
+ last_used_at: datetime | None
+ activated: bool
+ usable: bool
+
+
+class ManagedIdentityMetadata(BaseModel):
+ subject: str
+ slug: str
+ source: Literal["managed"] = "managed"
+ role: Role
+ enabled: bool
+ version: int
+ created_at: datetime
+ credentials: list[ManagedCredentialMetadata]
+
+
+class DeploymentIdentityMetadata(BaseModel):
+ subject: str
+ role: Role
+ credential_id: str
+ expires_at: datetime
+ source: Literal["deployment"] = "deployment"
+
+
+class ManagedIdentityList(BaseModel):
+ revision: int
+ initialized: bool
+ identities: list[ManagedIdentityMetadata]
+ deployment_identities: list[DeploymentIdentityMetadata]
+
+
+class ManagedIdentityResult(BaseModel):
+ revision: int
+ identity: ManagedIdentityMetadata
+
+
+class ManagedIdentityIssued(ManagedIdentityResult):
+ one_time_key: str = Field(repr=False)
diff --git a/backend/src/modelforge_api/domain/model_changes.py b/backend/src/modelforge_api/domain/model_changes.py
new file mode 100644
index 0000000..f8726cb
--- /dev/null
+++ b/backend/src/modelforge_api/domain/model_changes.py
@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+ModelChangeStatus = Literal[
+ "searching",
+ "found",
+ "source_checked",
+ "downloaded",
+ "ready_to_test",
+ "tested",
+ "comparison_available",
+ "migration_required",
+ "ready_to_activate",
+ "active",
+ "checked",
+ "rolled_back",
+ "rejected",
+]
+
+
+class Contract(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class ModelChangeCreate(Contract):
+ project_binding_id: UUID
+ reason: str = Field(default="Model verbeteren", min_length=1, max_length=500)
+
+
+class ModelChangeCandidateLink(Contract):
+ expected_version: int = Field(ge=1)
+ candidate_model_id: UUID | None = None
+ candidate_revision_id: UUID | None = None
+ candidate_artifact_set_id: UUID | None = None
+ candidate_deployment_id: UUID | None = None
+ discovery_assessment_id: UUID | None = None
+ download_plan_id: UUID | None = None
+ artifact_job_id: UUID | None = None
+
+ @model_validator(mode="after")
+ def at_least_one_identity(self) -> ModelChangeCandidateLink:
+ if not any((
+ self.candidate_model_id,
+ self.candidate_revision_id,
+ self.candidate_artifact_set_id,
+ self.candidate_deployment_id,
+ self.discovery_assessment_id,
+ self.download_plan_id,
+ self.artifact_job_id,
+ )):
+ raise ValueError("at least one candidate or acquisition identity is required")
+ return self
+
+
+class ModelChangeEvidenceLink(Contract):
+ expected_version: int = Field(ge=1)
+ selected_comparison_id: UUID | None = None
+ migration_plan_id: UUID | None = None
+ lifecycle_plan_id: UUID | None = None
+
+ @model_validator(mode="after")
+ def at_least_one_evidence(self) -> ModelChangeEvidenceLink:
+ if not any((self.selected_comparison_id, self.migration_plan_id, self.lifecycle_plan_id)):
+ raise ValueError("at least one evidence reference is required")
+ return self
+
+
+class ModelChangeReject(Contract):
+ expected_version: int = Field(ge=1)
+ reason: str = Field(min_length=3, max_length=1000)
+
+
+class ChangeModelIdentity(Contract):
+ model_id: UUID | None = None
+ display_name: str | None = None
+ revision_id: UUID | None = None
+ resolved_commit_sha: str | None = None
+ artifact_set_id: UUID | None = None
+ deployment_id: UUID | None = None
+
+
+class ChangeAcquisition(Contract):
+ discovery_assessment_id: UUID | None = None
+ security_gate: str = "unknown"
+ license_gate: str = "unknown"
+ hardware_fit: str = "unknown"
+ storage_bytes: int | None = None
+ download_plan_id: UUID | None = None
+ artifact_job_id: UUID | None = None
+ download_status: str | None = None
+
+
+class ChangeImpact(Contract):
+ affected_projects: list[str] = Field(default_factory=list)
+ affected_project_count: int = 0
+ application_change_required: bool = False
+ migration_required: bool = False
+ migration_kind: str = "none"
+ downtime_expected: bool = False
+ rollback_available: bool = False
+ blockers: list[str] = Field(default_factory=list)
+
+
+class ChangeProgressStep(Contract):
+ key: str
+ label: str
+ state: Literal["done", "current", "upcoming", "blocked"]
+
+
+class ModelChangeResponse(Contract):
+ id: UUID
+ project_binding_id: UUID
+ project_id: UUID
+ project_key: str
+ project_name: str
+ capability: str
+ status: ModelChangeStatus
+ stored_status: ModelChangeStatus
+ status_reason: str
+ version: int
+ current: ChangeModelIdentity
+ candidate: ChangeModelIdentity | None = None
+ acquisition: ChangeAcquisition
+ selected_comparison_id: UUID | None = None
+ migration_plan_id: UUID | None = None
+ lifecycle_plan_id: UUID | None = None
+ impact: ChangeImpact
+ next_action: str
+ progress: list[ChangeProgressStep]
+ created_by: str
+ updated_by: str
+ activated_at: datetime | None = None
+ checked_at: datetime | None = None
+ closed_at: datetime | None = None
+ created_at: datetime
+ updated_at: datetime
+
+
+class ModelChangeList(Contract):
+ items: list[ModelChangeResponse]
+ total: int
diff --git a/backend/src/modelforge_api/domain/model_updates.py b/backend/src/modelforge_api/domain/model_updates.py
new file mode 100644
index 0000000..1d14f95
--- /dev/null
+++ b/backend/src/modelforge_api/domain/model_updates.py
@@ -0,0 +1,51 @@
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class UpdateModel(BaseModel):
+ model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class ModelWatchCreate(UpdateModel):
+ model_id: UUID
+ baseline_revision_id: UUID
+ upstream_ref: str = Field(default="main", min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._/-]*$")
+
+
+class ModelWatchResponse(UpdateModel):
+ id: UUID
+ model_id: UUID
+ baseline_revision_id: UUID
+ baseline_commit_sha: str
+ upstream_ref: str
+ last_checked_at: datetime | None
+ last_error_code: str | None
+ enabled: bool
+ version: int
+
+
+class ModelWatchUpdate(UpdateModel):
+ expected_version: int = Field(ge=1)
+ expected_baseline_revision_id: UUID
+ baseline_revision_id: UUID
+ enabled: bool
+ reason: str = Field(min_length=1, max_length=500, pattern=r"\S")
+
+
+class ModelUpdateMonitorResponse(UpdateModel):
+ enabled: bool
+ check_interval_seconds: int
+
+
+class ModelUpdateNoticeResponse(UpdateModel):
+ id: UUID
+ watch_id: UUID
+ model_id: UUID
+ baseline_commit_sha: str
+ candidate_revision_id: UUID
+ candidate_commit_sha: str
+ status: Literal["new", "dismissed"]
+ discovered_at: datetime
diff --git a/backend/src/modelforge_api/domain/node_lifecycle.py b/backend/src/modelforge_api/domain/node_lifecycle.py
new file mode 100644
index 0000000..7fc2199
--- /dev/null
+++ b/backend/src/modelforge_api/domain/node_lifecycle.py
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import uuid
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class NodeLifecycleModel(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class NodeLifecyclePreviewRequest(NodeLifecycleModel):
+ target_enabled: bool
+
+
+class NodeLifecycleChange(NodeLifecycleModel):
+ target_enabled: bool
+ expected_generation: int = Field(ge=1)
+ dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+ operator: str = Field(min_length=1, max_length=255)
+ reason: str = Field(min_length=10, max_length=2000)
+ confirmation: str = Field(min_length=1, max_length=255)
+
+
+class NodeLifecycleBlocker(NodeLifecycleModel):
+ code: str
+ message: str
+ record_type: str
+ count: int = Field(ge=1)
+ resource_ids: list[str] = Field(default_factory=list)
+
+
+class NodeLifecyclePreview(NodeLifecycleModel):
+ node_id: uuid.UUID
+ persisted_identity: str
+ hostname: str
+ display_name: str
+ current_enabled: bool
+ current_liveness: str
+ target_enabled: bool
+ node_generation: int = Field(ge=1)
+ dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+ can_apply: bool
+ blockers: list[NodeLifecycleBlocker]
+ agent_evidence_after_enable: Literal["unchanged", "awaiting_fresh_authenticated_heartbeat"]
+
+
+class NodeLifecycleResult(NodeLifecycleModel):
+ node_id: uuid.UUID
+ enabled: bool
+ generation: int = Field(ge=1)
+ liveness: Literal["offline", "disabled"]
+ agent_evidence_state: Literal["unchanged", "awaiting_fresh_authenticated_heartbeat"]
diff --git a/backend/src/modelforge_api/domain/observability.py b/backend/src/modelforge_api/domain/observability.py
index f50368f..8eb88e2 100644
--- a/backend/src/modelforge_api/domain/observability.py
+++ b/backend/src/modelforge_api/domain/observability.py
@@ -363,6 +363,34 @@ class CapacitySnapshotResponse(OperationalModel):
freshness_seconds: float
+class ActiveSchedulerLeaseResponse(OperationalModel):
+ lease_id: uuid.UUID
+ project_id: uuid.UUID | None
+ node_id: uuid.UUID | None
+ accelerator_id: uuid.UUID
+ deployment_id: uuid.UUID
+ lease_state: Literal["PENDING", "GRANTED", "ACTIVE", "RELEASING"]
+ workload_status: Literal["QUEUED", "RUNNING", "UNKNOWN"]
+ priority: str
+ lease_type: Literal[
+ "RESIDENCY_RESERVATION", "REQUEST_EXECUTION", "LOAD_TRANSITION", "UNKNOWN"
+ ]
+ reserved_vram_bytes: int
+ materialized_vram_bytes: int
+ created_at: datetime
+ acquired_at: datetime | None
+ heartbeat_at: datetime | None
+ expires_at: datetime
+
+
+class ActiveSchedulerLeaseInventoryResponse(OperationalModel):
+ observed_at: datetime
+ completeness: Literal["COMPLETE", "TRUNCATED"]
+ limit: int
+ returned_count: int
+ leases: list[ActiveSchedulerLeaseResponse]
+
+
class TrendResponse(OperationalModel):
subject: str
sample_count: int
diff --git a/backend/src/modelforge_api/domain/operator_identity.py b/backend/src/modelforge_api/domain/operator_identity.py
new file mode 100644
index 0000000..0fe854b
--- /dev/null
+++ b/backend/src/modelforge_api/domain/operator_identity.py
@@ -0,0 +1,57 @@
+"""Deployment-owned, hash-only human identities; independent of machine credentials."""
+
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from typing import Literal
+
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+
+class OperatorIdentity(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True)
+
+ credential_id: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$")
+ subject: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._@-]{0,127}$")
+ role: Literal["viewer", "operator", "admin"]
+ token_sha256: str = Field(pattern=r"^[0-9a-f]{64}$", repr=False)
+ expires_at: AwareDatetime
+
+
+def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+ result: dict[str, object] = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValueError("Duplicate identity field")
+ result[key] = value
+ return result
+
+
+@lru_cache(maxsize=8)
+def parse_operator_identities(document: str) -> tuple[OperatorIdentity, ...]:
+ """Bound and validate once per deployment snapshot; never echo rejected input."""
+ try:
+ if len(document) > 65_536:
+ raise ValueError("Identity document too large")
+ payload = json.loads(document, object_pairs_hook=_unique_object)
+ if not isinstance(payload, list) or not 1 <= len(payload) <= 100:
+ raise ValueError("Identity list required")
+ identities = TypeAdapter(tuple[OperatorIdentity, ...]).validate_python(payload)
+ identifiers: set[str] = set()
+ digests: set[str] = set()
+ roles: dict[str, str] = {}
+ for identity in identities:
+ if (
+ identity.subject == "legacy-operator"
+ or identity.credential_id in identifiers
+ or identity.token_sha256 in digests
+ or roles.get(identity.subject, identity.role) != identity.role
+ ):
+ raise ValueError("Ambiguous identity configuration")
+ identifiers.add(identity.credential_id)
+ digests.add(identity.token_sha256)
+ roles[identity.subject] = identity.role
+ return identities
+ except (ValueError, TypeError, ValidationError, RecursionError) as exc:
+ raise ValueError("Invalid operator identities; use the documented hash-only format") from exc
diff --git a/backend/src/modelforge_api/domain/portfolio.py b/backend/src/modelforge_api/domain/portfolio.py
new file mode 100644
index 0000000..9d3fd30
--- /dev/null
+++ b/backend/src/modelforge_api/domain/portfolio.py
@@ -0,0 +1,197 @@
+"""Typed, secret-free contracts for repository inventory reconciliation."""
+
+from __future__ import annotations
+
+import re
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+class StrictModel(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+
+
+RepositoryClassification = Literal[
+ "local_model",
+ "hosted_model",
+ "gateway",
+ "mixed",
+ "model_management",
+ "runtime_observer",
+ "non_ai",
+ "unknown",
+]
+PROJECT_LINK_CLASSIFICATIONS = frozenset({"local_model", "gateway", "mixed"})
+
+
+class PortfolioCodeEvidence(StrictModel):
+ """A bounded source-code signal; source text and arbitrary notes are never accepted."""
+
+ path: str = Field(min_length=1, max_length=512)
+ signal: Literal[
+ "provider_dependency",
+ "inference_callsite",
+ "model_configuration",
+ "capability_client",
+ "runtime_declaration",
+ "model_management",
+ "runtime_observer",
+ ]
+ capabilities: list[str] = Field(default_factory=list, max_length=8)
+
+ @field_validator("path")
+ @classmethod
+ def validate_repository_relative_path(cls, value: str) -> str:
+ if (
+ value.startswith(("/", "\\"))
+ or "\\" in value
+ or ":" in value
+ or any(ord(character) < 32 for character in value)
+ or any(part in {"", ".", ".."} for part in value.split("/"))
+ ):
+ raise ValueError("path must be a normalized repository-relative file path")
+ return value
+
+ @field_validator("capabilities")
+ @classmethod
+ def validate_capabilities(cls, values: list[str]) -> list[str]:
+ pattern = re.compile(r"^[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*)+(?:@[1-9][0-9]*)?$")
+ if any(not pattern.fullmatch(value) for value in values):
+ raise ValueError("capabilities must be safe capability identifiers")
+ if len(values) != len(set(values)):
+ raise ValueError("capabilities must be unique")
+ return values
+
+
+class PortfolioRuntimeEvidence(StrictModel):
+ """A runtime observation separate from source-code classification."""
+
+ component: str = Field(pattern=r"^[a-z][a-z0-9-]{0,63}$")
+ state: Literal["running", "stopped", "unknown"]
+ observed_at: datetime
+ observation_source: Literal["container_inventory", "health_probe"]
+ git_revision: str | None = Field(default=None, pattern=r"^(?:[0-9a-f]{7,40})$")
+ image_digest: str | None = Field(default=None, pattern=r"^sha256:[0-9a-f]{64}$")
+
+ @field_validator("observed_at")
+ @classmethod
+ def require_timezone(cls, value: datetime) -> datetime:
+ if value.tzinfo is None or value.utcoffset() is None:
+ raise ValueError("runtime observation time must include a timezone")
+ return value.astimezone(UTC)
+
+
+class PortfolioRepositoryInput(StrictModel):
+ external_repository_id: str = Field(pattern=r"^[a-z0-9][a-z0-9._:-]{0,127}$")
+ project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]{0,127}$")
+ slug: str = Field(min_length=3, max_length=255)
+ default_branch: str = Field(min_length=1, max_length=255)
+ head_sha: str = Field(pattern=r"^[0-9a-f]{40}$")
+ classification: RepositoryClassification
+ accept_head_update: bool = False
+ code_evidence: list[PortfolioCodeEvidence] = Field(default_factory=list, max_length=128)
+ runtime_evidence: list[PortfolioRuntimeEvidence] = Field(default_factory=list, max_length=64)
+
+ @field_validator("slug")
+ @classmethod
+ def validate_slug(cls, value: str) -> str:
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", value):
+ raise ValueError("slug must be an owner/repository identifier, not a URL")
+ return value
+
+ @field_validator("default_branch")
+ @classmethod
+ def validate_branch(cls, value: str) -> str:
+ if (
+ not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,254}", value)
+ or ".." in value
+ or "//" in value
+ or value.endswith(("/", ".", ".lock"))
+ ):
+ raise ValueError("default branch must be a safe Git branch name")
+ return value
+
+ @model_validator(mode="after")
+ def validate_runtime_components(self) -> PortfolioRepositoryInput:
+ if self.classification in PROJECT_LINK_CLASSIFICATIONS and self.project_key is None:
+ raise ValueError("project_key is required for a local model consumer or provider")
+ components = [item.component for item in self.runtime_evidence]
+ if len(components) != len(set(components)):
+ raise ValueError("runtime components must be unique within a repository observation")
+ return self
+
+
+class PortfolioReconcileRequest(StrictModel):
+ source_system: Literal["gitea"] = "gitea"
+ snapshot_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+ observed_at: datetime
+ complete: bool = False
+ repositories: list[PortfolioRepositoryInput] = Field(max_length=256)
+
+ @field_validator("observed_at")
+ @classmethod
+ def require_timezone(cls, value: datetime) -> datetime:
+ if value.tzinfo is None or value.utcoffset() is None:
+ raise ValueError("snapshot observation time must include a timezone")
+ return value.astimezone(UTC)
+
+ @model_validator(mode="after")
+ def validate_repository_identity_uniqueness(self) -> PortfolioReconcileRequest:
+ if not self.repositories and not self.complete:
+ raise ValueError("an empty inventory is valid only as an explicit complete snapshot")
+ identities = [item.external_repository_id for item in self.repositories]
+ if len(identities) != len(set(identities)):
+ raise ValueError("a snapshot may contain each external repository only once")
+ return self
+
+
+class PortfolioRepositoryView(StrictModel):
+ id: str
+ project_key: str | None
+ project_name: str | None
+ project_id: str | None
+ source_system: str
+ external_repository_id: str
+ slug: str
+ default_branch: str
+ head_sha: str
+ classification: RepositoryClassification
+ code_evidence: list[PortfolioCodeEvidence]
+ runtime_evidence: list[PortfolioRuntimeEvidence]
+ observed_at: datetime
+ snapshot_id: str
+ snapshot_digest: str
+ pending_head_sha: str | None
+ pending_default_branch: str | None
+ pending_project_key: str | None
+ pending_classification: RepositoryClassification | None
+ pending_code_evidence: list[PortfolioCodeEvidence]
+ pending_observed_at: datetime | None
+ pending_snapshot_id: str | None
+ pending_snapshot_digest: str | None
+ last_seen_at: datetime
+ stale_at: datetime | None
+
+
+class PortfolioInventoryResponse(StrictModel):
+ repositories: list[PortfolioRepositoryView]
+
+
+class PortfolioReconcileResponse(StrictModel):
+ source_system: str
+ snapshot_id: str
+ snapshot_digest: str
+ observed_at: datetime
+ complete: bool
+ replayed: bool = False
+ stale_snapshot: bool = False
+ projects_created: int = 0
+ repositories_created: int = 0
+ repositories_updated: int = 0
+ stale_observations: int = 0
+ repositories_marked_stale: int = 0
+ runtime_observations_updated: int = 0
+ head_changes_review_required: int = 0
+ head_changes_accepted: int = 0
diff --git a/backend/src/modelforge_api/domain/projects.py b/backend/src/modelforge_api/domain/projects.py
new file mode 100644
index 0000000..e872aaa
--- /dev/null
+++ b/backend/src/modelforge_api/domain/projects.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from modelforge_api.domain.serving import ServiceClientResponse
+
+
+class ProjectCreate(BaseModel):
+ model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+ key: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$")
+ name: str = Field(min_length=1, max_length=255)
+ description: str = Field(min_length=1, max_length=2000)
+ owner: str = Field(min_length=1, max_length=128)
+ environment: Literal["lab", "shadow", "evaluation", "production"] = "lab"
+ labels: list[str] = Field(default_factory=list, max_length=20)
+
+
+class ProjectConnectivityResponse(BaseModel):
+ client_id: uuid.UUID
+ project_key: str | None
+ capability: str
+ connected: Literal[True] = True
+ correlation_id: str
+
+
+class ProjectRequirementCreate(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ capability: str = Field(pattern=r"^[a-z][a-z0-9.]*(?:@[1-9][0-9]*)$", max_length=128)
+ migration_support: Literal["none", "reindex", "schema"] = "none"
+ priority: Literal["interactive", "background", "benchmark"] = "interactive"
+
+
+class ProjectServingModelResponse(BaseModel):
+ """Exact model identity resolved from the stable deployment used by a project requirement."""
+
+ deployment_id: uuid.UUID
+ artifact_set_id: uuid.UUID
+ model_id: uuid.UUID
+ model_key: str
+ display_name: str
+ upstream_source: str
+ revision_id: uuid.UUID
+ upstream_revision: str
+ resolved_commit_sha: str
+ runtime_profile_id: uuid.UUID
+ compute_node_id: uuid.UUID
+ channel: str
+ production: bool
+ artifact_availability: str
+
+
+class ProjectRequirementResponse(BaseModel):
+ id: uuid.UUID
+ capability: str
+ channel: str
+ priority: str
+ migration_support: str
+ deprecated: bool
+ deployment_id: uuid.UUID | None
+ deployment_health: str
+ serving_model: ProjectServingModelResponse | None = None
+ credential_ready: bool
+ connectivity_checked_at: datetime | None
+ capability_checked_at: datetime | None
+ capability_check_request_id: str | None
+ state: str
+ blockers: list[str]
+
+
+class ProjectWorkspaceResponse(BaseModel):
+ id: uuid.UUID
+ key: str
+ name: str
+ description: str
+ active: bool
+ owner: str | None
+ environment: str | None
+ labels: list[str]
+ source: Literal["operator", "manifest"]
+ production_eligible: bool | None
+ state: str
+ requirements: list[ProjectRequirementResponse]
+ clients: list[ServiceClientResponse]
diff --git a/backend/src/modelforge_api/domain/recovery.py b/backend/src/modelforge_api/domain/recovery.py
index 4da9534..7114725 100644
--- a/backend/src/modelforge_api/domain/recovery.py
+++ b/backend/src/modelforge_api/domain/recovery.py
@@ -292,6 +292,32 @@ class BackupSetResponse(RecoveryModel):
entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
+class RecoveryRetentionDecision(RecoveryModel):
+ backup_id: str
+ current_state: BackupState
+ decision: Literal["RETAIN", "EXPIRE"]
+ reason: Literal[
+ "ACTIVE_RESTORE_IN_PROGRESS",
+ "LEGAL_HOLD",
+ "MILESTONE",
+ "MINIMUM_VERIFIED_BACKUPS",
+ "RETENTION_NOT_REACHED",
+ "RETENTION_ELIGIBLE",
+ ]
+ expires_at: datetime | None
+
+
+class RecoveryRetentionPreview(RecoveryModel):
+ observed_at: datetime
+ policy_key: str
+ policy_revision: int
+ minimum_verified_backups: int
+ active_restore_in_progress: bool
+ retained_backup_ids: list[str]
+ expiring_backup_ids: list[str]
+ decisions: list[RecoveryRetentionDecision]
+
+
class RestorePlanCreate(RecoveryModel):
backup_set_id: uuid.UUID
mode: RestoreMode
diff --git a/backend/src/modelforge_api/domain/release.py b/backend/src/modelforge_api/domain/release.py
index f7adffd..9f3f2cb 100644
--- a/backend/src/modelforge_api/domain/release.py
+++ b/backend/src/modelforge_api/domain/release.py
@@ -14,6 +14,7 @@ support is a far better outcome than one that starts and writes rows the next ve
from __future__ import annotations
+import re
from dataclasses import dataclass
from datetime import UTC, datetime
from enum import StrEnum
@@ -23,7 +24,7 @@ from pathlib import Path
#: Schema revisions this application version can run against, oldest first. The application refuses
#: to serve against anything outside this list rather than guessing.
-SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260830_0024",)
+SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260924_0039",)
#: The revision a clean installation and a completed upgrade must both arrive at.
TARGET_SCHEMA_REVISION = SUPPORTED_SCHEMA_REVISIONS[-1]
@@ -35,6 +36,21 @@ SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS: tuple[str, ...] = (
"20260827_0021",
"20260828_0022",
"20260830_0023",
+ "20260830_0024",
+ "20260907_0025",
+ "20260908_0026",
+ "20260908_0027",
+ "20260908_0028",
+ "20260909_0029",
+ "20260909_0030",
+ "20260909_0031",
+ "20260909_0032",
+ "20260909_0033",
+ "20260910_0034",
+ "20260911_0035",
+ "20260923_0036",
+ "20260923_0037",
+ "20260924_0038",
TARGET_SCHEMA_REVISION,
)
@@ -47,14 +63,36 @@ SUPPORTED_AGENT_PROTOCOL_VERSIONS: tuple[int, ...] = (1,)
#: The protocol version this control plane itself speaks.
CURRENT_AGENT_PROTOCOL_VERSION = SUPPORTED_AGENT_PROTOCOL_VERSIONS[-1]
-#: Release channels. v1 ships one; the enum exists so adding another is a typed change.
-RELEASE_CHANNEL = "stable"
-
#: Minimum PostgreSQL major version. The schema uses partial unique indexes and generated columns
#: that older majors either lack or plan differently.
MINIMUM_POSTGRES_MAJOR = 16
+_PEP440_NUMBERED_RC = re.compile(
+ r"^(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)\."
+ r"(?P0|[1-9][0-9]*)rc(?P0|[1-9][0-9]*)$"
+)
+
+
+def _canonicalize_packaged_version(value: str) -> str:
+ """Convert the one PEP 440 form produced for a numbered SemVer RC back to SemVer.
+
+ Build backends correctly normalize ``1.3.0-rc.1`` to ``1.3.0rc1`` in installed
+ distribution metadata. The release contract, image labels, and API intentionally expose the
+ reviewed SemVer spelling, so installed images must restore it rather than reporting a second
+ version representation. All other metadata is left untouched and remains subject to the
+ strict semantic-version validation below.
+ """
+
+ match = _PEP440_NUMBERED_RC.fullmatch(value)
+ if match is None:
+ return value
+ return (
+ f"{match.group('major')}.{match.group('minor')}.{match.group('patch')}"
+ f"-rc.{match.group('number')}"
+ )
+
+
def _read_version_file() -> str:
"""Read the repository's VERSION file, falling back to the packaged distribution metadata."""
@@ -68,7 +106,7 @@ def _read_version_file() -> str:
from importlib.metadata import PackageNotFoundError, version
try:
- return version("modelforge-api")
+ return _canonicalize_packaged_version(version("modelforge-api"))
except PackageNotFoundError: # pragma: no cover - only in a broken install
return "0.0.0"
@@ -89,9 +127,13 @@ class SemanticVersion:
@classmethod
def parse(cls, value: str) -> SemanticVersion:
- core, _, prerelease = value.partition("-")
+ core, separator, prerelease = value.partition("-")
parts = core.split(".")
- if len(parts) != 3 or not all(part.isdigit() for part in parts):
+ if (
+ len(parts) != 3
+ or not all(part.isdigit() for part in parts)
+ or (separator and not prerelease)
+ ):
raise ValueError(f"{value!r} is not a MAJOR.MINOR.PATCH version")
major, minor, patch = (int(part) for part in parts)
return cls(major, minor, patch, prerelease or None)
@@ -105,6 +147,26 @@ class SemanticVersion:
return self.prerelease is not None
+def release_channel_for(version: str) -> str:
+ """Derive the reported channel from the single product-version source.
+
+ A release candidate must never identify itself as ``stable`` merely because it shares the
+ executable with the eventual stable release. Unknown prereleases stay visibly non-stable too,
+ rather than acquiring the RC label without an explicit ``rc.`` version.
+ """
+
+ prerelease = SemanticVersion.parse(version).prerelease
+ if prerelease is None:
+ return "stable"
+ if prerelease.startswith("rc.") and prerelease.removeprefix("rc.").isdigit():
+ return "rc"
+ return "prerelease"
+
+
+#: The channel is derived from VERSION rather than maintained as a second, drift-prone constant.
+RELEASE_CHANNEL = release_channel_for(PRODUCT_VERSION)
+
+
# --------------------------------------------------------------------------- compatibility
diff --git a/backend/src/modelforge_api/domain/secret_slots.py b/backend/src/modelforge_api/domain/secret_slots.py
new file mode 100644
index 0000000..51fb73f
--- /dev/null
+++ b/backend/src/modelforge_api/domain/secret_slots.py
@@ -0,0 +1,79 @@
+"""Contracts for the sole supported retrievable platform secret."""
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator
+
+SLOT = "huggingface_control_plane"
+
+
+class SecretSlotRequest(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True, hide_input_in_errors=True)
+ expected_revision: int = Field(ge=0)
+ reason: str = Field(min_length=8, max_length=500)
+
+ @field_validator("reason")
+ @classmethod
+ def safe_reason(cls, value: str) -> str:
+ value = value.strip()
+ if len(value) < 8 or any(prefix in value for prefix in ("hf_", "mfop", "mfsvc_", "mfnode_", "mfenroll_")):
+ raise ValueError("Provide a non-secret explanation of at least eight characters")
+ return value
+
+
+class SecretSlotSet(SecretSlotRequest):
+ value: SecretStr = Field(min_length=1, max_length=4096)
+
+ @field_validator("value")
+ @classmethod
+ def valid_token(cls, value: SecretStr) -> SecretStr:
+ raw = value.get_secret_value()
+ if not raw.startswith("hf_") or not raw.isascii() or not all(c.isalnum() or c in "_-" for c in raw) or len(raw) < 8:
+ raise ValueError("Provide a valid Hugging Face token without whitespace")
+ return value
+
+
+class SecretSlotVersionRequest(SecretSlotRequest):
+ version: int = Field(ge=1)
+
+
+class SecretSlotRevoke(SecretSlotRequest):
+ confirm_slot: Literal["huggingface_control_plane"]
+
+
+class SecretVersionMetadata(BaseModel):
+ version: int
+ key_id: str
+ created_at: datetime
+ revoked_at: datetime | None
+ test_status: Literal["untested", "passed", "failed"]
+ tested_at: datetime | None
+ test_code: str | None
+
+
+class SecretSlotMetadata(BaseModel):
+ slot: Literal["huggingface_control_plane"] = "huggingface_control_plane"
+ revision: int
+ source: Literal["deployment", "managed"]
+ state: Literal["deployment", "pending", "active", "revoked", "quarantined"]
+ configured: bool
+ consumer: Literal["huggingface_control_plane_metadata"] = "huggingface_control_plane_metadata"
+ application_mode: Literal["new_requests"] = "new_requests"
+ root_key_available: bool
+ candidate_version: int | None
+ active_version: int | None
+ versions: list[SecretVersionMetadata]
+ node_credentials_managed: Literal[False] = False
+
+
+class DeploymentSecretMetadata(BaseModel):
+ key: str
+ configured: bool
+ consumer: str
+ application_mode: Literal["restart_required"] = "restart_required"
+ editable: Literal[False] = False
+
+
+class SecretSlotCatalog(BaseModel):
+ slot: SecretSlotMetadata
+ deployment_secrets: list[DeploymentSecretMetadata]
diff --git a/backend/src/modelforge_api/domain/selected_model_comparison.py b/backend/src/modelforge_api/domain/selected_model_comparison.py
new file mode 100644
index 0000000..c38a41f
--- /dev/null
+++ b/backend/src/modelforge_api/domain/selected_model_comparison.py
@@ -0,0 +1,106 @@
+"""Bounded operator intent and designated-consumer evidence contracts."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+
+from modelforge_api.domain.evaluation import EvaluationCaseDefinitionResponse, EvaluationRunComplete
+
+
+class ComparisonModel(BaseModel):
+ model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class ComparisonCreate(ComparisonModel):
+ project_binding_id: UUID
+ consumer_client_id: UUID
+ suite_revision_id: UUID
+ current_deployment_id: UUID
+ candidate_deployment_id: UUID
+ candidate_route_id: UUID
+ current_route_id: UUID | None = None
+ corpus_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+ retrieval_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+ current_index_ref: str = Field(min_length=1, max_length=512)
+ candidate_index_ref: str = Field(min_length=1, max_length=512)
+ idempotency_key: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$")
+
+
+class ComparisonEvaluationSummary(ComparisonModel):
+ comparability: str
+ eligibility: str
+ metric_deltas: dict[str, float]
+ critical_regressions: int
+
+
+class ComparisonResponse(ComparisonModel):
+ id: UUID
+ project_binding_id: UUID
+ consumer_client_id: UUID
+ suite_revision_id: UUID
+ current_deployment_id: UUID
+ candidate_deployment_id: UUID
+ candidate_route_id: UUID
+ actor_id: str
+ pins: dict[str, Any]
+ pins_digest: str
+ status: str
+ version: int
+ expected_cases: int
+ current_completed_cases: int
+ candidate_completed_cases: int
+ claimed_at: datetime | None
+ deadline_at: datetime | None
+ cancel_requested_at: datetime | None
+ finished_at: datetime | None
+ error_code: str | None
+ baseline_run_id: UUID | None
+ candidate_run_id: UUID | None
+ evaluation_comparison_id: UUID | None
+ created_at: datetime
+ updated_at: datetime
+ evaluation_summary: ComparisonEvaluationSummary | None = None
+ production_promotion_performed: Literal[False] = False
+
+
+class ComparisonList(ComparisonModel):
+ items: list[ComparisonResponse]
+ total: int
+
+
+class ComparisonCancel(ComparisonModel):
+ expected_version: int = Field(ge=1)
+
+
+class ComparisonClaim(ComparisonModel):
+ comparison: ComparisonResponse
+ claim_token: str
+ claim_expires_at: datetime
+ cases: list[EvaluationCaseDefinitionResponse]
+
+
+class ComparisonClaimRequest(ComparisonModel):
+ claim_token: SecretStr = Field(min_length=32, max_length=128)
+
+
+class ComparisonProgress(ComparisonClaimRequest):
+ current_completed_cases: int = Field(ge=0, le=100)
+ candidate_completed_cases: int = Field(ge=0, le=100)
+
+
+class ComparisonResult(ComparisonClaimRequest):
+ pins_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+ consumer_revision_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+ current: EvaluationRunComplete
+ candidate: EvaluationRunComplete
+ current_gateway_requests: dict[UUID, UUID] = Field(max_length=100)
+ candidate_gateway_requests: dict[UUID, UUID] = Field(max_length=100)
+
+
+class ComparisonFinish(ComparisonClaimRequest):
+ outcome: Literal["failed", "cancelled"]
+ error_code: str | None = Field(default=None, max_length=64, pattern=r"^[A-Z][A-Z0-9_]*$")
diff --git a/backend/src/modelforge_api/domain/selected_replacement.py b/backend/src/modelforge_api/domain/selected_replacement.py
new file mode 100644
index 0000000..7727fe8
--- /dev/null
+++ b/backend/src/modelforge_api/domain/selected_replacement.py
@@ -0,0 +1,49 @@
+"""Read-only, advisory inputs to the existing approved lifecycle workflow."""
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict
+
+from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
+
+
+class PreviewModel(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class ReplacementBlocker(PreviewModel):
+ code: str
+ message: str
+ project_binding_id: UUID | None = None
+
+
+class ReplacementBinding(PreviewModel):
+ project_binding_id: UUID
+ project_id: UUID
+ project_name: str
+ comparison_id: UUID | None
+ comparison_status: str | None
+ eligible: bool
+
+
+class ReplacementApproval(PreviewModel):
+ id: UUID
+ approved_by: str
+ approved_at: datetime
+ evidence_fingerprint: str
+
+
+class ReplacementPreview(PreviewModel):
+ comparison_id: UUID
+ capability_contract_id: UUID | None
+ current_deployment_id: UUID
+ candidate_deployment_id: UUID
+ route_generation: int | None
+ same_embedding_space: bool
+ affected_bindings: list[ReplacementBinding]
+ production_approvals: list[ReplacementApproval]
+ blockers: list[ReplacementBlocker]
+ selection: ServingTransitionSelection | None
+ production_validation_required: Literal["SATISFIED"] = "SATISFIED"
+ production_promotion_performed: Literal[False] = False
diff --git a/backend/src/modelforge_api/domain/self_check.py b/backend/src/modelforge_api/domain/self_check.py
new file mode 100644
index 0000000..6de02e4
--- /dev/null
+++ b/backend/src/modelforge_api/domain/self_check.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field, JsonValue
+
+
+class SelfCheckStatus(StrEnum):
+ PASS = "pass" # noqa: S105 - diagnostic status, not a credential
+ WARN = "warn"
+ FAIL = "fail"
+ UNKNOWN = "unknown"
+
+
+class SelfCheckKey(StrEnum):
+ API = "api"
+ DATABASE = "database"
+ REDIS = "redis"
+ MANIFESTS = "manifests"
+ STORAGE = "storage"
+ NODES = "nodes"
+ SCHEDULER = "scheduler"
+ DEPLOYMENTS = "deployments"
+ BACKUP = "backup"
+ CREDENTIALS = "credentials"
+ CONFIGURATION = "configuration"
+ RELEASE = "release"
+
+
+class SelfCheckActionCode(StrEnum):
+ NONE_REQUIRED = "none_required"
+ REVIEW_API_IDENTITY = "review_api_identity"
+ CHECK_DATABASE = "check_database"
+ CHECK_REDIS = "check_redis"
+ REVIEW_MANIFESTS = "review_manifests"
+ REVIEW_STORAGE = "review_storage"
+ REVIEW_NODE_FRESHNESS = "review_node_freshness"
+ REVIEW_SCHEDULER_INPUTS = "review_scheduler_inputs"
+ REVIEW_DEPLOYMENTS = "review_deployments"
+ VERIFY_BACKUP_PAYLOAD = "verify_backup_payload"
+ REVIEW_CREDENTIALS = "review_credentials"
+ COMPLETE_CONFIGURATION = "complete_configuration"
+ VERIFY_RELEASE_IDENTITY = "verify_release_identity"
+
+
+class SelfCheckOverall(StrEnum):
+ HEALTHY = "healthy"
+ DEGRADED = "degraded"
+ UNHEALTHY = "unhealthy"
+
+
+class SelfCheckNextAction(BaseModel):
+ code: SelfCheckActionCode
+ summary: str
+
+
+class SelfCheckResult(BaseModel):
+ key: SelfCheckKey
+ label: str
+ status: SelfCheckStatus
+ summary: str
+ detail: str | None = None
+ evidence: dict[str, JsonValue] = Field(default_factory=dict)
+ next_action: SelfCheckNextAction
+
+
+class SelfCheckReport(BaseModel):
+ observed_at: datetime
+ overall: SelfCheckOverall
+ read_only: bool = True
+ checks: list[SelfCheckResult] = Field(default_factory=list)
diff --git a/backend/src/modelforge_api/domain/serving.py b/backend/src/modelforge_api/domain/serving.py
index dd335a8..9c90133 100644
--- a/backend/src/modelforge_api/domain/serving.py
+++ b/backend/src/modelforge_api/domain/serving.py
@@ -8,7 +8,7 @@ import uuid
from datetime import datetime
from typing import Any, Literal
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator
CAPABILITY_VERSION_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+@[1-9][0-9]*$")
ServingOperation = Literal["load", "invoke", "health", "drain", "unload"]
@@ -119,6 +119,7 @@ class ResidencyResponse(ServingModel):
class CapabilityDeploymentResponse(ServingModel):
id: uuid.UUID
+ route_generation: int = Field(ge=0)
capability: str
contract_version: int
deployment_candidate_id: uuid.UUID
@@ -159,7 +160,8 @@ class ServiceClientCreate(ServingModel):
project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]*$")
integration_environment: Literal["production", "shadow", "evaluation", "lab"] | None = None
purpose: str | None = Field(default=None, min_length=3, max_length=500)
- credential_expires_at: datetime | None = None
+ credential_expires_at: AwareDatetime | None = None
+ project_binding_id: uuid.UUID | None = None
@field_validator("allowed_capabilities")
@classmethod
@@ -176,9 +178,13 @@ class ServiceClientCreate(ServingModel):
if any(value is not None for value in project_fields) and not all(
value is not None for value in project_fields
):
- raise ValueError("project_key, integration_environment and purpose are required together")
+ raise ValueError(
+ "project_key, integration_environment and purpose are required together"
+ )
if self.project_key is not None and len(self.allowed_capabilities) != 1:
raise ValueError("a project service identity must bind exactly one capability")
+ if self.project_binding_id is not None and self.project_key is None:
+ raise ValueError("an explicit binding requires a project identity")
return self
@@ -195,6 +201,7 @@ class ServiceClientResponse(ServingModel):
integration_environment: str | None
purpose: str | None
credential_prefix: str | None
+ credential_id: uuid.UUID | None = None
credential_created_at: datetime | None
credential_expires_at: datetime | None
credential_revoked_at: datetime | None
@@ -206,20 +213,35 @@ class ServiceClientCreated(ServiceClientResponse):
credential: str
+class ServiceCredentialRotate(ServingModel):
+ expected_credential_id: uuid.UUID
+ overlap_seconds: int = Field(default=0, strict=True, ge=0, le=86400)
+ expires_at: AwareDatetime
+
+
+class ServiceCredentialRevoke(ServingModel):
+ expected_credential_id: uuid.UUID
+ confirmation: str = Field(min_length=1, max_length=255)
+
+
+class ServiceCredentialResponse(ServingModel):
+ id: uuid.UUID
+ secret_prefix: str
+ created_at: datetime
+ expires_at: datetime | None
+ superseded_at: datetime | None
+ revoked_at: datetime | None
+ last_used_at: datetime | None
+
+
class ProjectFitEvidenceCreate(ServingModel):
project_key: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
capability: str = Field(pattern=r"^[a-z][a-z0-9.]*@[1-9][0-9]*$")
environment: Literal["production", "shadow", "evaluation", "lab"]
- recommendation: Literal[
- "PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"
- ]
- evidence_class: Literal[
- "OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"
- ]
+ recommendation: Literal["PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"]
+ evidence_class: Literal["OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"]
engineering_integration: Literal["PASS", "INCOMPLETE", "BLOCKED"]
- production_validation: Literal[
- "DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"
- ]
+ production_validation: Literal["DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"]
production_action: Literal["NONE"] = "NONE"
case_count: int = Field(ge=0, le=1_000_000)
metric_values: dict[str, float | int | str | bool | None]
@@ -693,6 +715,7 @@ class AgentServingJobLease(ServingModel):
top_n: int | None = Field(default=None, ge=1, le=10)
modality_payload: dict[str, Any] | None = None
request_id: uuid.UUID | None = None
+ lab_run_id: uuid.UUID | None = None
@model_validator(mode="after")
def validate_capability_payload(self) -> AgentServingJobLease:
@@ -736,6 +759,20 @@ class AgentServingJobFailure(ServingModel):
details: dict[str, Any] = Field(default_factory=dict)
+class AgentServingJobCancellationCheck(ServingModel):
+ lease_token: str = Field(min_length=32, max_length=512)
+
+
+class AgentServingJobCancellationStatus(ServingModel):
+ cancel_requested: bool
+
+
+class AgentServingJobCancellationAck(ServingModel):
+ lease_token: str = Field(min_length=32, max_length=512)
+ worker_instance_id: str = Field(min_length=1, max_length=255)
+ state: Literal["cold", "warm", "busy", "loading", "draining", "unloading", "failed"]
+
+
class AgentResidentState(ServingModel):
deployment_id: uuid.UUID
artifact_set_id: uuid.UUID
diff --git a/backend/src/modelforge_api/domain/update_preflight.py b/backend/src/modelforge_api/domain/update_preflight.py
new file mode 100644
index 0000000..54a4c91
--- /dev/null
+++ b/backend/src/modelforge_api/domain/update_preflight.py
@@ -0,0 +1,115 @@
+"""Read-only evidence contract for planning a ModelForge release update."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+EvidenceStatus = Literal["KNOWN", "UNKNOWN"]
+
+
+class UpdateEvidence(BaseModel):
+ status: EvidenceStatus
+ value: str | bool | None = None
+ reason: str | None = None
+
+
+class CurrentRelease(BaseModel):
+ version: str
+ channel: str
+ schema_revision: str = Field(
+ description="Schema target advertised by this application build; not the applied database revision."
+ )
+ agent_protocol_version: int
+
+
+class ApprovedRelease(BaseModel):
+ status: Literal["UNKNOWN"] = "UNKNOWN"
+ version: str | None = None
+ notes: UpdateEvidence
+ schema_revision: UpdateEvidence
+
+
+class BuildProvenance(BaseModel):
+ status: Literal["AVAILABLE", "UNKNOWN"]
+ source_commit: str | None = None
+ built_at: str | None = None
+ image_digest: str | None = None
+ reason: str | None = None
+
+
+class UpdateImpact(BaseModel):
+ schema_impact: UpdateEvidence = Field(alias="schema")
+ breaking: UpdateEvidence
+
+
+class BackupRequirement(BaseModel):
+ required: UpdateEvidence
+
+
+class UpdateExecution(BaseModel):
+ available: Literal[False] = False
+ control: Literal["HUMAN_CONTROLLED"] = "HUMAN_CONTROLLED"
+
+
+class UpdatePreflight(BaseModel):
+ observed_at: datetime
+ status: Literal["INCOMPLETE"] = "INCOMPLETE"
+ read_only: Literal[True] = True
+ current: CurrentRelease
+ approved: ApprovedRelease
+ impact: UpdateImpact
+ backup: BackupRequirement
+ provenance: dict[Literal["current", "approved"], BuildProvenance]
+ execution: UpdateExecution
+
+
+def update_preflight(
+ *,
+ version: str,
+ channel: str,
+ schema_revision: str,
+ agent_protocol_version: int,
+ source_commit: str | None,
+ built_at: str | None,
+ image_digest: str | None,
+) -> UpdatePreflight:
+ """Build a plan view from local release identity without probing or mutating other systems."""
+
+ provenance_complete = all((source_commit, built_at, image_digest))
+ provenance_status: Literal["AVAILABLE", "UNKNOWN"] = "AVAILABLE" if provenance_complete else "UNKNOWN"
+ unknown_reason = "No approved update metadata is configured; this value is not inferred."
+
+ return UpdatePreflight(
+ observed_at=datetime.now(UTC),
+ current=CurrentRelease(
+ version=version,
+ channel=channel,
+ schema_revision=schema_revision,
+ agent_protocol_version=agent_protocol_version,
+ ),
+ approved=ApprovedRelease(
+ notes=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+ schema_revision=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+ ),
+ impact=UpdateImpact(
+ schema=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+ breaking=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+ ),
+ backup=BackupRequirement(
+ required=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
+ ),
+ provenance={
+ "current": BuildProvenance(
+ status=provenance_status,
+ source_commit=source_commit,
+ built_at=built_at,
+ image_digest=image_digest,
+ reason=None if provenance_complete else "One or more build provenance fields are unavailable.",
+ ),
+ "approved": BuildProvenance(status="UNKNOWN", reason=unknown_reason),
+ },
+ execution=UpdateExecution(),
+ )
diff --git a/backend/src/modelforge_api/main.py b/backend/src/modelforge_api/main.py
index 7524f05..d6f93e7 100644
--- a/backend/src/modelforge_api/main.py
+++ b/backend/src/modelforge_api/main.py
@@ -18,8 +18,10 @@ from sqlalchemy.orm import Session
from modelforge_api import __version__
from modelforge_api.api.authorization import (
AccessBoundary,
+ Principal,
access_boundary_for_request,
authenticate_operator_token,
+ operator_token_from_headers,
request_body_limit_bytes,
required_capability_for_request,
)
@@ -27,21 +29,35 @@ from modelforge_api.api.request_limits import RequestBodyLimitMiddleware
from modelforge_api.api.routes import (
acquisition,
agent,
+ audit_timeline,
catalog,
+ configuration,
evaluation,
+ external_creative,
hardware,
health,
+ identity,
+ lab,
lifecycle,
migrations,
+ model_updates,
observability,
+ operator_login,
+ portfolio,
+ projects,
recovery,
registry,
runtime,
+ selected_model_comparison,
+ selected_replacement,
serving,
+ update_preflight,
)
from modelforge_api.db import engine, get_session
from modelforge_api.domain.observability import metrics, route_class
from modelforge_api.domain.release import build_identity
+from modelforge_api.services.alert_notifications import AlertNotificationError
+from modelforge_api.services.configuration_changes import ConfigurationError
from modelforge_api.services.evaluation import EvaluationError
from modelforge_api.services.hardware_polling import HardwarePollingService
from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
@@ -50,6 +66,7 @@ from modelforge_api.services.migration_engine import (
MigrationEngineError,
MigrationEngineService,
)
+from modelforge_api.services.model_update_polling import ModelUpdatePollingService
from modelforge_api.services.node_agent import (
AgentProtocolError,
NodeAgentService,
@@ -59,9 +76,14 @@ from modelforge_api.services.node_decommission import NodeDecommissionError
from modelforge_api.services.node_liveness import NodeLivenessPollingService
from modelforge_api.services.observability import ObservabilityError, ObservabilityService
from modelforge_api.services.observability_polling import ObservabilityPollingService
+from modelforge_api.services.operator_identities import (
+ ManagedIdentityError,
+ authenticate_managed_operator,
+)
from modelforge_api.services.project_registry import sync_project_registry
from modelforge_api.services.recovery import RecoveryError, RecoveryService
from modelforge_api.services.registry import RegistryError, seed_candidate_registry
+from modelforge_api.services.secret_slots import SecretSlotError
from modelforge_api.services.serving import (
CapabilityAuthenticationEvidence,
ServingError,
@@ -151,6 +173,18 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
"project_bindings_waiting_for_contracts",
bindings=sync_result.unavailable_contracts,
)
+ if sync_result.unavailable_bindings:
+ logger.warning(
+ "project_bindings_unavailable_for_declared_stability",
+ bindings=[
+ {
+ "project": item.project_key,
+ "capability": f"{item.capability}@{item.contract_version}",
+ "declared_stability": item.declared_stability,
+ }
+ for item in sync_result.unavailable_bindings
+ ],
+ )
poller = HardwarePollingService(settings)
hardware_task = (
asyncio.create_task(poller.run()) if settings.hardware_refresh_on_startup else None
@@ -166,15 +200,24 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
else None
)
observability_poller = ObservabilityPollingService(settings, engine)
- if settings.observability_monitor_enabled:
+ if settings.observability_monitor_enabled or settings.alert_notifications_enabled:
with Session(engine) as session:
ObservabilityService(session, settings).ensure_defaults()
observability_task = (
asyncio.create_task(observability_poller.run())
- if settings.observability_monitor_enabled
+ if settings.observability_monitor_enabled or settings.alert_notifications_enabled
+ else None
+ )
+ model_update_poller = ModelUpdatePollingService(settings, engine)
+ model_update_task = (
+ asyncio.create_task(model_update_poller.run())
+ if settings.env == "production" and settings.model_update_monitor_enabled
else None
)
yield
+ if model_update_task:
+ model_update_poller.stop()
+ await model_update_task
if observability_task:
observability_poller.stop()
await observability_task
@@ -199,19 +242,51 @@ app = FastAPI(
redoc_url="/redoc" if interactive_api_enabled else None,
openapi_url="/openapi.json" if interactive_api_enabled else None,
)
+app.include_router(operator_login.router)
app.include_router(health.router)
app.include_router(lifecycle.router)
app.include_router(migrations.router)
app.include_router(observability.router)
app.include_router(recovery.router)
app.include_router(agent.router)
+app.include_router(audit_timeline.router)
app.include_router(hardware.router)
app.include_router(catalog.router)
+app.include_router(configuration.router)
+app.include_router(identity.router)
+app.include_router(portfolio.router)
+app.include_router(projects.router)
app.include_router(registry.router)
app.include_router(acquisition.router)
+app.include_router(model_updates.router)
+app.include_router(selected_model_comparison.router)
+app.include_router(selected_replacement.router)
app.include_router(runtime.router)
+app.include_router(lab.router)
app.include_router(serving.router)
+app.include_router(external_creative.router)
app.include_router(evaluation.router)
+app.include_router(update_preflight.router)
+
+
+@app.exception_handler(ConfigurationError)
+async def configuration_error(request: Request, exc: ConfigurationError) -> JSONResponse:
+ return JSONResponse(status_code=exc.status_code, content={"error": {
+ "code": exc.code, "message": exc.message,
+ "correlation_id": getattr(request.state, "correlation_id", "unknown"), "details": {},
+ }})
+
+
+@app.exception_handler(ManagedIdentityError)
+async def managed_identity_error(request: Request, exc: ManagedIdentityError) -> JSONResponse:
+ return _authentication_error(exc.status_code, exc.code, exc.message,
+ getattr(request.state, "correlation_id", "unknown"))
+
+
+@app.exception_handler(SecretSlotError)
+async def secret_slot_error(request: Request, exc: SecretSlotError) -> JSONResponse:
+ return _authentication_error(exc.status_code, exc.code, exc.message,
+ getattr(request.state, "correlation_id", "unknown"))
@app.exception_handler(LifecycleError)
@@ -277,6 +352,24 @@ async def observability_error(request: Request, exc: ObservabilityError) -> JSON
)
+@app.exception_handler(AlertNotificationError)
+async def alert_notification_error(
+ request: Request, exc: AlertNotificationError
+) -> JSONResponse:
+ correlation_id = getattr(request.state, "correlation_id", "unknown")
+ return JSONResponse(
+ status_code=exc.status_code,
+ content={
+ "error": {
+ "code": exc.code,
+ "message": exc.message,
+ "correlation_id": correlation_id,
+ "details": {},
+ }
+ },
+ )
+
+
@contextmanager
def _authentication_session(application: FastAPI) -> Iterator[Session]:
"""Use a test/application session override when present, otherwise a short owned session."""
@@ -316,6 +409,7 @@ def _authenticate_node_before_body(
service.session.commit()
return evidence
+
with _authentication_session(application) as session:
service = NodeAgentService(session, request_settings)
_identity, evidence = service.authenticate_with_evidence(authorization)
@@ -323,6 +417,13 @@ def _authenticate_node_before_body(
return evidence
+def _authenticate_managed_before_body(application: FastAPI, token: str) -> Principal:
+ with _authentication_session(application) as session:
+ principal = authenticate_managed_operator(session, token)
+ session.commit()
+ return principal
+
+
def _authenticate_capability_before_body(
application: FastAPI,
request_settings: Settings,
@@ -389,12 +490,31 @@ async def correlation_middleware(
boundary,
)
response: Response | None = None
+ if request.method == "POST" and request.url.path == "/api/v1/operator/login":
+ request.state.request_body_limit_bytes = 8192
+ try:
+ await run_in_threadpool(operator_login.limit_login, request, request_settings)
+ except HTTPException as exc:
+ response = _authentication_error(
+ exc.status_code, f"http_{exc.status_code}", str(exc.detail), correlation_id
+ )
if boundary is AccessBoundary.CONTROL_PLANE:
try:
- request.state.principal = authenticate_operator_token(
- request_settings,
+ operator_token = operator_token_from_headers(
request.headers.get("x-modelforge-admin-token"),
+ request.headers.get("authorization"),
)
+ if operator_token is not None and operator_token.startswith("mfopm_"):
+ request.state.principal = await run_in_threadpool(
+ _authenticate_managed_before_body, request.app, operator_token,
+ )
+ else:
+ request.state.principal = authenticate_operator_token(request_settings, operator_token)
+ except ManagedIdentityError as exc:
+ response = _authentication_error(exc.status_code, exc.code, exc.message, correlation_id)
+ except SQLAlchemyError:
+ response = _authentication_error(503, "MANAGED_AUTHENTICATION_UNAVAILABLE",
+ "managed authentication backend is unavailable", correlation_id)
except HTTPException as exc:
response = _authentication_error(
exc.status_code,
@@ -461,15 +581,15 @@ async def correlation_middleware(
response = await call_next(request)
duration_seconds = time.perf_counter() - started
response.headers["x-correlation-id"] = correlation_id
+ if boundary is AccessBoundary.CONTROL_PLANE or request.url.path == "/api/v1/operator/login":
+ response.headers["Cache-Control"] = "no-store"
body_error_status_code = getattr(
request.state,
"request_body_error_status_code",
None,
)
effective_status_code = (
- body_error_status_code
- if isinstance(body_error_status_code, int)
- else response.status_code
+ body_error_status_code if isinstance(body_error_status_code, int) else response.status_code
)
labels = {
"method": request.method,
@@ -536,6 +656,20 @@ def _renderable(value: object) -> object:
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
correlation_id = getattr(request.state, "correlation_id", "unknown")
+ errors = exc.errors()
+ if request.url.path.startswith("/api/v1/selected-model-comparisons"):
+ # Claim secrets, dataset content and arbitrary discriminator locations never echo.
+ errors = [{"type": error["type"], "msg": "Invalid comparison request field"} for error in errors]
+ elif request.url.path.startswith("/api/v1/lab/"):
+ # Discriminator messages and locations can contain caller-controlled payload data.
+ errors = [{"type": error["type"], "msg": "Invalid lab request field"} for error in errors]
+ elif request.url.path.startswith("/api/v1/portfolio"):
+ # Portfolio snapshots are intentionally secret-free; never echo body values or paths.
+ errors = [{"type": error["type"], "msg": "Invalid portfolio request field"} for error in errors]
+ if request.url.path == "/api/v1/operator/login":
+ return _authentication_error(
+ 422, "request_validation_failed", "Invalid login request", correlation_id
+ )
return JSONResponse(
status_code=422,
content={
@@ -545,8 +679,16 @@ async def validation_error(request: Request, exc: RequestValidationError) -> JSO
"correlation_id": correlation_id,
"details": {
"errors": [
- {key: _renderable(value) for key, value in error.items() if key != "ctx"}
- for error in exc.errors()
+ {key: _renderable(value) for key, value in error.items()
+ if key != "ctx" and not (
+ (request.url.path.startswith("/api/v1/configuration/")
+ or request.url.path.startswith("/api/v1/operator/managed-identities"))
+ and key in {"input", "loc"}
+ ) and not (
+ request.url.path.startswith("/api/v1/portfolio")
+ and key in {"input", "loc"}
+ )}
+ for error in errors
]
},
}
diff --git a/backend/src/modelforge_api/persistence/__init__.py b/backend/src/modelforge_api/persistence/__init__.py
index acd647e..22c517e 100644
--- a/backend/src/modelforge_api/persistence/__init__.py
+++ b/backend/src/modelforge_api/persistence/__init__.py
@@ -1,3 +1,4 @@
+from .model_change import ModelChangeCase
from .models import Base
-__all__ = ["Base"]
+__all__ = ["Base", "ModelChangeCase"]
diff --git a/backend/src/modelforge_api/persistence/hardware_repository.py b/backend/src/modelforge_api/persistence/hardware_repository.py
index a8d071b..dcb1837 100644
--- a/backend/src/modelforge_api/persistence/hardware_repository.py
+++ b/backend/src/modelforge_api/persistence/hardware_repository.py
@@ -40,7 +40,10 @@ class HardwareRepository:
def node_for_update(self, node_id: uuid.UUID) -> ComputeNode | None:
return self.session.scalar(
- select(ComputeNode).where(ComputeNode.id == node_id).with_for_update()
+ select(ComputeNode)
+ .where(ComputeNode.id == node_id)
+ .with_for_update()
+ .execution_options(populate_existing=True)
)
def accelerators(self, node_id: uuid.UUID | None = None) -> list[Accelerator]:
diff --git a/backend/src/modelforge_api/persistence/model_change.py b/backend/src/modelforge_api/persistence/model_change.py
new file mode 100644
index 0000000..b78dc0d
--- /dev/null
+++ b/backend/src/modelforge_api/persistence/model_change.py
@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+import sqlalchemy as sa
+from sqlalchemy.orm import Mapped, mapped_column
+
+from modelforge_api.persistence.models import Base, IdMixin, TimestampMixin
+
+MODEL_CHANGE_STATUSES = (
+ "searching",
+ "found",
+ "source_checked",
+ "downloaded",
+ "ready_to_test",
+ "tested",
+ "comparison_available",
+ "migration_required",
+ "ready_to_activate",
+ "active",
+ "checked",
+ "rolled_back",
+ "rejected",
+)
+
+
+class ModelChangeCase(Base, IdMixin, TimestampMixin):
+ """User-facing workflow shell that only references authoritative ModelForge evidence."""
+
+ __tablename__ = "model_change_cases"
+ __table_args__ = (
+ sa.CheckConstraint("version >= 1", name="ck_model_change_case_version"),
+ sa.CheckConstraint(
+ "status IN (" + ",".join(repr(value) for value in MODEL_CHANGE_STATUSES) + ")",
+ name="ck_model_change_case_status",
+ ),
+ sa.Index("ix_model_change_cases_binding", "project_binding_id"),
+ sa.Index("ix_model_change_cases_status", "status"),
+ sa.Index(
+ "uq_model_change_cases_open_binding",
+ "project_binding_id",
+ unique=True,
+ postgresql_where=sa.text("closed_at IS NULL"),
+ sqlite_where=sa.text("closed_at IS NULL"),
+ ),
+ )
+
+ project_binding_id: Mapped[uuid.UUID] = mapped_column(
+ sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"), nullable=False
+ )
+ current_deployment_id: Mapped[uuid.UUID] = mapped_column(
+ sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"), nullable=False
+ )
+ candidate_model_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("models.id", ondelete="RESTRICT")
+ )
+ candidate_revision_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("model_revisions.id", ondelete="RESTRICT")
+ )
+ candidate_artifact_set_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("artifact_sets.id", ondelete="RESTRICT")
+ )
+ candidate_deployment_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT")
+ )
+ discovery_assessment_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("discovery_candidate_assessments.id", ondelete="RESTRICT")
+ )
+ download_plan_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("download_plans.id", ondelete="RESTRICT")
+ )
+ artifact_job_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("artifact_jobs.id", ondelete="RESTRICT")
+ )
+ selected_comparison_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("selected_model_comparisons.id", ondelete="RESTRICT")
+ )
+ migration_plan_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("migration_plans.id", ondelete="RESTRICT")
+ )
+ lifecycle_plan_id: Mapped[uuid.UUID | None] = mapped_column(
+ sa.ForeignKey("lifecycle_promotion_plans.id", ondelete="RESTRICT")
+ )
+ status: Mapped[str] = mapped_column(sa.String(32), nullable=False, default="searching")
+ status_reason: Mapped[str] = mapped_column(sa.Text, nullable=False, default="")
+ version: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=1)
+ created_by: Mapped[str] = mapped_column(sa.String(255), nullable=False)
+ updated_by: Mapped[str] = mapped_column(sa.String(255), nullable=False)
+ context: Mapped[dict[str, Any]] = mapped_column(sa.JSON, nullable=False, default=dict)
+ activated_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
+ checked_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
+ closed_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
diff --git a/backend/src/modelforge_api/persistence/models.py b/backend/src/modelforge_api/persistence/models.py
index be1b74c..cc6a712 100644
--- a/backend/src/modelforge_api/persistence/models.py
+++ b/backend/src/modelforge_api/persistence/models.py
@@ -58,6 +58,123 @@ class Base(DeclarativeBase):
pass
+class SecretSlotState(Base):
+ __tablename__ = "platform_secret_state"
+ __table_args__ = (CheckConstraint("singleton_id = 1", name="ck_secret_state_singleton"), CheckConstraint("revision >= 0", name="ck_secret_state_revision"))
+ singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+ revision: Mapped[int] = mapped_column(Integer, nullable=False)
+ initialized: Mapped[bool] = mapped_column(Boolean, nullable=False)
+
+
+@event.listens_for(SecretSlotState.__table__, "after_create")
+def _seed_secret_slot_state(_table: Any, connection: Connection, **_kwargs: Any) -> None:
+ connection.execute(insert(SecretSlotState).values(singleton_id=1, revision=0, initialized=False))
+
+
+class SecretSlotVersion(Base):
+ __tablename__ = "platform_secret_versions"
+ __table_args__ = (CheckConstraint("version > 0", name="ck_secret_version_positive"),)
+ version: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False)
+ key_id: Mapped[str] = mapped_column(String(64), nullable=False)
+ secret_ciphertext: Mapped[str] = mapped_column(Text, nullable=False)
+ secret_nonce: Mapped[str] = mapped_column(String(32), nullable=False)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class SecretSlotTest(Base):
+ __tablename__ = "platform_secret_tests"
+ version: Mapped[int] = mapped_column(ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), primary_key=True)
+ passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
+ tested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ code: Mapped[str] = mapped_column(String(64), nullable=False)
+
+
+class SecretSlotActivation(Base):
+ __tablename__ = "platform_secret_activations"
+ __table_args__ = (CheckConstraint("singleton_id = 1", name="ck_secret_activation_singleton"),)
+ singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+ version: Mapped[int] = mapped_column(ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), nullable=False)
+ activated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ManagedOperatorState(Base):
+ __tablename__ = "managed_operator_state"
+ __table_args__ = (
+ CheckConstraint("singleton_id = 1", name="ck_managed_operator_singleton"),
+ CheckConstraint("revision >= 0", name="ck_managed_operator_revision"),
+ )
+ singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
+ revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+
+
+@event.listens_for(ManagedOperatorState.__table__, "after_create")
+def _seed_managed_operator_state(_table: Any, connection: Connection, **_kwargs: Any) -> None:
+ connection.execute(insert(ManagedOperatorState).values(singleton_id=1, revision=0))
+
+
+class ManagedOperatorIdentity(Base):
+ __tablename__ = "managed_operator_identities"
+ __table_args__ = (
+ CheckConstraint("role IN ('viewer','operator','admin')", name="ck_managed_operator_role"),
+ CheckConstraint("version >= 1", name="ck_managed_operator_version"),
+ )
+ slug: Mapped[str] = mapped_column(String(63), primary_key=True)
+ role: Mapped[str] = mapped_column(String(16), nullable=False)
+ enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+ version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ManagedOperatorCredential(Base):
+ __tablename__ = "managed_operator_credentials"
+ __table_args__ = (
+ Index("uq_managed_operator_current", "identity_slug", unique=True,
+ postgresql_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
+ sqlite_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL")),
+ )
+ id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
+ identity_slug: Mapped[str] = mapped_column(
+ ForeignKey("managed_operator_identities.slug", ondelete="RESTRICT"), index=True,
+ )
+ token_sha256: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
+ expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ superseded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ManagedOperatorCredentialActivation(Base):
+ """Rebuildable admission evidence, deliberately cleared by supported recovery."""
+ __tablename__ = "managed_operator_credential_activations"
+ credential_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("managed_operator_credentials.id", ondelete="RESTRICT"), primary_key=True,
+ )
+ activated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class ConfigurationState(Base):
+ __tablename__ = "configuration_state"
+ __table_args__ = (CheckConstraint("id = 1"), CheckConstraint("revision >= 0"))
+
+ id: Mapped[int] = mapped_column(Integer, primary_key=True)
+ revision: Mapped[int] = mapped_column(Integer, nullable=False)
+ overrides: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+
+
+class ConfigurationRevision(Base):
+ __tablename__ = "configuration_revisions"
+
+ revision: Mapped[int] = mapped_column(Integer, primary_key=True)
+ previous_revision: Mapped[int] = mapped_column(Integer, nullable=False)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+ actor: Mapped[str] = mapped_column(String(255), nullable=False)
+ correlation_id: Mapped[str] = mapped_column(String(255), nullable=False)
+ overrides: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+ preview_digest: Mapped[str] = mapped_column(String(64), nullable=False)
+
+
_PROTECTED_AUDIT_TABLES = frozenset({"audit_events", "audit_chain_heads"})
@@ -974,12 +1091,14 @@ class CapabilityContract(Base, IdMixin, TimestampMixin):
__tablename__ = "capability_contracts"
__table_args__ = (
UniqueConstraint("capability_id", "version", name="uq_capability_contract_version"),
+ CheckConstraint("serving_generation >= 0", name="ck_capability_serving_generation"),
)
capability_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("capabilities.id", ondelete="RESTRICT"), index=True
)
version: Mapped[int] = mapped_column(Integer)
+ serving_generation: Mapped[int] = mapped_column(BigInteger, default=0, server_default="0")
input_schema: Mapped[dict[str, Any]] = mapped_column(JSON)
output_schema: Mapped[dict[str, Any]] = mapped_column(JSON)
contract: Mapped[dict[str, Any]] = mapped_column(JSON)
@@ -1000,6 +1119,87 @@ class Project(Base, IdMixin, TimestampMixin):
bindings: Mapped[list[ProjectBinding]] = relationship(back_populates="project")
+class PortfolioRepositoryObservation(Base, IdMixin, TimestampMixin):
+ """Latest source inventory state, optionally linked to a ModelForge project identity."""
+
+ __tablename__ = "portfolio_repository_observations"
+ __table_args__ = (
+ UniqueConstraint(
+ "source_system", "external_repository_id", name="uq_portfolio_repo_external_id"
+ ),
+ UniqueConstraint("source_system", "project_key", name="uq_portfolio_source_project_key"),
+ UniqueConstraint("project_id", name="uq_portfolio_repository_project_id"),
+ CheckConstraint("length(head_sha) = 40", name="ck_portfolio_repo_head_sha_length"),
+ CheckConstraint(
+ "pending_head_sha IS NULL OR length(pending_head_sha) = 40",
+ name="ck_portfolio_repo_pending_sha_length",
+ ),
+ CheckConstraint(
+ "classification IN ('local_model', 'hosted_model', 'gateway', 'mixed', "
+ "'model_management', 'runtime_observer', 'non_ai', 'unknown')",
+ name="ck_portfolio_repo_classification",
+ ),
+ )
+
+ project_id: Mapped[uuid.UUID | None] = mapped_column(
+ ForeignKey("projects.id", ondelete="RESTRICT"), index=True, nullable=True
+ )
+ project_key: Mapped[str | None] = mapped_column(String(128))
+ source_system: Mapped[str] = mapped_column(String(16))
+ external_repository_id: Mapped[str] = mapped_column(String(128))
+ slug: Mapped[str] = mapped_column(String(255))
+ default_branch: Mapped[str] = mapped_column(String(255))
+ head_sha: Mapped[str] = mapped_column(String(40))
+ classification: Mapped[str] = mapped_column(String(32))
+ code_evidence: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list)
+ runtime_evidence: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list)
+ source_observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+ source_snapshot_id: Mapped[str] = mapped_column(String(128))
+ source_snapshot_digest: Mapped[str] = mapped_column(String(64))
+ pending_head_sha: Mapped[str | None] = mapped_column(String(40))
+ pending_default_branch: Mapped[str | None] = mapped_column(String(255))
+ pending_project_key: Mapped[str | None] = mapped_column(String(128))
+ pending_classification: Mapped[str | None] = mapped_column(String(32))
+ pending_code_evidence: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON)
+ pending_observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ pending_snapshot_id: Mapped[str | None] = mapped_column(String(128))
+ pending_snapshot_digest: Mapped[str | None] = mapped_column(String(64))
+ last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+ stale_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class PortfolioReconciliationRun(Base, IdMixin, TimestampMixin):
+ """Idempotency ledger for exact replay and conflict detection of source snapshots."""
+
+ __tablename__ = "portfolio_reconciliation_runs"
+ __table_args__ = (
+ UniqueConstraint("source_system", "snapshot_id", name="uq_portfolio_snapshot_identity"),
+ CheckConstraint("length(payload_digest) = 64", name="ck_portfolio_snapshot_digest_length"),
+ CheckConstraint(
+ "projects_created >= 0 AND repositories_created >= 0 "
+ "AND repositories_updated >= 0 AND stale_observations >= 0 "
+ "AND repositories_marked_stale >= 0 AND runtime_observations_updated >= 0 "
+ "AND head_changes_review_required >= 0 AND head_changes_accepted >= 0",
+ name="ck_portfolio_snapshot_counts_nonnegative",
+ ),
+ )
+
+ source_system: Mapped[str] = mapped_column(String(16))
+ snapshot_id: Mapped[str] = mapped_column(String(128))
+ payload_digest: Mapped[str] = mapped_column(String(64))
+ observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+ complete: Mapped[bool] = mapped_column(Boolean, default=False)
+ stale_snapshot: Mapped[bool] = mapped_column(Boolean, default=False)
+ projects_created: Mapped[int] = mapped_column(Integer, default=0)
+ repositories_created: Mapped[int] = mapped_column(Integer, default=0)
+ repositories_updated: Mapped[int] = mapped_column(Integer, default=0)
+ stale_observations: Mapped[int] = mapped_column(Integer, default=0)
+ repositories_marked_stale: Mapped[int] = mapped_column(Integer, default=0)
+ runtime_observations_updated: Mapped[int] = mapped_column(Integer, default=0)
+ head_changes_review_required: Mapped[int] = mapped_column(Integer, default=0)
+ head_changes_accepted: Mapped[int] = mapped_column(Integer, default=0)
+
+
class ProjectBinding(Base, IdMixin, TimestampMixin):
__tablename__ = "project_bindings"
__table_args__ = (
@@ -1026,6 +1226,18 @@ class ProjectBinding(Base, IdMixin, TimestampMixin):
project: Mapped[Project] = relationship(back_populates="bindings")
+class ProjectOnboarding(Base):
+ __tablename__ = "project_onboarding"
+
+ project_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("projects.id", ondelete="RESTRICT"), primary_key=True
+ )
+ owner: Mapped[str] = mapped_column(String(128))
+ environment: Mapped[str] = mapped_column(String(32))
+ labels: Mapped[list[str]] = mapped_column(JSON, default=list)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+
+
class ComputeNode(Base, IdMixin, TimestampMixin):
__tablename__ = "compute_nodes"
@@ -1335,9 +1547,28 @@ class BenchmarkSuite(Base, IdMixin, TimestampMixin):
class BenchmarkRun(Base, IdMixin):
__tablename__ = "benchmark_runs"
- deployment_id: Mapped[uuid.UUID] = mapped_column(
- ForeignKey("deployments.id", ondelete="RESTRICT"), index=True
+ __table_args__ = (
+ UniqueConstraint(
+ "actor_subject", "idempotency_key", name="uq_benchmark_runs_actor_idempotency"
+ ),
+ CheckConstraint(
+ "(actor_subject IS NULL AND idempotency_key IS NULL AND "
+ "request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
+ "(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
+ "request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
+ name="ck_benchmark_runs_operator_identity",
+ ),
)
+ deployment_id: Mapped[uuid.UUID | None] = mapped_column(
+ ForeignKey("deployments.id", ondelete="RESTRICT"), index=True, nullable=True
+ )
+ capability_deployment_id: Mapped[uuid.UUID | None] = mapped_column(
+ ForeignKey("capability_deployments.id", ondelete="RESTRICT")
+ )
+ target_snapshot: Mapped[dict[str, Any] | None] = mapped_column(JSON)
+ actor_subject: Mapped[str | None] = mapped_column(String(255))
+ idempotency_key: Mapped[str | None] = mapped_column(String(128))
+ request_fingerprint: Mapped[str | None] = mapped_column(String(64))
suite_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("benchmark_suites.id", ondelete="RESTRICT"), index=True
)
@@ -1354,10 +1585,16 @@ class BenchmarkRun(Base, IdMixin):
seed: Mapped[int | None] = mapped_column(Integer)
results: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
status: Mapped[str] = mapped_column(String(32))
+ failure_code: Mapped[str | None] = mapped_column(String(64))
comparable: Mapped[bool] = mapped_column(Boolean, default=True)
stale_reason: Mapped[str | None] = mapped_column(Text)
+ cancellation_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ generation: Mapped[int] = mapped_column(Integer, default=1, server_default="1", nullable=False)
+
+ __mapper_args__ = {"version_id_col": generation} # noqa: RUF012
class Experiment(Base, IdMixin, TimestampMixin):
@@ -1621,8 +1858,8 @@ class ServiceCredential(Base, IdMixin):
"uq_active_service_client_credential",
"service_client_id",
unique=True,
- postgresql_where=sa_text("revoked_at IS NULL"),
- sqlite_where=sa_text("revoked_at IS NULL"),
+ postgresql_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
+ sqlite_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
),
)
@@ -1631,12 +1868,115 @@ class ServiceCredential(Base, IdMixin):
)
secret_hash: Mapped[str] = mapped_column(String(64), unique=True)
secret_prefix: Mapped[str] = mapped_column(String(16))
+ superseded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+class ExternalCreativeAllocation(Base, IdMixin):
+ """A service-client-owned reservation for a creative workload run outside ModelForge."""
+
+ __tablename__ = "external_creative_allocations"
+ __table_args__ = (
+ UniqueConstraint(
+ "service_client_id", "idempotency_key", name="uq_external_creative_idempotency"
+ ),
+ CheckConstraint("required_vram_bytes > 0", name="ck_external_creative_vram_positive"),
+ CheckConstraint("ttl_seconds >= 30 AND ttl_seconds <= 3600", name="ck_external_creative_ttl"),
+ )
+
+ service_client_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("service_clients.id", ondelete="RESTRICT"), index=True
+ )
+ project_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("projects.id", ondelete="RESTRICT"), index=True
+ )
+ accelerator_id: Mapped[uuid.UUID | None] = mapped_column(
+ ForeignKey("accelerators.id", ondelete="RESTRICT"), index=True
+ )
+ request_id: Mapped[str] = mapped_column(String(128), index=True)
+ idempotency_key: Mapped[str] = mapped_column(String(128))
+ request_sha256: Mapped[str] = mapped_column(String(64))
+ workload: Mapped[str] = mapped_column(String(32))
+ resource_ref: Mapped[str] = mapped_column(String(255))
+ provider_resource_id: Mapped[str] = mapped_column(String(128))
+ required_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+ priority: Mapped[str] = mapped_column(String(32))
+ ttl_seconds: Mapped[int] = mapped_column(Integer)
+ state: Mapped[str] = mapped_column(String(32), index=True)
+ decision_code: Mapped[str] = mapped_column(String(64))
+ decision_evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ required_actions: Mapped[list[str]] = mapped_column(JSON, default=list)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+ updated_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+ )
+ acquired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+ uncertain_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ released_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class ExternalCreativeResource(Base, IdMixin):
+ __tablename__ = "external_creative_resources"
+ __table_args__ = (
+ UniqueConstraint("project_id", "resource_ref", "workload", "provider_resource_id",
+ name="uq_external_creative_resource_binding"),
+ CheckConstraint("configured_vram_bytes > 0", name="ck_external_resource_vram_positive"),
+ Index("ix_external_creative_resources_project", "project_id", "approved"),
+ )
+ project_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("projects.id", ondelete="RESTRICT"))
+ resource_ref: Mapped[str] = mapped_column(String(255))
+ workload: Mapped[str] = mapped_column(String(32))
+ provider_resource_id: Mapped[str] = mapped_column(String(128))
+ approved: Mapped[bool] = mapped_column(Boolean, server_default="false")
+ configured_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+ measured_peak_vram_bytes: Mapped[int | None] = mapped_column(BigInteger)
+ measurement_count: Mapped[int] = mapped_column(Integer, server_default="0")
+ updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class ExternalCreativeActionAttestation(Base, IdMixin):
+ __tablename__ = "external_creative_action_attestations"
+ __table_args__ = (
+ UniqueConstraint("allocation_id", "action", name="uq_external_creative_action_attestation"),
+ CheckConstraint("before_vram_bytes >= 0 AND after_vram_bytes >= 0",
+ name="ck_external_action_vram_nonnegative"),
+ )
+ allocation_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"))
+ action: Mapped[str] = mapped_column(String(64))
+ before_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+ after_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+ observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+ validated: Mapped[bool] = mapped_column(Boolean)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class ExternalCreativeProfileObservation(Base, IdMixin):
+ __tablename__ = "external_creative_profile_observations"
+ __table_args__ = (
+ UniqueConstraint("allocation_id", name="uq_external_creative_profile_observation_allocation"),
+ CheckConstraint("observed_peak_vram_bytes > 0 AND reserved_vram_bytes > 0",
+ name="ck_external_profile_vram_positive"),
+ CheckConstraint("duration_ms >= 0 AND duration_ms <= 604800000",
+ name="ck_external_profile_duration"),
+ )
+ allocation_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"))
+ resource_ref: Mapped[str] = mapped_column(String(255))
+ provider_resource_id: Mapped[str] = mapped_column(String(128))
+ workload: Mapped[str] = mapped_column(String(32))
+ observed_peak_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+ reserved_vram_bytes: Mapped[int] = mapped_column(BigInteger)
+ duration_ms: Mapped[int] = mapped_column(BigInteger)
+ outcome: Mapped[str] = mapped_column(String(16))
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
class ResidencyAllocation(Base, IdMixin, TimestampMixin):
__tablename__ = "residency_allocations"
__table_args__ = (UniqueConstraint("capability_deployment_id", name="uq_residency_deployment"),)
@@ -1739,6 +2079,9 @@ class ServingJob(Base, IdMixin, TimestampMixin):
gateway_request_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("gateway_requests.id", ondelete="SET NULL"), index=True
)
+ lab_execution_run_id: Mapped[uuid.UUID | None] = mapped_column(
+ ForeignKey("lab_execution_runs.id", ondelete="SET NULL"), index=True
+ )
operation: Mapped[str] = mapped_column(String(32), index=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
priority: Mapped[str] = mapped_column(String(32), default="production")
@@ -1755,6 +2098,53 @@ class ServingJob(Base, IdMixin, TimestampMixin):
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+class LabExecutionRun(Base, IdMixin, TimestampMixin):
+ __tablename__ = "lab_execution_runs"
+ __table_args__ = (
+ UniqueConstraint("actor_subject", "idempotency_key", name="uq_lab_run_actor_idempotency"),
+ CheckConstraint(
+ "status IN ('accepted', 'preparing', 'queued', 'running', 'cancel_requested', "
+ "'completed', 'failed', 'cancelled', 'interrupted')",
+ name="ck_lab_execution_run_status",
+ ),
+ CheckConstraint("input_count BETWEEN 1 AND 8", name="ck_lab_execution_run_input_count"),
+ CheckConstraint("timeout_seconds BETWEEN 5 AND 300", name="ck_lab_execution_run_timeout"),
+ CheckConstraint(
+ "queue_timeout_seconds BETWEEN 1 AND 60", name="ck_lab_execution_run_queue_timeout"
+ ),
+ CheckConstraint(
+ "retention_seconds BETWEEN 60 AND 3600", name="ck_lab_execution_run_retention"
+ ),
+ )
+
+ actor_subject: Mapped[str] = mapped_column(String(255), index=True)
+ idempotency_key: Mapped[str] = mapped_column(String(128))
+ request_fingerprint: Mapped[str] = mapped_column(String(64))
+ status: Mapped[str] = mapped_column(String(24), default="accepted", index=True)
+ capability: Mapped[str] = mapped_column(String(128), default="rag.embedding@1")
+ target_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON)
+ input_sha256: Mapped[str] = mapped_column(String(64))
+ input_count: Mapped[int] = mapped_column(Integer)
+ input_encoded_bytes: Mapped[int] = mapped_column(Integer)
+ payload_reference: Mapped[str | None] = mapped_column(String(128))
+ input_payload_expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+ gateway_request_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("gateway_requests.id", ondelete="RESTRICT"), unique=True, index=True
+ )
+ timeout_seconds: Mapped[int] = mapped_column(Integer, default=60)
+ queue_timeout_seconds: Mapped[int] = mapped_column(Integer, default=10)
+ retention_seconds: Mapped[int] = mapped_column(Integer, default=300)
+ output_metadata: Mapped[dict[str, Any] | None] = mapped_column(JSON)
+ worker_observations: Mapped[dict[str, Any] | None] = mapped_column(JSON)
+ failure_code: Mapped[str | None] = mapped_column(String(64))
+ started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ dispatch_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ cancellation_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ cancellation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
class SchedulerPolicyRevision(Base, IdMixin):
__tablename__ = "scheduler_policy_revisions"
@@ -2439,6 +2829,12 @@ Index("ix_migrations_binding_status", Migration.project_binding_id, Migration.st
Index(
"ix_benchmark_runs_deployment_completed", BenchmarkRun.deployment_id, BenchmarkRun.completed_at
)
+Index("ix_benchmark_runs_actor_subject", BenchmarkRun.actor_subject)
+Index(
+ "ix_benchmark_runs_capability_deployment_completed",
+ BenchmarkRun.capability_deployment_id,
+ BenchmarkRun.completed_at,
+)
Index(
"ix_capability_deployments_contract_channel_status",
CapabilityDeployment.capability_contract_id,
@@ -3085,7 +3481,7 @@ def _append_canonical_audit_event(
"""
if not isinstance(details, dict):
- raise ValueError("canonical audit details must be an object")
+ raise TypeError("canonical audit details must be an object")
if expected_event_count != expected_last_sequence:
raise ValueError("canonical audit append requires a coherent predecessor count")
if expected_hash_format != AUDIT_CURRENT_HASH_FORMAT:
@@ -3518,6 +3914,43 @@ class AlertHistoryEvent(Base, IdMixin):
)
+class AlertNotificationDelivery(Base, IdMixin):
+ __tablename__ = "alert_notification_deliveries"
+ __table_args__ = (
+ UniqueConstraint("idempotency_key", name="uq_alert_notification_idempotency"),
+ CheckConstraint(
+ "event_kind IN ('FIRING', 'RESOLVED', 'TEST')",
+ name="ck_alert_notification_event_kind",
+ ),
+ CheckConstraint(
+ "status IN ('PENDING', 'DISPATCHING', 'RETRY_WAIT', 'DELIVERED', 'FAILED', 'HELD_FOR_REVIEW')",
+ name="ck_alert_notification_status",
+ ),
+ CheckConstraint("attempt_count >= 0", name="ck_alert_notification_attempt_count"),
+ )
+
+ alert_history_event_id: Mapped[uuid.UUID | None] = mapped_column(
+ ForeignKey("alert_history_events.id", ondelete="RESTRICT"), index=True
+ )
+ route_key: Mapped[str] = mapped_column(String(64), default="default")
+ idempotency_key: Mapped[str] = mapped_column(String(160), nullable=False)
+ event_kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
+ severity: Mapped[str | None] = mapped_column(String(16), index=True)
+ payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
+ status: Mapped[str] = mapped_column(String(24), nullable=False, index=True)
+ attempt_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+ next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+ last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ delivered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+ claim_token_hash: Mapped[str | None] = mapped_column(String(64))
+ last_http_status: Mapped[int | None] = mapped_column(Integer)
+ last_error_code: Mapped[str | None] = mapped_column(String(64))
+ created_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), server_default=func.now(), nullable=False
+ )
+
+
class MaintenanceWindow(Base, IdMixin):
__tablename__ = "maintenance_windows"
@@ -4289,3 +4722,76 @@ def _prevent_immutable_field_changes(_mapper: Any, _connection: Any, target: Bas
for _immutable_type in IMMUTABLE_FIELDS:
event.listen(_immutable_type, "before_update", _prevent_immutable_field_changes)
+
+
+class ModelUpdateWatch(Base, IdMixin):
+ __tablename__ = "model_update_watches"
+ __table_args__ = (UniqueConstraint("model_id", "upstream_ref", name="uq_model_watch_ref"),)
+ model_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("models.id", ondelete="RESTRICT"))
+ baseline_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_revisions.id", ondelete="RESTRICT"))
+ baseline_commit_sha: Mapped[str] = mapped_column(String(64))
+ upstream_ref: Mapped[str] = mapped_column(String(255))
+ enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
+ version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
+ last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ last_error_code: Mapped[str | None] = mapped_column(String(64))
+ check_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ check_token: Mapped[uuid.UUID | None] = mapped_column(Uuid)
+
+
+class ModelUpdateNotice(Base, IdMixin):
+ __tablename__ = "model_update_notices"
+ __table_args__ = (
+ UniqueConstraint("watch_id", "candidate_commit_sha", name="uq_model_notice_commit"),
+ CheckConstraint("status IN ('new', 'dismissed')", name="ck_model_notice_status"),
+ )
+ watch_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_update_watches.id", ondelete="RESTRICT"))
+ model_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("models.id", ondelete="RESTRICT"))
+ baseline_commit_sha: Mapped[str] = mapped_column(String(64))
+ candidate_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_revisions.id", ondelete="RESTRICT"))
+ candidate_commit_sha: Mapped[str] = mapped_column(String(64))
+ status: Mapped[str] = mapped_column(String(16), default="new")
+ discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+
+
+class SelectedModelComparison(Base, IdMixin, TimestampMixin):
+ """One explicitly delegated current/candidate comparison; no implicit replay."""
+
+ __tablename__ = "selected_model_comparisons"
+ __table_args__ = (
+ UniqueConstraint("project_binding_id", "idempotency_key", name="uq_selected_comparison_key"),
+ Index("uq_selected_comparison_active_binding", "project_binding_id", unique=True,
+ postgresql_where=sa_text("status IN ('pending', 'running', 'cancel_requested')"),
+ sqlite_where=sa_text("status IN ('pending', 'running', 'cancel_requested')")),
+ CheckConstraint("status IN ('pending', 'running', 'cancel_requested', 'completed', 'failed', 'cancelled', 'interrupted')", name="ck_selected_comparison_status"),
+ CheckConstraint("expected_cases BETWEEN 1 AND 100", name="ck_selected_comparison_cases"),
+ CheckConstraint("current_completed_cases BETWEEN 0 AND expected_cases AND candidate_completed_cases BETWEEN 0 AND expected_cases", name="ck_selected_comparison_progress"),
+ CheckConstraint("version > 0", name="ck_selected_comparison_version"),
+ CheckConstraint("current_deployment_id <> candidate_deployment_id", name="ck_selected_comparison_distinct_targets"),
+ )
+ project_binding_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("project_bindings.id", ondelete="RESTRICT"))
+ consumer_client_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("service_clients.id", ondelete="RESTRICT"))
+ suite_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("evaluation_suite_revisions.id", ondelete="RESTRICT"))
+ current_deployment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_deployments.id", ondelete="RESTRICT"))
+ candidate_deployment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_deployments.id", ondelete="RESTRICT"))
+ candidate_route_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_experiment_routes.id", ondelete="RESTRICT"))
+ idempotency_key: Mapped[str] = mapped_column(String(128))
+ request_digest: Mapped[str] = mapped_column(String(64))
+ actor_id: Mapped[str] = mapped_column(String(128))
+ pins: Mapped[dict[str, Any]] = mapped_column(JSON)
+ status: Mapped[str] = mapped_column(String(32), default="pending")
+ version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
+ current_completed_cases: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+ candidate_completed_cases: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+ expected_cases: Mapped[int] = mapped_column(Integer)
+ claim_token_hash: Mapped[str | None] = mapped_column(String(64))
+ claim_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ deadline_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ error_code: Mapped[str | None] = mapped_column(String(64))
+ result_digest: Mapped[str | None] = mapped_column(String(64))
+ baseline_run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_runs.id", ondelete="RESTRICT"))
+ candidate_run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_runs.id", ondelete="RESTRICT"))
+ evaluation_comparison_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_comparisons.id", ondelete="RESTRICT"))
diff --git a/backend/src/modelforge_api/persistence/registry_repository.py b/backend/src/modelforge_api/persistence/registry_repository.py
index a6822b9..a8fb3eb 100644
--- a/backend/src/modelforge_api/persistence/registry_repository.py
+++ b/backend/src/modelforge_api/persistence/registry_repository.py
@@ -13,6 +13,8 @@ from modelforge_api.persistence.models import (
Model,
ModelArtifact,
ModelRevision,
+ ModelUpdateNotice,
+ ModelUpdateWatch,
RuntimeProfile,
StorageRoot,
)
@@ -171,6 +173,20 @@ class RegistryRepository:
def dependencies(self, resource_type: str, resource_id: uuid.UUID) -> list[dict[str, str]]:
dependencies: list[dict[str, str]] = []
+ if resource_type in {"model", "model_revision"}:
+ for entity, relation, column in (
+ (ModelUpdateWatch, "update_tracking_baseline",
+ ModelUpdateWatch.model_id if resource_type == "model"
+ else ModelUpdateWatch.baseline_revision_id),
+ (ModelUpdateNotice, "update_candidate_history",
+ ModelUpdateNotice.model_id if resource_type == "model"
+ else ModelUpdateNotice.candidate_revision_id),
+ ):
+ dependencies.extend(
+ {"resource_type": "model_update_watch" if entity is ModelUpdateWatch
+ else "model_update_notice", "resource_id": str(item), "relation": relation}
+ for item in self.session.scalars(select(entity.id).where(column == resource_id))
+ )
if resource_type == "model":
ids = self.session.scalars(
select(ModelRevision.id).where(ModelRevision.model_id == resource_id)
diff --git a/backend/src/modelforge_api/persistence/runtime_engine.py b/backend/src/modelforge_api/persistence/runtime_engine.py
index 4c6371a..31dc712 100644
--- a/backend/src/modelforge_api/persistence/runtime_engine.py
+++ b/backend/src/modelforge_api/persistence/runtime_engine.py
@@ -20,4 +20,3 @@ def register_application_engine(engine: Engine) -> Engine:
def is_application_connection(connection: Connection) -> bool:
return connection.engine in _APPLICATION_ENGINES
-
diff --git a/backend/src/modelforge_api/persistence/serving_repository.py b/backend/src/modelforge_api/persistence/serving_repository.py
index 6e21fab..23650ce 100644
--- a/backend/src/modelforge_api/persistence/serving_repository.py
+++ b/backend/src/modelforge_api/persistence/serving_repository.py
@@ -53,6 +53,15 @@ class ServingRepository:
self.session.flush()
return entity
+ def lock_contract_route(self, contract_id: uuid.UUID) -> CapabilityContract | None:
+ """Serialize stable-route writers; caller retains this lock through commit."""
+ return self.session.scalar(
+ select(CapabilityContract)
+ .where(CapabilityContract.id == contract_id)
+ .with_for_update()
+ .execution_options(populate_existing=True)
+ )
+
def capability(self, key: str) -> Capability | None:
return self.session.scalar(select(Capability).where(Capability.key == key))
@@ -251,6 +260,7 @@ class ServingRepository:
select(ServiceCredential).where(
ServiceCredential.service_client_id == client_id,
ServiceCredential.revoked_at.is_(None),
+ ServiceCredential.superseded_at.is_(None),
)
)
@@ -342,7 +352,9 @@ class ServingRepository:
ServingJob.compute_node_id == node_id,
or_(
ServingJob.status == "queued",
- (ServingJob.status == "running") & (ServingJob.lease_expires_at < now),
+ (ServingJob.status == "running")
+ & ServingJob.lab_execution_run_id.is_(None)
+ & (ServingJob.lease_expires_at < now),
),
)
.order_by(priority_order, ServingJob.created_at)
diff --git a/backend/src/modelforge_api/providers/huggingface.py b/backend/src/modelforge_api/providers/huggingface.py
index 19c91a1..c05d8ee 100644
--- a/backend/src/modelforge_api/providers/huggingface.py
+++ b/backend/src/modelforge_api/providers/huggingface.py
@@ -4,7 +4,7 @@ from collections.abc import Iterable
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import PurePosixPath
-from typing import Any, Protocol
+from typing import Any, Literal, Protocol
import httpx
from huggingface_hub import HfApi
@@ -116,8 +116,10 @@ class OfficialHuggingFaceProvider:
"""Narrow adapter over the official huggingface_hub client; no HTML scraping."""
def __init__(self, *, token: str | None = None, timeout: float = 30) -> None:
- self.api = HfApi(token=token)
- self.token = token
+ # Explicit False disables ambient HF_TOKEN/cache credentials. The official public
+ # provider endpoint is fixed; HF_ENDPOINT must not redirect credentials elsewhere.
+ self.api = HfApi(endpoint="https://huggingface.co", token=token or False)
+ self.token: str | Literal[False] = token or False
self.timeout = timeout
@staticmethod
diff --git a/backend/src/modelforge_api/services/acquisition.py b/backend/src/modelforge_api/services/acquisition.py
index 1e654ab..ba705e2 100644
--- a/backend/src/modelforge_api/services/acquisition.py
+++ b/backend/src/modelforge_api/services/acquisition.py
@@ -4,6 +4,7 @@ import hashlib
import json
import secrets
import uuid
+from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -126,13 +127,20 @@ class AcquisitionService:
*,
actor_type: str = "operator",
actor_id: str = "local-operator",
+ credential_configured: bool | Callable[[], bool] | None = None,
) -> None:
self.session = session
self.settings = settings
self.provider = provider
+ self._credential_configured = bool(settings.hf_token) if credential_configured is None else credential_configured
self.repo = AcquisitionRepository(session)
self.audit = AuditWriter(session, actor_type, actor_id)
+ @property
+ def credential_configured(self) -> bool:
+ value = self._credential_configured
+ return value() if callable(value) else value
+
def _commit(self) -> None:
self.session.commit()
@@ -386,7 +394,7 @@ class AcquisitionService:
revision = self.session.get(ModelRevision, artifact_set.revision_id)
if not snapshot or not revision:
raise AcquisitionError("artifact set provenance is incomplete")
- if snapshot.access_state != "public" and not self.settings.hf_token:
+ if snapshot.access_state != "public" and not self.credential_configured:
raise AcquisitionError(
f"repository access is {snapshot.access_state}; Hugging Face authentication is required"
)
diff --git a/backend/src/modelforge_api/services/alert_notifications.py b/backend/src/modelforge_api/services/alert_notifications.py
new file mode 100644
index 0000000..1f2d389
--- /dev/null
+++ b/backend/src/modelforge_api/services/alert_notifications.py
@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import hashlib
+import secrets
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpx
+import structlog
+from sqlalchemy import or_, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal
+from modelforge_api.domain.alert_notifications import (
+ NotificationDeliveryResponse,
+ NotificationKind,
+ NotificationStatus,
+ NotificationTestResponse,
+)
+from modelforge_api.persistence.models import (
+ AlertHistoryEvent,
+ AlertNotificationDelivery,
+ OperationalAlert,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings
+
+logger = structlog.get_logger()
+_SEVERITY_RANK = {"INFO": 0, "WARNING": 1, "CRITICAL": 2}
+_ACTIVE_STATES = ("PENDING", "DISPATCHING", "RETRY_WAIT")
+
+
+class AlertNotificationError(RuntimeError):
+ def __init__(self, status_code: int, code: str, message: str) -> None:
+ super().__init__(message)
+ self.status_code = status_code
+ self.code = code
+ self.message = message
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _route_ready(settings: Settings) -> bool:
+ if not settings.alert_notifications_enabled or settings.alert_webhook_url is None:
+ return False
+ try:
+ parsed = urlsplit(settings.alert_webhook_url.get_secret_value())
+ hostname = (parsed.hostname or "").casefold().rstrip(".")
+ allowed = set(settings.alert_webhook_allowed_hosts.split(","))
+ return (
+ parsed.scheme.casefold() == "https"
+ and hostname in allowed
+ and parsed.username is None
+ and parsed.password is None
+ and not parsed.fragment
+ and parsed.port in (None, 443)
+ )
+ except ValueError:
+ return False
+
+
+def _payload(alert: OperationalAlert, event: AlertHistoryEvent) -> dict[str, Any]:
+ return {
+ "schema_version": 1,
+ "event_id": str(event.id),
+ "event": event.to_state,
+ "alert_id": str(alert.id),
+ "alert_type": alert.alert_type[:64],
+ "severity": alert.severity,
+ "subject_type": alert.subject_type[:64],
+ "subject_ref": alert.subject_ref[:255],
+ "summary": alert.summary[:500],
+ "occurred_at": event.occurred_at.isoformat(),
+ }
+
+
+def enqueue_alert_transition(
+ session: Session,
+ settings: Settings,
+ alert: OperationalAlert,
+ event: AlertHistoryEvent,
+) -> None:
+ """Add one route delivery in the caller's alert/history transaction."""
+ kind = event.to_state
+ if (
+ not settings.alert_notifications_enabled
+ or kind not in {NotificationKind.FIRING.value, NotificationKind.RESOLVED.value}
+ or _SEVERITY_RANK.get(alert.severity, -1)
+ < _SEVERITY_RANK[settings.alert_webhook_min_severity]
+ ):
+ return
+ session.add(
+ AlertNotificationDelivery(
+ alert_history_event_id=event.id,
+ route_key="default",
+ idempotency_key=f"alert:{event.id}:default",
+ event_kind=kind,
+ severity=alert.severity,
+ payload=_payload(alert, event),
+ status=NotificationStatus.PENDING.value,
+ attempt_count=0,
+ next_attempt_at=None,
+ )
+ )
+
+
+class AlertNotificationService:
+ def __init__(self, session: Session, settings: Settings) -> None:
+ self.session = session
+ self.settings = settings
+
+ def queue_test(self, principal: Principal, correlation_id: str | None) -> NotificationTestResponse:
+ if not _route_ready(self.settings):
+ raise AlertNotificationError(
+ 503,
+ "NOTIFICATION_ROUTE_UNAVAILABLE",
+ "The deployment-managed alert notification route is unavailable",
+ )
+ event_id = secrets.token_hex(16)
+ row = AlertNotificationDelivery(
+ alert_history_event_id=None,
+ route_key="default",
+ idempotency_key=f"test:{event_id}:default",
+ event_kind=NotificationKind.TEST.value,
+ severity=None,
+ payload={
+ "schema_version": 1,
+ "event_id": event_id,
+ "event": "TEST",
+ "summary": "ModelForge operational alert webhook test",
+ "occurred_at": _now().isoformat(),
+ },
+ status=NotificationStatus.PENDING.value,
+ attempt_count=0,
+ next_attempt_at=None,
+ )
+ self.session.add(row)
+ self.session.flush()
+ AuditWriter(
+ self.session,
+ context=AuditContext("operator", principal.subject, correlation_id),
+ ).write(
+ "ALERT_NOTIFICATION_TEST_QUEUED",
+ "alert_notification_delivery",
+ str(row.id),
+ {"route_key": "default", "delivery_id": str(row.id)},
+ )
+ self.session.commit()
+ return NotificationTestResponse(
+ delivery=NotificationDeliveryResponse.model_validate(row),
+ message="Test notification queued for asynchronous delivery",
+ )
+
+ def deliveries(self, limit: int) -> list[NotificationDeliveryResponse]:
+ rows = self.session.scalars(
+ select(AlertNotificationDelivery)
+ .order_by(AlertNotificationDelivery.created_at.desc(), AlertNotificationDelivery.id.desc())
+ .limit(limit)
+ )
+ return [NotificationDeliveryResponse.model_validate(row) for row in rows]
+
+
+def _claim(
+ session: Session, settings: Settings, now: datetime
+) -> list[tuple[Any, str, str, dict[str, Any], int]]:
+ due = or_(
+ AlertNotificationDelivery.status == NotificationStatus.PENDING.value,
+ (
+ (AlertNotificationDelivery.status == NotificationStatus.RETRY_WAIT.value)
+ & (AlertNotificationDelivery.next_attempt_at <= now)
+ ),
+ (
+ (AlertNotificationDelivery.status == NotificationStatus.DISPATCHING.value)
+ & (AlertNotificationDelivery.lease_expires_at <= now)
+ ),
+ )
+ rows = list(
+ session.scalars(
+ select(AlertNotificationDelivery)
+ .where(due)
+ .order_by(AlertNotificationDelivery.created_at, AlertNotificationDelivery.id)
+ .limit(settings.alert_notification_batch_size)
+ .with_for_update(skip_locked=True)
+ )
+ )
+ claims: list[tuple[Any, str, str, dict[str, Any], int]] = []
+ for row in rows:
+ token = secrets.token_urlsafe(32)
+ row.status = NotificationStatus.DISPATCHING.value
+ row.attempt_count += 1
+ row.last_attempt_at = now
+ row.lease_expires_at = now + timedelta(seconds=settings.alert_notification_lease_seconds)
+ row.claim_token_hash = hashlib.sha256(token.encode()).hexdigest()
+ claims.append((row.id, row.idempotency_key, token, dict(row.payload), row.attempt_count))
+ session.commit()
+ return claims
+
+
+def _retry_delay(settings: Settings, attempt: int) -> int:
+ multiplier = 1 << min(max(attempt - 1, 0), 10)
+ return min(settings.alert_notification_retry_base_seconds * multiplier, 3600)
+
+
+def _safe_result(
+ status_code: int | None,
+ error_code: str | None,
+ *,
+ settings: Settings,
+ attempt: int,
+) -> tuple[str, datetime | None, int | None, str | None]:
+ if status_code is not None and 200 <= status_code < 300:
+ return NotificationStatus.DELIVERED.value, None, status_code, None
+ retryable = (
+ (status_code is None and error_code != "DESTINATION_REJECTED")
+ or status_code in {408, 425, 429}
+ or (status_code is not None and status_code >= 500)
+ )
+ safe_code = error_code or (
+ "REMOTE_RETRYABLE" if retryable else "REMOTE_REJECTED"
+ )
+ if status_code is not None and 300 <= status_code < 400:
+ retryable = False
+ safe_code = "REDIRECT_REJECTED"
+ if retryable and attempt < settings.alert_notification_max_attempts:
+ due = _now() + timedelta(seconds=_retry_delay(settings, attempt))
+ return NotificationStatus.RETRY_WAIT.value, due, status_code, safe_code
+ return NotificationStatus.FAILED.value, None, status_code, safe_code
+
+
+def _post_once(
+ settings: Settings,
+ payload: dict[str, Any],
+ idempotency_key: str,
+ transport: httpx.BaseTransport | None,
+) -> tuple[int | None, str | None]:
+ webhook = settings.alert_webhook_url
+ if not _route_ready(settings) or webhook is None:
+ return None, "DESTINATION_REJECTED"
+ url = webhook.get_secret_value()
+ try:
+ with httpx.Client(
+ timeout=httpx.Timeout(settings.alert_notification_timeout_seconds),
+ follow_redirects=False,
+ trust_env=False,
+ transport=transport,
+ ) as client, client.stream(
+ "POST",
+ url,
+ json=payload,
+ headers={
+ "Accept": "application/json",
+ "Content-Type": "application/json",
+ "Idempotency-Key": idempotency_key,
+ },
+ ) as response:
+ return response.status_code, None
+ except httpx.TimeoutException:
+ return None, "DELIVERY_TIMEOUT"
+ except httpx.HTTPError:
+ return None, "DELIVERY_TRANSPORT_ERROR"
+ except (ValueError, OSError, RuntimeError):
+ return None, "DELIVERY_TRANSPORT_ERROR"
+
+
+def dispatch_pending(
+ session: Session,
+ settings: Settings,
+ *,
+ now: datetime | None = None,
+ transport: httpx.BaseTransport | None = None,
+) -> int:
+ """Claim and deliver one bounded batch; network calls happen outside the claim transaction."""
+ if not settings.alert_notifications_enabled:
+ return 0
+ claims = _claim(session, settings, now or _now())
+ for row_id, idempotency_key, token, payload, attempt in claims:
+ event_id = str(payload.get("event_id", ""))[:64]
+ status_code, error_code = _post_once(
+ settings, payload, idempotency_key, transport
+ )
+ status, next_attempt, status_code, safe_code = _safe_result(
+ status_code, error_code, settings=settings, attempt=attempt
+ )
+ token_hash = hashlib.sha256(token.encode()).hexdigest()
+ values: dict[str, Any] = {
+ "status": status,
+ "next_attempt_at": next_attempt,
+ "lease_expires_at": None,
+ "claim_token_hash": None,
+ "last_http_status": status_code,
+ "last_error_code": safe_code,
+ }
+ if status == NotificationStatus.DELIVERED.value:
+ values["delivered_at"] = _now()
+ session.execute(
+ update(AlertNotificationDelivery)
+ .where(
+ AlertNotificationDelivery.id == row_id,
+ AlertNotificationDelivery.status == NotificationStatus.DISPATCHING.value,
+ AlertNotificationDelivery.claim_token_hash == token_hash,
+ )
+ .values(**values)
+ )
+ session.commit()
+ logger.info(
+ "alert_notification_delivery_attempted",
+ delivery_id=str(row_id),
+ event_id=event_id,
+ status=status,
+ error_code=safe_code,
+ http_status=status_code,
+ attempt=attempt,
+ )
+ return len(claims)
diff --git a/backend/src/modelforge_api/services/audit.py b/backend/src/modelforge_api/services/audit.py
index 3f66296..343fe6f 100644
--- a/backend/src/modelforge_api/services/audit.py
+++ b/backend/src/modelforge_api/services/audit.py
@@ -126,7 +126,8 @@ def _decode_canonical_audit_payload(value: str) -> dict[str, Any]:
decoded = json.loads(value, object_pairs_hook=reject_duplicate_keys)
if not isinstance(decoded, dict):
- raise ValueError("canonical audit payload is not an object")
+ # The string is valid JSON but an invalid persisted audit value, not a caller type error.
+ raise ValueError("canonical audit payload is not an object") # noqa: TRY004
return decoded
diff --git a/backend/src/modelforge_api/services/audit_timeline.py b/backend/src/modelforge_api/services/audit_timeline.py
new file mode 100644
index 0000000..2e116d8
--- /dev/null
+++ b/backend/src/modelforge_api/services/audit_timeline.py
@@ -0,0 +1,209 @@
+"""Read-only, bounded queries over append-only audit events."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import json
+import re
+import uuid
+from dataclasses import dataclass
+from datetime import UTC
+
+from sqlalchemy import and_, or_, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.audit_timeline import (
+ AuditTimelineEvent,
+ AuditTimelineRequest,
+ AuditTimelineResponse,
+)
+from modelforge_api.persistence.models import AuditEvent
+
+_CURSOR_VERSION = 1
+_CURSOR_MAX_LENGTH = 256
+
+
+class AuditTimelineCursorError(ValueError):
+ """A cursor is malformed or was created for a different filter set."""
+
+
+@dataclass(frozen=True, slots=True)
+class _AuditCursor:
+ sequence: int
+ event_id: uuid.UUID
+
+
+def _filters_digest(request: AuditTimelineRequest) -> str:
+ values: dict[str, object] = {
+ "principal": request.principal,
+ "action": request.action,
+ "resource": request.resource,
+ "outcome": request.outcome,
+ "occurred_after": (
+ request.occurred_after.astimezone(UTC).isoformat()
+ if request.occurred_after is not None
+ else None
+ ),
+ "occurred_before": (
+ request.occurred_before.astimezone(UTC).isoformat()
+ if request.occurred_before is not None
+ else None
+ ),
+ "correlation_id": request.correlation_id,
+ }
+ encoded = json.dumps(values, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ return hashlib.sha256(encoded).hexdigest()
+
+
+def _encode_cursor(sequence: int, event_id: uuid.UUID, filters_digest: str) -> str:
+ payload = json.dumps(
+ {
+ "v": _CURSOR_VERSION,
+ "sequence": sequence,
+ "event_id": str(event_id),
+ "filters": filters_digest,
+ },
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+ return base64.urlsafe_b64encode(payload).rstrip(b"=").decode("ascii")
+
+
+def _decode_cursor(value: str, filters_digest: str) -> _AuditCursor:
+ if len(value) > _CURSOR_MAX_LENGTH or not re.fullmatch(r"[A-Za-z0-9_-]+", value):
+ raise AuditTimelineCursorError("invalid audit timeline cursor")
+ padded = value + "=" * (-len(value) % 4)
+ try:
+ decoded = base64.b64decode(padded, altchars=b"-_", validate=True)
+ payload = json.loads(decoded)
+ except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError, ValueError):
+ raise AuditTimelineCursorError("invalid audit timeline cursor") from None
+ if not isinstance(payload, dict) or set(payload) != {"v", "sequence", "event_id", "filters"}:
+ raise AuditTimelineCursorError("invalid audit timeline cursor")
+ sequence = payload["sequence"]
+ event_id_value = payload["event_id"]
+ cursor_version = payload["v"]
+ cursor_filters = payload["filters"]
+ if (
+ type(cursor_version) is not int
+ or cursor_version != _CURSOR_VERSION
+ or type(sequence) is not int
+ or sequence < 1
+ or not isinstance(event_id_value, str)
+ or not isinstance(cursor_filters, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", cursor_filters)
+ ):
+ raise AuditTimelineCursorError("invalid audit timeline cursor")
+ try:
+ event_id = uuid.UUID(event_id_value)
+ except (AttributeError, TypeError, ValueError):
+ raise AuditTimelineCursorError("invalid audit timeline cursor") from None
+ if str(event_id) != event_id_value or cursor_filters != filters_digest:
+ raise AuditTimelineCursorError("audit timeline cursor does not match the supplied filters")
+ return _AuditCursor(sequence=sequence, event_id=event_id)
+
+
+class AuditTimelineService:
+ """Read a bounded page without returning arbitrary event details or changing the chain."""
+
+ def __init__(self, session: Session) -> None:
+ self.session = session
+
+ def list_events(self, request: AuditTimelineRequest) -> AuditTimelineResponse:
+ filters_digest = _filters_digest(request)
+ cursor = _decode_cursor(request.cursor, filters_digest) if request.cursor else None
+ predicates = []
+ if request.principal:
+ predicates.append(
+ or_(
+ AuditEvent.actor_id.icontains(request.principal, autoescape=True),
+ AuditEvent.actor_type.icontains(request.principal, autoescape=True),
+ )
+ )
+ if request.action:
+ predicates.append(AuditEvent.action.icontains(request.action, autoescape=True))
+ if request.resource:
+ predicates.append(
+ or_(
+ AuditEvent.resource_type.icontains(request.resource, autoescape=True),
+ AuditEvent.resource_id.icontains(request.resource, autoescape=True),
+ )
+ )
+ if request.outcome:
+ predicates.append(AuditEvent.outcome.icontains(request.outcome, autoescape=True))
+ if request.occurred_after is not None:
+ predicates.append(AuditEvent.occurred_at >= request.occurred_after)
+ if request.occurred_before is not None:
+ predicates.append(AuditEvent.occurred_at <= request.occurred_before)
+ if request.correlation_id:
+ predicates.append(
+ AuditEvent.correlation_id.icontains(request.correlation_id, autoescape=True)
+ )
+ if cursor is not None:
+ predicates.append(
+ or_(
+ AuditEvent.sequence < cursor.sequence,
+ and_(
+ AuditEvent.sequence == cursor.sequence,
+ AuditEvent.id < cursor.event_id,
+ ),
+ )
+ )
+
+ statement = (
+ select(
+ AuditEvent.id,
+ AuditEvent.sequence,
+ AuditEvent.occurred_at,
+ AuditEvent.correlation_id,
+ AuditEvent.actor_type,
+ AuditEvent.actor_id,
+ AuditEvent.action,
+ AuditEvent.resource_type,
+ AuditEvent.resource_id,
+ AuditEvent.outcome,
+ )
+ .where(*predicates)
+ .order_by(AuditEvent.sequence.desc(), AuditEvent.id.desc())
+ .limit(request.limit + 1)
+ )
+ rows = self.session.execute(statement).all()
+ has_more = len(rows) > request.limit
+ page = rows[: request.limit]
+ events = [
+ AuditTimelineEvent(
+ id=event_id,
+ sequence=sequence,
+ occurred_at=(
+ occurred_at.replace(tzinfo=UTC)
+ if occurred_at.tzinfo is None
+ else occurred_at.astimezone(UTC)
+ ),
+ correlation_id=correlation_id,
+ actor_type=actor_type,
+ actor_id=actor_id,
+ action=action,
+ resource_type=resource_type,
+ resource_id=resource_id,
+ outcome=outcome,
+ )
+ for (
+ event_id,
+ sequence,
+ occurred_at,
+ correlation_id,
+ actor_type,
+ actor_id,
+ action,
+ resource_type,
+ resource_id,
+ outcome,
+ ) in page
+ ]
+ next_cursor = None
+ if has_more and page:
+ last_event_id = uuid.UUID(str(page[-1][0]))
+ next_cursor = _encode_cursor(page[-1][1], last_event_id, filters_digest)
+ return AuditTimelineResponse(events=events, next_cursor=next_cursor, limit=request.limit)
diff --git a/backend/src/modelforge_api/services/benchmark_runs.py b/backend/src/modelforge_api/services/benchmark_runs.py
new file mode 100644
index 0000000..3f34ee5
--- /dev/null
+++ b/backend/src/modelforge_api/services/benchmark_runs.py
@@ -0,0 +1,1222 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import json
+import math
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, cast
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+from sqlalchemy import and_, or_, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm.exc import StaleDataError
+
+from modelforge_api.domain.benchmarking import (
+ BenchmarkRunHistoryPage,
+ BenchmarkRunProgress,
+ BenchmarkRunResponse,
+ BenchmarkRunStartRequest,
+ BenchmarkRunStatus,
+ BenchmarkSuiteResponse,
+)
+from modelforge_api.domain.contracts import BenchmarkEnvironmentFingerprint, BenchmarkRunContract
+from modelforge_api.domain.lab import (
+ LabEmbeddingInput,
+ LabExecutionStartRequest,
+)
+from modelforge_api.domain.serving import EmbeddingInvokeRequest
+from modelforge_api.persistence.models import (
+ Accelerator,
+ ArtifactSet,
+ BenchmarkRun,
+ BenchmarkSuite,
+ Capability,
+ CapabilityContract,
+ GatewayRequest,
+ LabExecutionRun,
+ ModelRevision,
+ RuntimeEnvironment,
+ RuntimeProfile,
+)
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingError, ServingService
+
+_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
+_SUITE_KEY = "modelforge.rag-embedding.operator-performance"
+_SUITE_REVISION = "1"
+_DATASET_REVISION = "builtin:rag-embedding-performance-v1"
+_SUITE_ID = uuid5(NAMESPACE_URL, f"{_SUITE_KEY}@{_SUITE_REVISION}")
+_SUITE_IMMUTABLE_AT = datetime(2026, 9, 24, tzinfo=UTC)
+_BENCHMARK_INPUTS = (
+ "ModelForge embedding performance sample one.",
+ "ModelForge embedding performance sample two.",
+ "ModelForge embedding performance sample three.",
+ "ModelForge embedding performance sample four.",
+)
+_HISTORY_CURSOR_VERSION = 1
+_HISTORY_CURSOR_MAX_LENGTH = 512
+
+
+@dataclass(frozen=True, slots=True)
+class _BenchmarkHistoryCursor:
+ created_at: datetime
+ run_id: UUID
+
+
+def _encode_history_cursor(run: BenchmarkRun, actor_subject: str) -> str:
+ payload = json.dumps(
+ {
+ "v": _HISTORY_CURSOR_VERSION,
+ "created_at": _aware(run.created_at).astimezone(UTC).isoformat(),
+ "run_id": str(run.id),
+ "actor": hashlib.sha256(actor_subject.encode("utf-8")).hexdigest(),
+ },
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+ return base64.urlsafe_b64encode(payload).rstrip(b"=").decode("ascii")
+
+
+def _decode_history_cursor(value: str, actor_subject: str) -> _BenchmarkHistoryCursor:
+ def invalid() -> ServingError:
+ return ServingError(422, "BENCHMARK_CURSOR_INVALID", "benchmark history cursor is invalid")
+
+ if len(value) > _HISTORY_CURSOR_MAX_LENGTH or not re.fullmatch(r"[A-Za-z0-9_-]+", value):
+ raise invalid()
+ padded = value + "=" * (-len(value) % 4)
+ try:
+ decoded = base64.b64decode(padded, altchars=b"-_", validate=True)
+ payload = json.loads(decoded)
+ except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError, ValueError):
+ raise invalid() from None
+ if not isinstance(payload, dict) or set(payload) != {"v", "created_at", "run_id", "actor"}:
+ raise invalid()
+ cursor_version = payload["v"]
+ created_at_value = payload["created_at"]
+ run_id_value = payload["run_id"]
+ actor_digest = payload["actor"]
+ if (
+ type(cursor_version) is not int
+ or cursor_version != _HISTORY_CURSOR_VERSION
+ or not isinstance(created_at_value, str)
+ or not isinstance(run_id_value, str)
+ or not isinstance(actor_digest, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", actor_digest)
+ or actor_digest != hashlib.sha256(actor_subject.encode("utf-8")).hexdigest()
+ ):
+ raise invalid()
+ try:
+ created_at = datetime.fromisoformat(created_at_value)
+ run_id = UUID(run_id_value)
+ except (TypeError, ValueError):
+ raise invalid() from None
+ if (
+ created_at.tzinfo is None
+ or created_at.utcoffset() is None
+ or str(run_id) != run_id_value
+ ):
+ raise invalid()
+ return _BenchmarkHistoryCursor(created_at=created_at.astimezone(UTC), run_id=run_id)
+_BENCHMARK_CASES = tuple(f"request-{index + 1}" for index in range(len(_BENCHMARK_INPUTS)))
+_TERMINAL_RUNS = frozenset({"completed", "failed", "cancelled", "interrupted"})
+_TERMINAL_BENCHMARKS = frozenset({"completed", "failed", "cancelled", "interrupted"})
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _aware(value: datetime) -> datetime:
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+
+
+def _canonical_json(value: Any) -> bytes:
+ return json.dumps(
+ value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
+ ).encode("utf-8")
+
+
+def _sha256(value: Any) -> str:
+ return hashlib.sha256(_canonical_json(value)).hexdigest()
+
+
+def _finite_nonnegative(value: Any) -> float | None:
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ return None
+ number = float(value)
+ return number if math.isfinite(number) and number >= 0 else None
+
+
+def _percentile(values: list[float], fraction: float) -> float | None:
+ if not values:
+ return None
+ ordered = sorted(values)
+ index = max(0, min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1))
+ return ordered[index]
+
+
+def _suite_definition() -> dict[str, Any]:
+ return {
+ "schema_version": 1,
+ "capability": "rag.embedding@1",
+ "execution": {"concurrency": 1, "max_invocations": len(_BENCHMARK_INPUTS)},
+ "cases": [
+ {"case_id": case_id, "input_sha256": _sha256([value])}
+ for case_id, value in zip(_BENCHMARK_CASES, _BENCHMARK_INPUTS, strict=True)
+ ],
+ "reported_metrics": [
+ "initial_request_latency_ms",
+ "cold_load_latency_ms",
+ "warm_latency_ms",
+ "throughput",
+ "failure_rate",
+ "gpu_vram_telemetry",
+ ],
+ "quality_metrics": {"status": "unknown", "reason": "PROJECT_CORPUS_NOT_INCLUDED"},
+ }
+
+
+class BenchmarkRunService:
+ """Drive small operator benchmarks through the durable LAB and private-worker path."""
+
+ def __init__(self, serving: ServingService) -> None:
+ self.serving = serving
+ self.session = serving.session
+
+ def suites(self) -> list[BenchmarkSuiteResponse]:
+ definition = _suite_definition()
+ contract = self.session.scalar(
+ select(CapabilityContract)
+ .join(Capability, Capability.id == CapabilityContract.capability_id)
+ .where(Capability.key == "rag.embedding", CapabilityContract.version == 1)
+ )
+ if contract is None:
+ raise ServingError(
+ 503,
+ "BENCHMARK_CAPABILITY_UNAVAILABLE",
+ "the declared rag.embedding@1 contract is unavailable",
+ )
+ suite = self.session.scalar(
+ select(BenchmarkSuite).where(
+ BenchmarkSuite.key == _SUITE_KEY,
+ BenchmarkSuite.revision == _SUITE_REVISION,
+ )
+ )
+ if suite is not None and (
+ suite.id != _SUITE_ID
+ or suite.immutable_at is None
+ or suite.dataset_revision != _DATASET_REVISION
+ or suite.definition != definition
+ or suite.capability_contract_id != contract.id
+ ):
+ raise ServingError(
+ 409, "BENCHMARK_SUITE_INVALID", "the immutable benchmark suite is inconsistent"
+ )
+ if suite is not None:
+ return [self._suite_response(suite)]
+ return [
+ BenchmarkSuiteResponse(
+ id=_SUITE_ID,
+ key=_SUITE_KEY,
+ revision=_SUITE_REVISION,
+ capability="rag.embedding@1",
+ dataset_revision=_DATASET_REVISION,
+ definition=definition,
+ immutable_at=_SUITE_IMMUTABLE_AT,
+ )
+ ]
+
+ @staticmethod
+ def _suite_response(suite: BenchmarkSuite) -> BenchmarkSuiteResponse:
+ return BenchmarkSuiteResponse(
+ id=suite.id,
+ key=suite.key,
+ revision=suite.revision,
+ capability="rag.embedding@1",
+ dataset_revision=suite.dataset_revision,
+ definition=suite.definition,
+ immutable_at=suite.immutable_at or _now(),
+ )
+
+ def _suite(self, suite_id: UUID) -> BenchmarkSuite:
+ if suite_id != _SUITE_ID:
+ raise ServingError(404, "BENCHMARK_SUITE_NOT_FOUND", "benchmark suite was not found")
+ suite = self.session.scalar(
+ select(BenchmarkSuite).where(
+ BenchmarkSuite.id == _SUITE_ID,
+ BenchmarkSuite.id == suite_id,
+ BenchmarkSuite.key == _SUITE_KEY,
+ BenchmarkSuite.revision == _SUITE_REVISION,
+ )
+ )
+ if suite is None:
+ contract = self.session.scalar(
+ select(CapabilityContract)
+ .join(Capability, Capability.id == CapabilityContract.capability_id)
+ .where(Capability.key == "rag.embedding", CapabilityContract.version == 1)
+ )
+ if contract is None:
+ raise ServingError(
+ 503,
+ "BENCHMARK_CAPABILITY_UNAVAILABLE",
+ "the declared rag.embedding@1 contract is unavailable",
+ )
+ suite = BenchmarkSuite(
+ id=_SUITE_ID,
+ key=_SUITE_KEY,
+ revision=_SUITE_REVISION,
+ capability_contract_id=contract.id,
+ dataset_revision=_DATASET_REVISION,
+ definition=_suite_definition(),
+ immutable_at=_SUITE_IMMUTABLE_AT,
+ )
+ self.session.add(suite)
+ try:
+ self.session.flush()
+ self.session.commit()
+ except IntegrityError:
+ self.session.rollback()
+ suite = self.session.get(BenchmarkSuite, _SUITE_ID)
+ if (
+ suite is None
+ or suite.immutable_at is None
+ or suite.dataset_revision != _DATASET_REVISION
+ or suite.definition != _suite_definition()
+ ):
+ raise ServingError(
+ 409, "BENCHMARK_SUITE_INVALID", "the immutable benchmark suite is inconsistent"
+ )
+ contract = self.session.get(CapabilityContract, suite.capability_contract_id)
+ capability = self.session.get(Capability, contract.capability_id) if contract else None
+ if (
+ contract is None
+ or contract.version != 1
+ or capability is None
+ or capability.key != "rag.embedding"
+ ):
+ raise ServingError(409, "BENCHMARK_SUITE_INVALID", "suite capability identity changed")
+ return suite
+
+ def _provenance(
+ self, request: BenchmarkRunStartRequest
+ ) -> tuple[dict[str, Any], str, str, BenchmarkEnvironmentFingerprint, bool]:
+ sample = LabEmbeddingInput(
+ capability="rag.embedding",
+ payload=EmbeddingInvokeRequest(input=_BENCHMARK_INPUTS[0]),
+ )
+ deployment, route_key = self.serving._admit_operator_embedding(
+ sample, request.deployment_id, request.experiment_route_id
+ )
+ snapshot = self.serving._operator_lab_target_snapshot(
+ deployment, request.experiment_route_id
+ )
+ expected = request.target.model_dump(mode="json")
+ actual = {
+ "deployment_candidate_id": snapshot["candidate_id"],
+ "artifact_set_id": snapshot["artifact_set_id"],
+ "runtime_profile_id": snapshot["runtime_profile_id"],
+ "compute_node_id": snapshot["compute_node_id"],
+ "execution_approval_id": snapshot["execution_approval_id"],
+ }
+ if expected != actual:
+ raise ServingError(
+ 409, "BENCHMARK_TARGET_MISMATCH", "selected benchmark identities do not match"
+ )
+ if route_key == "":
+ raise ServingError(
+ 409, "BENCHMARK_TARGET_MISMATCH", "selected experiment route is invalid"
+ )
+
+ profile = self.session.get(RuntimeProfile, deployment.runtime_profile_id)
+ artifact_set = self.session.get(ArtifactSet, deployment.artifact_set_id)
+ accelerator = self.session.get(Accelerator, deployment.accelerator_id)
+ node = self.serving.repo.node(deployment.compute_node_id)
+ environment = (
+ self.session.get(RuntimeEnvironment, profile.runtime_environment_id)
+ if profile and profile.runtime_environment_id
+ else None
+ )
+ revision = (
+ self.session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
+ )
+ if (
+ profile is None
+ or artifact_set is None
+ or accelerator is None
+ or node is None
+ or environment is None
+ or revision is None
+ or len(profile.artifact_sha256) != 64
+ ):
+ raise ServingError(
+ 409,
+ "BENCHMARK_PROVENANCE_INCOMPLETE",
+ "exact benchmark model, artifact, runtime or hardware provenance is unavailable",
+ )
+ launch_arguments = profile.launch_arguments or {}
+ if self._contains_secret_argument(launch_arguments):
+ raise ServingError(
+ 409,
+ "BENCHMARK_RUNTIME_ARGUMENTS_UNSAFE",
+ "runtime arguments cannot be recorded safely for benchmark provenance",
+ )
+ context_length = next(
+ (
+ value
+ for value in (
+ profile.context_policy.get("context_length"),
+ profile.context_policy.get("max_sequence_length"),
+ profile.context_policy.get("max_model_len"),
+ )
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0
+ ),
+ None,
+ )
+ os_name = node.os_name
+ operating_system = (
+ f"{os_name} {node.os_version}" if os_name and node.os_version else os_name
+ )
+ fingerprint = BenchmarkEnvironmentFingerprint(
+ runtime_type=profile.runtime_type,
+ runtime_version=profile.runtime_version,
+ runtime_image_digest=profile.runtime_image_digest or environment.image_digest,
+ launch_arguments=launch_arguments,
+ cuda_version=environment.cuda_runtime_version,
+ driver_version=accelerator.driver_version,
+ accelerator_name=accelerator.name,
+ accelerator_uuid=accelerator.device_uuid,
+ context_length=context_length,
+ concurrency=profile.concurrency,
+ seed=None,
+ operating_system=operating_system,
+ python_version=environment.python_version,
+ )
+ complete = all(
+ value is not None and value != ""
+ for value in (
+ fingerprint.runtime_image_digest,
+ fingerprint.cuda_version,
+ fingerprint.driver_version,
+ fingerprint.accelerator_name,
+ fingerprint.accelerator_uuid,
+ fingerprint.operating_system,
+ fingerprint.python_version,
+ )
+ )
+ return (
+ snapshot,
+ revision.resolved_commit_sha,
+ profile.artifact_sha256,
+ fingerprint,
+ complete,
+ )
+
+ @staticmethod
+ def _contains_secret_argument(value: Any) -> bool:
+ sensitive = {"credential", "password", "secret", "token", "private_key", "api_key"}
+ if isinstance(value, dict):
+ for key, item in value.items():
+ normalized = re.sub(r"[^a-z0-9]+", "_", str(key).lower()).strip("_")
+ if any(word in normalized.split("_") for word in sensitive):
+ return True
+ if BenchmarkRunService._contains_secret_argument(item):
+ return True
+ elif isinstance(value, list):
+ return any(BenchmarkRunService._contains_secret_argument(item) for item in value)
+ elif isinstance(value, str):
+ normalized = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
+ return any(word in normalized.split("_") for word in sensitive)
+ return False
+
+ def start(
+ self,
+ request: BenchmarkRunStartRequest,
+ *,
+ actor_subject: str,
+ idempotency_key: str,
+ ) -> BenchmarkRunResponse:
+ if not actor_subject or len(actor_subject) > 255:
+ raise ServingError(
+ 401, "OPERATOR_CONTEXT_REQUIRED", "operator principal is unavailable"
+ )
+ if not _IDEMPOTENCY_KEY.fullmatch(idempotency_key):
+ raise ServingError(
+ 422, "BENCHMARK_IDEMPOTENCY_KEY_INVALID", "idempotency key is invalid"
+ )
+ fingerprint_hash = _sha256(request.model_dump(mode="json"))
+ existing = self.session.scalar(
+ select(BenchmarkRun).where(
+ BenchmarkRun.actor_subject == actor_subject,
+ BenchmarkRun.idempotency_key == idempotency_key,
+ )
+ )
+ if existing is not None:
+ if existing.request_fingerprint != fingerprint_hash:
+ raise ServingError(
+ 409,
+ "BENCHMARK_IDEMPOTENCY_CONFLICT",
+ "idempotency key was already used for a different benchmark request",
+ )
+ return self._response(existing)
+
+ suite = self._suite(request.suite_id)
+ snapshot, revision_sha, artifact_sha, environment, comparable = self._provenance(request)
+ now = _now()
+ run = BenchmarkRun(
+ deployment_id=None,
+ capability_deployment_id=request.deployment_id,
+ actor_subject=actor_subject,
+ idempotency_key=idempotency_key,
+ request_fingerprint=fingerprint_hash,
+ suite_id=suite.id,
+ target_snapshot=snapshot,
+ model_revision_sha=revision_sha,
+ artifact_sha256=artifact_sha,
+ runtime_type=environment.runtime_type,
+ runtime_version=environment.runtime_version,
+ runtime_image_digest=environment.runtime_image_digest,
+ launch_arguments=environment.launch_arguments,
+ suite_revision=suite.revision,
+ dataset_revision=suite.dataset_revision,
+ environment_fingerprint=environment.model_dump(mode="json"),
+ environment_digest=environment.digest,
+ seed=environment.seed,
+ results=self._initial_results(comparable),
+ status="accepted",
+ comparable=comparable,
+ stale_reason=None if comparable else "ENVIRONMENT_PROVENANCE_INCOMPLETE",
+ started_at=now,
+ created_at=now,
+ )
+ self.session.add(run)
+ try:
+ self.session.commit()
+ except IntegrityError:
+ self.session.rollback()
+ winner = self.session.scalar(
+ select(BenchmarkRun).where(
+ BenchmarkRun.actor_subject == actor_subject,
+ BenchmarkRun.idempotency_key == idempotency_key,
+ )
+ )
+ if winner is not None and winner.request_fingerprint == fingerprint_hash:
+ return self._response(winner)
+ if winner is not None:
+ raise ServingError(
+ 409,
+ "BENCHMARK_IDEMPOTENCY_CONFLICT",
+ "idempotency key was already used for a different benchmark request",
+ ) from None
+ raise ServingError(
+ 409, "STATE_CONFLICT", "benchmark run could not be persisted"
+ ) from None
+ self._advance(run.id)
+ return self.get(run.id, actor_subject=actor_subject)
+
+ @staticmethod
+ def _initial_results(comparable: bool) -> dict[str, Any]:
+ return {
+ "schema_version": 1,
+ "execution": {
+ "next_case_index": 0,
+ "active_case": None,
+ "active_lab_run_id": None,
+ "attempts": [],
+ },
+ "metrics": {
+ "initial_request_latency_ms": None,
+ "cold_load_latency_ms": {"p50": None, "p95": None, "samples": 0},
+ "warm_latency_ms": {"p50": None, "p95": None, "samples": 0},
+ "total_latency_ms": {"p50": None, "p95": None, "samples": 0},
+ "throughput": {
+ "successful_vectors_per_inference_second": None,
+ "inference_samples": 0,
+ "measured_vectors": 0,
+ "successful_vectors_coverage": None,
+ "successful_calls_per_observed_second": None,
+ "observed_elapsed_ms": None,
+ },
+ "failures": {
+ "attempted": 0,
+ "failed": 0,
+ "rate": None,
+ },
+ "gpu_vram": {
+ "telemetry_status": "UNKNOWN",
+ "vram_status": "UNKNOWN",
+ "gpu_utilization_status": "UNKNOWN",
+ "source": None,
+ "measured_samples": 0,
+ "peak_used_vram_bytes": None,
+ "peak_process_used_vram_bytes": None,
+ "minimum_free_vram_bytes": None,
+ "peak_gpu_utilization_percent": None,
+ "target_matched_samples": 0,
+ "target_accelerator_uuid": None,
+ },
+ "quality": {
+ "status": "UNKNOWN",
+ "metrics": {"recall_at_k": None, "mrr": None, "ndcg_at_k": None},
+ "reason": "PROJECT_CORPUS_NOT_INCLUDED_IN_PERFORMANCE_SUITE",
+ },
+ "environment_comparable": comparable,
+ },
+ }
+
+ def _response(self, run: BenchmarkRun) -> BenchmarkRunResponse:
+ suite = self.session.get(BenchmarkSuite, run.suite_id)
+ if suite is None or run.capability_deployment_id is None:
+ raise ServingError(
+ 500, "BENCHMARK_PROVENANCE_MISSING", "benchmark provenance is incomplete"
+ )
+ try:
+ environment = BenchmarkEnvironmentFingerprint.model_validate(
+ run.environment_fingerprint
+ )
+ except ValueError as error:
+ raise ServingError(
+ 500, "BENCHMARK_PROVENANCE_INVALID", "benchmark environment is invalid"
+ ) from error
+ contract = BenchmarkRunContract(
+ deployment_id=str(run.capability_deployment_id),
+ model_revision_sha=run.model_revision_sha,
+ artifact_sha256=run.artifact_sha256,
+ suite_key=suite.key,
+ suite_revision=run.suite_revision,
+ dataset_revision=run.dataset_revision,
+ environment=environment,
+ started_at=_aware(run.started_at),
+ completed_at=_aware(run.completed_at) if run.completed_at else None,
+ )
+ results = run.results if isinstance(run.results, dict) else {}
+ execution_value = results.get("execution")
+ execution = execution_value if isinstance(execution_value, dict) else {}
+ attempts = execution.get("attempts", [])
+ completed = len(attempts) if isinstance(attempts, list) else 0
+ active_case = execution.get("active_case")
+ return BenchmarkRunResponse(
+ id=run.id,
+ capability="rag.embedding@1",
+ suite_id=suite.id,
+ capability_deployment_id=run.capability_deployment_id,
+ status=cast(BenchmarkRunStatus, run.status),
+ contract=contract,
+ target_snapshot=run.target_snapshot or {},
+ progress=BenchmarkRunProgress(
+ completed_invocations=completed,
+ total_invocations=len(_BENCHMARK_CASES),
+ active_case=active_case if isinstance(active_case, str) else None,
+ ),
+ results=results,
+ failure_code=run.failure_code,
+ cancellation_requested_at=(
+ _aware(run.cancellation_requested_at) if run.cancellation_requested_at else None
+ ),
+ created_at=_aware(run.created_at),
+ completed_at=_aware(run.completed_at) if run.completed_at else None,
+ )
+
+ def _owned(self, run_id: UUID, actor_subject: str) -> BenchmarkRun:
+ run = self.session.scalar(
+ select(BenchmarkRun).where(
+ BenchmarkRun.id == run_id,
+ BenchmarkRun.actor_subject == actor_subject,
+ )
+ )
+ if run is None:
+ raise ServingError(404, "BENCHMARK_RUN_NOT_FOUND", "benchmark run was not found")
+ return run
+
+ def get(self, run_id: UUID, *, actor_subject: str) -> BenchmarkRunResponse:
+ return self._response(self._owned(run_id, actor_subject))
+
+ def list_page(
+ self,
+ *,
+ actor_subject: str,
+ limit: int = 50,
+ cursor: str | None = None,
+ ) -> BenchmarkRunHistoryPage:
+ if not actor_subject:
+ raise ServingError(
+ 401, "OPERATOR_CONTEXT_REQUIRED", "operator principal is unavailable"
+ )
+ if not 1 <= limit <= 100:
+ raise ServingError(422, "BENCHMARK_LIMIT_INVALID", "benchmark history limit is invalid")
+ decoded_cursor = (
+ _decode_history_cursor(cursor, actor_subject) if cursor is not None else None
+ )
+ predicates = [BenchmarkRun.actor_subject == actor_subject]
+ if decoded_cursor is not None:
+ predicates.append(
+ or_(
+ BenchmarkRun.created_at < decoded_cursor.created_at,
+ and_(
+ BenchmarkRun.created_at == decoded_cursor.created_at,
+ BenchmarkRun.id < decoded_cursor.run_id,
+ ),
+ )
+ )
+ rows = list(
+ self.session.scalars(
+ select(BenchmarkRun)
+ .where(*predicates)
+ .order_by(BenchmarkRun.created_at.desc(), BenchmarkRun.id.desc())
+ .limit(limit + 1)
+ )
+ )
+ has_more = len(rows) > limit
+ page_rows = rows[:limit]
+ next_cursor = _encode_history_cursor(page_rows[-1], actor_subject) if has_more else None
+ return BenchmarkRunHistoryPage(
+ runs=[self._response(row) for row in page_rows],
+ next_cursor=next_cursor,
+ limit=limit,
+ )
+
+ def list(self, *, actor_subject: str, limit: int = 50) -> list[BenchmarkRunResponse]:
+ """Return the original list contract for callers that do not page history."""
+ return self.list_page(actor_subject=actor_subject, limit=limit).runs
+
+ def cancel(self, run_id: UUID, *, actor_subject: str) -> BenchmarkRunResponse:
+ self._owned(run_id, actor_subject)
+ run = self.session.scalar(
+ select(BenchmarkRun)
+ .where(BenchmarkRun.id == run_id, BenchmarkRun.actor_subject == actor_subject)
+ .with_for_update()
+ )
+ if run is None:
+ raise ServingError(404, "BENCHMARK_RUN_NOT_FOUND", "benchmark run was not found")
+ if run.status in _TERMINAL_BENCHMARKS:
+ self.session.commit()
+ return self._response(run)
+ run.cancellation_requested_at = run.cancellation_requested_at or _now()
+ run.status = "cancel_requested"
+ results = (
+ run.results if isinstance(run.results, dict) else self._initial_results(run.comparable)
+ )
+ execution = results.get("execution")
+ active_id = execution.get("active_lab_run_id") if isinstance(execution, dict) else None
+ if not active_id:
+ run.status = "cancelled"
+ run.completed_at = _now()
+ self.session.commit()
+ if active_id:
+ try:
+ LabRunService(self.serving).cancel(
+ UUID(str(active_id)), actor_subject=actor_subject
+ )
+ except ServingError as error:
+ if error.code != "LAB_RUN_NOT_FOUND":
+ raise
+ self.reconcile_one(run_id)
+ return self.get(run_id, actor_subject=actor_subject)
+
+ def _lab_request(self, parent: BenchmarkRun, index: int) -> LabExecutionStartRequest:
+ if parent.capability_deployment_id is None:
+ raise ServingError(
+ 409, "BENCHMARK_TARGET_MISSING", "benchmark target identity is missing"
+ )
+ snapshot = parent.target_snapshot
+ if not isinstance(snapshot, dict):
+ raise ServingError(
+ 409, "BENCHMARK_TARGET_MISSING", "benchmark target snapshot is missing"
+ )
+ text = _BENCHMARK_INPUTS[index]
+ return LabExecutionStartRequest.model_validate(
+ {
+ "target": {
+ "deployment_candidate_id": snapshot["candidate_id"],
+ "artifact_set_id": snapshot["artifact_set_id"],
+ "runtime_profile_id": snapshot["runtime_profile_id"],
+ "compute_node_id": snapshot["compute_node_id"],
+ "execution_approval_id": snapshot["execution_approval_id"],
+ },
+ "input": {"capability": "rag.embedding", "payload": {"input": text}},
+ "limits": {
+ "timeout_seconds": 60,
+ "queue_timeout_seconds": 10,
+ "retention_seconds": 60,
+ "max_output_bytes": 65536,
+ "concurrency": 1,
+ },
+ "deployment_id": str(parent.capability_deployment_id),
+ "experiment_route_id": snapshot["experiment_route_id"],
+ }
+ )
+
+ @staticmethod
+ def _child_idempotency_key(run_id: UUID, index: int) -> str:
+ return f"benchmark-{run_id.hex}-{index:02d}"
+
+ def _advance(self, run_id: UUID) -> bool:
+ run = self.session.scalar(
+ select(BenchmarkRun).where(BenchmarkRun.id == run_id).with_for_update(skip_locked=True)
+ )
+ if run is None or run.status in _TERMINAL_BENCHMARKS:
+ self.session.rollback()
+ return False
+ if run.actor_subject is None:
+ run.status = "failed"
+ run.failure_code = "BENCHMARK_ACTOR_MISSING"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ results = (
+ run.results if isinstance(run.results, dict) else self._initial_results(run.comparable)
+ )
+ execution = results.get("execution")
+ if not isinstance(execution, dict):
+ run.status = "failed"
+ run.failure_code = "BENCHMARK_EXECUTION_STATE_INVALID"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ active_id = execution.get("active_lab_run_id")
+ child: LabExecutionRun | None = None
+ if active_id:
+ child = self.session.get(LabExecutionRun, UUID(str(active_id)))
+ if child is None:
+ run.status = "interrupted"
+ run.failure_code = "BENCHMARK_LINKED_LAB_RUN_MISSING"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ if child.status not in _TERMINAL_RUNS:
+ if run.cancellation_requested_at is not None:
+ self.session.commit()
+ LabRunService(self.serving).cancel(child.id, actor_subject=run.actor_subject)
+ return True
+ run.status = "running"
+ self.session.commit()
+ return False
+ index = execution.get("next_case_index")
+ if not isinstance(index, int) or not 0 <= index < len(_BENCHMARK_CASES):
+ run.status = "interrupted"
+ run.failure_code = "BENCHMARK_CASE_INDEX_INVALID"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ try:
+ attempt = self._attempt(child, index)
+ except ServingError as error:
+ run.status = "interrupted"
+ run.failure_code = error.code
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ attempts = execution.get("attempts")
+ if not isinstance(attempts, list):
+ attempts = []
+ execution = {
+ **execution,
+ "next_case_index": index + 1,
+ "active_case": None,
+ "active_lab_run_id": None,
+ "attempts": [*attempts, attempt],
+ }
+ results = {**results, "execution": execution}
+ results = self._refresh_metrics(results, run.environment_fingerprint)
+ run.results = results
+ if child.status == "interrupted":
+ run.status = "interrupted"
+ run.failure_code = child.failure_code or "LAB_WORKER_INTERRUPTED"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ if child.status == "cancelled" or run.cancellation_requested_at is not None:
+ run.status = "cancelled"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ if index + 1 >= len(_BENCHMARK_CASES):
+ run.status = "completed"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ run.status = "running"
+ self.session.commit()
+ return self._launch_next(run_id)
+
+ if run.cancellation_requested_at is not None:
+ run.status = "cancelled"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ next_index = execution.get("next_case_index", 0)
+ if not isinstance(next_index, int) or not 0 <= next_index < len(_BENCHMARK_CASES):
+ run.status = "interrupted"
+ run.failure_code = "BENCHMARK_CASE_INDEX_INVALID"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ self.session.commit()
+ return self._launch_next(run_id, next_index)
+
+ def _launch_next(self, run_id: UUID, index: int | None = None) -> bool:
+ run = self.session.get(BenchmarkRun, run_id)
+ if run is None or run.status in _TERMINAL_BENCHMARKS or run.actor_subject is None:
+ return False
+ actor_subject = run.actor_subject
+ results = (
+ run.results if isinstance(run.results, dict) else self._initial_results(run.comparable)
+ )
+ execution = results.get("execution")
+ if not isinstance(execution, dict) or execution.get("active_lab_run_id"):
+ return False
+ chosen = execution.get("next_case_index", 0) if index is None else index
+ if not isinstance(chosen, int) or not 0 <= chosen < len(_BENCHMARK_CASES):
+ return False
+ if run.cancellation_requested_at is not None:
+ run.status = "cancelled"
+ run.completed_at = _now()
+ self.session.commit()
+ return True
+ try:
+ child_response = LabRunService(self.serving).start(
+ self._lab_request(run, chosen),
+ actor_subject=actor_subject,
+ idempotency_key=self._child_idempotency_key(run.id, chosen),
+ )
+ except ServingError as error:
+ current = self.session.scalar(
+ select(BenchmarkRun).where(BenchmarkRun.id == run_id).with_for_update()
+ )
+ if current is None:
+ self.session.rollback()
+ return False
+ current.status = "failed"
+ current.failure_code = error.code
+ current.completed_at = _now()
+ self.session.commit()
+ return True
+ current = self.session.scalar(
+ select(BenchmarkRun).where(BenchmarkRun.id == run_id).with_for_update()
+ )
+ if current is None:
+ self.session.rollback()
+ return False
+ current_results = (
+ current.results
+ if isinstance(current.results, dict)
+ else self._initial_results(current.comparable)
+ )
+ current_execution = current_results.get("execution")
+ if not isinstance(current_execution, dict):
+ current.status = "interrupted"
+ current.failure_code = "BENCHMARK_EXECUTION_STATE_INVALID"
+ current.completed_at = _now()
+ self.session.commit()
+ try:
+ LabRunService(self.serving).cancel(child_response.id, actor_subject=actor_subject)
+ except ServingError as error:
+ if error.code != "LAB_RUN_NOT_FOUND":
+ raise
+ return True
+ # The child idempotency key is deterministic, so two reconcilers racing here can only
+ # converge on the same LAB run even on databases without row-level lock support.
+ cancel_child = False
+ if not current_execution.get("active_lab_run_id"):
+ same_case = current_execution.get("next_case_index", 0) == chosen
+ if same_case:
+ current_execution = {
+ **current_execution,
+ "active_case": (
+ None
+ if current.status in _TERMINAL_BENCHMARKS
+ else _BENCHMARK_CASES[chosen]
+ ),
+ "active_lab_run_id": str(child_response.id),
+ "next_case_index": chosen,
+ }
+ current.results = {**current_results, "execution": current_execution}
+ if not same_case:
+ cancel_child = True
+ elif current.status in _TERMINAL_BENCHMARKS:
+ # Cancellation can commit while LAB admission is in progress. Preserve that
+ # terminal truth and stop the just-created child instead of resurrecting it.
+ cancel_child = True
+ elif (
+ current.cancellation_requested_at is not None
+ or current.status == "cancel_requested"
+ ):
+ current.cancellation_requested_at = current.cancellation_requested_at or _now()
+ current.status = "cancel_requested"
+ cancel_child = True
+ else:
+ current.status = "running"
+ elif current_execution.get("active_lab_run_id") != str(child_response.id):
+ # Progress moved while this reconciler was waiting on LAB admission. The
+ # deterministic child key prevents duplicate rows; cancel any obsolete child.
+ cancel_child = child_response.status not in _TERMINAL_RUNS
+ self.session.commit()
+ if cancel_child:
+ try:
+ LabRunService(self.serving).cancel(child_response.id, actor_subject=actor_subject)
+ except ServingError as error:
+ if error.code != "LAB_RUN_NOT_FOUND":
+ raise
+ return True
+
+ def _attempt(self, child: LabExecutionRun, index: int) -> dict[str, Any]:
+ gateway = self.session.get(GatewayRequest, child.gateway_request_id)
+ if gateway is None:
+ raise ServingError(
+ 409,
+ "BENCHMARK_GATEWAY_EVIDENCE_MISSING",
+ "linked Gateway request evidence is missing",
+ )
+ worker = child.worker_observations if isinstance(child.worker_observations, dict) else {}
+ return {
+ "case_id": _BENCHMARK_CASES[index],
+ "lab_run_id": str(child.id),
+ "status": child.status,
+ "failure_code": child.failure_code or gateway.failure_code,
+ "gateway": {
+ "cold": gateway.cold,
+ "queue_time_ms": _finite_nonnegative(gateway.queue_time_ms),
+ "load_time_ms": _finite_nonnegative(gateway.load_time_ms),
+ "inference_time_ms": _finite_nonnegative(gateway.inference_time_ms),
+ "total_latency_ms": _finite_nonnegative(gateway.total_latency_ms),
+ "created_at": _aware(gateway.created_at).isoformat(),
+ "finished_at": _aware(gateway.finished_at).isoformat()
+ if gateway.finished_at
+ else None,
+ "input_count": gateway.input_count,
+ },
+ "worker_observations": worker,
+ }
+
+ @staticmethod
+ def _refresh_metrics(results: dict[str, Any], environment: dict[str, Any]) -> dict[str, Any]:
+ execution = results.get("execution")
+ attempts = execution.get("attempts", []) if isinstance(execution, dict) else []
+ if not isinstance(attempts, list):
+ attempts = []
+ gateways = [item.get("gateway", {}) for item in attempts if isinstance(item, dict)]
+ total_latencies = [
+ value
+ for gateway in gateways
+ if isinstance(gateway, dict)
+ if (value := _finite_nonnegative(gateway.get("total_latency_ms"))) is not None
+ ]
+ warm_latencies = [
+ value
+ for gateway in gateways
+ if isinstance(gateway, dict) and gateway.get("cold") is False
+ if (value := _finite_nonnegative(gateway.get("total_latency_ms"))) is not None
+ ]
+ cold_load = [
+ value
+ for gateway in gateways
+ if isinstance(gateway, dict) and gateway.get("cold") is True
+ if (value := _finite_nonnegative(gateway.get("load_time_ms"))) is not None
+ ]
+ failures = sum(
+ 1 for item in attempts if isinstance(item, dict) and item.get("status") != "completed"
+ )
+ successful = [
+ item
+ for item in attempts
+ if isinstance(item, dict) and item.get("status") == "completed"
+ ]
+ measured_inference: list[tuple[int, float]] = []
+ successful_vectors = 0
+ for item in successful:
+ gateway = item.get("gateway")
+ if not isinstance(gateway, dict):
+ continue
+ input_count = gateway.get("input_count")
+ if (
+ isinstance(input_count, int)
+ and not isinstance(input_count, bool)
+ and input_count > 0
+ ):
+ successful_vectors += input_count
+ inference_ms_value = _finite_nonnegative(gateway.get("inference_time_ms"))
+ if (
+ inference_ms_value is not None
+ and inference_ms_value > 0
+ and isinstance(input_count, int)
+ and not isinstance(input_count, bool)
+ and input_count > 0
+ ):
+ measured_inference.append((input_count, inference_ms_value))
+ inference_ms = sum(value for _, value in measured_inference)
+ vectors = sum(count for count, _ in measured_inference)
+ times: list[datetime] = []
+ for gateway in gateways:
+ if not isinstance(gateway, dict):
+ continue
+ for key in ("created_at", "finished_at"):
+ value = gateway.get(key)
+ if isinstance(value, str):
+ try:
+ times.append(datetime.fromisoformat(value))
+ except ValueError:
+ continue
+ elapsed_ms = (
+ max(0.0, (_aware(max(times)) - _aware(min(times))).total_seconds() * 1000)
+ if len(times) >= 2
+ else None
+ )
+ samples: list[dict[str, float]] = []
+ for item in attempts:
+ if not isinstance(item, dict):
+ continue
+ observation = item.get("worker_observations")
+ if not isinstance(observation, dict):
+ continue
+ worker_metrics = observation.get("metrics")
+ if not isinstance(worker_metrics, dict):
+ continue
+ if observation.get("device_matches_target") is True:
+ safe_metrics = {
+ key: value
+ for key in (
+ "used_vram_bytes",
+ "process_used_vram_bytes",
+ "free_vram_bytes",
+ "gpu_utilization_percent",
+ )
+ if (value := _finite_nonnegative(worker_metrics.get(key))) is not None
+ and (key != "gpu_utilization_percent" or value <= 100)
+ }
+ if safe_metrics:
+ samples.append(safe_metrics)
+ observed_values = {
+ key: [
+ value
+ for item in samples
+ if (value := _finite_nonnegative(item.get(key))) is not None
+ ]
+ for key in (
+ "used_vram_bytes",
+ "process_used_vram_bytes",
+ "free_vram_bytes",
+ "gpu_utilization_percent",
+ )
+ }
+ initial_gateway = gateways[0] if gateways and isinstance(gateways[0], dict) else {}
+ metrics = {
+ "initial_request_latency_ms": _finite_nonnegative(
+ initial_gateway.get("total_latency_ms")
+ ),
+ "cold_load_latency_ms": {
+ "p50": _percentile(cold_load, 0.50),
+ "p95": _percentile(cold_load, 0.95),
+ "samples": len(cold_load),
+ },
+ "warm_latency_ms": {
+ "p50": _percentile(warm_latencies, 0.50),
+ "p95": _percentile(warm_latencies, 0.95),
+ "samples": len(warm_latencies),
+ },
+ "total_latency_ms": {
+ "p50": _percentile(total_latencies, 0.50),
+ "p95": _percentile(total_latencies, 0.95),
+ "samples": len(total_latencies),
+ },
+ "throughput": {
+ "successful_vectors_per_inference_second": (
+ vectors / (inference_ms / 1000) if inference_ms > 0 else None
+ ),
+ "inference_samples": len(measured_inference),
+ "measured_vectors": vectors,
+ "successful_vectors_coverage": (
+ vectors / successful_vectors if successful_vectors > 0 else None
+ ),
+ "successful_calls_per_observed_second": (
+ len(successful) / (elapsed_ms / 1000)
+ if elapsed_ms is not None and elapsed_ms > 0
+ else None
+ ),
+ "observed_elapsed_ms": elapsed_ms,
+ },
+ "failures": {
+ "attempted": len(attempts),
+ "failed": failures,
+ "rate": failures / len(attempts) if attempts else None,
+ },
+ "gpu_vram": {
+ "telemetry_status": "MEASURED" if samples else "UNKNOWN",
+ "vram_status": (
+ "MEASURED"
+ if any(
+ observed_values[key]
+ for key in (
+ "used_vram_bytes",
+ "process_used_vram_bytes",
+ "free_vram_bytes",
+ )
+ )
+ else "UNKNOWN"
+ ),
+ "gpu_utilization_status": (
+ "MEASURED" if observed_values["gpu_utilization_percent"] else "UNKNOWN"
+ ),
+ "source": "runtime-worker" if samples else None,
+ "measured_samples": len(samples),
+ "peak_used_vram_bytes": max(observed_values["used_vram_bytes"], default=None),
+ "peak_process_used_vram_bytes": max(
+ observed_values["process_used_vram_bytes"], default=None
+ ),
+ "minimum_free_vram_bytes": min(observed_values["free_vram_bytes"], default=None),
+ "peak_gpu_utilization_percent": max(
+ observed_values["gpu_utilization_percent"], default=None
+ ),
+ "target_matched_samples": len(samples),
+ "target_accelerator_uuid": environment.get("accelerator_uuid"),
+ },
+ }
+ current_metrics = results.get("metrics")
+ merged = {
+ **(current_metrics if isinstance(current_metrics, dict) else {}),
+ **metrics,
+ "environment_comparable": bool(
+ current_metrics.get("environment_comparable")
+ if isinstance(current_metrics, dict)
+ else False
+ ),
+ "quality": {
+ "status": "UNKNOWN",
+ "metrics": {"recall_at_k": None, "mrr": None, "ndcg_at_k": None},
+ "reason": "PROJECT_CORPUS_NOT_INCLUDED_IN_PERFORMANCE_SUITE",
+ },
+ }
+ return {**results, "metrics": merged}
+
+ def reconcile_one(self, run_id: UUID) -> bool:
+ try:
+ return self._advance(run_id)
+ except StaleDataError:
+ # The row generation is the portable fallback when a test/development SQLite
+ # database ignores SELECT FOR UPDATE. A concurrent reconciler will retry next tick.
+ self.session.rollback()
+ return False
+ except ServingError:
+ raise
+
+ def reconcile(self) -> dict[str, int]:
+ driven = 0
+ run_ids = list(
+ self.session.scalars(
+ select(BenchmarkRun.id)
+ .where(BenchmarkRun.status.in_(("accepted", "running", "cancel_requested")))
+ .order_by(BenchmarkRun.created_at)
+ .limit(100)
+ )
+ )
+ for run_id in run_ids:
+ try:
+ driven += int(self.reconcile_one(run_id))
+ except ServingError:
+ # Admission failures are persisted as typed terminal benchmark outcomes by
+ # _launch_next; unexpected errors remain visible to the normal reconciler.
+ self.session.rollback()
+ raise
+ return {"benchmark_runs_driven": driven}
diff --git a/backend/src/modelforge_api/services/configuration.py b/backend/src/modelforge_api/services/configuration.py
new file mode 100644
index 0000000..5c1ecf0
--- /dev/null
+++ b/backend/src/modelforge_api/services/configuration.py
@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlsplit, urlunsplit
+
+from pydantic import SecretStr
+
+from modelforge_api.domain.configuration import (
+ ConfigurationApplicationMode,
+ ConfigurationCatalogResponse,
+ ConfigurationSettingResponse,
+ ConfigurationSource,
+)
+from modelforge_api.domain.configuration_policy import CONFIGURATION_CONSUMER, SERVING_OVERRIDE_KEYS
+from modelforge_api.domain.configuration_reference import SETTING_DOCS, Sensitivity
+from modelforge_api.settings import Settings
+
+
+def _environment_variable_name(setting_key: str) -> str:
+ return f"MODELFORGE_{setting_key.upper()}"
+
+
+def _is_configured(value: object) -> bool:
+ if value is None:
+ return False
+ if isinstance(value, SecretStr):
+ return bool(value.get_secret_value())
+ if isinstance(value, str):
+ return bool(value)
+ return True
+
+
+def _redact_url_userinfo(value: str) -> str:
+ """Remove URL userinfo even when a legacy setting is not classified as SECRET."""
+
+ try:
+ parsed = urlsplit(value)
+ except ValueError:
+ return value
+ if not parsed.scheme or "@" not in parsed.netloc:
+ return value
+ # Keep useful topology without leaking a username or password. rsplit handles userinfo that
+ # legitimately contains percent-encoded separators while preserving the complete host/port.
+ safe_netloc = parsed.netloc.rsplit("@", 1)[-1]
+ return urlunsplit((parsed.scheme, safe_netloc, parsed.path, parsed.query, parsed.fragment))
+
+
+def _safe_effective_value(
+ value: object, sensitivity: Sensitivity
+) -> str | int | float | bool | None:
+ if sensitivity is Sensitivity.SECRET:
+ return None
+ if value is None:
+ return None
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, str):
+ return _redact_url_userinfo(value)
+ if isinstance(value, int | float):
+ return value
+ if isinstance(value, Path):
+ return str(value)
+ # Future settings may gain structured values. A read-only catalog must not guess a wire format
+ # or accidentally serialise an object with surprising/revealing repr semantics.
+ return None
+
+
+def build_configuration_catalog(
+ settings: Settings,
+ *,
+ revision: int = 0,
+ overrides: dict[str, int | float] | None = None,
+) -> ConfigurationCatalogResponse:
+ """Return the runtime configuration reference without exposing credential material."""
+
+ setting_fields = set(type(settings).model_fields)
+ documented_fields = set(SETTING_DOCS)
+ if setting_fields != documented_fields:
+ missing_docs = sorted(setting_fields - documented_fields)
+ unknown_docs = sorted(documented_fields - setting_fields)
+ raise RuntimeError(
+ "configuration reference is out of sync with Settings: "
+ f"missing_docs={missing_docs}, unknown_docs={unknown_docs}"
+ )
+
+ active = overrides or {}
+ schema = Settings.model_json_schema()["properties"]
+ entries: list[ConfigurationSettingResponse] = []
+ for key in type(settings).model_fields:
+ documentation = SETTING_DOCS[key]
+ environment_variable = _environment_variable_name(key)
+ value = getattr(settings, key)
+ field = type(settings).model_fields[key]
+ rules = {
+ name: rule
+ for name, rule in schema[key].items()
+ if name in {"minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "maxLength"}
+ }
+ editable = key in SERVING_OVERRIDE_KEYS
+ if key == "scheduler_safety_reserve_percentage":
+ rules.pop("maximum", None)
+ rules["exclusiveMaximum"] = 0.5
+ if key == "gateway_queue_timeout_seconds":
+ rules["less_than_setting"] = "gateway_request_timeout_seconds"
+ entries.append(
+ ConfigurationSettingResponse(
+ key=key,
+ display_name=key.replace("_", " ").capitalize(),
+ category=key.split("_", 1)[0],
+ data_type=schema[key].get("type", "optional"),
+ default_value=_safe_effective_value(field.default, documentation.sensitivity),
+ validation_rules=rules,
+ editable=editable,
+ consumer=CONFIGURATION_CONSUMER if editable else None,
+ rollback=(
+ "Preview a previous revision or remove the override, then apply."
+ if editable
+ else "Restore deployment configuration and restart the component."
+ ),
+ environment_variable=environment_variable,
+ description=documentation.description,
+ sensitivity=documentation.sensitivity,
+ required_in_production=documentation.required_in_production,
+ application_mode=(
+ ConfigurationApplicationMode.DYNAMIC
+ if editable
+ else ConfigurationApplicationMode.RESTART_REQUIRED
+ ),
+ configured=_is_configured(value),
+ source=(
+ ConfigurationSource.OPERATOR_OVERRIDE
+ if key in active
+ else ConfigurationSource(settings.setting_source(key))
+ ),
+ value=_safe_effective_value(value, documentation.sensitivity),
+ )
+ )
+
+ return ConfigurationCatalogResponse(
+ environment=settings.env,
+ settings=entries,
+ read_only=False,
+ revision=revision,
+ overrides=active,
+ )
diff --git a/backend/src/modelforge_api/services/configuration_changes.py b/backend/src/modelforge_api/services/configuration_changes.py
new file mode 100644
index 0000000..0e50b00
--- /dev/null
+++ b/backend/src/modelforge_api/services/configuration_changes.py
@@ -0,0 +1,326 @@
+"""Atomic, preview-bound safe overrides; all persisted payloads are numeric and allowlisted."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import math
+from datetime import UTC, datetime
+
+from pydantic import ValidationError
+from sqlalchemy import select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.db import rows_affected
+from modelforge_api.domain.configuration import (
+ ConfigurationApplied,
+ ConfigurationApply,
+ ConfigurationChange,
+ ConfigurationDifference,
+ ConfigurationExport,
+ ConfigurationImport,
+ ConfigurationPreview,
+ ConfigurationRevisionResponse,
+ ConfigurationSource,
+)
+from modelforge_api.domain.configuration_policy import (
+ CONFIGURATION_CONSUMER,
+ CONFIGURATION_FORMAT,
+ SERVING_OVERRIDE_KEYS,
+)
+from modelforge_api.domain.configuration_reference import SETTING_DOCS, Sensitivity
+from modelforge_api.domain.release import PRODUCT_VERSION
+from modelforge_api.persistence.models import ConfigurationRevision, ConfigurationState
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings
+
+
+class ConfigurationError(Exception):
+ def __init__(self, code: str, message: str, status_code: int = 409) -> None:
+ super().__init__(message)
+ self.code = code
+ self.message = message
+ self.status_code = status_code
+
+
+def _validate_overrides(settings: Settings, overrides: dict[str, int | float]) -> Settings:
+ for key, value in overrides.items():
+ if key not in SERVING_OVERRIDE_KEYS:
+ raise ConfigurationError(
+ "CONFIGURATION_KEY_NOT_EDITABLE",
+ "Only the documented serving settings can be changed here.",
+ 422,
+ )
+ if type(value) not in (int, float) or (
+ isinstance(value, float) and not math.isfinite(value)
+ ):
+ raise ConfigurationError("CONFIGURATION_INVALID", "A finite number is required.", 422)
+ if Settings.model_fields[key].annotation is int and type(value) is not int:
+ raise ConfigurationError("CONFIGURATION_INVALID", f"{key} requires an integer.", 422)
+ try:
+ candidate = Settings.model_validate(settings.model_dump() | overrides)
+ candidate._inherit_setting_sources(settings)
+ except ValidationError as exc:
+ # Validation input may contain credentials from the deployment settings. Never echo it.
+ raise ConfigurationError(
+ "CONFIGURATION_INVALID",
+ "A value is outside the documented validation bounds.",
+ 422,
+ ) from exc
+ if candidate.gateway_queue_timeout_seconds >= candidate.gateway_request_timeout_seconds:
+ raise ConfigurationError(
+ "CONFIGURATION_INVALID",
+ "Queue timeout must be lower than request timeout.",
+ 422,
+ )
+ if candidate.scheduler_safety_reserve_percentage >= 0.5:
+ raise ConfigurationError(
+ "CONFIGURATION_INVALID",
+ "The GPU safety reserve must be below 50%.",
+ 422,
+ )
+ return candidate
+
+
+def configuration_state(session: Session) -> tuple[int, dict[str, int | float]]:
+ row = session.execute(
+ select(
+ ConfigurationState.revision,
+ ConfigurationState.overrides,
+ ).where(ConfigurationState.id == 1)
+ ).first()
+ if row is None:
+ # Empty create_all test databases use the same first-write CAS as a fresh registry.
+ return 0, {}
+ return row.revision, dict(row.overrides)
+
+
+def resolve_serving_settings(session: Session, settings: Settings) -> Settings:
+ return resolved_configuration(session, settings)[2]
+
+
+def resolved_configuration(
+ session: Session,
+ settings: Settings,
+) -> tuple[int, dict[str, int | float], Settings]:
+ revision, overrides = configuration_state(session)
+ return revision, overrides, _validate_overrides(settings, overrides) if overrides else settings
+
+
+class ConfigurationService:
+ def __init__(self, session: Session, settings: Settings, context: AuditContext) -> None:
+ self.session = session
+ self.settings = settings
+ self.context = context
+
+ def preview(self, request: ConfigurationChange) -> ConfigurationPreview:
+ revision, current = configuration_state(self.session)
+ if request.expected_revision != revision:
+ raise ConfigurationError(
+ "CONFIGURATION_CONFLICT", "Configuration changed; refresh and preview again."
+ )
+ desired = dict(current)
+ for key, value in request.changes.items():
+ if key not in SERVING_OVERRIDE_KEYS:
+ raise ConfigurationError(
+ "CONFIGURATION_KEY_NOT_EDITABLE",
+ "The requested setting is deployment-managed.",
+ 422,
+ )
+ if value is None:
+ desired.pop(key, None)
+ else:
+ desired[key] = value
+ before = _validate_overrides(self.settings, current)
+ after = _validate_overrides(self.settings, desired)
+ differences = [
+ ConfigurationDifference(
+ key=key,
+ before=getattr(before, key),
+ after=getattr(after, key),
+ source_before=(
+ ConfigurationSource.OPERATOR_OVERRIDE
+ if key in current
+ else ConfigurationSource(self.settings.setting_source(key))
+ ),
+ source_after=(
+ ConfigurationSource.OPERATOR_OVERRIDE
+ if key in desired
+ else ConfigurationSource(self.settings.setting_source(key))
+ ),
+ )
+ for key in sorted(request.changes)
+ if getattr(before, key) != getattr(after, key) or (key in current) != (key in desired)
+ ]
+ if not differences:
+ raise ConfigurationError(
+ "CONFIGURATION_NO_CHANGE", "The proposal does not change configuration.", 422
+ )
+ # Include every editable base value: deployment changes invalidate earlier previews, even
+ # if the operator revision has not advanced. No credential enters this digest.
+ baseline = {key: getattr(self.settings, key) for key in sorted(SERVING_OVERRIDE_KEYS)}
+ payload = {
+ "format": CONFIGURATION_FORMAT,
+ "revision": revision,
+ "base": baseline,
+ "current": current,
+ "desired": desired,
+ }
+ digest = hashlib.sha256(
+ json.dumps(
+ payload,
+ sort_keys=True,
+ separators=(",", ":"),
+ allow_nan=False,
+ ).encode()
+ ).hexdigest()
+ return ConfigurationPreview(
+ expected_revision=revision,
+ preview_digest=digest,
+ differences=differences,
+ affected_components=[CONFIGURATION_CONSUMER],
+ overrides=desired,
+ impact="New serving requests use this snapshot. In-flight requests retain their settings. "
+ "Admission and queue changes may reject new work; no model is promoted or worker restarted.",
+ rollback="Preview a retained revision or remove an override, then explicitly apply it as a new revision.",
+ )
+
+ def apply(self, request: ConfigurationApply) -> ConfigurationApplied:
+ preview = self.preview(request)
+ if not hmac.compare_digest(request.preview_digest, preview.preview_digest):
+ raise ConfigurationError(
+ "CONFIGURATION_PREVIEW_STALE", "The proposal changed; preview it again."
+ )
+ next_revision = request.expected_revision + 1
+ try:
+ result = self.session.execute(
+ update(ConfigurationState)
+ .where(
+ ConfigurationState.id == 1,
+ ConfigurationState.revision == request.expected_revision,
+ )
+ .values(revision=next_revision, overrides=preview.overrides)
+ )
+ if rows_affected(result) != 1:
+ if request.expected_revision != 0:
+ raise ConfigurationError(
+ "CONFIGURATION_CONFLICT", "A concurrent configuration change won."
+ )
+ self.session.add(ConfigurationState(id=1, revision=1, overrides=preview.overrides))
+ self.session.flush() # Unique singleton also protects the empty-registry race.
+ self.session.add(
+ ConfigurationRevision(
+ revision=next_revision,
+ previous_revision=request.expected_revision,
+ created_at=datetime.now(UTC),
+ actor=self.context.actor_id,
+ correlation_id=self.context.correlation_id or "unknown",
+ overrides=preview.overrides,
+ preview_digest=preview.preview_digest,
+ )
+ )
+ self.session.flush()
+ # Exercise the exact reader used by every new ServingService before acknowledging.
+ resolve_serving_settings(self.session, self.settings)
+ AuditWriter(self.session, context=self.context).write(
+ "configuration.applied",
+ "configuration_revision",
+ str(next_revision),
+ {
+ "previous_revision": request.expected_revision,
+ "preview_digest": preview.preview_digest,
+ "differences": [item.model_dump() for item in preview.differences],
+ "acknowledged_by": CONFIGURATION_CONSUMER,
+ },
+ )
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise ConfigurationError(
+ "CONFIGURATION_CONFLICT", "A concurrent configuration change won."
+ ) from exc
+ except Exception:
+ self.session.rollback()
+ raise
+ return ConfigurationApplied(
+ revision=next_revision,
+ acknowledged_by=CONFIGURATION_CONSUMER,
+ preview_digest=preview.preview_digest,
+ )
+
+ def history(self, limit: int = 30) -> list[ConfigurationRevisionResponse]:
+ rows = self.session.scalars(
+ select(ConfigurationRevision)
+ .order_by(
+ ConfigurationRevision.revision.desc(),
+ )
+ .limit(limit)
+ )
+ return [
+ ConfigurationRevisionResponse.model_validate(row, from_attributes=True) for row in rows
+ ]
+
+ def rollback_change(self, revision: int, expected_revision: int) -> ConfigurationChange:
+ _, current = configuration_state(self.session)
+ if revision == 0:
+ desired: dict[str, int | float] = {}
+ else:
+ row = self.session.get(ConfigurationRevision, revision)
+ if row is None:
+ raise ConfigurationError(
+ "CONFIGURATION_REVISION_NOT_FOUND", "Configuration revision was not found.", 404
+ )
+ desired = row.overrides
+ if desired == current:
+ raise ConfigurationError("CONFIGURATION_NO_CHANGE", "That revision is already effective.", 422)
+ return ConfigurationChange(
+ expected_revision=expected_revision,
+ changes={key: desired.get(key) for key in current.keys() | desired.keys()},
+ )
+
+ def export(self) -> ConfigurationExport:
+ _, overrides = configuration_state(self.session)
+ _validate_overrides(self.settings, overrides)
+ return ConfigurationExport(
+ format=CONFIGURATION_FORMAT,
+ source_version=PRODUCT_VERSION,
+ overrides=overrides,
+ secret_slots=sorted({"huggingface_control_plane"} | {
+ key for key, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET
+ }),
+ )
+
+ def import_change(self, request: ConfigurationImport) -> ConfigurationChange:
+ document = request.document
+ if document.format != CONFIGURATION_FORMAT:
+ raise ConfigurationError(
+ "CONFIGURATION_FORMAT_UNSUPPORTED", "Unsupported configuration format.", 422
+ )
+ if document.source_version.split(".")[0] != PRODUCT_VERSION.split(".")[0]:
+ raise ConfigurationError(
+ "CONFIGURATION_VERSION_UNSUPPORTED", "Incompatible product major version.", 422
+ )
+ expected_slots = {"huggingface_control_plane"} | {
+ key for key, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET
+ }
+ if (
+ len(document.secret_slots) != len(set(document.secret_slots))
+ or set(document.secret_slots) != expected_slots
+ ):
+ raise ConfigurationError(
+ "CONFIGURATION_SLOTS_INVALID",
+ "Secret slot references do not match this product.",
+ 422,
+ )
+ _validate_overrides(self.settings, document.overrides)
+ _, current = configuration_state(self.session)
+ changes = {
+ key: document.overrides.get(key) for key in current.keys() | document.overrides.keys()
+ }
+ if not changes:
+ raise ConfigurationError(
+ "CONFIGURATION_NO_CHANGE", "The import contains no overrides to change.", 422
+ )
+ return ConfigurationChange(expected_revision=request.expected_revision, changes=changes)
diff --git a/backend/src/modelforge_api/services/evaluation.py b/backend/src/modelforge_api/services/evaluation.py
index 77818f3..8877a0e 100644
--- a/backend/src/modelforge_api/services/evaluation.py
+++ b/backend/src/modelforge_api/services/evaluation.py
@@ -130,8 +130,16 @@ def _percentile(values: list[float], fraction: float) -> float:
class EvaluationService:
- def __init__(self, session: Session) -> None:
+ def __init__(self, session: Session, *, commit: bool = True) -> None:
self.session = session
+ self._commit_results = commit
+
+ def _finish_result(self, row: EvaluationRun | EvaluationComparison) -> None:
+ if self._commit_results:
+ self.session.commit()
+ else:
+ self.session.flush()
+ self.session.refresh(row)
def create_suite(self, request: EvaluationSuiteCreate) -> EvaluationSuiteResponse:
if self.session.get(Project, request.project_id) is None:
@@ -220,6 +228,8 @@ class EvaluationService:
"project or suite revision not found",
status_code=404,
)
+ if not self._suite_belongs_to_project(revision.id, request.project_id):
+ raise EvaluationError("suite_project_mismatch", "suite revision belongs to another project")
cases = list(
self.session.scalars(
select(EvaluationCase).where(EvaluationCase.suite_revision_id == revision.id)
@@ -254,8 +264,7 @@ class EvaluationService:
started_at=datetime.now(UTC),
)
self.session.add(run)
- self.session.commit()
- self.session.refresh(run)
+ self._finish_result(run)
return EvaluationRunResponse.model_validate(run)
def complete_run(
@@ -266,6 +275,8 @@ class EvaluationService:
raise EvaluationError("run_not_found", "evaluation run not found", status_code=404)
if run.status != "running":
raise EvaluationError("run_not_active", "only a running evaluation can complete")
+ if not self._suite_belongs_to_project(run.suite_revision_id, run.project_id):
+ raise EvaluationError("suite_project_mismatch", "suite revision belongs to another project")
cases = {
case.id: case
for case in self.session.scalars(
@@ -310,10 +321,17 @@ class EvaluationService:
run.aggregate_metrics = {name: totals[name] / len(request.results) for name in METRICS}
run.status = "completed" if errors == 0 else "completed_with_errors"
run.completed_at = datetime.now(UTC)
- self.session.commit()
- self.session.refresh(run)
+ self._finish_result(run)
return EvaluationRunResponse.model_validate(run)
+ def _suite_belongs_to_project(self, revision_id: uuid.UUID, project_id: uuid.UUID) -> bool:
+ owner = self.session.scalar(
+ select(EvaluationSuite.project_id)
+ .join(EvaluationSuiteRevision, EvaluationSuiteRevision.suite_id == EvaluationSuite.id)
+ .where(EvaluationSuiteRevision.id == revision_id)
+ )
+ return owner == project_id
+
def runs(self) -> list[EvaluationRunResponse]:
return [
EvaluationRunResponse.model_validate(item)
@@ -758,6 +776,8 @@ class EvaluationService:
raise EvaluationError("run_incomplete", "both runs must be complete")
factors = {
"project": baseline.project_id == candidate.project_id,
+ "baseline_suite_project": self._suite_belongs_to_project(baseline.suite_revision_id, baseline.project_id),
+ "candidate_suite_project": self._suite_belongs_to_project(candidate.suite_revision_id, candidate.project_id),
"suite_revision": baseline.suite_revision_id == candidate.suite_revision_id,
"corpus_revision": baseline.corpus_revision == candidate.corpus_revision,
"retrieval_config": baseline.retrieval_config_digest
@@ -765,7 +785,8 @@ class EvaluationService:
}
if all(factors.values()):
comparability = "comparable"
- elif factors["project"] and factors["suite_revision"]:
+ elif (factors["project"] and factors["suite_revision"]
+ and factors["baseline_suite_project"] and factors["candidate_suite_project"]):
comparability = "partially_comparable"
else:
comparability = "not_comparable"
@@ -817,8 +838,10 @@ class EvaluationService:
for name in METRICS
}
error_rate = candidate.error_count / candidate.expected_cases
+ baseline_error_free = baseline.status == "completed" and baseline.error_count == 0
eligible = (
comparability == "comparable"
+ and baseline_error_free
and error_rate <= request.maximum_error_rate
and deltas["recall_at_10"] >= request.minimum_recall_at_10_delta
and (not request.require_no_critical_regressions or critical_regressions == 0)
@@ -843,13 +866,14 @@ class EvaluationService:
promotion_eligibility=eligibility,
eligibility_evidence={
"error_rate": error_rate,
+ "baseline_error_count": baseline.error_count,
+ "baseline_error_free": baseline_error_free,
"criteria": request.model_dump(mode="json"),
"promotion_performed": False,
},
)
self.session.add(comparison)
- self.session.commit()
- self.session.refresh(comparison)
+ self._finish_result(comparison)
return EvaluationComparisonResponse.model_validate(comparison)
def comparison(self, comparison_id: uuid.UUID) -> EvaluationComparisonResponse:
diff --git a/backend/src/modelforge_api/services/external_creative.py b/backend/src/modelforge_api/services/external_creative.py
new file mode 100644
index 0000000..cd85ec9
--- /dev/null
+++ b/backend/src/modelforge_api/services/external_creative.py
@@ -0,0 +1,770 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import cast
+
+import sqlalchemy as sa
+from sqlalchemy import func, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.external_creative import (
+ CreativeAllocationState,
+ CreativeRequiredAction,
+ CreativeWorkload,
+ ExternalActionAttestation,
+ ExternalAllocationCreate,
+ ExternalAllocationList,
+ ExternalAllocationResponse,
+ ExternalProfileObservation,
+ ExternalResourceApproval,
+ ExternalResourceList,
+ ExternalResourceResponse,
+)
+from modelforge_api.persistence.models import (
+ Accelerator,
+ ExternalCreativeActionAttestation,
+ ExternalCreativeAllocation,
+ ExternalCreativeProfileObservation,
+ ExternalCreativeResource,
+ Project,
+ ProjectBinding,
+ ServiceClient,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.serving import ServingError, ServingService
+
+ACTIVE = {"granted", "uncertain"}
+PRIORITY = {"production": 0, "interactive": 1, "background": 2, "benchmark": 3}
+resources = cast(sa.Table, ExternalCreativeResource.__table__)
+attestations = cast(sa.Table, ExternalCreativeActionAttestation.__table__)
+observations = cast(sa.Table, ExternalCreativeProfileObservation.__table__)
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _digest(value: ExternalAllocationCreate) -> str:
+ return hashlib.sha256(
+ json.dumps(value.model_dump(mode="json"), sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+
+
+class ExternalCreativeService:
+ def __init__(self, session: Session, serving: ServingService):
+ self.session, self.serving = session, serving
+
+ def _lock_accelerators(self) -> None:
+ self.session.scalars(select(Accelerator).order_by(Accelerator.id).with_for_update()).all()
+
+ def _project(self, client: ServiceClient) -> uuid.UUID:
+ fresh = self.session.get(
+ ServiceClient, client.id, populate_existing=True, with_for_update=True
+ )
+ binding = (
+ self.session.get(
+ ProjectBinding,
+ fresh.project_binding_id,
+ populate_existing=True,
+ with_for_update=True,
+ )
+ if fresh and fresh.project_binding_id
+ else None
+ )
+ project = (
+ self.session.get(
+ Project, binding.project_id, populate_existing=True, with_for_update=True
+ )
+ if binding
+ else None
+ )
+ if (
+ fresh is None
+ or fresh.status != "active"
+ or fresh.disabled_at is not None
+ or "creative.resource@1" not in fresh.allowed_capabilities
+ or binding is None
+ or binding.deprecated_at is not None
+ or project is None
+ or not project.active
+ ):
+ raise ServingError(
+ 403,
+ "PROJECT_BINDING_REQUIRED",
+ "creative coordination requires an active project-bound client",
+ )
+ return binding.project_id
+
+ def _priority(self, client: ServiceClient, requested: str) -> str:
+ binding = self.session.get(ProjectBinding, client.project_binding_id)
+ allowed = max(
+ PRIORITY.get(client.workload_priority, 99),
+ PRIORITY.get(binding.priority, 99) if binding else 99,
+ )
+ if PRIORITY.get(requested, -1) < allowed:
+ raise ServingError(
+ 403,
+ "ALLOCATION_PRIORITY_FORBIDDEN",
+ "requested priority exceeds client or project policy",
+ )
+ return requested
+
+ def _expire(self, row: ExternalCreativeAllocation) -> bool:
+ expiry = row.expires_at
+ if row.state == "granted" and (
+ expiry is None
+ or (expiry.astimezone(UTC) if expiry.tzinfo else expiry.replace(tzinfo=UTC)) <= _now()
+ ):
+ row.state, row.uncertain_at, row.updated_at = "uncertain", _now(), _now()
+ row.expires_at = None
+ row.decision_code = "ALLOCATION_EXPIRED_UNCERTAIN"
+ self._audit("system", "external-creative-expiry").write(
+ "EXTERNAL_CREATIVE_ALLOCATION_EXPIRED",
+ "external_creative_allocation",
+ str(row.id),
+ {"state": row.state, "reservation_retained_bytes": row.required_vram_bytes},
+ )
+ return True
+ return False
+
+ def _expire_grants(self) -> None:
+ for row in self.session.scalars(
+ select(ExternalCreativeAllocation)
+ .where(ExternalCreativeAllocation.state == "granted")
+ .order_by(ExternalCreativeAllocation.id)
+ .with_for_update()
+ .execution_options(populate_existing=True)
+ ):
+ self._expire(row)
+ self.session.flush()
+
+ def _queued_authority(self, row: ExternalCreativeAllocation) -> bool:
+ client = self.session.get(ServiceClient, row.service_client_id, populate_existing=True)
+ try:
+ if client is None or self._project(client) != row.project_id:
+ return False
+ self._priority(client, row.priority)
+ except ServingError:
+ return False
+ resource = (
+ self.session.execute(
+ select(resources)
+ .where(
+ resources.c.project_id == row.project_id,
+ resources.c.resource_ref == row.resource_ref,
+ resources.c.workload == row.workload,
+ resources.c.provider_resource_id == row.provider_resource_id,
+ )
+ .with_for_update()
+ )
+ .mappings()
+ .one_or_none()
+ )
+ if resource is None or not resource["approved"]:
+ return False
+ row.required_vram_bytes = max(
+ row.required_vram_bytes,
+ resource["configured_vram_bytes"],
+ resource["measured_peak_vram_bytes"] or 0,
+ )
+ return True
+
+ def _audit(
+ self, client_or_type: ServiceClient | str, actor_id: str | None = None
+ ) -> AuditWriter:
+ if isinstance(client_or_type, ServiceClient):
+ return AuditWriter(self.session, "service_client", str(client_or_type.id))
+ return AuditWriter(self.session, client_or_type, actor_id or "unknown")
+
+ def _response(
+ self, row: ExternalCreativeAllocation, *, as_of: datetime | None = None
+ ) -> ExternalAllocationResponse:
+ expired = False
+ if as_of is not None and row.state == "granted":
+ expiry = row.expires_at
+ expired = (
+ expiry is None
+ or (expiry.astimezone(UTC) if expiry.tzinfo else expiry.replace(tzinfo=UTC))
+ <= as_of
+ )
+ return ExternalAllocationResponse(
+ allocation_id=row.id,
+ project_id=row.project_id,
+ request_id=row.request_id,
+ installation_id=row.decision_evidence.get("installation_id", "unknown"),
+ workload=cast(CreativeWorkload, row.workload),
+ resource_ref=row.resource_ref,
+ provider_resource_id=row.provider_resource_id,
+ required_vram_bytes=row.required_vram_bytes,
+ priority=row.priority,
+ state=cast(CreativeAllocationState, "uncertain" if expired else row.state),
+ decision_code=("ALLOCATION_EXPIRED_UNCERTAIN" if expired else row.decision_code),
+ required_actions=cast(list[CreativeRequiredAction], row.required_actions or []),
+ accelerator_id=row.accelerator_id,
+ acquired_at=row.acquired_at,
+ heartbeat_at=row.heartbeat_at,
+ expires_at=None if expired else row.expires_at,
+ uncertain_at=row.uncertain_at,
+ released_at=row.released_at,
+ decision_evidence=row.decision_evidence or {},
+ queue_position=(row.decision_evidence or {}).get("queue_position"),
+ )
+
+ def _promote_queue(self) -> None:
+ self._lock_accelerators()
+ self._expire_grants()
+ queued = list(
+ self.session.scalars(
+ select(ExternalCreativeAllocation)
+ .where(ExternalCreativeAllocation.state == "queued")
+ .order_by(ExternalCreativeAllocation.created_at, ExternalCreativeAllocation.id)
+ .with_for_update()
+ )
+ )
+ queued.sort(
+ key=lambda item: (PRIORITY.get(item.priority, 9), item.created_at, str(item.id))
+ )
+ # Snapshot already includes all existing external reservations.
+ reserved: dict[uuid.UUID, int] = {}
+ budgets, now = self.serving.scheduler_overview(commit=False), _now()
+ remaining_position = 0
+ for item in queued:
+ if not self._queued_authority(item):
+ item.state, item.decision_code = "blocked", "QUEUED_AUTHORITY_REVOKED"
+ item.decision_evidence = {**item.decision_evidence, "queue_position": None}
+ self._audit("system", "external-creative-admission").write(
+ "EXTERNAL_CREATIVE_ALLOCATION_BLOCKED",
+ "external_creative_allocation",
+ str(item.id),
+ {"decision_code": item.decision_code},
+ )
+ continue
+ chosen = next(
+ (
+ budget
+ for budget in budgets
+ if budget.attribution_confidence != "UNKNOWN"
+ and budget.pressure_state not in {"HIGH", "CRITICAL"}
+ and budget.schedulable_free_vram_bytes - reserved.get(budget.accelerator_id, 0)
+ >= item.required_vram_bytes
+ ),
+ None,
+ )
+ evidence = dict(item.decision_evidence or {})
+ if chosen is None:
+ remaining_position += 1
+ evidence["queue_position"] = remaining_position
+ item.decision_evidence = evidence
+ continue
+ item.state, item.decision_code, item.accelerator_id = (
+ "granted",
+ "CAPACITY_GRANTED_FROM_QUEUE",
+ chosen.accelerator_id,
+ )
+ item.acquired_at = item.heartbeat_at = item.updated_at = now
+ item.expires_at = now + timedelta(seconds=item.ttl_seconds)
+ # This grant is based on observed schedulable free VRAM. A forced
+ # unload would discard a reusable resident model despite admission
+ # already succeeding. Preparation is only meaningful when a future
+ # decision explicitly accounts for reclaimable provider memory.
+ item.required_actions = []
+ evidence.update(
+ {
+ "queue_position": None,
+ "promoted_at": now.isoformat(),
+ "policy_revision": chosen.policy_revision,
+ }
+ )
+ item.decision_evidence = evidence
+ self._audit("system", "external-creative-admission").write(
+ "EXTERNAL_CREATIVE_ALLOCATION_GRANTED",
+ "external_creative_allocation",
+ str(item.id),
+ {
+ "accelerator_id": str(chosen.accelerator_id),
+ "reserved_vram_bytes": item.required_vram_bytes,
+ },
+ )
+ reserved[chosen.accelerator_id] = (
+ reserved.get(chosen.accelerator_id, 0) + item.required_vram_bytes
+ )
+
+ def acquire(
+ self, request: ExternalAllocationCreate, client: ServiceClient
+ ) -> ExternalAllocationResponse:
+ self._lock_accelerators()
+ self._expire_grants()
+ project_id = self._project(client)
+ self._priority(client, request.priority)
+ digest = _digest(request)
+ previous = self.session.scalar(
+ select(ExternalCreativeAllocation).where(
+ ExternalCreativeAllocation.service_client_id == client.id,
+ ExternalCreativeAllocation.idempotency_key == request.idempotency_key,
+ )
+ )
+ if previous:
+ if previous.request_sha256 != digest:
+ raise ServingError(
+ 409,
+ "IDEMPOTENCY_CONFLICT",
+ "idempotency key belongs to another allocation request",
+ )
+ self.session.commit()
+ return self._response(previous)
+
+ binding = (
+ self.session.execute(
+ select(resources)
+ .where(
+ resources.c.project_id == project_id,
+ resources.c.resource_ref == request.resource_ref,
+ resources.c.workload == request.workload,
+ resources.c.provider_resource_id == request.provider_resource_id,
+ )
+ .with_for_update()
+ )
+ .mappings()
+ .one_or_none()
+ )
+ if binding is None or not binding["approved"]:
+ raise ServingError(
+ 403,
+ "RESOURCE_NOT_APPROVED",
+ "resource and provider binding is not approved for this project",
+ )
+ profiled = binding["measured_peak_vram_bytes"] if request.use_measured_profile else None
+ required_vram = max(
+ request.required_vram_bytes, profiled or 0, binding["configured_vram_bytes"]
+ )
+
+ # Give already-queued work first access to newly available capacity, while
+ # allowing requests that fit to bypass queue entries that still cannot.
+ self._promote_queue()
+ budgets = self.serving.scheduler_overview(commit=False)
+ chosen = next(
+ (
+ b
+ for b in budgets
+ if b.attribution_confidence != "UNKNOWN"
+ and b.pressure_state not in {"HIGH", "CRITICAL"}
+ and b.schedulable_free_vram_bytes >= required_vram
+ ),
+ None,
+ )
+ queued = list(
+ self.session.scalars(
+ select(ExternalCreativeAllocation).where(
+ ExternalCreativeAllocation.state == "queued"
+ )
+ )
+ )
+ queue_position = 1 + sum(
+ PRIORITY.get(item.priority, 9) <= PRIORITY[request.priority] for item in queued
+ )
+ state, code = ("granted", "CAPACITY_GRANTED") if chosen else ("queued", "CAPACITY_QUEUED")
+ observed_budget = chosen or max(
+ budgets, key=lambda budget: budget.schedulable_free_vram_bytes, default=None
+ )
+ actions: list[str] = []
+ evidence = {
+ "installation_id": request.installation_id,
+ "policy_revision": observed_budget.policy_revision
+ if observed_budget
+ else "unavailable",
+ "observed_at": observed_budget.observed_at.isoformat()
+ if observed_budget and observed_budget.observed_at
+ else None,
+ "headroom_before_bytes": (
+ observed_budget.schedulable_free_vram_bytes if observed_budget else 0
+ ),
+ "profile_source": "measured" if profiled else "configured",
+ "requested_vram_bytes": request.required_vram_bytes,
+ "queue_position": None if chosen else queue_position,
+ }
+ now = _now()
+ row = ExternalCreativeAllocation(
+ service_client_id=client.id,
+ project_id=project_id,
+ accelerator_id=chosen.accelerator_id if chosen else None,
+ request_id=request.request_id,
+ idempotency_key=request.idempotency_key,
+ request_sha256=digest,
+ workload=request.workload,
+ resource_ref=request.resource_ref,
+ provider_resource_id=request.provider_resource_id,
+ required_vram_bytes=required_vram,
+ priority=request.priority,
+ ttl_seconds=request.ttl_seconds,
+ state=state,
+ decision_code=code,
+ decision_evidence=evidence,
+ required_actions=actions if chosen else [],
+ acquired_at=now if chosen else None,
+ heartbeat_at=now if chosen else None,
+ expires_at=now + timedelta(seconds=request.ttl_seconds) if chosen else None,
+ )
+ self.session.add(row)
+ try:
+ self.session.flush()
+ self._audit(client).write(
+ "EXTERNAL_CREATIVE_ALLOCATION_DECIDED",
+ "external_creative_allocation",
+ str(row.id),
+ {
+ "project_id": str(project_id),
+ "workload": row.workload,
+ "resource_ref": row.resource_ref,
+ "state": row.state,
+ "decision_code": row.decision_code,
+ "accelerator_id": str(row.accelerator_id) if row.accelerator_id else None,
+ },
+ )
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise ServingError(
+ 409,
+ "ALLOCATION_CONFLICT",
+ "allocation was concurrently created; retry with the same idempotency key",
+ ) from exc
+ self.session.refresh(row)
+ return self._response(row)
+
+ def get(self, allocation_id: uuid.UUID, client: ServiceClient) -> ExternalAllocationResponse:
+ self._lock_accelerators()
+ self._project(client)
+ self._promote_queue()
+ row = self.session.get(ExternalCreativeAllocation, allocation_id)
+ if row is None or row.service_client_id != client.id:
+ raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
+ self.session.commit()
+ return self._response(row)
+
+ def action(
+ self,
+ allocation_id: uuid.UUID,
+ request_id: str,
+ action: str,
+ client: ServiceClient,
+ observed_peak_vram_bytes: int | None = None,
+ attestation: ExternalActionAttestation | None = None,
+ ) -> ExternalAllocationResponse:
+ self._lock_accelerators()
+ row = self.session.get(
+ ExternalCreativeAllocation,
+ allocation_id,
+ with_for_update=True,
+ populate_existing=True,
+ )
+ if row is None or row.service_client_id != client.id or row.request_id != request_id:
+ raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
+ if self._expire(row) and action == "heartbeat":
+ self.session.commit()
+ raise ServingError(
+ 409, "ALLOCATION_EXPIRED", "expired allocation retains an uncertain reservation"
+ )
+ if action == "heartbeat":
+ self._project(client)
+ if action == "attest" and attestation is None:
+ raise ServingError(422, "ATTESTATION_REQUIRED", "attestation is required")
+ now = _now()
+ if attestation is not None:
+ if attestation.action not in (row.required_actions or []):
+ raise ServingError(409, "ACTION_NOT_REQUIRED", "attested action was not required")
+ if attestation.observed_after > attestation.observed_before:
+ raise ServingError(
+ 422, "ATTESTATION_INVALID", "action attestation did not release VRAM"
+ )
+ self.session.execute(
+ attestations.insert().values(
+ id=uuid.uuid4(),
+ allocation_id=row.id,
+ action=attestation.action,
+ before_vram_bytes=attestation.observed_before,
+ after_vram_bytes=attestation.observed_after,
+ observed_at=attestation.completed_at,
+ validated=True,
+ created_at=now,
+ )
+ )
+ if action == "release":
+ if row.state != "released":
+ row.state, row.released_at, row.updated_at = "released", now, now
+ elif action == "uncertain":
+ if row.state != "released":
+ row.state, row.uncertain_at, row.expires_at, row.updated_at = (
+ "uncertain",
+ now,
+ None,
+ now,
+ )
+ elif action == "heartbeat":
+ if row.state != "granted":
+ raise ServingError(
+ 409, "ALLOCATION_NOT_ACTIVE", "only granted allocations can be renewed"
+ )
+ row.heartbeat_at, row.expires_at, row.updated_at = (
+ now,
+ now + timedelta(seconds=row.ttl_seconds),
+ now,
+ )
+ self._audit(client).write(
+ f"EXTERNAL_CREATIVE_ALLOCATION_{action.upper()}",
+ "external_creative_allocation",
+ str(row.id),
+ {"state": row.state, "workload": row.workload},
+ )
+ if action == "release":
+ self.session.flush()
+ self._promote_queue()
+ self.session.commit()
+ self.session.refresh(row)
+ return self._response(row)
+
+ def observe(
+ self, body: ExternalProfileObservation, client: ServiceClient
+ ) -> ExternalResourceResponse:
+ self._lock_accelerators()
+ self._project(client)
+ row = self.session.get(
+ ExternalCreativeAllocation,
+ body.allocation_id,
+ with_for_update=True,
+ populate_existing=True,
+ )
+ if row is None or row.service_client_id != client.id:
+ raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
+ if self._expire(row):
+ self.session.commit()
+ if (
+ row.acquired_at is None
+ or row.accelerator_id is None
+ or row.state not in {"granted", "released"}
+ or body.reserved_vram_bytes != row.required_vram_bytes
+ ):
+ raise ServingError(
+ 409,
+ "PROFILE_GRANT_REQUIRED",
+ "observation requires an actual grant and exact reserved bytes",
+ )
+ if (row.resource_ref, row.provider_resource_id, row.workload) != (
+ body.resource_ref,
+ body.provider_resource_id,
+ body.workload,
+ ):
+ raise ServingError(
+ 409, "PROFILE_BINDING_MISMATCH", "observation does not match its allocation"
+ )
+ record = (
+ self.session.execute(
+ select(resources)
+ .where(
+ resources.c.project_id == row.project_id,
+ resources.c.resource_ref == row.resource_ref,
+ resources.c.workload == row.workload,
+ resources.c.provider_resource_id == row.provider_resource_id,
+ )
+ .with_for_update()
+ )
+ .mappings()
+ .one()
+ )
+ previous = self.session.execute(
+ select(observations.c.id).where(observations.c.allocation_id == row.id)
+ ).scalar_one_or_none()
+ if previous is not None:
+ return self._resource(record["id"])
+ try:
+ self.session.execute(
+ observations.insert().values(
+ id=uuid.uuid4(),
+ allocation_id=row.id,
+ resource_ref=body.resource_ref,
+ provider_resource_id=body.provider_resource_id,
+ workload=body.workload,
+ observed_peak_vram_bytes=body.observed_peak_vram_bytes,
+ reserved_vram_bytes=row.required_vram_bytes,
+ duration_ms=body.duration_ms,
+ outcome=body.outcome,
+ created_at=_now(),
+ )
+ )
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise ServingError(
+ 409, "PROFILE_OBSERVATION_EXISTS", "allocation already has a profile observation"
+ ) from exc
+ self.session.execute(
+ resources.update()
+ .where(resources.c.id == record["id"])
+ .values(
+ measured_peak_vram_bytes=max(
+ record["measured_peak_vram_bytes"] or 0, body.observed_peak_vram_bytes
+ ),
+ measurement_count=record["measurement_count"] + 1,
+ updated_at=_now(),
+ )
+ )
+ self._audit(client).write(
+ "EXTERNAL_CREATIVE_PROFILE_OBSERVED",
+ "external_creative_resource",
+ str(record["id"]),
+ {"outcome": body.outcome, "observed_peak_vram_bytes": body.observed_peak_vram_bytes},
+ )
+ self.session.commit()
+ return self._resource(record["id"])
+
+ def _resource(self, identifier: uuid.UUID) -> ExternalResourceResponse:
+ record = (
+ self.session.execute(select(resources).where(resources.c.id == identifier))
+ .mappings()
+ .one()
+ )
+ return ExternalResourceResponse(
+ resource_id=identifier,
+ **{
+ key: record[key]
+ for key in ExternalResourceResponse.model_fields
+ if key != "resource_id"
+ },
+ )
+
+ def upsert_resource(
+ self, body: ExternalResourceApproval, actor_id: str
+ ) -> ExternalResourceResponse:
+ record = (
+ self.session.execute(
+ select(resources)
+ .where(
+ resources.c.project_id == body.project_id,
+ resources.c.resource_ref == body.resource_ref,
+ resources.c.workload == body.workload,
+ resources.c.provider_resource_id == body.provider_resource_id,
+ )
+ .with_for_update()
+ )
+ .mappings()
+ .one_or_none()
+ )
+ identifier, now = (record["id"] if record else uuid.uuid4()), _now()
+ values = {**body.model_dump(), "updated_at": now}
+ if record:
+ self.session.execute(
+ resources.update().where(resources.c.id == identifier).values(**values)
+ )
+ else:
+ self.session.execute(
+ resources.insert().values(
+ id=identifier, measured_peak_vram_bytes=None, measurement_count=0, **values
+ )
+ )
+ self._audit("operator", actor_id).write(
+ "EXTERNAL_CREATIVE_RESOURCE_CONFIGURED",
+ "external_creative_resource",
+ str(identifier),
+ {"project_id": str(body.project_id), "approved": body.approved},
+ )
+ self.session.commit()
+ return self._resource(identifier)
+
+ def list_allocations(
+ self, state: str | None, limit: int, offset: int
+ ) -> ExternalAllocationList:
+ observed_at = _now()
+ expired_grant = sa.and_(
+ ExternalCreativeAllocation.state == "granted",
+ sa.or_(
+ ExternalCreativeAllocation.expires_at.is_(None),
+ ExternalCreativeAllocation.expires_at <= observed_at,
+ ),
+ )
+ effective_state = sa.case(
+ (expired_grant, "uncertain"), else_=ExternalCreativeAllocation.state
+ )
+ condition = effective_state == state if state else sa.true()
+ with self.session.no_autoflush:
+ total = (
+ self.session.scalar(
+ select(func.count()).select_from(ExternalCreativeAllocation).where(condition)
+ )
+ or 0
+ )
+ rows = self.session.scalars(
+ select(ExternalCreativeAllocation)
+ .where(condition)
+ .order_by(ExternalCreativeAllocation.created_at.desc())
+ .offset(offset)
+ .limit(limit)
+ ).all()
+ return ExternalAllocationList(
+ items=[self._response(row, as_of=observed_at) for row in rows], total=total
+ )
+
+ def list_resources(
+ self, project_id: uuid.UUID | None, limit: int, offset: int
+ ) -> ExternalResourceList:
+ condition = resources.c.project_id == project_id if project_id else sa.true()
+ total = (
+ self.session.scalar(select(func.count()).select_from(resources).where(condition)) or 0
+ )
+ records = (
+ self.session.execute(
+ select(resources)
+ .where(condition)
+ .order_by(resources.c.updated_at.desc())
+ .offset(offset)
+ .limit(limit)
+ )
+ .mappings()
+ .all()
+ )
+ return ExternalResourceList(
+ items=[
+ ExternalResourceResponse(
+ resource_id=record["id"],
+ **{
+ key: record[key]
+ for key in ExternalResourceResponse.model_fields
+ if key != "resource_id"
+ },
+ )
+ for record in records
+ ],
+ total=total,
+ )
+
+ def reconcile(
+ self, allocation_id: uuid.UUID, action: str, reason: str, actor_id: str
+ ) -> ExternalAllocationResponse:
+ self._lock_accelerators()
+ row = self.session.get(
+ ExternalCreativeAllocation,
+ allocation_id,
+ with_for_update=True,
+ populate_existing=True,
+ )
+ if row is None:
+ raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
+ self._expire(row)
+ if row.state != "uncertain":
+ raise ServingError(
+ 409, "ALLOCATION_NOT_UNCERTAIN", "only uncertain allocations can be reconciled"
+ )
+ if action == "confirm_released":
+ row.state, row.released_at, row.updated_at = "released", _now(), _now()
+ self.session.flush()
+ self._promote_queue()
+ self._audit("operator", actor_id).write(
+ "EXTERNAL_CREATIVE_ALLOCATION_RECONCILED",
+ "external_creative_allocation",
+ str(row.id),
+ {"action": action, "reason": reason},
+ )
+ self.session.commit()
+ self.session.refresh(row)
+ return self._response(row)
diff --git a/backend/src/modelforge_api/services/lab.py b/backend/src/modelforge_api/services/lab.py
new file mode 100644
index 0000000..4241c87
--- /dev/null
+++ b/backend/src/modelforge_api/services/lab.py
@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import hashlib
+import json
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import (
+ LabBinding,
+ LabEmbeddingInput,
+ LabInputSummary,
+ LabPreflightRequest,
+ LabPreflightResponse,
+ LabRerankingInput,
+ LabVisionInput,
+)
+from modelforge_api.persistence.models import DeploymentCandidate
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.registry import RegistryNotFound
+from modelforge_api.services.runtime import RuntimeService
+from modelforge_api.settings import Settings
+
+FAMILY_MODALITIES = {
+ "rag.embedding": "embedding",
+ "rag.reranking": "reranking",
+ "document.ocr": "document",
+ "vision.embedding": "vision",
+ "speech.transcription": "audio",
+}
+
+
+class LabPreflightService:
+ """Read-only snapshot validation, never an execution grant or resource reservation."""
+
+ def __init__(self, session: Session, settings: Settings) -> None:
+ self.session = session
+ self.settings = settings
+ self.runtime = RuntimeService(session, settings)
+
+ def preflight(self, request: LabPreflightRequest) -> LabPreflightResponse:
+ with self.session.no_autoflush:
+ return self._preflight(request)
+
+ def _preflight(self, request: LabPreflightRequest) -> LabPreflightResponse:
+ payload = request.input.payload.model_dump(mode="json")
+ encoded = json.dumps(
+ payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+ ).encode()
+ count = 1
+ blockers: list[str] = []
+ if isinstance(request.input, LabEmbeddingInput):
+ values = request.input.payload.inputs()
+ count = len(values)
+ if count > self.settings.gateway_max_batch_size or any(
+ len(value) > self.settings.gateway_max_input_characters for value in values
+ ):
+ blockers.append("INPUT_LIMIT_EXCEEDED")
+ elif isinstance(request.input, LabRerankingInput):
+ count = len(request.input.payload.documents)
+ elif isinstance(request.input, LabVisionInput):
+ count = len(request.input.payload.items)
+ if len(encoded) > self.settings.gateway_max_payload_bytes:
+ blockers.append("PAYLOAD_LIMIT_EXCEEDED")
+ summary = LabInputSummary(
+ sha256=hashlib.sha256(encoded).hexdigest(), item_count=count, encoded_bytes=len(encoded)
+ )
+ contract = next(
+ (
+ item
+ for item in ManifestRegistry(self.settings.config_root).capabilities()
+ if item.capability == request.input.capability
+ and item.version == request.contract_version
+ ),
+ None,
+ )
+ if contract is None:
+ blockers.append("CAPABILITY_CONTRACT_UNAVAILABLE")
+ binding: LabBinding | None = None
+ target = request.target
+ candidate = self.session.get(DeploymentCandidate, target.deployment_candidate_id)
+ if candidate is None:
+ blockers.append("CANDIDATE_NOT_FOUND")
+ else:
+ if (
+ candidate.production
+ or candidate.channel != "lab"
+ or candidate.status != "lab_ready"
+ ):
+ blockers.append("CANDIDATE_NOT_LAB_READY")
+ if (
+ candidate.artifact_set_id != target.artifact_set_id
+ or candidate.runtime_profile_id != target.runtime_profile_id
+ or candidate.compute_node_id != target.compute_node_id
+ ):
+ blockers.append("TARGET_IDENTITY_MISMATCH")
+ repo = self.runtime.repo
+ profile = repo.profile(candidate.runtime_profile_id)
+ node = repo.node(candidate.compute_node_id)
+ assessment = repo.assessment(candidate.compatibility_assessment_id)
+ probe = repo.probe(candidate.runtime_probe_id)
+ approval = repo.approval(target.execution_approval_id)
+ if not profile or not node or not assessment or not probe or not approval:
+ blockers.append("EVIDENCE_MISSING")
+ else:
+ if (
+ profile.artifact_set_id != candidate.artifact_set_id
+ or any(
+ item.artifact_set_id != candidate.artifact_set_id
+ or item.runtime_profile_id != candidate.runtime_profile_id
+ or item.compute_node_id != candidate.compute_node_id
+ for item in (assessment, probe)
+ )
+ or probe.compatibility_assessment_id != assessment.id
+ or probe.execution_approval_id != approval.id
+ or approval.artifact_set_id != candidate.artifact_set_id
+ ):
+ blockers.append("EVIDENCE_IDENTITY_MISMATCH")
+ if profile.modality != FAMILY_MODALITIES[request.input.capability]:
+ blockers.append("CAPABILITY_PROFILE_MISMATCH")
+ if profile.trust_remote_code or profile.network_egress:
+ blockers.append("RUNTIME_POLICY_UNSUPPORTED")
+ if (
+ not node.enabled
+ or not node.lab_eligible
+ or node.liveness_state != "online"
+ or not {
+ "deployment.load.v1",
+ "deployment.invoke.v1",
+ "deployment.unload.v1",
+ }.issubset(node.agent_capabilities)
+ ):
+ blockers.append("NODE_NOT_READY")
+ try:
+ if (
+ assessment.status not in {"compatible", "requires_probe"}
+ or assessment.blockers
+ or self.runtime.assessment_response(assessment).stale
+ ):
+ blockers.append("ASSESSMENT_STALE_OR_BLOCKED")
+ if (
+ self.runtime.approval_response(approval).stale
+ or approval.scope != "lab_execution"
+ ):
+ blockers.append("LAB_APPROVAL_STALE")
+ except RegistryNotFound:
+ blockers.append("ARTIFACT_EVIDENCE_MISSING")
+ if probe.status != "completed" or probe.cancel_requested or probe.failure_code:
+ blockers.append("PROBE_NOT_COMPLETED")
+ result = probe.inference_result
+ capability = request.input.capability
+ valid_output = False
+ if capability in {"rag.embedding", "vision.embedding"}:
+ dimension = result.get("dimension")
+ valid_output = (
+ isinstance(dimension, int)
+ and not isinstance(dimension, bool)
+ and 32 <= dimension <= 8192
+ and result.get("finite") is True
+ and result.get("output_type") in (None, "float_vector")
+ and (
+ not contract
+ or not contract.vector
+ or contract.vector.dimensionality is None
+ or dimension == contract.vector.dimensionality
+ )
+ )
+ elif capability == "rag.reranking":
+ valid_output = (
+ result.get("output_type") == "ranked_scores"
+ and result.get("finite") is True
+ and probe.runtime_facts.get("adapter") == "qwen3_reranker"
+ )
+ else:
+ expected = "ocr_text" if capability == "document.ocr" else "transcription"
+ valid_output = (
+ result.get("output_type") == expected and result.get("output_valid") is True
+ )
+ if not valid_output:
+ blockers.append("PROBE_OUTPUT_INVALID")
+ gpu_uuid = probe.runtime_facts.get("gpu_uuid")
+ accelerator = next(
+ (item for item in repo.accelerators(node.id) if item.device_uuid == gpu_uuid),
+ None,
+ )
+ if accelerator is None:
+ blockers.append("PROBED_ACCELERATOR_MISSING")
+ # Only expose a coherent server-resolved binding, never echo a mismatched target.
+ if not {"TARGET_IDENTITY_MISMATCH", "EVIDENCE_IDENTITY_MISMATCH"}.intersection(
+ blockers
+ ):
+ binding = LabBinding(
+ **target.model_dump(),
+ compatibility_assessment_id=assessment.id,
+ runtime_probe_id=probe.id,
+ capability=capability,
+ runtime_profile_fingerprint=profile.fingerprint,
+ assessment_fingerprint=assessment.environment_fingerprint,
+ approval_evidence_fingerprint=approval.evidence_fingerprint,
+ probe_environment_fingerprint=probe.environment_fingerprint,
+ accelerator_id=accelerator.id if accelerator else None,
+ )
+ return LabPreflightResponse(
+ input_summary=summary,
+ binding=binding,
+ limits=request.limits,
+ blockers=list(dict.fromkeys(blockers)),
+ checks_passed=not blockers,
+ )
diff --git a/backend/src/modelforge_api/services/lab_runs.py b/backend/src/modelforge_api/services/lab_runs.py
new file mode 100644
index 0000000..7689a36
--- /dev/null
+++ b/backend/src/modelforge_api/services/lab_runs.py
@@ -0,0 +1,805 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import UUID, uuid4
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.lab import (
+ LabEmbeddingInput,
+ LabExecutionRunResponse,
+ LabExecutionStartRequest,
+ LabInputSummary,
+)
+from modelforge_api.persistence.models import (
+ CapabilityDeployment,
+ GatewayRequest,
+ LabExecutionRun,
+ ServingGpuLease,
+ ServingJob,
+)
+from modelforge_api.services.gpu_scheduler import PlacementVerdict
+from modelforge_api.services.serving import ServingError, ServingService
+
+_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
+_ACTIVE_STATUSES = frozenset({"accepted", "preparing", "queued", "running", "cancel_requested"})
+_TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "interrupted"})
+_RESIDENT_STATES = frozenset({"warm", "idle"})
+
+
+def _canonical_json(value: Any) -> bytes:
+ return json.dumps(
+ value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
+ ).encode("utf-8")
+
+
+def _sha256(value: Any) -> str:
+ return hashlib.sha256(_canonical_json(value)).hexdigest()
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _aware(value: datetime) -> datetime:
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+
+
+class LabRunService:
+ """Persist actor-scoped identities for bounded, selected embedding runs."""
+
+ def __init__(self, serving: ServingService) -> None:
+ self.serving = serving
+ self.session = serving.session
+
+ def _owned(self, run_id: UUID, actor_subject: str) -> LabExecutionRun:
+ run = self.session.scalar(
+ select(LabExecutionRun).where(
+ LabExecutionRun.id == run_id,
+ LabExecutionRun.actor_subject == actor_subject,
+ )
+ )
+ if run is None:
+ raise ServingError(404, "LAB_RUN_NOT_FOUND", "LAB run was not found")
+ return run
+
+ @staticmethod
+ def _response(run: LabExecutionRun) -> LabExecutionRunResponse:
+ return LabExecutionRunResponse(
+ id=run.id,
+ status=run.status, # type: ignore[arg-type]
+ target_snapshot=run.target_snapshot,
+ input_summary=LabInputSummary(
+ sha256=run.input_sha256,
+ item_count=run.input_count,
+ encoded_bytes=run.input_encoded_bytes,
+ ),
+ output_metadata=run.output_metadata,
+ worker_observations=run.worker_observations,
+ failure_code=run.failure_code,
+ input_payload_expires_at=run.input_payload_expires_at,
+ retention_seconds=run.retention_seconds,
+ created_at=run.created_at,
+ started_at=run.started_at,
+ cancellation_requested_at=run.cancellation_requested_at,
+ cancellation_acknowledged_at=run.cancellation_acknowledged_at,
+ finished_at=run.finished_at,
+ )
+
+ def _fingerprint(self, request: LabExecutionStartRequest) -> str:
+ return _sha256(request.model_dump(mode="json"))
+
+ def _selected_target(
+ self, request: LabExecutionStartRequest
+ ) -> tuple[CapabilityDeployment, dict[str, Any], LabEmbeddingInput]:
+ deployment = self.serving.repo.deployment(request.deployment_id)
+ if deployment is None:
+ raise ServingError(404, "LAB_TARGET_NOT_FOUND", "selected LAB target was not found")
+ target = request.target
+ if (
+ deployment.deployment_candidate_id != target.deployment_candidate_id
+ or deployment.artifact_set_id != target.artifact_set_id
+ or deployment.runtime_profile_id != target.runtime_profile_id
+ or deployment.compute_node_id != target.compute_node_id
+ or deployment.execution_approval_id != target.execution_approval_id
+ ):
+ raise ServingError(
+ 409, "LAB_TARGET_MISMATCH", "selected LAB target identity does not match"
+ )
+ if not isinstance(request.input, LabEmbeddingInput):
+ raise ServingError(
+ 422, "LAB_INPUT_NOT_SUPPORTED", "operator execution supports text embeddings only"
+ )
+ selected, _route_key = self.serving._admit_operator_embedding(
+ request.input, deployment.id, request.experiment_route_id
+ )
+ return (
+ selected,
+ self.serving._operator_lab_target_snapshot(selected, request.experiment_route_id),
+ request.input,
+ )
+
+ def _gateway(self, run: LabExecutionRun) -> GatewayRequest | None:
+ return self.session.get(GatewayRequest, run.gateway_request_id)
+
+ def _set_gateway_state(
+ self,
+ run: LabExecutionRun,
+ status: str,
+ *,
+ failure_code: str | None = None,
+ ) -> None:
+ gateway = self._gateway(run)
+ if gateway is None:
+ return
+ gateway.status = status
+ if status in {"completed", "failed", "cancelled", "interrupted"}:
+ gateway.finished_at = _now()
+ if failure_code:
+ gateway.failure_code = failure_code
+ evidence = gateway.decision_evidence or {}
+ lab_evidence = evidence.get("lab_run_evidence", {})
+ if isinstance(lab_evidence, dict):
+ updated = {**lab_evidence, "status": status}
+ if failure_code:
+ updated["failure_code"] = failure_code
+ if run.output_metadata:
+ updated["output"] = run.output_metadata
+ if run.worker_observations:
+ updated["worker_observations"] = run.worker_observations
+ gateway.decision_evidence = {**evidence, "lab_run_evidence": updated}
+
+ def _delete_payload(self, run: LabExecutionRun) -> None:
+ if run.payload_reference:
+ self.serving.payload_store.delete(run.payload_reference)
+ run.payload_reference = None
+
+ def _release_reservation(self, run: LabExecutionRun, *, failed: bool = False) -> None:
+ gateway = self._gateway(run)
+ if gateway is None:
+ return
+ now = _now()
+ request_id = gateway.request_id
+ lease = self.session.scalar(
+ select(ServingGpuLease)
+ .where(ServingGpuLease.request_id == request_id)
+ .order_by(ServingGpuLease.created_at.desc())
+ .limit(1)
+ )
+ if lease is None:
+ return
+ if lease and lease.state not in {"released", "expired", "failed"}:
+ lease.state = "failed" if failed else "released"
+ lease.released_at = now
+ deployment = (
+ self.serving.repo.deployment(gateway.capability_deployment_id)
+ if gateway.capability_deployment_id
+ else None
+ )
+ residency = self.serving.repo.residency(deployment.id, lock=True) if deployment else None
+ if residency:
+ residency.active_requests = max(0, residency.active_requests - 1)
+ if residency.state == "busy":
+ residency.state = "warm"
+ elif residency.state == "loading":
+ residency.state = "cold"
+ residency.transition_reason = "LAB_RUN_CANCELLED_BEFORE_LOAD"
+ residency.last_used_at = now
+
+ def _finish(
+ self,
+ run: LabExecutionRun,
+ status: str,
+ *,
+ failure_code: str | None = None,
+ ) -> None:
+ run.status = status
+ run.failure_code = failure_code
+ run.finished_at = _now()
+ self._set_gateway_state(run, status, failure_code=failure_code)
+ self._delete_payload(run)
+
+ def start(
+ self,
+ request: LabExecutionStartRequest,
+ *,
+ actor_subject: str,
+ idempotency_key: str,
+ ) -> LabExecutionRunResponse:
+ if not actor_subject or len(actor_subject) > 255:
+ raise ServingError(
+ 401, "OPERATOR_CONTEXT_REQUIRED", "operator principal is unavailable"
+ )
+ if not _IDEMPOTENCY_KEY.fullmatch(idempotency_key):
+ raise ServingError(422, "LAB_IDEMPOTENCY_KEY_INVALID", "idempotency key is invalid")
+ fingerprint = self._fingerprint(request)
+ existing = self.session.scalar(
+ select(LabExecutionRun).where(
+ LabExecutionRun.actor_subject == actor_subject,
+ LabExecutionRun.idempotency_key == idempotency_key,
+ )
+ )
+ if existing is not None:
+ if existing.request_fingerprint != fingerprint:
+ raise ServingError(
+ 409,
+ "LAB_IDEMPOTENCY_CONFLICT",
+ "idempotency key was already used for a different LAB request",
+ )
+ return self._response(existing)
+
+ deployment, target_snapshot, embedding_input = self._selected_target(request)
+ payload = embedding_input.payload
+ inputs = payload.inputs()
+ input_sha256 = _sha256(inputs)
+ encoded_input = _canonical_json(payload.model_dump(mode="json"))
+ run_id = uuid4()
+ context = DispatchContext.for_operator(actor_subject, run_id)
+ now = _now()
+ payload_reference = self.serving.payload_store.put(
+ {"input": inputs, "input_type": payload.input_type},
+ request.limits.retention_seconds,
+ )
+ gateway_request = GatewayRequest(
+ request_id=uuid4(),
+ service_client_id=None,
+ capability_key="rag.embedding",
+ capability_version=1,
+ capability_deployment_id=deployment.id,
+ compute_node_id=deployment.compute_node_id,
+ status="queued",
+ priority="lab",
+ input_sha256=input_sha256,
+ input_count=len(inputs),
+ decision_evidence={
+ "dispatch_context": context.provenance(),
+ "operator_route_id": str(request.experiment_route_id),
+ "input_persisted": False,
+ "input_type": payload.input_type,
+ "lab_run_evidence": {
+ "schema_version": 1,
+ "status": "accepted",
+ "capability": "rag.embedding@1",
+ "operation_id": str(run_id),
+ "target": target_snapshot,
+ "input": {
+ "sha256": input_sha256,
+ "item_count": len(inputs),
+ "content_logged": False,
+ },
+ },
+ },
+ )
+ self.session.add(gateway_request)
+ try:
+ self.session.flush()
+ run = LabExecutionRun(
+ id=run_id,
+ actor_subject=actor_subject,
+ idempotency_key=idempotency_key,
+ request_fingerprint=fingerprint,
+ status="accepted",
+ capability="rag.embedding@1",
+ target_snapshot=target_snapshot,
+ input_sha256=input_sha256,
+ input_count=len(inputs),
+ input_encoded_bytes=len(encoded_input),
+ payload_reference=payload_reference,
+ input_payload_expires_at=now + timedelta(seconds=request.limits.retention_seconds),
+ gateway_request_id=gateway_request.id,
+ timeout_seconds=request.limits.timeout_seconds,
+ queue_timeout_seconds=request.limits.queue_timeout_seconds,
+ retention_seconds=request.limits.retention_seconds,
+ created_at=now,
+ )
+ self.session.add(run)
+ self.session.commit()
+ except IntegrityError:
+ self.session.rollback()
+ winner = self.session.scalar(
+ select(LabExecutionRun).where(
+ LabExecutionRun.actor_subject == actor_subject,
+ LabExecutionRun.idempotency_key == idempotency_key,
+ )
+ )
+ self.serving.payload_store.delete(payload_reference)
+ if winner is not None and winner.request_fingerprint == fingerprint:
+ return self._response(winner)
+ if winner is not None:
+ raise ServingError(
+ 409,
+ "LAB_IDEMPOTENCY_CONFLICT",
+ "idempotency key was already used for a different LAB request",
+ ) from None
+ raise ServingError(409, "STATE_CONFLICT", "LAB run could not be persisted") from None
+ except Exception:
+ self.session.rollback()
+ self.serving.payload_store.delete(payload_reference)
+ raise
+ self.session.refresh(run)
+ return self._response(run)
+
+ def _active_job(self, run: LabExecutionRun) -> ServingJob | None:
+ return self.session.scalar(
+ select(ServingJob)
+ .where(ServingJob.lab_execution_run_id == run.id)
+ .order_by(ServingJob.created_at.desc())
+ .limit(1)
+ )
+
+ def _input(self, run: LabExecutionRun) -> LabEmbeddingInput | None:
+ if not run.payload_reference or _aware(run.input_payload_expires_at) <= _now():
+ return None
+ payload = self.serving.payload_store.get(run.payload_reference)
+ if not payload:
+ return None
+ values = payload.get("input")
+ input_type = payload.get("input_type", "raw")
+ if (
+ not isinstance(values, list)
+ or not values
+ or len(values) != run.input_count
+ or any(not isinstance(value, str) for value in values)
+ or input_type not in {"raw", "query", "document"}
+ or _sha256(values) != run.input_sha256
+ ):
+ return None
+ try:
+ return LabEmbeddingInput.model_validate(
+ {
+ "capability": "rag.embedding",
+ "payload": {"input": values, "input_type": input_type},
+ }
+ )
+ except ValueError:
+ return None
+
+ def _revalidate_target(
+ self, run: LabExecutionRun, input_value: LabEmbeddingInput
+ ) -> CapabilityDeployment:
+ deployment_id = UUID(str(run.target_snapshot["deployment_id"]))
+ route_id = UUID(str(run.target_snapshot["experiment_route_id"]))
+ deployment, _ = self.serving._admit_operator_embedding(input_value, deployment_id, route_id)
+ current_snapshot = self.serving._operator_lab_target_snapshot(deployment, route_id)
+ if current_snapshot != run.target_snapshot:
+ raise ServingError(
+ 409, "LAB_TARGET_CHANGED", "selected LAB target changed after run acceptance"
+ )
+ return deployment
+
+ def _add_job(
+ self,
+ run: LabExecutionRun,
+ deployment: CapabilityDeployment,
+ gateway: GatewayRequest,
+ operation: str,
+ *,
+ now: datetime,
+ ) -> ServingJob:
+ key = _sha256({"run_id": str(run.id), "operation": operation})
+ existing = self.session.scalar(select(ServingJob).where(ServingJob.idempotency_key == key))
+ if existing is not None:
+ return existing
+ job = ServingJob(
+ capability_deployment_id=deployment.id,
+ compute_node_id=deployment.compute_node_id,
+ gateway_request_id=gateway.id,
+ lab_execution_run_id=run.id,
+ operation=operation,
+ status="queued",
+ priority="lab",
+ idempotency_key=key,
+ payload_reference=(run.payload_reference if operation == "invoke" else None),
+ attempt_count=0,
+ result_summary={},
+ created_at=now,
+ updated_at=now,
+ )
+ self.session.add(job)
+ self.session.flush()
+ return job
+
+ def _queue_invoke(
+ self,
+ run: LabExecutionRun,
+ deployment: CapabilityDeployment,
+ gateway: GatewayRequest,
+ *,
+ now: datetime,
+ ) -> bool:
+ if self._input(run) is None:
+ self._finish(run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
+ self._release_reservation(run, failed=True)
+ return False
+ if self.serving.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
+ return False
+ if (
+ self.serving.repo.active_gateway_requests()
+ > self.serving.settings.scheduler_global_queue_limit
+ ):
+ return False
+ job = self._add_job(run, deployment, gateway, "invoke", now=now)
+ lease = self.session.scalar(
+ select(ServingGpuLease)
+ .where(ServingGpuLease.request_id == gateway.request_id)
+ .order_by(ServingGpuLease.created_at.desc())
+ .limit(1)
+ )
+ if lease:
+ lease.serving_job_id = job.id
+ run.status = "queued"
+ gateway.status = "running"
+ self._set_gateway_state(run, "queued")
+ self.session.commit()
+ return True
+
+ def _admit_and_queue(self, run: LabExecutionRun, now: datetime) -> bool:
+ input_value = self._input(run)
+ if input_value is None:
+ self._finish(run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
+ self.session.commit()
+ return True
+ try:
+ deployment = self._revalidate_target(run, input_value)
+ except ServingError as exc:
+ self._finish(run, "failed", failure_code=exc.code)
+ self.session.commit()
+ return True
+ gateway = self._gateway(run)
+ if gateway is None:
+ self._finish(run, "failed", failure_code="LAB_EVIDENCE_MISSING")
+ self.session.commit()
+ return True
+ if self.serving.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
+ return False
+ if (
+ self.serving.repo.active_gateway_requests()
+ > self.serving.settings.scheduler_global_queue_limit
+ ):
+ return False
+ self.serving.repo.lock_accelerator(deployment.accelerator_id)
+ residency = self.serving.repo.residency(deployment.id, lock=True)
+ if residency is None:
+ self._finish(run, "failed", failure_code="SCHEDULER_STATE_STALE")
+ self.session.commit()
+ return True
+ if residency.active_requests >= deployment.max_concurrency:
+ return False
+ if residency.state not in _RESIDENT_STATES and residency.state != "cold":
+ if residency.state in {"loading", "busy", "unloading", "draining"}:
+ return False
+ self._finish(run, "failed", failure_code="LAB_TARGET_UNAVAILABLE")
+ self.session.commit()
+ return True
+ plan = self.serving._placement_plan(
+ deployment,
+ priority="lab",
+ deadline_remaining_ms=float(run.timeout_seconds * 1000),
+ request_id=gateway.request_id,
+ dry_run=True,
+ persist=False,
+ commit=False,
+ )
+ if plan.verdict == PlacementVerdict.QUEUE.value:
+ self.session.commit()
+ return False
+ if plan.verdict == PlacementVerdict.ADMIT_AFTER_EVICTION.value:
+ self._finish(run, "failed", failure_code="LAB_EVICTION_REQUIRED")
+ self.session.commit()
+ return True
+ if plan.verdict != PlacementVerdict.ADMIT.value:
+ code = {
+ PlacementVerdict.REJECT_HEALTH.value: "LAB_TARGET_UNAVAILABLE",
+ PlacementVerdict.REJECT_POLICY.value: "LAB_POLICY_BLOCKED",
+ PlacementVerdict.REJECT_CAPACITY.value: "LAB_CAPACITY_BLOCKED",
+ }.get(plan.verdict, "LAB_ADMISSION_BLOCKED")
+ self._finish(run, "failed", failure_code=code)
+ self.session.commit()
+ return True
+
+ context = DispatchContext.for_operator(run.actor_subject, run.id)
+ deadline = _aware(run.created_at) + timedelta(seconds=run.timeout_seconds)
+ lease = ServingGpuLease(
+ accelerator_id=deployment.accelerator_id,
+ capability_deployment_id=deployment.id,
+ request_id=gateway.request_id,
+ reserved_vram_bytes=plan.required_vram_bytes,
+ priority="lab",
+ state="active",
+ owner=context.lease_owner,
+ created_at=now,
+ acquired_at=now,
+ heartbeat_at=now,
+ expires_at=deadline,
+ lease_type=(
+ "request_execution"
+ if residency.state in _RESIDENT_STATES
+ else "residency_reservation"
+ ),
+ materialized_vram_bytes=(
+ residency.measured_resident_vram_bytes if residency.state in _RESIDENT_STATES else 0
+ ),
+ generation=residency.generation,
+ )
+ self.session.add(lease)
+ gateway.status = "running"
+ gateway.decision_evidence = {
+ **(gateway.decision_evidence or {}),
+ "dispatch_context": context.provenance(),
+ "placement": {
+ "verdict": plan.verdict,
+ "decision_fingerprint": plan.decision_fingerprint,
+ "required_vram_bytes": plan.required_vram_bytes,
+ "reason_codes": plan.reason_codes,
+ },
+ }
+ run.dispatch_started_at = now
+ if residency.state == "cold":
+ residency.state = "loading"
+ residency.transition_started_at = now
+ residency.transition_reason = "LAB_RUN_LOAD"
+ residency.generation += 1
+ operation = "load"
+ run.status = "preparing"
+ else:
+ residency.state = "busy"
+ operation = "invoke"
+ run.status = "queued"
+ residency.active_requests += 1
+ residency.last_used_at = now
+ job = self._add_job(run, deployment, gateway, operation, now=now)
+ self.session.flush()
+ lease.serving_job_id = job.id
+ lab_evidence = (gateway.decision_evidence or {}).get("lab_run_evidence", {})
+ if isinstance(lab_evidence, dict):
+ gateway.decision_evidence = {
+ **gateway.decision_evidence,
+ "lab_run_evidence": {**lab_evidence, "status": run.status},
+ }
+ self.session.commit()
+ return True
+
+ def _drive_preparing(self, run: LabExecutionRun, now: datetime) -> bool:
+ gateway = self._gateway(run)
+ deployment = (
+ self.serving.repo.deployment(gateway.capability_deployment_id)
+ if gateway and gateway.capability_deployment_id
+ else None
+ )
+ if gateway is None or deployment is None:
+ self._finish(run, "failed", failure_code="LAB_EVIDENCE_MISSING")
+ self.session.commit()
+ return True
+ jobs = list(
+ self.session.scalars(
+ select(ServingJob)
+ .where(ServingJob.lab_execution_run_id == run.id)
+ .order_by(ServingJob.created_at.desc())
+ )
+ )
+ latest = jobs[0] if jobs else None
+ if latest is None:
+ self._finish(run, "interrupted", failure_code="LAB_DISPATCH_STATE_INCOMPLETE")
+ self.session.commit()
+ return True
+ if latest.status in {"queued", "running"}:
+ return False
+ if latest.operation != "load" or latest.status != "completed":
+ self._finish(run, "failed", failure_code=latest.failure_code or "RESIDENCY_LOAD_FAILED")
+ self._release_reservation(run, failed=True)
+ self.session.commit()
+ return True
+ input_value = self._input(run)
+ if input_value is None:
+ self._finish(run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
+ self._release_reservation(run, failed=True)
+ self.session.commit()
+ return True
+ try:
+ selected = self._revalidate_target(run, input_value)
+ except ServingError as exc:
+ self._finish(run, "failed", failure_code=exc.code)
+ self._release_reservation(run, failed=True)
+ self.session.commit()
+ return True
+ if _aware(run.created_at) + timedelta(seconds=run.timeout_seconds) <= now:
+ self._finish(run, "failed", failure_code="REQUEST_TIMEOUT")
+ self._release_reservation(run, failed=True)
+ self.session.commit()
+ return True
+ self._queue_invoke(run, selected, gateway, now=now)
+ return run.status == "queued"
+
+ def _expire_run(self, run: LabExecutionRun, now: datetime) -> bool:
+ gateway = self._gateway(run)
+ if gateway is None:
+ self._finish(run, "failed", failure_code="LAB_EVIDENCE_MISSING")
+ self.session.commit()
+ return True
+ job = self._active_job(run)
+ if job and job.status == "running":
+ run.status = "cancel_requested"
+ run.failure_code = "REQUEST_TIMEOUT"
+ run.cancellation_requested_at = run.cancellation_requested_at or now
+ self._set_gateway_state(run, "cancel_requested", failure_code="REQUEST_TIMEOUT")
+ self._delete_payload(run)
+ self.session.commit()
+ return True
+ # Do not lock/update a queued job here: claim and completion lock job then run.
+ # The terminal run state fences a concurrent claim, which will mark that job
+ # failed after acquiring the same locks in the established order.
+ self._release_reservation(run, failed=True)
+ self._finish(run, "failed", failure_code="REQUEST_TIMEOUT")
+ self.session.commit()
+ return True
+
+ def _interrupt_stale_job(self, job_id: UUID, now: datetime) -> bool:
+ job = self.session.scalar(
+ select(ServingJob).where(ServingJob.id == job_id).with_for_update()
+ )
+ if (
+ job is None
+ or job.lab_execution_run_id is None
+ or job.status != "running"
+ or job.lease_expires_at is None
+ or _aware(job.lease_expires_at) > now
+ ):
+ self.session.rollback()
+ return False
+ run = self.session.scalar(
+ select(LabExecutionRun)
+ .where(LabExecutionRun.id == job.lab_execution_run_id)
+ .with_for_update()
+ )
+ if run is None or run.status in _TERMINAL_STATUSES:
+ self.session.rollback()
+ return False
+ job.status = "failed"
+ job.failure_code = "LAB_WORKER_LEASE_STALE"
+ job.finished_at = now
+ job.lease_token_hash = None
+ job.lease_expires_at = None
+ gateway = self._gateway(run)
+ deployment = (
+ self.serving.repo.deployment(gateway.capability_deployment_id)
+ if gateway and gateway.capability_deployment_id
+ else None
+ )
+ if deployment:
+ deployment.health_status = "unavailable"
+ residency = self.serving.repo.residency(deployment.id, lock=True)
+ if residency:
+ residency.state = "failed"
+ residency.failure_code = "LAB_WORKER_LEASE_STALE"
+ residency.failure_message = None
+ residency.transition_reason = "LAB_WORKER_LEASE_STALE"
+ self._finish(run, "interrupted", failure_code="LAB_WORKER_LEASE_STALE")
+ # The GPU reservation is intentionally left until its own expiry. A missing
+ # worker heartbeat is not proof that the process has stopped using the device.
+ self.session.commit()
+ return True
+
+ def reconcile(self) -> dict[str, int]:
+ now = _now()
+ interrupted = 0
+ stale_job_ids = list(
+ self.session.scalars(
+ select(ServingJob.id)
+ .where(
+ ServingJob.lab_execution_run_id.is_not(None),
+ ServingJob.status == "running",
+ ServingJob.lease_expires_at <= now,
+ )
+ .order_by(ServingJob.lease_expires_at)
+ .limit(100)
+ )
+ )
+ for job_id in stale_job_ids:
+ interrupted += self._interrupt_stale_job(job_id, now)
+ driven = 0
+ candidate_ids = list(
+ self.session.scalars(
+ select(LabExecutionRun.id)
+ .where(LabExecutionRun.status.in_(("accepted", "preparing", "queued")))
+ .order_by(LabExecutionRun.created_at)
+ .limit(100)
+ )
+ )
+ for run_id in candidate_ids:
+ run = self.session.scalar(
+ select(LabExecutionRun)
+ .where(
+ LabExecutionRun.id == run_id,
+ LabExecutionRun.status.in_(("accepted", "preparing", "queued")),
+ )
+ .with_for_update(skip_locked=True)
+ )
+ if run is None:
+ self.session.rollback()
+ continue
+ now = _now()
+ if _aware(run.created_at) + timedelta(seconds=run.timeout_seconds) <= now:
+ driven += self._expire_run(run, now)
+ continue
+ if run.status in {"accepted", "queued"} and (
+ _aware(run.created_at) + timedelta(seconds=run.queue_timeout_seconds) <= now
+ ):
+ self._expire_run(run, now)
+ driven += 1
+ continue
+ if run.status == "accepted":
+ driven += self._admit_and_queue(run, now)
+ elif run.status == "preparing":
+ driven += self._drive_preparing(run, now)
+ elif run.status == "queued":
+ job = self._active_job(run)
+ if job is None or job.status not in {"queued", "running"}:
+ self._finish(run, "interrupted", failure_code="LAB_DISPATCH_STATE_INCOMPLETE")
+ self.session.commit()
+ driven += 1
+ else:
+ self.session.rollback()
+ return {"runs_driven": driven, "stale_runs_interrupted": interrupted}
+
+ def get(self, run_id: UUID, *, actor_subject: str) -> LabExecutionRunResponse:
+ return self._response(self._owned(run_id, actor_subject))
+
+ def cancel(self, run_id: UUID, *, actor_subject: str) -> LabExecutionRunResponse:
+ # Check ownership before taking locks so a foreign id stays indistinguishable from a
+ # missing run. Lock order is job then run, matching claim and worker completion.
+ self._owned(run_id, actor_subject)
+ job = self.session.scalar(
+ select(ServingJob)
+ .where(
+ ServingJob.lab_execution_run_id == run_id,
+ ServingJob.status.in_(("queued", "running")),
+ )
+ .order_by(ServingJob.created_at.desc())
+ .limit(1)
+ .with_for_update()
+ )
+ run = self.session.scalar(
+ select(LabExecutionRun)
+ .where(
+ LabExecutionRun.id == run_id,
+ LabExecutionRun.actor_subject == actor_subject,
+ )
+ .with_for_update()
+ )
+ if run is None:
+ raise ServingError(404, "LAB_RUN_NOT_FOUND", "LAB run was not found")
+ if run.status in _TERMINAL_STATUSES:
+ self.session.commit()
+ return self._response(run)
+ now = _now()
+ if job is not None and job.status == "running":
+ # Inference is synchronous inside the worker. The run remains nonterminal until
+ # the worker reaches a safe boundary and acknowledges cancellation.
+ run.status = "cancel_requested"
+ run.cancellation_requested_at = run.cancellation_requested_at or now
+ self._delete_payload(run)
+ self._set_gateway_state(run, "cancel_requested")
+ else:
+ if job is not None:
+ job.status = "cancelled"
+ job.failure_code = "CANCELLED"
+ job.finished_at = now
+ job.lease_token_hash = None
+ job.lease_expires_at = None
+ job.payload_reference = None
+ self._release_reservation(run)
+ self._finish(run, "cancelled")
+ run.cancellation_requested_at = run.cancellation_requested_at or now
+ run.cancellation_acknowledged_at = None
+ self.session.commit()
+ if job is not None and job.status == "cancelled":
+ # The worker never claimed this job, so no worker acknowledgement is expected.
+ pass
+ self.session.refresh(run)
+ return self._response(run)
diff --git a/backend/src/modelforge_api/services/lifecycle.py b/backend/src/modelforge_api/services/lifecycle.py
index 29beac7..5b2c6f6 100644
--- a/backend/src/modelforge_api/services/lifecycle.py
+++ b/backend/src/modelforge_api/services/lifecycle.py
@@ -75,6 +75,9 @@ from modelforge_api.persistence.models import (
RuntimeProbe,
RuntimeProfile,
)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.lifecycle_serving import LifecycleServing
+from modelforge_api.services.serving import ServingError
class LifecycleError(RuntimeError):
@@ -94,6 +97,13 @@ def _now() -> datetime:
return datetime.now(UTC)
+def _serving_changed_at(operation: LifecycleOperation) -> str:
+ if operation.finished_at is None:
+ raise LifecycleError(409, "SERVING_JOURNAL_INVALID", "committed serving timestamp is missing")
+ value = operation.finished_at
+ return (value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)).isoformat()
+
+
DEFAULT_POLICIES: tuple[dict[str, Any], ...] = (
{
"key": "lab-promotion",
@@ -305,6 +315,18 @@ class LifecycleService:
"SUBJECT_TARGET_MISMATCH",
"approval target does not match lifecycle subject",
)
+ if request.evidence.serving_transition is not None:
+ selected = request.evidence.serving_transition
+ if request.evidence.production_validation != "SATISFIED":
+ raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
+ if (request.environment != "PRODUCTION" or policy.scope != "CAPABILITY_PRODUCTION"
+ or request.target_type != "CAPABILITY_DEPLOYMENT"
+ or request.target_ref != str(selected.candidate_deployment_id)
+ or request.requested_transition != "STABLE"
+ or request.evidence.capability_deployment_id != selected.candidate_deployment_id
+ or request.evidence.production_execution_approval_id != selected.production_approval_id
+ or request.evidence.rollback_target_ref != str(selected.current_deployment_id)):
+ raise LifecycleError(422, "SERVING_APPROVAL_MISMATCH", "serving transition requires exact production approval scope and targets")
resolved = self._resolve_evidence(request.evidence.model_dump(mode="json"))
snapshot = {
"declared": request.evidence.model_dump(mode="json"),
@@ -400,7 +422,9 @@ class LifecycleService:
item.status = ApprovalStatus.REJECTED
event_type = "APPROVAL_REJECTED"
elif action == "revoke":
- if item.status != ApprovalStatus.APPROVED:
+ if item.status != ApprovalStatus.APPROVED and not (
+ item.status == ApprovalStatus.STALE and item.approved_by is not None
+ ):
raise LifecycleError(409, "APPROVAL_NOT_APPROVED", "only approval can be revoked")
item.status = ApprovalStatus.REVOKED
event_type = "APPROVAL_REVOKED"
@@ -468,13 +492,29 @@ class LifecycleService:
"APPROVAL_MIGRATION_MISMATCH",
"promotion plan migration differs from approved evidence",
)
+ serving_contract = approval.evidence_snapshot.get("resolved", {}).get("serving_transition")
+ if "serving_transition" in request.affected_identities:
+ raise LifecycleError(422, "SERVING_CONTRACT_SERVER_OWNED", "serving contract is derived from approved evidence")
+ if serving_contract is not None:
+ if (request.canary_strategy or request.migration_class not in {"transparent", "behavioral"}
+ or subject.target_type != "CAPABILITY_DEPLOYMENT" or subject.environment != "PRODUCTION"
+ or subject.target_ref != str(request.candidate_deployment_id)
+ or subject.state not in {"LAB_READY", "PROMOTION_ELIGIBLE"}
+ or request.desired_state != "STABLE"):
+ raise LifecycleError(422, "SERVING_PLAN_UNSUPPORTED", "only reviewed same-space stable replacement is supported")
+ contract = LifecycleServing(self.session).validate(serving_contract)
+ request = request.model_copy(update={
+ "affected_identities": {**request.affected_identities, "serving_transition": serving_contract},
+ "project_consumers": sorted({value["project_id"] for value in contract.bindings.values()}),
+ })
try:
intermediate = (
DeploymentLifecycleState.CANARY
if request.canary_strategy
else request.desired_state
)
- assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
+ if serving_contract is None:
+ assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
except ValueError as exc:
raise LifecycleError(422, "INVALID_LIFECYCLE_TRANSITION", str(exc)) from exc
if request.migration_class == "requires_reindex":
@@ -635,6 +675,11 @@ class LifecycleService:
raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
if subject.version != request.expected_subject_version:
raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently")
+ serving_contract = plan.affected_identities.get("serving_transition")
+ if serving_contract is not None:
+ LifecycleServing(self.session).validate(serving_contract)
+ elif subject.environment == "PRODUCTION" and subject.target_type == "CAPABILITY_DEPLOYMENT" and plan.desired_state == "STABLE":
+ raise LifecycleError(409, "SERVING_TRANSITION_REQUIRED", "operational stable promotion requires the approved serving adapter")
plan_version = plan.version
snapshot_payload = {
"subject_id": str(subject.id),
@@ -645,7 +690,10 @@ class LifecycleService:
"version": subject.version,
"rollback_target_ref": plan.rollback_target_ref,
"affected_identities": plan.affected_identities,
+ "superseded_by_ref": subject.superseded_by_ref,
}
+ if serving_contract is not None:
+ snapshot_payload["serving_transition"] = serving_contract
snapshot = LifecycleRollbackSnapshot(
promotion_plan_id=plan.id,
snapshot=snapshot_payload,
@@ -741,7 +789,8 @@ class LifecycleService:
return self._operation_response(operation)
def observe_canary(
- self, operation_id: uuid.UUID, observation: CanaryObservation
+ self, operation_id: uuid.UUID, observation: CanaryObservation,
+ *, audit_context: AuditContext | None = None,
) -> LifecycleOperationResponse:
operation = self._operation(operation_id)
if operation.stage != OperationStage.VERIFYING:
@@ -751,16 +800,32 @@ class LifecycleService:
)
if not canary or canary.status != CanaryStatus.RUNNING:
raise LifecycleError(409, "CANARY_NOT_RUNNING", "canary is not running")
+ plan = self._plan(operation.promotion_plan_id)
+ subject = self._require_manual_canary_rehearsal(plan)
+ context = audit_context or AuditContext(
+ actor_type="manual_lab_rehearsal", actor_id="local-lab-rehearsal",
+ )
canary.request_count += observation.request_count
canary.error_count += observation.error_count
canary.latency_p95_ms = observation.latency_p95_ms
- canary.result = observation.model_dump(mode="json")
+ canary.result = {
+ **observation.model_dump(mode="json"),
+ "provenance": {
+ "source": "MANUAL_LAB_REHEARSAL",
+ "authoritative": False,
+ "actor": context.actor_id,
+ "correlation_id": context.correlation_id,
+ },
+ }
trigger = self._canary_trigger(canary, observation)
if trigger:
canary.status = CanaryStatus.ABORTED
canary.abort_trigger = trigger
canary.finished_at = _now()
- self._rollback(operation, trigger, "SYSTEM", "Automatic canary abort")
+ self._rollback(
+ operation, trigger, context.actor_id,
+ "Automatic abort from manual LAB rehearsal observation",
+ )
event_type = "CANARY_ABORTED"
elif canary.request_count >= canary.target_request_count:
canary.status = CanaryStatus.READY_FOR_PROMOTION_REVIEW
@@ -768,18 +833,14 @@ class LifecycleService:
event_type = "CANARY_READY_FOR_PROMOTION_REVIEW"
else:
event_type = "CANARY_OBSERVED"
- plan = self._plan(operation.promotion_plan_id)
- subject = self.session.get(LifecycleSubject, plan.subject_id)
- if subject is None:
- raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
self._event(
event_type,
subject.target_type,
subject.target_ref,
subject.state,
canary.status,
- "lifecycle-engine",
- "SYSTEM",
+ context.actor_id,
+ "MANUAL_LAB_REHEARSAL",
self._approval(plan.approval_request_id).policy_revision_id,
[str(canary.id)],
trigger or "Bounded canary observation recorded without payload content",
@@ -790,14 +851,64 @@ class LifecycleService:
},
change_id=str(operation.id),
)
+ AuditWriter(self.session, context=context).write(
+ action="lifecycle.manual_lab_canary_observed",
+ resource_type="lifecycle_operation",
+ resource_id=str(operation.id),
+ details={
+ "source": "MANUAL_LAB_REHEARSAL",
+ "canary_id": str(canary.id),
+ "status": canary.status,
+ "authoritative": False,
+ },
+ )
self.session.commit()
return self._operation_response(operation)
+ def _require_manual_canary_rehearsal(
+ self, plan: LifecyclePromotionPlan,
+ ) -> LifecycleSubject:
+ """No manual or legacy canary evidence can qualify an operational promotion."""
+ subject = self.session.get(LifecycleSubject, plan.subject_id)
+ approval = self.session.get(LifecycleApprovalRequest, plan.approval_request_id)
+ policy = self.session.get(LifecyclePolicyRevision, approval.policy_revision_id) if (
+ approval is not None
+ ) else None
+ if (
+ subject is None or approval is None or policy is None
+ or subject.target_type != "LAB_REHEARSAL"
+ or subject.environment != "LAB"
+ or approval.environment != "LAB"
+ or policy.scope != "LAB_PROMOTION"
+ or plan.desired_state != "LAB_STABLE"
+ or approval.subject_id != subject.id
+ or approval.target_type != subject.target_type
+ or approval.target_ref != subject.target_ref
+ or approval.requested_transition != plan.desired_state
+ ):
+ raise LifecycleError(
+ 409, "MANUAL_CANARY_REHEARSAL_ONLY",
+ "Canary observations and commit require an explicit LAB rehearsal; "
+ "authoritative operational canary evidence ingestion is unavailable",
+ )
+ return subject
+
def commit_operation(
self, operation_id: uuid.UUID, decision: ApprovalDecision
) -> LifecycleOperationResponse:
operation = self._operation(operation_id)
plan = self._plan(operation.promotion_plan_id)
+ if operation.stage == OperationStage.COMMITTED:
+ transition = plan.affected_identities.get("serving_transition")
+ if transition is not None:
+ snapshot = self.session.scalar(select(LifecycleRollbackSnapshot).where(
+ LifecycleRollbackSnapshot.promotion_plan_id == plan.id))
+ if (snapshot is None or _hash(snapshot.snapshot) != snapshot.snapshot_digest
+ or snapshot.snapshot.get("serving_transition") != transition):
+ raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact installed serving snapshot is required")
+ LifecycleServing(self.session).installed(transition, _serving_changed_at(operation))
+ self.session.commit()
+ return self._operation_response(operation)
if operation.stage != OperationStage.VERIFYING:
raise LifecycleError(
409, "OPERATION_NOT_REVIEWABLE", "operation is not ready for review"
@@ -805,6 +916,8 @@ class LifecycleService:
canary = self.session.scalar(
select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id)
)
+ if canary or plan.canary_strategy:
+ self._require_manual_canary_rehearsal(plan)
if canary and canary.status != CanaryStatus.READY_FOR_PROMOTION_REVIEW:
raise LifecycleError(
409,
@@ -823,14 +936,26 @@ class LifecycleService:
raise LifecycleError(
409, "ROLLBACK_SNAPSHOT_MISSING", "immutable rollback snapshot is unavailable"
)
+ serving_contract = plan.affected_identities.get("serving_transition")
+ serving_changed_at = None
+ if serving_contract is not None:
+ approval = self._approval(plan.approval_request_id)
+ self._refresh_approval(approval)
+ if approval.status != ApprovalStatus.APPROVED or _hash(approval.evidence_snapshot) != approval.evidence_fingerprint:
+ raise LifecycleError(409, "APPROVAL_STALE", "serving approval is no longer current")
+ if snapshot.snapshot.get("serving_transition") != serving_contract or _hash(snapshot.snapshot) != snapshot.snapshot_digest:
+ raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact immutable serving snapshot is required")
+ serving_changed_at = _now()
+ LifecycleServing(self.session).switch(serving_contract, serving_changed_at.isoformat())
expected_state = (
DeploymentLifecycleState.CANARY if plan.canary_strategy else plan.current_state
)
try:
- assert_deployment_transition(
- DeploymentLifecycleState(expected_state),
- DeploymentLifecycleState(plan.desired_state),
- )
+ if serving_contract is None:
+ assert_deployment_transition(
+ DeploymentLifecycleState(expected_state),
+ DeploymentLifecycleState(plan.desired_state),
+ )
except ValueError as exc:
raise LifecycleError(409, "STATE_CHANGED", str(exc)) from exc
previous = str(expected_state)
@@ -862,7 +987,7 @@ class LifecycleService:
)
self.session.refresh(subject)
operation.stage = OperationStage.COMMITTED
- operation.finished_at = _now()
+ operation.finished_at = serving_changed_at or _now()
operation.version += 1
plan.status = PromotionPlanStatus.COMMITTED
plan.version += 1
@@ -1142,6 +1267,14 @@ class LifecycleService:
def _resolve_evidence(self, declared: dict[str, Any]) -> dict[str, Any]:
facts: dict[str, Any] = {}
+ if declared.get("serving_transition") is not None:
+ from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
+
+ if declared.get("production_validation") != "SATISFIED":
+ raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
+ facts["serving_transition"] = LifecycleServing(self.session).prepare(
+ ServingTransitionSelection.model_validate(declared["serving_transition"])
+ ).model_dump(mode="json")
for field, model in (
("artifact_set_id", ArtifactSet),
("runtime_profile_id", RuntimeProfile),
@@ -1359,8 +1492,21 @@ class LifecycleService:
change_id=str(item.id),
)
return
- current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
- if _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
+ stale_details: dict[str, Any] = {}
+ current_resolved: dict[str, Any] | None
+ try:
+ current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
+ except (LifecycleError, ServingError) as exc:
+ # A historical serving approval must remain inspectable after its route
+ # changes. Expected qualification failures invalidate authority; internal
+ # errors still propagate rather than being disguised as stale evidence.
+ if (item.evidence_snapshot["declared"].get("serving_transition") is None
+ or exc.status_code not in {404, 409, 422}):
+ raise
+ current_resolved = None
+ stale_details = {"validation_code": exc.code, "validation_reason": str(exc)}
+ item.blockers = sorted(set(item.blockers) | {exc.code})
+ if current_resolved is None or _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
previous = item.status
item.status = ApprovalStatus.STALE
item.stale_at = now
@@ -1375,7 +1521,7 @@ class LifecycleService:
item.policy_revision_id,
[],
"Referenced evidence changed after approval request creation",
- {},
+ stale_details,
change_id=str(item.id),
)
@@ -1461,6 +1607,7 @@ class LifecycleService:
) -> None:
started = time.perf_counter()
plan = self._plan(operation.promotion_plan_id)
+ self._approval(plan.approval_request_id) # Same approval-before-route lock order as commit.
snapshot = self.session.scalar(
select(LifecycleRollbackSnapshot).where(
LifecycleRollbackSnapshot.promotion_plan_id == plan.id
@@ -1492,6 +1639,21 @@ class LifecycleService:
)
return
snapshot_version = int(snapshot.snapshot["version"])
+ serving_contract = plan.affected_identities.get("serving_transition")
+ was_committed = operation.stage == OperationStage.COMMITTED
+ if serving_contract is not None:
+ if (snapshot.snapshot.get("serving_transition") != serving_contract
+ or _hash(snapshot.snapshot) != snapshot.snapshot_digest):
+ raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact serving snapshot is unavailable")
+ # A subject CAS failure below rolls back the route in the same transaction.
+ # A later route is preserved rather than guessed away.
+ try:
+ LifecycleServing(self.session).restore(serving_contract,
+ _serving_changed_at(operation) if was_committed else _now().isoformat(),
+ committed=was_committed)
+ except Exception:
+ self.session.rollback()
+ raise
expected_version = snapshot_version + (
2 if operation.stage == OperationStage.COMMITTED else 1
)
@@ -1511,6 +1673,9 @@ class LifecycleService:
)
)
if rows_affected(restored) != 1:
+ if serving_contract is not None:
+ self.session.rollback()
+ raise LifecycleError(409, "ROLLBACK_VERSION_CONFLICT", "lifecycle subject changed; serving state was preserved")
operation.stage = OperationStage.FAILED
operation.failure_code = "ROLLBACK_VERSION_CONFLICT"
operation.failure_details = {
@@ -1581,6 +1746,26 @@ class LifecycleService:
500, "RETENTION_POLICY_MISSING", "rollback retention policy missing"
)
retained_until = _now() + timedelta(days=policy.minimum_rollback_days)
+ transition = plan.affected_identities.get("serving_transition")
+ if transition is not None:
+ prior = transition["current"]
+ for kind, ref in (("ARTIFACT_SET", prior["artifact_set_id"]),
+ ("RUNTIME_PROFILE", prior["runtime_profile_id"]),
+ ("CAPABILITY_DEPLOYMENT", prior["id"])):
+ retained = self.session.scalar(select(LifecycleRetentionRecord).where(
+ LifecycleRetentionRecord.target_type == kind,
+ LifecycleRetentionRecord.target_ref == ref))
+ if retained is None:
+ self.session.add(LifecycleRetentionRecord(
+ policy_revision_id=policy.id, target_type=kind, target_ref=ref,
+ state=RetentionState.ROLLBACK_RETAINED, retained_until=retained_until,
+ legal_hold=False, reason="Exact prior serving identity retained for approved rollback",
+ created_at=_now()))
+ else:
+ previous_until = retained.retained_until
+ if previous_until is None or previous_until.replace(tzinfo=UTC) < retained_until:
+ retained.retained_until = retained_until
+ retained.state = RetentionState.ROLLBACK_RETAINED
existing = self.session.scalar(
select(LifecycleRetentionRecord).where(
LifecycleRetentionRecord.target_type == "ROLLBACK_TARGET",
@@ -1856,19 +2041,19 @@ class LifecycleService:
)
def _approval(self, approval_id: uuid.UUID) -> LifecycleApprovalRequest:
- item = self.session.get(LifecycleApprovalRequest, approval_id)
+ item = self.session.get(LifecycleApprovalRequest, approval_id, with_for_update=True, populate_existing=True)
if not item:
raise LifecycleError(404, "APPROVAL_NOT_FOUND", "approval request not found")
return item
def _plan(self, plan_id: uuid.UUID) -> LifecyclePromotionPlan:
- item = self.session.get(LifecyclePromotionPlan, plan_id)
+ item = self.session.get(LifecyclePromotionPlan, plan_id, with_for_update=True, populate_existing=True)
if not item:
raise LifecycleError(404, "PROMOTION_PLAN_NOT_FOUND", "promotion plan not found")
return item
def _operation(self, operation_id: uuid.UUID) -> LifecycleOperation:
- item = self.session.get(LifecycleOperation, operation_id)
+ item = self.session.get(LifecycleOperation, operation_id, with_for_update=True, populate_existing=True)
if not item:
raise LifecycleError(404, "LIFECYCLE_OPERATION_NOT_FOUND", "operation not found")
return item
diff --git a/backend/src/modelforge_api/services/lifecycle_serving.py b/backend/src/modelforge_api/services/lifecycle_serving.py
new file mode 100644
index 0000000..dd36fc7
--- /dev/null
+++ b/backend/src/modelforge_api/services/lifecycle_serving.py
@@ -0,0 +1,337 @@
+"""Transaction-only serving adapter for approved lifecycle plans.
+
+This adapter never commits, invokes inference, or changes a consumer index.
+The lifecycle engine owns the approval, journal, transaction and rollback.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime
+from typing import Any, NoReturn
+from uuid import UUID
+
+from sqlalchemy import select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lifecycle_contracts import (
+ ServingTransitionContract,
+ ServingTransitionSelection,
+)
+from modelforge_api.domain.serving import ProductionApprovalCreate, SupplyChainReview
+from modelforge_api.persistence.models import (
+ Accelerator,
+ ArtifactSet,
+ CapabilityContract,
+ CapabilityDeployment,
+ CapabilityExperimentRoute,
+ CapabilityResourceEnvelope,
+ ComputeNode,
+ DeploymentCandidate,
+ EvaluationComparison,
+ EvaluationRun,
+ ProductionExecutionApproval,
+ Project,
+ ProjectBinding,
+ ProjectFitEvidence,
+ ResidencyAllocation,
+ RuntimeCompatibilityAssessment,
+ RuntimeProbe,
+ SelectedModelComparison,
+ ServiceClient,
+)
+from modelforge_api.persistence.serving_repository import ServingRepository
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
+from modelforge_api.services.serving import ServingService
+from modelforge_api.settings import get_settings
+
+
+def _fail(code: str, message: str) -> NoReturn:
+ # The parent service imports this adapter; defer its public boundary error.
+ from modelforge_api.services.lifecycle import LifecycleError
+
+ raise LifecycleError(409, code, message)
+
+
+def _json(value: Any) -> Any:
+ if isinstance(value, datetime):
+ return value.replace(tzinfo=UTC).isoformat() if value.tzinfo is None else value.astimezone(UTC).isoformat()
+ return str(value) if isinstance(value, UUID) else value
+
+
+def digest(value: Any) -> str:
+ return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
+ default=_json, allow_nan=False).encode()).hexdigest()
+
+
+ROUTE_FIELDS = (
+ "id", "capability_contract_id", "deployment_candidate_id", "production_approval_id",
+ "execution_approval_id", "embedding_space_id", "artifact_set_id", "runtime_profile_id",
+ "compute_node_id", "accelerator_id", "channel", "status", "production", "routing_weight",
+ "fallback_policy", "residency_policy", "keep_warm_seconds", "max_concurrency", "max_queue_depth",
+ "config_fingerprint", "provenance", "rollback_policy", "promoted_at", "draining_at", "deprecated_at",
+)
+
+
+class LifecycleServing:
+ def __init__(self, session: Session) -> None:
+ self.session = session
+ self.repo = ServingRepository(session)
+
+ @staticmethod
+ def route_snapshot(row: CapabilityDeployment) -> dict[str, Any]:
+ return {field: _json(getattr(row, field)) for field in ROUTE_FIELDS}
+
+ def _deployment(self, identifier: UUID) -> CapabilityDeployment:
+ row = self.session.get(CapabilityDeployment, identifier, with_for_update=True, populate_existing=True)
+ if row is None:
+ _fail("SERVING_TARGET_MISSING", "exact serving deployment is unavailable")
+ return row
+
+ def _lock_accelerators(self, selection: ServingTransitionSelection) -> None:
+ node_ids = list(self.session.scalars(select(CapabilityDeployment.compute_node_id).where(
+ CapabilityDeployment.id.in_((selection.current_deployment_id, selection.candidate_deployment_id)))))
+ # Hardware inventory flushes compute_nodes before updating accelerators.
+ # Native admission does not acquire a node lock while holding its accelerator.
+ self.session.scalars(select(ComputeNode).where(ComputeNode.id.in_(node_ids)).order_by(
+ ComputeNode.id).with_for_update().execution_options(populate_existing=True)).all()
+ identifiers = list(self.session.scalars(select(CapabilityDeployment.accelerator_id).where(
+ CapabilityDeployment.id.in_((selection.current_deployment_id, selection.candidate_deployment_id)))))
+ self.session.scalars(select(Accelerator).where(Accelerator.id.in_(identifiers)).order_by(
+ Accelerator.id).with_for_update()).all()
+
+ def _bindings(self, contract_id: UUID) -> dict[str, dict[str, Any]]:
+ # Lock even deprecated bindings: a concurrent reactivation changes the blast radius.
+ bindings = list(self.session.scalars(select(ProjectBinding).where(
+ ProjectBinding.capability_contract_id == contract_id
+ ).order_by(ProjectBinding.id).with_for_update().execution_options(populate_existing=True)))
+ result = {}
+ for binding in bindings:
+ project = self.session.get(Project, binding.project_id, with_for_update=True, populate_existing=True)
+ clients = list(self.session.scalars(select(ServiceClient).where(
+ ServiceClient.project_binding_id == binding.id
+ ).order_by(ServiceClient.id).with_for_update().execution_options(populate_existing=True)))
+ if binding.channel != "stable" or binding.deprecated_at is not None:
+ continue
+ if project is None:
+ _fail("SERVING_BINDING_MISSING", "binding project disappeared")
+ result[str(binding.id)] = {
+ "project_id": str(binding.project_id), "project_active": project.active,
+ "priority": binding.priority, "optional": binding.optional,
+ "fallback_policy": binding.fallback_policy, "migration_support": binding.migration_support,
+ "slo": binding.slo, "benchmark_requirements": binding.benchmark_requirements,
+ "clients": [{"id": str(client.id), "status": client.status,
+ "disabled_at": _json(client.disabled_at),
+ "capabilities": client.allowed_capabilities} for client in clients],
+ }
+ if not result or len(result) > 1000:
+ _fail("SERVING_BINDINGS_UNBOUNDED", "transition requires one to 1000 exact stable bindings")
+ return result
+
+ def _experiment_routes(self, candidate_id: UUID) -> dict[str, dict[str, Any]]:
+ return {str(row.id): {"route_key": row.route_key, "status": row.status,
+ "capability_contract_id": str(row.capability_contract_id)}
+ for row in self.session.scalars(select(CapabilityExperimentRoute).where(
+ CapabilityExperimentRoute.capability_deployment_id == candidate_id
+ ).order_by(CapabilityExperimentRoute.id).with_for_update().execution_options(populate_existing=True))}
+
+ def prepare(self, selection: ServingTransitionSelection) -> ServingTransitionContract:
+ # Obtain the contract before any deployment locks to match other route writers.
+ candidate_ref = self.session.get(CapabilityDeployment, selection.candidate_deployment_id)
+ if candidate_ref is None:
+ _fail("SERVING_TARGET_MISSING", "candidate deployment is unavailable")
+ contract = self.repo.lock_contract_route(candidate_ref.capability_contract_id)
+ if contract is None or contract.deprecated_at is not None:
+ _fail("SERVING_CONTRACT_UNAVAILABLE", "capability contract is unavailable")
+ if contract.serving_generation != selection.expected_route_generation:
+ _fail("SERVING_ROUTE_CHANGED", "stable route generation changed after selection")
+ # Match inventory node->accelerator and admission accelerator->residency/binding.
+ self._lock_accelerators(selection)
+ current = self._deployment(selection.current_deployment_id)
+ candidate = self._deployment(selection.candidate_deployment_id)
+ if (current.id == candidate.id or current.capability_contract_id != contract.id
+ or not current.production or current.status != "stable" or current.channel != "stable"
+ or candidate.production or candidate.status != "candidate" or candidate.channel != "experiment"):
+ _fail("SERVING_SELECTION_INVALID", "exact stable current and nonproduction candidate are required")
+ if current.embedding_space_id is None or current.embedding_space_id != candidate.embedding_space_id:
+ _fail("SERVING_REINDEX_REQUIRED", "changed embedding identity requires trusted consumer reindex and cutover")
+ experiment_routes = self._experiment_routes(candidate.id)
+ bindings = self._bindings(contract.id)
+ if set(bindings) != {str(identifier) for identifier in selection.comparison_ids}:
+ _fail("SERVING_BINDING_EVIDENCE_REQUIRED", "every affected stable binding requires its own comparison")
+ if len(set(selection.comparison_ids.values())) != len(bindings):
+ _fail("SERVING_COMPARISON_REUSED", "one project comparison cannot authorize another binding")
+ comparison_service = SelectedModelComparisonService(self.session, AuditContext("system", "lifecycle-serving"))
+ evidence = {}
+ for binding_id, comparison_id in sorted(selection.comparison_ids.items(), key=lambda item: str(item[0])):
+ comparison = self.session.get(SelectedModelComparison, comparison_id, with_for_update=True, populate_existing=True)
+ latest = self.session.scalar(select(SelectedModelComparison.id).where(
+ SelectedModelComparison.project_binding_id == binding_id
+ ).order_by(SelectedModelComparison.created_at.desc(), SelectedModelComparison.id.desc()).limit(1))
+ if (comparison is None or comparison.status != "completed" or comparison.id != latest
+ or comparison.project_binding_id != binding_id
+ or comparison.current_deployment_id != current.id or comparison.candidate_deployment_id != candidate.id
+ or comparison.finished_at is None or comparison.result_digest is None):
+ _fail("SERVING_COMPARISON_STALE", "latest completed exact current/candidate comparison is required")
+ comparison_service._revalidate(comparison)
+ evaluated = self.session.get(EvaluationComparison, comparison.evaluation_comparison_id)
+ if (evaluated is None or evaluated.comparability != "comparable"
+ or evaluated.promotion_eligibility != "eligible" or evaluated.critical_regressions
+ or evaluated.baseline_run_id != comparison.baseline_run_id
+ or evaluated.candidate_run_id != comparison.candidate_run_id):
+ _fail("SERVING_COMPARISON_NOT_ELIGIBLE", "project comparison does not qualify replacement")
+ for run_id in (comparison.baseline_run_id, comparison.candidate_run_id):
+ run = self.session.get(EvaluationRun, run_id)
+ if run is None or run.status != "completed" or run.error_count or run.completed_cases != run.expected_cases:
+ _fail("SERVING_RUN_NOT_COMPLETE", "both comparison legs must be complete and error-free")
+ fit = self.session.scalar(select(ProjectFitEvidence).where(
+ ProjectFitEvidence.project_binding_id == binding_id,
+ ProjectFitEvidence.capability_deployment_id == candidate.id,
+ ).order_by(ProjectFitEvidence.created_at.desc(), ProjectFitEvidence.id.desc()).limit(1)
+ .with_for_update().execution_options(populate_existing=True))
+ if fit is not None and (fit.recommendation != "PROMOTION_ELIGIBLE"
+ or fit.production_validation != "SATISFIED" or fit.engineering_integration != "PASS"
+ or fit.critical_errors or fit.blockers or fit.case_count < 1):
+ _fail("SERVING_PROJECT_FIT_BLOCKED", "latest exact binding/candidate project-fit evidence blocks production")
+ evidence[str(binding_id)] = {"id": str(comparison.id), "pins": comparison.pins,
+ "result_digest": comparison.result_digest, "evaluation_comparison_id": str(evaluated.id),
+ "baseline_run_id": str(evaluated.baseline_run_id), "candidate_run_id": str(evaluated.candidate_run_id),
+ "project_fit": ({"id": str(fit.id), "evidence_digest": fit.evidence_digest,
+ "record_digest": digest({column.name: _json(getattr(fit, column.name))
+ for column in fit.__table__.columns})} if fit is not None else None),
+ "evaluation_digest": digest({column.name: _json(getattr(evaluated, column.name))
+ for column in evaluated.__table__.columns})}
+ approval = self._production_evidence(candidate, contract, selection.production_approval_id)
+ return ServingTransitionContract(selection=selection, capability_contract_id=contract.id,
+ current=self.route_snapshot(current), candidate=self.route_snapshot(candidate),
+ experiment_routes=experiment_routes,
+ bindings=bindings, comparisons=evidence, production_approval_fingerprint=approval.evidence_fingerprint)
+
+ def _production_evidence(self, candidate: CapabilityDeployment, contract: CapabilityContract,
+ approval_id: UUID) -> ProductionExecutionApproval:
+ approval = self.session.get(ProductionExecutionApproval, approval_id,
+ with_for_update=True, populate_existing=True)
+ if (approval is None or approval.status != "approved" or approval.revoked_at is not None
+ or approval.deployment_candidate_id != candidate.deployment_candidate_id
+ or approval.capability_contract_id != contract.id
+ or approval.artifact_set_id != candidate.artifact_set_id
+ or approval.runtime_profile_id != candidate.runtime_profile_id
+ or approval.compute_node_id != candidate.compute_node_id):
+ _fail("SERVING_PRODUCTION_APPROVAL_INVALID", "exact current production execution approval is required")
+ # Immutable identity fields are already sealed. Mutable qualification/revocation
+ # fields must remain locked until the containing lifecycle transaction commits.
+ source = self.session.get(DeploymentCandidate, candidate.deployment_candidate_id,
+ with_for_update=True, populate_existing=True)
+ if source is None:
+ _fail("SERVING_TARGET_MISSING", "candidate provenance disappeared")
+ self.session.get(RuntimeProbe, source.runtime_probe_id, with_for_update=True, populate_existing=True)
+ self.session.get(RuntimeCompatibilityAssessment, source.compatibility_assessment_id,
+ with_for_update=True, populate_existing=True)
+ self.session.get(ArtifactSet, candidate.artifact_set_id, with_for_update=True, populate_existing=True)
+ self.session.get(ComputeNode, candidate.compute_node_id, with_for_update=True, populate_existing=True)
+ for artifact in self.repo.set_artifacts(candidate.artifact_set_id):
+ self.session.refresh(artifact, with_for_update=True)
+ # Reuse the existing production validation engine, without its create/promote commits.
+ settings = get_settings()
+ serving = ServingService(self.session, settings, ManifestRegistry(settings.config_root), None)
+ facts, _, probe, _, _ = serving._approval_facts(candidate.deployment_candidate_id, contract,
+ ProductionApprovalCreate(capability="rag.embedding", contract_version=1,
+ approved_by=approval.approved_by, reason=approval.reason,
+ supply_chain_review=SupplyChainReview.model_validate(approval.supply_chain_evidence),
+ deployment_config=approval.deployment_config))
+ if digest(facts) != approval.evidence_fingerprint:
+ _fail("SERVING_PRODUCTION_APPROVAL_STALE", "production probe or supply-chain evidence changed")
+ envelope = self.session.scalar(select(CapabilityResourceEnvelope).where(
+ CapabilityResourceEnvelope.capability_deployment_id == candidate.id
+ ).with_for_update().execution_options(populate_existing=True))
+ if envelope is None or envelope.stale or envelope.runtime_probe_id != probe.id:
+ _fail("SERVING_ENVELOPE_STALE", "candidate requires its exact current measured envelope")
+ residency = self.session.scalar(select(ResidencyAllocation).where(
+ ResidencyAllocation.capability_deployment_id == candidate.id
+ ).with_for_update().execution_options(populate_existing=True))
+ if (residency is None or residency.compute_node_id != candidate.compute_node_id
+ or residency.accelerator_id != candidate.accelerator_id
+ or residency.state not in {"cold", "ready", "warm"}):
+ _fail("SERVING_RESIDENCY_UNAVAILABLE", "candidate has no settled schedulable runtime residency")
+ return approval
+
+ def validate(self, payload: dict[str, Any]) -> ServingTransitionContract:
+ expected = ServingTransitionContract.model_validate(payload)
+ actual = self.prepare(expected.selection)
+ if actual != expected:
+ _fail("SERVING_PLAN_STALE", "approved serving identities or affected bindings changed")
+ return expected
+
+ def _generation(self, contract_id: UUID, expected: int) -> None:
+ changed = self.session.execute(update(CapabilityContract).where(
+ CapabilityContract.id == contract_id, CapabilityContract.serving_generation == expected
+ ).values(serving_generation=expected + 1).returning(CapabilityContract.id)).scalar_one_or_none()
+ if changed is None:
+ _fail("SERVING_ROUTE_CHANGED", "stable route generation changed concurrently")
+
+ @staticmethod
+ def after(contract: ServingTransitionContract, changed_at: str) -> tuple[dict[str, Any], dict[str, Any]]:
+ old = {**contract.current, "status": "draining", "draining_at": changed_at, "routing_weight": 0}
+ new = {**contract.candidate, "status": "stable", "channel": "stable", "production": True,
+ "production_approval_id": str(contract.selection.production_approval_id),
+ "promoted_at": changed_at, "routing_weight": 100}
+ return old, new
+
+ def switch(self, payload: dict[str, Any], changed_at: str) -> None:
+ contract = self.validate(payload)
+ self._generation(contract.capability_contract_id, contract.selection.expected_route_generation)
+ current = self._deployment(contract.selection.current_deployment_id)
+ candidate = self._deployment(contract.selection.candidate_deployment_id)
+ current.status, current.draining_at = "draining", datetime.fromisoformat(changed_at)
+ current.routing_weight = 0
+ self.session.flush() # Release the one-stable partial unique index before activating the next.
+ candidate.status, candidate.channel, candidate.production = "stable", "stable", True
+ candidate.production_approval_id = contract.selection.production_approval_id
+ candidate.routing_weight = 100
+ candidate.promoted_at = datetime.fromisoformat(changed_at)
+ self.session.flush()
+
+ def installed(self, payload: dict[str, Any], changed_at: str, *, committed: bool = True) -> tuple[ServingTransitionContract, CapabilityDeployment, CapabilityDeployment]:
+ """Verify exact persisted result for retry/rollback, without switching anything."""
+ contract = ServingTransitionContract.model_validate(payload)
+ route = self.repo.lock_contract_route(contract.capability_contract_id)
+ expected_generation = contract.selection.expected_route_generation + int(committed)
+ if route is None or route.serving_generation != expected_generation:
+ _fail("SERVING_ROLLBACK_CONFLICT", "later stable route generation must be preserved")
+ self._lock_accelerators(contract.selection)
+ current = self._deployment(contract.selection.current_deployment_id)
+ candidate = self._deployment(contract.selection.candidate_deployment_id)
+ expected = self.after(contract, changed_at) if committed else (contract.current, contract.candidate)
+ if ((self.route_snapshot(current), self.route_snapshot(candidate)) != expected
+ or self._bindings(contract.capability_contract_id) != contract.bindings
+ or self._experiment_routes(candidate.id) != contract.experiment_routes):
+ _fail("SERVING_ROLLBACK_CONFLICT", "serving state or affected bindings changed after the snapshot")
+ return contract, current, candidate
+
+ def restore(self, payload: dict[str, Any], changed_at: str, *, committed: bool) -> None:
+ contract, current, candidate = self.installed(payload, changed_at, committed=committed)
+ if not committed:
+ return
+ route = self.session.get(CapabilityContract, contract.capability_contract_id)
+ if route is None or current.production_approval_id is None:
+ _fail("SERVING_ROLLBACK_APPROVAL_MISSING", "prior serving authority is unavailable")
+ self._production_evidence(current, route, current.production_approval_id)
+ expected_generation = contract.selection.expected_route_generation + 1
+ self._generation(contract.capability_contract_id, expected_generation)
+ candidate.status = str(contract.candidate["status"])
+ candidate.channel = str(contract.candidate["channel"])
+ candidate.production = bool(contract.candidate["production"])
+ candidate.routing_weight = int(contract.candidate["routing_weight"])
+ candidate.production_approval_id = (UUID(contract.candidate["production_approval_id"])
+ if contract.candidate["production_approval_id"] else None)
+ candidate.promoted_at = (datetime.fromisoformat(contract.candidate["promoted_at"])
+ if contract.candidate["promoted_at"] else None)
+ self.session.flush()
+ current.status = str(contract.current["status"])
+ current.routing_weight = int(contract.current["routing_weight"])
+ current.draining_at = (datetime.fromisoformat(contract.current["draining_at"])
+ if contract.current["draining_at"] else None)
+ self.session.flush()
diff --git a/backend/src/modelforge_api/services/manifest_registry.py b/backend/src/modelforge_api/services/manifest_registry.py
index 82f5c98..4b45c08 100644
--- a/backend/src/modelforge_api/services/manifest_registry.py
+++ b/backend/src/modelforge_api/services/manifest_registry.py
@@ -73,6 +73,10 @@ class ManifestRegistry:
ProjectManifest.model_validate(self._read_yaml(path))
for path in sorted((self.config_root / "projects").glob("*.yaml"))
]
+ project_ids = [item.project.id for item in manifests]
+ if len(project_ids) != len(set(project_ids)):
+ raise ValueError("duplicate project manifest identity")
+
capabilities = self.capabilities()
by_identity = {(item.capability, item.version): item for item in capabilities}
capability_versions = set(by_identity)
diff --git a/backend/src/modelforge_api/services/migration_engine.py b/backend/src/modelforge_api/services/migration_engine.py
index 5f09ac1..b16e4dd 100644
--- a/backend/src/modelforge_api/services/migration_engine.py
+++ b/backend/src/modelforge_api/services/migration_engine.py
@@ -459,11 +459,13 @@ class MigrationEngineService:
}
return set()
- def plans(self) -> list[MigrationPlanResponse]:
+ def plans(self, limit: int = 200) -> list[MigrationPlanResponse]:
return [
self._plan_response(item)
for item in self.session.scalars(
- select(MigrationPlan).order_by(MigrationPlan.created_at.desc())
+ select(MigrationPlan)
+ .order_by(MigrationPlan.created_at.desc(), MigrationPlan.id.desc())
+ .limit(limit)
)
]
diff --git a/backend/src/modelforge_api/services/model_changes.py b/backend/src/modelforge_api/services/model_changes.py
new file mode 100644
index 0000000..2e44e97
--- /dev/null
+++ b/backend/src/modelforge_api/services/model_changes.py
@@ -0,0 +1,813 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Literal, cast
+from uuid import UUID
+
+from sqlalchemy import func, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.model_changes import (
+ ChangeAcquisition,
+ ChangeImpact,
+ ChangeModelIdentity,
+ ChangeProgressStep,
+ ModelChangeCandidateLink,
+ ModelChangeCreate,
+ ModelChangeEvidenceLink,
+ ModelChangeList,
+ ModelChangeReject,
+ ModelChangeResponse,
+ ModelChangeStatus,
+)
+from modelforge_api.domain.selected_replacement import ReplacementPreview
+from modelforge_api.persistence.model_change import ModelChangeCase
+from modelforge_api.persistence.models import (
+ ArtifactJob,
+ ArtifactSet,
+ Capability,
+ CapabilityContract,
+ CapabilityDeployment,
+ DiscoveryCandidateAssessment,
+ DownloadPlan,
+ GatewayRequest,
+ LifecyclePromotionPlan,
+ MigrationPlan,
+ Model,
+ ModelRevision,
+ Project,
+ ProjectBinding,
+ SelectedModelComparison,
+ ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.evaluation import EvaluationError
+from modelforge_api.services.lifecycle import LifecycleError
+from modelforge_api.services.selected_replacement import SelectedReplacementService
+from modelforge_api.services.serving import ServingError
+
+ProgressState = Literal["done", "current", "upcoming", "blocked"]
+
+
+class ModelChangeError(ValueError):
+ def __init__(self, code: str, message: str, status_code: int = 409) -> None:
+ super().__init__(message)
+ self.code = code
+ self.message = message
+ self.status_code = status_code
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _gate(value: object) -> str:
+ if not isinstance(value, dict):
+ return "unknown"
+ text = " ".join(
+ str(value.get(key, "")) for key in ("status", "state", "verdict", "result")
+ ).lower()
+ if any(word in text for word in ("block", "reject", "deny", "fail", "incompat")):
+ return "blocked"
+ if any(
+ word in text for word in ("approved", "clear", "verified", "compatible", "pass", "safe")
+ ):
+ return "clear"
+ return "review"
+
+
+class ModelChangeService:
+ """User workflow over authoritative ModelForge evidence; never executes lifecycle work itself."""
+
+ def __init__(self, session: Session, audit_context: AuditContext) -> None:
+ self.session = session
+ self.audit_context = audit_context
+
+ def _error(self, code: str, message: str, status: int = 409) -> ModelChangeError:
+ return ModelChangeError(code, message, status)
+
+ def _audit(
+ self,
+ row: ModelChangeCase,
+ action: str,
+ details: dict[str, object] | None = None,
+ ) -> None:
+ AuditWriter(self.session, context=self.audit_context).write(
+ action,
+ "model_change_case",
+ str(row.id),
+ {
+ "project_binding_id": str(row.project_binding_id),
+ "current_deployment_id": str(row.current_deployment_id),
+ "candidate_deployment_id": (
+ str(row.candidate_deployment_id) if row.candidate_deployment_id else None
+ ),
+ "status": row.status,
+ "version": row.version,
+ **(details or {}),
+ },
+ )
+
+ def _row(self, identifier: UUID, *, lock: bool = False) -> ModelChangeCase:
+ query = select(ModelChangeCase).where(ModelChangeCase.id == identifier)
+ if lock:
+ query = query.with_for_update().execution_options(populate_existing=True)
+ row = self.session.scalar(query)
+ if row is None:
+ raise self._error("MODEL_CHANGE_NOT_FOUND", "Model change case was not found", 404)
+ return row
+
+ def _binding_context(
+ self, binding_id: UUID
+ ) -> tuple[ProjectBinding, Project, CapabilityContract, Capability]:
+ binding = self.session.get(ProjectBinding, binding_id)
+ project = self.session.get(Project, binding.project_id) if binding else None
+ contract = (
+ self.session.get(CapabilityContract, binding.capability_contract_id) if binding else None
+ )
+ capability = self.session.get(Capability, contract.capability_id) if contract else None
+ if (
+ binding is None
+ or binding.deprecated_at is not None
+ or binding.channel != "stable"
+ or project is None
+ or not project.active
+ or contract is None
+ or contract.deprecated_at is not None
+ or capability is None
+ ):
+ raise self._error(
+ "MODEL_CHANGE_BINDING_INVALID",
+ "The selected project task is not an active stable binding",
+ 422,
+ )
+ return binding, project, contract, capability
+
+ def _stable_deployment(self, contract_id: UUID) -> CapabilityDeployment | None:
+ return self.session.scalar(
+ select(CapabilityDeployment)
+ .where(
+ CapabilityDeployment.capability_contract_id == contract_id,
+ CapabilityDeployment.production.is_(True),
+ CapabilityDeployment.status == "stable",
+ CapabilityDeployment.deprecated_at.is_(None),
+ )
+ .order_by(
+ CapabilityDeployment.promoted_at.desc(),
+ CapabilityDeployment.created_at.desc(),
+ )
+ .limit(1)
+ )
+
+ def _identity(
+ self,
+ *,
+ model_id: UUID | None = None,
+ revision_id: UUID | None = None,
+ artifact_set_id: UUID | None = None,
+ deployment_id: UUID | None = None,
+ ) -> ChangeModelIdentity | None:
+ deployment = self.session.get(CapabilityDeployment, deployment_id) if deployment_id else None
+ if deployment is not None:
+ artifact_set_id = deployment.artifact_set_id
+ artifact = self.session.get(ArtifactSet, artifact_set_id) if artifact_set_id else None
+ if artifact is not None:
+ revision_id = artifact.revision_id
+ revision = self.session.get(ModelRevision, revision_id) if revision_id else None
+ if revision is not None:
+ model_id = revision.model_id
+ model = self.session.get(Model, model_id) if model_id else None
+ if not any((deployment, artifact, revision, model)):
+ return None
+ return ChangeModelIdentity(
+ model_id=model.id if model else model_id,
+ display_name=model.display_name if model else None,
+ revision_id=revision.id if revision else revision_id,
+ resolved_commit_sha=revision.resolved_commit_sha if revision else None,
+ artifact_set_id=artifact.id if artifact else artifact_set_id,
+ deployment_id=deployment.id if deployment else deployment_id,
+ )
+
+ def _current_identity(self, row: ModelChangeCase) -> ChangeModelIdentity:
+ identity = self._identity(deployment_id=row.current_deployment_id)
+ if identity is None:
+ raise self._error(
+ "MODEL_CHANGE_CURRENT_IDENTITY_MISSING",
+ "The exact model identity behind the current deployment is incomplete",
+ )
+ return identity
+
+ def _candidate_identity(self, row: ModelChangeCase) -> ChangeModelIdentity | None:
+ return self._identity(
+ model_id=row.candidate_model_id,
+ revision_id=row.candidate_revision_id,
+ artifact_set_id=row.candidate_artifact_set_id,
+ deployment_id=row.candidate_deployment_id,
+ )
+
+ def _acquisition(self, row: ModelChangeCase) -> ChangeAcquisition:
+ assessment = (
+ self.session.get(DiscoveryCandidateAssessment, row.discovery_assessment_id)
+ if row.discovery_assessment_id
+ else None
+ )
+ plan = self.session.get(DownloadPlan, row.download_plan_id) if row.download_plan_id else None
+ job = self.session.get(ArtifactJob, row.artifact_job_id) if row.artifact_job_id else None
+ storage_bytes = plan.total_size_bytes if plan else None
+ if storage_bytes is None and row.candidate_artifact_set_id:
+ artifact = self.session.get(ArtifactSet, row.candidate_artifact_set_id)
+ storage_bytes = artifact.total_size_bytes if artifact else None
+ fit = "unknown"
+ if assessment is not None and isinstance(assessment.gpu_fit, dict):
+ fit = str(
+ assessment.gpu_fit.get("fit")
+ or assessment.gpu_fit.get("status")
+ or assessment.gpu_fit.get("verdict")
+ or "unknown"
+ )
+ return ChangeAcquisition(
+ discovery_assessment_id=assessment.id if assessment else None,
+ security_gate=_gate(assessment.security_state) if assessment else "unknown",
+ license_gate=_gate(assessment.license_state) if assessment else "unknown",
+ hardware_fit=fit,
+ storage_bytes=storage_bytes,
+ download_plan_id=plan.id if plan else None,
+ artifact_job_id=job.id if job else None,
+ download_status=job.status if job else (plan.status if plan else None),
+ )
+
+ def _successful_project_request_after(
+ self, binding_id: UUID, deployment_id: UUID, since: datetime | None
+ ) -> bool:
+ clients = list(
+ self.session.scalars(
+ select(ServiceClient.id).where(
+ ServiceClient.project_binding_id == binding_id,
+ ServiceClient.status == "active",
+ ServiceClient.disabled_at.is_(None),
+ )
+ )
+ )
+ if not clients:
+ return False
+ query = select(GatewayRequest.id).where(
+ GatewayRequest.service_client_id.in_(clients),
+ GatewayRequest.capability_deployment_id == deployment_id,
+ GatewayRequest.status == "completed",
+ GatewayRequest.finished_at.is_not(None),
+ )
+ if since is not None:
+ query = query.where(GatewayRequest.finished_at >= since)
+ return (
+ self.session.scalar(query.order_by(GatewayRequest.finished_at.desc()).limit(1))
+ is not None
+ )
+
+ def _preview(self, row: ModelChangeCase) -> ReplacementPreview | None:
+ if row.selected_comparison_id is None:
+ return None
+ try:
+ return SelectedReplacementService(self.session).preview(row.selected_comparison_id)
+ except (EvaluationError, LifecycleError, ServingError):
+ # The detailed subsystem remains authoritative. A stale preview simply keeps this
+ # user projection from claiming readiness until the linked evidence is refreshed.
+ return None
+
+ def _derived_status(self, row: ModelChangeCase) -> ModelChangeStatus:
+ if row.closed_at is not None and row.status in {"rolled_back", "rejected"}:
+ return cast(ModelChangeStatus, row.status)
+ binding, _project, contract, _capability = self._binding_context(row.project_binding_id)
+ lifecycle = (
+ self.session.get(LifecyclePromotionPlan, row.lifecycle_plan_id)
+ if row.lifecycle_plan_id
+ else None
+ )
+ if lifecycle is not None and lifecycle.status == "ROLLED_BACK":
+ return "rolled_back"
+ stable = self._stable_deployment(contract.id)
+ if row.candidate_deployment_id and stable and stable.id == row.candidate_deployment_id:
+ since = stable.promoted_at or row.activated_at or row.updated_at
+ if self._successful_project_request_after(binding.id, stable.id, since):
+ return "checked"
+ return "active"
+ if lifecycle is not None and lifecycle.status == "COMMITTED":
+ return "active"
+ if lifecycle is not None and lifecycle.status == "APPROVED":
+ return "ready_to_activate"
+ comparison = (
+ self.session.get(SelectedModelComparison, row.selected_comparison_id)
+ if row.selected_comparison_id
+ else None
+ )
+ if comparison is not None:
+ if comparison.status == "completed":
+ preview = self._preview(row)
+ if preview is not None and not preview.same_embedding_space:
+ return "migration_required"
+ if preview is not None and preview.selection is not None:
+ return "ready_to_activate"
+ return "comparison_available"
+ if comparison.status in {"pending", "running", "cancel_requested"}:
+ return "tested"
+ if row.candidate_deployment_id:
+ return "ready_to_test"
+ job = self.session.get(ArtifactJob, row.artifact_job_id) if row.artifact_job_id else None
+ if job is not None and job.status == "completed":
+ return "downloaded"
+ plan = self.session.get(DownloadPlan, row.download_plan_id) if row.download_plan_id else None
+ if plan is not None and plan.status == "completed":
+ return "downloaded"
+ assessment = (
+ self.session.get(DiscoveryCandidateAssessment, row.discovery_assessment_id)
+ if row.discovery_assessment_id
+ else None
+ )
+ if assessment is not None and assessment.status.lower() not in {
+ "blocked",
+ "rejected",
+ "failed",
+ }:
+ return "source_checked"
+ if any((row.candidate_model_id, row.candidate_revision_id, row.candidate_artifact_set_id)):
+ return "found"
+ return "searching"
+
+ def _impact(self, row: ModelChangeCase) -> ChangeImpact:
+ _binding, project, _contract, _capability = self._binding_context(row.project_binding_id)
+ affected = [project.name]
+ blockers: list[str] = []
+ migration_required = False
+ migration_kind = "none"
+ rollback_available = bool(row.candidate_deployment_id)
+ preview = self._preview(row)
+ if preview is not None:
+ affected = list(dict.fromkeys(item.project_name for item in preview.affected_bindings)) or affected
+ blockers = list(dict.fromkeys(item.message for item in preview.blockers))
+ migration_required = not preview.same_embedding_space
+ migration_kind = "reindex" if migration_required else "none"
+ rollback_available = preview.current_deployment_id is not None
+ migration = (
+ self.session.get(MigrationPlan, row.migration_plan_id) if row.migration_plan_id else None
+ )
+ if migration is not None:
+ migration_kind = str(migration.migration_class).lower()
+ migration_required = migration_kind not in {"transparent", "none"}
+ lifecycle = (
+ self.session.get(LifecyclePromotionPlan, row.lifecycle_plan_id)
+ if row.lifecycle_plan_id
+ else None
+ )
+ downtime = False
+ application_change = False
+ if lifecycle is not None:
+ details = lifecycle.impact_analysis if isinstance(lifecycle.impact_analysis, dict) else {}
+ downtime = bool(details.get("downtime_expected", details.get("downtime", False)))
+ application_change = str(lifecycle.migration_class).lower() == "schema_breaking"
+ rollback_available = bool(lifecycle.rollback_target_ref)
+ if lifecycle.project_consumers:
+ affected = list(dict.fromkeys([*affected, *lifecycle.project_consumers]))
+ return ChangeImpact(
+ affected_projects=affected,
+ affected_project_count=len(affected),
+ application_change_required=application_change,
+ migration_required=migration_required,
+ migration_kind=migration_kind,
+ downtime_expected=downtime,
+ rollback_available=rollback_available,
+ blockers=blockers,
+ )
+
+ def _next_action(self, status: ModelChangeStatus) -> str:
+ return {
+ "searching": "Kandidaat zoeken",
+ "found": "Bron controleren",
+ "source_checked": "Downloaden voor test",
+ "downloaded": "Klaarzetten om te testen",
+ "ready_to_test": "Test uitvoeren",
+ "tested": "Vergelijk met huidig model",
+ "comparison_available": "Voorbereiden om te wisselen",
+ "migration_required": "Herindexering of migratie voorbereiden",
+ "ready_to_activate": "Veilig activeren",
+ "active": "Gezondheid controleren",
+ "checked": "Geen actie nodig",
+ "rolled_back": "Afgerond — teruggerold",
+ "rejected": "Afgerond — afgewezen",
+ }[status]
+
+ def _progress(self, status: ModelChangeStatus) -> list[ChangeProgressStep]:
+ labels = [
+ ("candidate", "Kandidaat"),
+ ("source", "Broncontrole"),
+ ("download", "Download"),
+ ("test", "Test"),
+ ("compare", "Vergelijking"),
+ ("impact", "Impact & migratie"),
+ ("activate", "Activeren"),
+ ("verify", "Controle"),
+ ]
+ rank = {
+ "searching": 0,
+ "found": 0,
+ "source_checked": 1,
+ "downloaded": 2,
+ "ready_to_test": 3,
+ "tested": 4,
+ "comparison_available": 5,
+ "migration_required": 5,
+ "ready_to_activate": 6,
+ "active": 7,
+ "checked": 8,
+ "rolled_back": 8,
+ "rejected": 8,
+ }[status]
+ steps: list[ChangeProgressStep] = []
+ for index, (key, label) in enumerate(labels):
+ state: ProgressState = (
+ "done"
+ if index < rank
+ else "current"
+ if index == rank and rank < len(labels)
+ else "upcoming"
+ )
+ if status == "migration_required" and key == "impact":
+ state = "blocked"
+ steps.append(ChangeProgressStep(key=key, label=label, state=state))
+ return steps
+
+ def _response(self, row: ModelChangeCase) -> ModelChangeResponse:
+ binding, project, contract, capability = self._binding_context(row.project_binding_id)
+ status = self._derived_status(row)
+ return ModelChangeResponse(
+ id=row.id,
+ project_binding_id=binding.id,
+ project_id=project.id,
+ project_key=project.key,
+ project_name=project.name,
+ capability=f"{capability.key}@{contract.version}",
+ status=status,
+ stored_status=cast(ModelChangeStatus, row.status),
+ status_reason=row.status_reason,
+ version=row.version,
+ current=self._current_identity(row),
+ candidate=self._candidate_identity(row),
+ acquisition=self._acquisition(row),
+ selected_comparison_id=row.selected_comparison_id,
+ migration_plan_id=row.migration_plan_id,
+ lifecycle_plan_id=row.lifecycle_plan_id,
+ impact=self._impact(row),
+ next_action=self._next_action(status),
+ progress=self._progress(status),
+ created_by=row.created_by,
+ updated_by=row.updated_by,
+ activated_at=row.activated_at,
+ checked_at=row.checked_at,
+ closed_at=row.closed_at,
+ created_at=row.created_at,
+ updated_at=row.updated_at,
+ )
+
+ def _remember_derived_status(self, row: ModelChangeCase) -> None:
+ status = self._derived_status(row)
+ if status != row.status:
+ row.status = status
+ row.status_reason = self._next_action(status)
+
+ def create(self, request: ModelChangeCreate) -> ModelChangeResponse:
+ binding, project, contract, capability = self._binding_context(request.project_binding_id)
+ self.session.scalar(
+ select(ProjectBinding).where(ProjectBinding.id == binding.id).with_for_update()
+ )
+ existing = self.session.scalar(
+ select(ModelChangeCase).where(
+ ModelChangeCase.project_binding_id == binding.id,
+ ModelChangeCase.closed_at.is_(None),
+ )
+ )
+ if existing is not None:
+ return self._response(existing)
+ current = self._stable_deployment(contract.id)
+ if current is None:
+ raise self._error(
+ "MODEL_CHANGE_CURRENT_UNAVAILABLE",
+ "No healthy stable model deployment is available for this project task",
+ 422,
+ )
+ row = ModelChangeCase(
+ project_binding_id=binding.id,
+ current_deployment_id=current.id,
+ status="searching",
+ status_reason=request.reason,
+ version=1,
+ created_by=self.audit_context.actor_id,
+ updated_by=self.audit_context.actor_id,
+ context={
+ "project_id": str(project.id),
+ "project_key": project.key,
+ "capability": f"{capability.key}@{contract.version}",
+ "reason": request.reason,
+ },
+ )
+ self.session.add(row)
+ try:
+ self.session.flush()
+ self._audit(row, "MODEL_CHANGE_CASE_CREATED")
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise self._error(
+ "MODEL_CHANGE_ALREADY_OPEN",
+ "Another model change is already open for this project task",
+ ) from exc
+ self.session.refresh(row)
+ return self._response(row)
+
+ def get(self, identifier: UUID) -> ModelChangeResponse:
+ return self._response(self._row(identifier))
+
+ def list(
+ self,
+ *,
+ project_key: str | None = None,
+ status: str | None = None,
+ project_binding_id: UUID | None = None,
+ open_only: bool = False,
+ limit: int = 100,
+ offset: int = 0,
+ ) -> ModelChangeList:
+ if not 1 <= limit <= 200:
+ raise self._error("MODEL_CHANGE_PAGE_INVALID", "Limit must be between 1 and 200", 422)
+ if offset < 0:
+ raise self._error("MODEL_CHANGE_OFFSET_INVALID", "Offset must be zero or greater", 422)
+ query = select(ModelChangeCase)
+ if project_key:
+ query = (
+ query.join(ProjectBinding, ProjectBinding.id == ModelChangeCase.project_binding_id)
+ .join(Project, Project.id == ProjectBinding.project_id)
+ .where(Project.key == project_key)
+ )
+ if status:
+ query = query.where(ModelChangeCase.status == status)
+ if project_binding_id is not None:
+ query = query.where(ModelChangeCase.project_binding_id == project_binding_id)
+ if open_only:
+ query = query.where(ModelChangeCase.closed_at.is_(None))
+ total = self.session.scalar(select(func.count()).select_from(query.subquery())) or 0
+ rows = self.session.scalars(
+ query.order_by(ModelChangeCase.updated_at.desc(), ModelChangeCase.id.asc())
+ .offset(offset)
+ .limit(limit)
+ ).all()
+ return ModelChangeList(items=[self._response(row) for row in rows], total=total)
+
+ def link_candidate(
+ self, identifier: UUID, request: ModelChangeCandidateLink
+ ) -> ModelChangeResponse:
+ row = self._row(identifier, lock=True)
+ if row.closed_at is not None:
+ raise self._error("MODEL_CHANGE_CLOSED", "This model change is already closed")
+ if row.version != request.expected_version:
+ raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before editing it")
+ _binding, _project, contract, _capability = self._binding_context(row.project_binding_id)
+ deployment = (
+ self.session.get(CapabilityDeployment, request.candidate_deployment_id)
+ if request.candidate_deployment_id
+ else None
+ )
+ if deployment is not None and (
+ deployment.capability_contract_id != contract.id
+ or deployment.id == row.current_deployment_id
+ or deployment.production
+ or deployment.channel not in {"experiment", "lab", "candidate"}
+ or deployment.status not in {"candidate", "approved", "lab_ready"}
+ or deployment.deprecated_at is not None
+ ):
+ raise self._error(
+ "MODEL_CHANGE_CANDIDATE_DEPLOYMENT_INVALID",
+ "The candidate deployment is not a safe non-production target for this project task",
+ 422,
+ )
+
+ artifact_set_id = request.candidate_artifact_set_id or (
+ deployment.artifact_set_id if deployment else None
+ )
+ plan = self.session.get(DownloadPlan, request.download_plan_id) if request.download_plan_id else None
+ if plan is not None:
+ if artifact_set_id is not None and plan.artifact_set_id != artifact_set_id:
+ raise self._error(
+ "MODEL_CHANGE_DOWNLOAD_MISMATCH", "Download plan belongs to another candidate", 422
+ )
+ artifact_set_id = plan.artifact_set_id
+ job = self.session.get(ArtifactJob, request.artifact_job_id) if request.artifact_job_id else None
+ if job is not None:
+ if plan is None:
+ plan = self.session.get(DownloadPlan, job.plan_id)
+ if plan is None or job.plan_id != plan.id or (
+ artifact_set_id is not None and plan.artifact_set_id != artifact_set_id
+ ):
+ raise self._error(
+ "MODEL_CHANGE_JOB_MISMATCH", "Download job belongs to another candidate", 422
+ )
+ artifact_set_id = plan.artifact_set_id
+
+ artifact = self.session.get(ArtifactSet, artifact_set_id) if artifact_set_id else None
+ if artifact_set_id and artifact is None:
+ raise self._error("MODEL_CHANGE_ARTIFACT_NOT_FOUND", "Candidate artifact set was not found", 404)
+ revision_id = request.candidate_revision_id or (artifact.revision_id if artifact else None)
+ revision = self.session.get(ModelRevision, revision_id) if revision_id else None
+ if revision_id and revision is None:
+ raise self._error("MODEL_CHANGE_REVISION_NOT_FOUND", "Candidate version was not found", 404)
+ model_id = request.candidate_model_id or (revision.model_id if revision else None)
+ model = self.session.get(Model, model_id) if model_id else None
+ if model_id and model is None:
+ raise self._error("MODEL_CHANGE_MODEL_NOT_FOUND", "Candidate model was not found", 404)
+ if revision and model_id and revision.model_id != model_id:
+ raise self._error(
+ "MODEL_CHANGE_IDENTITY_MISMATCH", "Candidate version belongs to another model", 422
+ )
+ if artifact and revision_id and artifact.revision_id != revision_id:
+ raise self._error(
+ "MODEL_CHANGE_IDENTITY_MISMATCH", "Candidate files belong to another version", 422
+ )
+
+ assessment = (
+ self.session.get(DiscoveryCandidateAssessment, request.discovery_assessment_id)
+ if request.discovery_assessment_id
+ else None
+ )
+ if request.discovery_assessment_id and assessment is None:
+ raise self._error(
+ "MODEL_CHANGE_ASSESSMENT_NOT_FOUND", "Candidate source review was not found", 404
+ )
+ if assessment is not None:
+ if assessment.model_id is not None and model_id is not None and assessment.model_id != model_id:
+ raise self._error(
+ "MODEL_CHANGE_ASSESSMENT_MISMATCH", "Source review belongs to another model", 422
+ )
+ if revision is not None and assessment.resolved_commit_sha != revision.resolved_commit_sha:
+ raise self._error(
+ "MODEL_CHANGE_ASSESSMENT_MISMATCH",
+ "Source review belongs to another exact version",
+ 422,
+ )
+
+ row.candidate_model_id = model_id
+ row.candidate_revision_id = revision_id
+ row.candidate_artifact_set_id = artifact_set_id
+ row.candidate_deployment_id = (
+ deployment.id if deployment else request.candidate_deployment_id or row.candidate_deployment_id
+ )
+ row.discovery_assessment_id = (
+ assessment.id if assessment else request.discovery_assessment_id or row.discovery_assessment_id
+ )
+ row.download_plan_id = plan.id if plan else request.download_plan_id or row.download_plan_id
+ row.artifact_job_id = job.id if job else request.artifact_job_id or row.artifact_job_id
+ row.version += 1
+ row.updated_by = self.audit_context.actor_id
+ self._remember_derived_status(row)
+ self._audit(row, "MODEL_CHANGE_CANDIDATE_LINKED")
+ self.session.commit()
+ self.session.refresh(row)
+ return self._response(row)
+
+ def link_evidence(
+ self, identifier: UUID, request: ModelChangeEvidenceLink
+ ) -> ModelChangeResponse:
+ row = self._row(identifier, lock=True)
+ if row.closed_at is not None:
+ raise self._error("MODEL_CHANGE_CLOSED", "This model change is already closed")
+ if row.version != request.expected_version:
+ raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before editing it")
+ binding, project, _contract, _capability = self._binding_context(row.project_binding_id)
+ comparison = (
+ self.session.get(SelectedModelComparison, request.selected_comparison_id)
+ if request.selected_comparison_id
+ else None
+ )
+ if request.selected_comparison_id:
+ if comparison is None:
+ raise self._error("MODEL_CHANGE_COMPARISON_NOT_FOUND", "Comparison was not found", 404)
+ if (
+ comparison.project_binding_id != binding.id
+ or comparison.current_deployment_id != row.current_deployment_id
+ or row.candidate_deployment_id is None
+ or comparison.candidate_deployment_id != row.candidate_deployment_id
+ ):
+ raise self._error(
+ "MODEL_CHANGE_COMPARISON_MISMATCH",
+ "Comparison does not belong to this project and exact current/candidate pair",
+ 422,
+ )
+
+ migration = (
+ self.session.get(MigrationPlan, request.migration_plan_id)
+ if request.migration_plan_id
+ else None
+ )
+ if request.migration_plan_id:
+ if migration is None:
+ raise self._error("MODEL_CHANGE_MIGRATION_NOT_FOUND", "Migration plan was not found", 404)
+ if migration.project_binding_id != binding.id or migration.project_id != project.id:
+ raise self._error(
+ "MODEL_CHANGE_MIGRATION_MISMATCH", "Migration plan belongs to another project task", 422
+ )
+ if row.candidate_deployment_id:
+ candidate = self.session.get(CapabilityDeployment, row.candidate_deployment_id)
+ if (
+ candidate
+ and candidate.embedding_space_id
+ and migration.target_space_id != candidate.embedding_space_id
+ ):
+ raise self._error(
+ "MODEL_CHANGE_MIGRATION_MISMATCH",
+ "Migration target does not match the candidate embedding space",
+ 422,
+ )
+
+ lifecycle = (
+ self.session.get(LifecyclePromotionPlan, request.lifecycle_plan_id)
+ if request.lifecycle_plan_id
+ else None
+ )
+ if request.lifecycle_plan_id:
+ if lifecycle is None:
+ raise self._error("MODEL_CHANGE_LIFECYCLE_NOT_FOUND", "Activation plan was not found", 404)
+ if (
+ row.candidate_deployment_id is None
+ or lifecycle.candidate_deployment_id != row.candidate_deployment_id
+ ):
+ raise self._error(
+ "MODEL_CHANGE_LIFECYCLE_MISMATCH", "Activation plan belongs to another candidate", 422
+ )
+ if (
+ str(project.id) not in {str(value) for value in lifecycle.project_consumers}
+ and project.name not in lifecycle.project_consumers
+ ):
+ impacted = lifecycle.impact_analysis if isinstance(lifecycle.impact_analysis, dict) else {}
+ project_ids = impacted.get("project_ids", [])
+ project_refs = (
+ {str(value) for value in project_ids} if isinstance(project_ids, list) else set()
+ )
+ if project_refs and str(project.id) not in project_refs:
+ raise self._error(
+ "MODEL_CHANGE_LIFECYCLE_MISMATCH",
+ "Activation plan does not affect this project",
+ 422,
+ )
+
+ if comparison is not None:
+ row.selected_comparison_id = comparison.id
+ if migration is not None:
+ row.migration_plan_id = migration.id
+ if lifecycle is not None:
+ row.lifecycle_plan_id = lifecycle.id
+ row.version += 1
+ row.updated_by = self.audit_context.actor_id
+ self._remember_derived_status(row)
+ self._audit(row, "MODEL_CHANGE_EVIDENCE_LINKED")
+ self.session.commit()
+ self.session.refresh(row)
+ return self._response(row)
+
+ def refresh(self, identifier: UUID, expected_version: int) -> ModelChangeResponse:
+ row = self._row(identifier, lock=True)
+ if row.version != expected_version:
+ raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before updating status")
+ before = row.status
+ derived = self._derived_status(row)
+ if derived != row.status:
+ row.status = derived
+ row.status_reason = self._next_action(derived)
+ row.version += 1
+ row.updated_by = self.audit_context.actor_id
+ if derived in {"active", "checked"} and row.activated_at is None:
+ row.activated_at = _now()
+ if derived == "checked" and row.checked_at is None:
+ row.checked_at = _now()
+ if derived == "rolled_back":
+ row.closed_at = _now()
+ self._audit(row, "MODEL_CHANGE_STATUS_REFRESHED", {"from": before, "to": derived})
+ self.session.commit()
+ self.session.refresh(row)
+ return self._response(row)
+
+ def reject(self, identifier: UUID, request: ModelChangeReject) -> ModelChangeResponse:
+ row = self._row(identifier, lock=True)
+ if row.version != request.expected_version:
+ raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before rejecting it")
+ if row.closed_at is not None:
+ return self._response(row)
+ if self._derived_status(row) in {"active", "checked"}:
+ raise self._error(
+ "MODEL_CHANGE_ALREADY_ACTIVE",
+ "An active change must be rolled back through the controlled lifecycle flow",
+ )
+ row.status = "rejected"
+ row.status_reason = request.reason
+ row.closed_at = _now()
+ row.updated_by = self.audit_context.actor_id
+ row.version += 1
+ self._audit(row, "MODEL_CHANGE_CASE_REJECTED", {"reason": request.reason})
+ self.session.commit()
+ self.session.refresh(row)
+ return self._response(row)
diff --git a/backend/src/modelforge_api/services/model_update_polling.py b/backend/src/modelforge_api/services/model_update_polling.py
new file mode 100644
index 0000000..be402aa
--- /dev/null
+++ b/backend/src/modelforge_api/services/model_update_polling.py
@@ -0,0 +1,76 @@
+"""Low-volume metadata checks for explicitly selected models; never model execution."""
+
+import asyncio
+import threading
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import structlog
+from sqlalchemy import Engine, or_, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import ModelUpdateWatch
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.services.registry import RegistryConflict
+from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
+from modelforge_api.settings import Settings
+
+logger = structlog.get_logger()
+
+
+class ModelUpdatePollingService:
+ """One due watch per minute per API process, with database-fenced per-watch admission."""
+
+ def __init__(self, settings: Settings, engine: Engine) -> None:
+ self.settings = settings
+ self.engine = engine
+ self._stop = threading.Event()
+
+ def stop(self) -> None:
+ self._stop.set()
+
+ def service(self, session: Session) -> ModelUpdateService:
+ provider = SecretAwareHuggingFaceProvider(session, self.settings)
+ acquisition = AcquisitionService(
+ session, self.settings, provider,
+ credential_configured=provider.credential_configured,
+ )
+ return ModelUpdateService(session, acquisition, AuditContext(
+ "automation", "selected-model-update-monitor", str(uuid.uuid4()),
+ ))
+
+ def run_once(self) -> None:
+ if self._stop.is_set() or not self.settings.model_update_monitor_enabled:
+ return
+ now = datetime.now(UTC)
+ cutoff = now - timedelta(seconds=self.settings.model_update_check_interval_seconds)
+ try:
+ with Session(self.engine) as session:
+ watch_id = session.scalar(select(ModelUpdateWatch.id).where(
+ ModelUpdateWatch.enabled.is_(True),
+ or_(ModelUpdateWatch.last_checked_at.is_(None),
+ ModelUpdateWatch.last_checked_at <= cutoff),
+ or_(ModelUpdateWatch.check_until.is_(None),
+ ModelUpdateWatch.check_until < now),
+ ).order_by(ModelUpdateWatch.last_checked_at.asc().nulls_first(),
+ ModelUpdateWatch.id).limit(1))
+ if watch_id is None:
+ return
+ # The service rechecks due/enabled state in its atomic claim, so another
+ # API process completing this watch cannot cause an immediate second poll.
+ self.service(session).check(watch_id, if_checked_before=cutoff)
+ except RegistryConflict:
+ # A manual action or another process won admission. Its state remains authoritative.
+ return
+ except Exception as error: # noqa: BLE001 - keep the background loop alive; never log credentials
+ logger.error("model_update_monitor_check_failed", error_type=type(error).__name__)
+
+ async def run(self) -> None:
+ while not self._stop.is_set():
+ await asyncio.to_thread(self.run_once)
+ for _ in range(60):
+ if self._stop.is_set():
+ return
+ await asyncio.sleep(1)
diff --git a/backend/src/modelforge_api/services/model_updates.py b/backend/src/modelforge_api/services/model_updates.py
new file mode 100644
index 0000000..f6eaa7b
--- /dev/null
+++ b/backend/src/modelforge_api/services/model_updates.py
@@ -0,0 +1,195 @@
+import uuid
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import func, or_, select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.model_updates import (
+ ModelUpdateNoticeResponse,
+ ModelWatchCreate,
+ ModelWatchResponse,
+ ModelWatchUpdate,
+)
+from modelforge_api.persistence.models import (
+ Model,
+ ModelRevision,
+ ModelUpdateNotice,
+ ModelUpdateWatch,
+)
+from modelforge_api.services.acquisition import AcquisitionService, ProviderBoundaryError
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.registry import RegistryConflict, RegistryNotFound
+
+
+class ModelUpdateService:
+ def __init__(self, session: Session, acquisition: AcquisitionService, context: AuditContext) -> None:
+ self.session = session
+ self.acquisition = acquisition
+ self.audit = AuditWriter(session, context=context)
+ self.acquisition.audit = self.audit
+
+ def watches(self, limit: int = 50, offset: int = 0) -> list[ModelWatchResponse]:
+ return [ModelWatchResponse.model_validate(item) for item in self.session.scalars(
+ select(ModelUpdateWatch).order_by(ModelUpdateWatch.id).limit(limit).offset(offset)
+ )]
+
+ def create(self, request: ModelWatchCreate) -> ModelWatchResponse:
+ model = self.session.get(Model, request.model_id)
+ revision = self.session.get(ModelRevision, request.baseline_revision_id)
+ if model is None or revision is None:
+ raise RegistryNotFound("model or baseline revision not found")
+ if model.source_type != "huggingface" or revision.model_id != model.id:
+ raise RegistryConflict("watch requires a Hugging Face model and its own exact revision")
+ existing = self.session.scalar(select(ModelUpdateWatch).where(
+ ModelUpdateWatch.model_id == model.id,
+ ModelUpdateWatch.upstream_ref == request.upstream_ref,
+ ))
+ if existing:
+ if existing.baseline_revision_id != revision.id:
+ raise RegistryConflict("watch baseline requires an explicit tracking update; review the existing watch")
+ return ModelWatchResponse.model_validate(existing)
+ item = ModelUpdateWatch(id=uuid.uuid4(), model_id=model.id,
+ baseline_revision_id=revision.id, baseline_commit_sha=revision.resolved_commit_sha,
+ upstream_ref=request.upstream_ref)
+ # The audit chain lock serializes admission, including the global watch limit.
+ self.audit.write("MODEL_WATCH_CREATED", "model_update_watch", str(item.id),
+ {"model_id": str(model.id), "baseline_revision_id": str(revision.id)})
+ if (self.session.scalar(select(func.count()).select_from(ModelUpdateWatch)) or 0) >= 100:
+ self.session.rollback()
+ raise RegistryConflict("maximum of 100 explicitly selected model watches reached")
+ self.session.add(item)
+ try:
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise RegistryConflict("watch already exists; refresh the watch list") from exc
+ return ModelWatchResponse.model_validate(item)
+
+ def change(self, watch_id: uuid.UUID, request: ModelWatchUpdate) -> ModelWatchResponse:
+ item = self.session.scalar(select(ModelUpdateWatch).where(
+ ModelUpdateWatch.id == watch_id).with_for_update().execution_options(populate_existing=True))
+ if item is None:
+ raise RegistryNotFound("model watch not found")
+ if item.version != request.expected_version or item.baseline_revision_id != request.expected_baseline_revision_id:
+ raise RegistryConflict("model watch changed; refresh before changing tracking")
+ revision = self.session.get(ModelRevision, request.baseline_revision_id)
+ if revision is None:
+ raise RegistryNotFound("baseline revision not found")
+ if revision.model_id != item.model_id:
+ raise RegistryConflict("watch baseline must belong to the same model")
+ baseline_changed = revision.id != item.baseline_revision_id
+ until = item.check_until
+ if until is not None and until.tzinfo is None:
+ until = until.replace(tzinfo=UTC)
+ if baseline_changed and item.check_token is not None and until is not None and until > datetime.now(UTC):
+ raise RegistryConflict("wait for the active check or pause the watch before changing its baseline")
+ if not baseline_changed and item.enabled == request.enabled:
+ return ModelWatchResponse.model_validate(item)
+ previous = {"baseline_revision_id": str(item.baseline_revision_id),
+ "baseline_commit_sha": item.baseline_commit_sha, "enabled": item.enabled,
+ "version": item.version}
+ item.enabled = request.enabled
+ item.baseline_revision_id = revision.id
+ item.baseline_commit_sha = revision.resolved_commit_sha
+ item.version += 1
+ if not item.enabled or baseline_changed:
+ # Any result from the old check must fail its token fence, even after resume.
+ item.check_token = None
+ item.check_until = None
+ if baseline_changed:
+ item.last_checked_at = None
+ item.last_error_code = None
+ self.audit.write("MODEL_WATCH_TRACKING_CHANGED", "model_update_watch", str(item.id), {
+ "previous": previous, "baseline_revision_id": str(revision.id),
+ "baseline_commit_sha": item.baseline_commit_sha, "enabled": item.enabled,
+ "version": item.version, "reason": request.reason.strip(),
+ })
+ self.session.commit()
+ return ModelWatchResponse.model_validate(item)
+
+ def check(self, watch_id: uuid.UUID, *, if_checked_before: datetime | None = None) -> ModelWatchResponse:
+ item = self.session.get(ModelUpdateWatch, watch_id)
+ if item is None:
+ raise RegistryNotFound("model watch not found")
+ now = datetime.now(UTC)
+ token = uuid.uuid4()
+ claim = update(ModelUpdateWatch).where(
+ ModelUpdateWatch.id == watch_id,
+ ModelUpdateWatch.enabled.is_(True),
+ or_(ModelUpdateWatch.check_until.is_(None), ModelUpdateWatch.check_until < now),
+ )
+ if if_checked_before is not None:
+ claim = claim.where(or_(ModelUpdateWatch.last_checked_at.is_(None),
+ ModelUpdateWatch.last_checked_at <= if_checked_before))
+ claimed = self.session.execute(claim.values(check_until=now + timedelta(minutes=5), check_token=token)
+ .execution_options(synchronize_session="fetch").returning(ModelUpdateWatch.id)).first()
+ if claimed is None:
+ self.session.rollback()
+ raise RegistryConflict("model watch is paused, not due, or a check is already in progress")
+ self.session.commit()
+ self.session.refresh(item)
+ if not item.enabled or item.check_token != token:
+ raise RegistryConflict("model watch check superseded; refresh its status")
+ model_id, upstream_ref, baseline_sha = item.model_id, item.upstream_ref, item.baseline_commit_sha
+ error = None
+ candidate = None
+ try:
+ snapshot = self.acquisition.refresh_model(model_id, upstream_ref)
+ candidate = self.session.scalar(select(ModelRevision).where(
+ ModelRevision.model_id == model_id,
+ ModelRevision.resolved_commit_sha == snapshot.resolved_commit_sha,
+ ))
+ if candidate is None:
+ raise RegistryConflict("upstream refresh did not resolve an exact revision")
+ except ProviderBoundaryError:
+ error = "upstream_unavailable"
+ except Exception:
+ self.session.rollback()
+ self.session.execute(update(ModelUpdateWatch).where(
+ ModelUpdateWatch.id == watch_id, ModelUpdateWatch.check_token == token,
+ ).values(check_token=None, check_until=None, last_checked_at=datetime.now(UTC),
+ last_error_code="check_failed"))
+ self.audit.write("MODEL_WATCH_CHECK_FAILED", "model_update_watch", str(watch_id),
+ {"error_code": "check_failed"}, outcome="blocked")
+ self.session.commit()
+ raise
+ # An expired check cannot overwrite a subsequent check's state or create a stale notice.
+ finished = self.session.execute(update(ModelUpdateWatch).where(
+ ModelUpdateWatch.id == watch_id, ModelUpdateWatch.check_token == token,
+ ).values(check_token=None, check_until=None, last_checked_at=datetime.now(UTC),
+ last_error_code=error).returning(ModelUpdateWatch.id)).first()
+ if finished is None:
+ self.session.rollback()
+ raise RegistryConflict("model watch check superseded; refresh its status")
+ if candidate and candidate.resolved_commit_sha != baseline_sha:
+ existing = self.session.scalar(select(ModelUpdateNotice).where(
+ ModelUpdateNotice.watch_id == watch_id,
+ ModelUpdateNotice.candidate_commit_sha == candidate.resolved_commit_sha,
+ ))
+ if existing is None:
+ self.session.add(ModelUpdateNotice(watch_id=watch_id, model_id=model_id,
+ baseline_commit_sha=baseline_sha,
+ candidate_revision_id=candidate.id, candidate_commit_sha=candidate.resolved_commit_sha))
+ self.audit.write("MODEL_WATCH_CHECKED", "model_update_watch", str(watch_id),
+ {"error_code": error}, outcome="blocked" if error else "success")
+ self.session.commit()
+ self.session.refresh(item)
+ return ModelWatchResponse.model_validate(item)
+
+ def notices(self, limit: int = 50, offset: int = 0) -> list[ModelUpdateNoticeResponse]:
+ return [ModelUpdateNoticeResponse.model_validate(item) for item in self.session.scalars(
+ select(ModelUpdateNotice).order_by(ModelUpdateNotice.discovered_at.desc(), ModelUpdateNotice.id)
+ .limit(limit).offset(offset)
+ )]
+
+ def dismiss(self, notice_id: uuid.UUID) -> ModelUpdateNoticeResponse:
+ item = self.session.scalar(select(ModelUpdateNotice).where(
+ ModelUpdateNotice.id == notice_id).with_for_update())
+ if item is None:
+ raise RegistryNotFound("model update notice not found")
+ if item.status != "dismissed":
+ item.status = "dismissed"
+ self.audit.write("MODEL_UPDATE_DISMISSED", "model_update_notice", str(item.id), {})
+ self.session.commit()
+ return ModelUpdateNoticeResponse.model_validate(item)
diff --git a/backend/src/modelforge_api/services/node_agent.py b/backend/src/modelforge_api/services/node_agent.py
index 3ea6646..ead1a89 100644
--- a/backend/src/modelforge_api/services/node_agent.py
+++ b/backend/src/modelforge_api/services/node_agent.py
@@ -361,6 +361,7 @@ class NodeAgentService:
raw: str,
*,
update_last_used: bool,
+ lock_node: bool = False,
) -> tuple[NodeCredential, ComputeNode]:
credential = self.repository.credential(credential_id)
if credential is None or credential.revoked_at is not None:
@@ -369,7 +370,11 @@ class NodeAgentService:
raise AgentAuthenticationError("invalid node credential")
if credential.scope != "node.publish":
raise AgentAuthorizationError("node credential scope is not authorized for publishing")
- node = self.repository.node(credential.compute_node_id)
+ node = (
+ self.repository.node_for_update(credential.compute_node_id)
+ if lock_node
+ else self.repository.node(credential.compute_node_id)
+ )
if node is None:
raise AgentAuthenticationError("credential node no longer exists")
if node.decommissioned_at is not None:
@@ -396,8 +401,13 @@ class NodeAgentService:
)
def authenticate(self, authorization: str | None) -> tuple[NodeCredential, ComputeNode]:
- identity, _evidence = self.authenticate_with_evidence(authorization)
- return identity
+ credential_id, raw = self._parse_credential(authorization)
+ return self._resolve_authenticated_node(
+ credential_id,
+ raw,
+ update_last_used=True,
+ lock_node=True,
+ )
def reuse_authentication(
self,
@@ -415,6 +425,7 @@ class NodeAgentService:
credential_id,
raw,
update_last_used=False,
+ lock_node=True,
)
if node.id != evidence.node_id:
raise AgentAuthenticationError("credential node identity changed")
@@ -430,8 +441,23 @@ class NodeAgentService:
if node.decommissioned_at is not None:
raise AgentConflictError("decommissioned compute node cannot publish agent state")
+ def _locked_publishable_node(self, node: ComputeNode) -> ComputeNode:
+ """Fence an authenticated agent write against concurrent lifecycle changes.
+
+ Agent routes also take this row lock while revalidating authentication. Keeping the same
+ node-first lock order here protects direct service callers and makes each state publisher
+ independently fail closed if its caller holds a stale ORM instance.
+ """
+ current = self.repository.node_for_update(node.id)
+ if current is None:
+ raise AgentNotFoundError("compute node no longer exists")
+ self._assert_not_decommissioned(current)
+ if not current.enabled:
+ raise AgentAuthorizationError("compute node is disabled")
+ return current
+
def heartbeat(self, node: ComputeNode, request: HeartbeatRequest) -> ObservationAck:
- self._assert_not_decommissioned(node)
+ node = self._locked_publishable_node(node)
self._assert_identity(node, request.identity_key)
if request.metadata.protocol_version != self.settings.agent_protocol_version:
raise AgentConflictError("agent protocol version is incompatible")
@@ -450,7 +476,7 @@ class NodeAgentService:
return ObservationAck(accepted=True, received_at=now)
def publish_inventory(self, node: ComputeNode, report: InventoryReport) -> ObservationAck:
- self._assert_not_decommissioned(node)
+ node = self._locked_publishable_node(node)
self._assert_identity(node, report.identity_key)
if report.host.identity_key != node.key:
raise AgentAuthorizationError("host inventory identity does not match credential node")
@@ -517,7 +543,7 @@ class NodeAgentService:
return observation.value if observation.availability.value == "known" else None
def publish_telemetry(self, node: ComputeNode, report: TelemetryReport) -> ObservationAck:
- self._assert_not_decommissioned(node)
+ node = self._locked_publishable_node(node)
self._assert_identity(node, report.identity_key)
if report.protocol_version != self.settings.agent_protocol_version:
raise AgentConflictError("agent protocol version is incompatible")
@@ -640,19 +666,12 @@ class NodeAgentService:
if node.decommissioned_at is not None:
raise AgentConflictError("decommissioned compute node cannot be updated")
values = update.model_dump(exclude_unset=True)
- was_enabled = node.enabled
+ if "enabled" in values:
+ raise AgentConflictError(
+ "node enabled state requires an impact preview and typed confirmation"
+ )
for key, value in values.items():
setattr(node, key, value)
- if "enabled" in values and values["enabled"] != was_enabled:
- node.liveness_state = (
- NodeLiveness.OFFLINE if values["enabled"] else NodeLiveness.DISABLED
- )
- self._audit(
- "NODE_ENABLED" if values["enabled"] else "NODE_DISABLED",
- "compute_node",
- str(node.id),
- {},
- )
self.session.commit()
return node
diff --git a/backend/src/modelforge_api/services/node_decommission.py b/backend/src/modelforge_api/services/node_decommission.py
index 3dd22ff..71661b8 100644
--- a/backend/src/modelforge_api/services/node_decommission.py
+++ b/backend/src/modelforge_api/services/node_decommission.py
@@ -4,7 +4,7 @@ import hashlib
import json
import uuid
from datetime import UTC, datetime
-from typing import Any
+from typing import Any, Literal
from sqlalchemy import delete, func, select, text, update
from sqlalchemy.exc import IntegrityError
@@ -18,6 +18,13 @@ from modelforge_api.domain.node_decommission import (
NodeDecommissionPreview,
NodeDecommissionResult,
)
+from modelforge_api.domain.node_lifecycle import (
+ NodeLifecycleBlocker,
+ NodeLifecycleChange,
+ NodeLifecyclePreview,
+ NodeLifecyclePreviewRequest,
+ NodeLifecycleResult,
+)
from modelforge_api.persistence.models import (
Accelerator,
AcceleratorTelemetryLatest,
@@ -102,7 +109,7 @@ class NodeDecommissionService:
def _node(self, node_id: uuid.UUID, *, lock: bool = False) -> ComputeNode:
statement = select(ComputeNode).where(ComputeNode.id == node_id)
if lock:
- statement = statement.with_for_update()
+ statement = statement.with_for_update().execution_options(populate_existing=True)
node = self.session.scalar(statement)
if node is None:
raise NodeDecommissionError(
@@ -740,6 +747,199 @@ class NodeDecommissionService:
dependent_records=dependent_records,
)
+ def _lifecycle_preview(
+ self, node: ComputeNode, request: NodeLifecyclePreviewRequest
+ ) -> NodeLifecyclePreview:
+ dependency_snapshot = self._snapshot(node)
+ blockers: list[NodeLifecycleBlocker] = []
+ active_credential_ids = sorted(
+ str(value)
+ for value in self.session.scalars(
+ select(NodeCredential.id).where(
+ NodeCredential.compute_node_id == node.id,
+ NodeCredential.revoked_at.is_(None),
+ )
+ )
+ )
+
+ if node.decommissioned_at is not None:
+ blockers.append(
+ NodeLifecycleBlocker(
+ code="node_decommissioned",
+ message="decommissioned node identities are terminal and cannot be enabled or disabled",
+ record_type="compute_node",
+ count=1,
+ resource_ids=[str(node.id)],
+ )
+ )
+ elif node.enabled == request.target_enabled:
+ blockers.append(
+ NodeLifecycleBlocker(
+ code="already_in_requested_state",
+ message="node is already in the requested enabled state",
+ record_type="compute_node",
+ count=1,
+ resource_ids=[str(node.id)],
+ )
+ )
+ elif not request.target_enabled:
+ # A live heartbeat is expected before an administrator disables a node. Every other
+ # decommission dependency is conservatively treated as a blocker for temporary disable.
+ blockers.extend(
+ NodeLifecycleBlocker.model_validate(item.model_dump())
+ for item in dependency_snapshot.blockers
+ if item.code != "node_online"
+ )
+ elif node.observation_source == "remote_agent" and not active_credential_ids:
+ blockers.append(
+ NodeLifecycleBlocker(
+ code="active_node_credential_missing",
+ message="remote node has no active credential; rotate or re-enroll before enabling",
+ record_type="node_credential",
+ count=1,
+ resource_ids=[],
+ )
+ )
+
+ digest_payload = {
+ "base_dependency_digest": dependency_snapshot.dependency_digest,
+ "node_id": str(node.id),
+ "target_enabled": request.target_enabled,
+ "active_credential_ids": active_credential_ids,
+ "blockers": [item.model_dump(mode="json") for item in blockers],
+ }
+ dependency_digest = hashlib.sha256(
+ json.dumps(digest_payload, sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+ return NodeLifecyclePreview(
+ node_id=node.id,
+ persisted_identity=node.key,
+ hostname=node.hostname,
+ display_name=node.display_name or node.hostname,
+ current_enabled=node.enabled,
+ current_liveness=node.liveness_state,
+ target_enabled=request.target_enabled,
+ node_generation=node.generation,
+ dependency_digest=dependency_digest,
+ can_apply=not blockers,
+ blockers=blockers,
+ agent_evidence_after_enable=(
+ "awaiting_fresh_authenticated_heartbeat"
+ if request.target_enabled and node.observation_source == "remote_agent"
+ else "unchanged"
+ ),
+ )
+
+ def preview_enabled_change(
+ self, node_id: uuid.UUID, request: NodeLifecyclePreviewRequest
+ ) -> NodeLifecyclePreview:
+ return self._lifecycle_preview(self._node(node_id), request)
+
+ def apply_enabled_change(
+ self, node_id: uuid.UUID, request: NodeLifecycleChange
+ ) -> NodeLifecycleResult:
+ node = self._node(node_id, lock=True)
+ if node.generation != request.expected_generation:
+ raise NodeDecommissionError(
+ 409,
+ "node_generation_conflict",
+ "node generation changed after lifecycle preview",
+ {"expected": request.expected_generation, "actual": node.generation},
+ )
+ if request.confirmation not in {node.key, node.hostname, node.display_name}:
+ raise NodeDecommissionError(
+ 409,
+ "node_lifecycle_confirmation_mismatch",
+ "typed confirmation must exactly match the persisted identity, hostname, or display name",
+ )
+
+ self._lock_dependency_tables()
+ preview = self._lifecycle_preview(
+ node, NodeLifecyclePreviewRequest(target_enabled=request.target_enabled)
+ )
+ if preview.dependency_digest != request.dependency_digest:
+ raise NodeDecommissionError(
+ 409,
+ "node_lifecycle_preview_stale",
+ "node state or dependencies changed after impact review; generate a fresh preview",
+ {"preview": preview.model_dump(mode="json")},
+ )
+ if not preview.can_apply:
+ raise NodeDecommissionError(
+ 409,
+ "node_lifecycle_blocked",
+ "node lifecycle transition is blocked until the listed dependencies are resolved",
+ {"blockers": [item.model_dump(mode="json") for item in preview.blockers]},
+ )
+
+ target_liveness: Literal["offline", "disabled"] = (
+ "offline" if request.target_enabled else "disabled"
+ )
+ values: dict[str, Any] = {
+ "enabled": request.target_enabled,
+ "generation": ComputeNode.generation + 1,
+ "liveness_state": target_liveness,
+ }
+ if request.target_enabled and node.observation_source == "remote_agent":
+ # Previous protocol/heartbeat facts must not make a re-enabled node look ready. A new
+ # authenticated, protocol-compatible heartbeat will publish current evidence again.
+ values.update(
+ agent_protocol_version=None,
+ agent_capabilities=[],
+ agent_version=None,
+ agent_started_at=None,
+ last_heartbeat_at=None,
+ last_connection_error="node enabled; awaiting fresh authenticated heartbeat",
+ )
+ claimed = rows_affected(
+ self.session.execute(
+ update(ComputeNode)
+ .where(
+ ComputeNode.id == node_id,
+ ComputeNode.generation == request.expected_generation,
+ ComputeNode.enabled == preview.current_enabled,
+ ComputeNode.decommissioned_at.is_(None),
+ )
+ .values(**values)
+ .execution_options(synchronize_session=False)
+ )
+ )
+ if not claimed:
+ self.session.rollback()
+ raise NodeDecommissionError(
+ 409,
+ "node_generation_conflict",
+ "node changed while the lifecycle transition was being claimed",
+ {"expected": request.expected_generation},
+ )
+
+ AuditWriter(self.session, "operator", request.operator).write(
+ "NODE_ENABLED" if request.target_enabled else "NODE_DISABLED",
+ "compute_node",
+ str(node.id),
+ {
+ "operator": request.operator,
+ "reason": request.reason,
+ "previous_enabled": preview.current_enabled,
+ "enabled": request.target_enabled,
+ "generation": request.expected_generation + 1,
+ "dependency_digest": request.dependency_digest,
+ "liveness": target_liveness,
+ },
+ )
+ self.session.commit()
+ return NodeLifecycleResult(
+ node_id=node.id,
+ enabled=request.target_enabled,
+ generation=request.expected_generation + 1,
+ liveness=target_liveness,
+ agent_evidence_state=(
+ "awaiting_fresh_authenticated_heartbeat"
+ if request.target_enabled and node.observation_source == "remote_agent"
+ else "unchanged"
+ ),
+ )
+
def preview(self, node_id: uuid.UUID) -> NodeDecommissionPreview:
return self._snapshot(self._node(node_id))
diff --git a/backend/src/modelforge_api/services/observability.py b/backend/src/modelforge_api/services/observability.py
index 215ef8d..c8d94a7 100644
--- a/backend/src/modelforge_api/services/observability.py
+++ b/backend/src/modelforge_api/services/observability.py
@@ -14,6 +14,8 @@ from sqlalchemy.orm import Session
from modelforge_api.db import rows_affected
from modelforge_api.domain.observability import (
+ ActiveSchedulerLeaseInventoryResponse,
+ ActiveSchedulerLeaseResponse,
AlertAction,
AlertResponse,
AlertRuleCreate,
@@ -74,6 +76,7 @@ from modelforge_api.persistence.models import (
StorageRoot,
StorageVolumeState,
)
+from modelforge_api.services.alert_notifications import enqueue_alert_transition
from modelforge_api.services.audit import AuditWriter
from modelforge_api.settings import Settings, get_settings
@@ -1157,6 +1160,133 @@ class ObservabilityService:
)
]
+ def active_scheduler_leases(
+ self,
+ *,
+ limit: int = 100,
+ sort_by: Literal["expires_at", "created_at", "priority"] = "expires_at",
+ order: Literal["asc", "desc"] = "asc",
+ observed_at: datetime | None = None,
+ ) -> ActiveSchedulerLeaseInventoryResponse:
+ """Read current, unexpired scheduler leases without touching reconciliation state."""
+
+ now = observed_at or _now()
+ sort_column = {
+ "expires_at": ServingGpuLease.expires_at,
+ "created_at": ServingGpuLease.created_at,
+ "priority": ServingGpuLease.priority,
+ }[sort_by]
+ primary_order = sort_column.asc() if order == "asc" else sort_column.desc()
+ rows = self.session.execute(
+ select(
+ ServingGpuLease,
+ Accelerator.compute_node_id,
+ GatewayRequest.status,
+ GatewayRequest.capability_deployment_id,
+ GatewayRequest.compute_node_id,
+ ProjectBinding.project_id,
+ )
+ .outerjoin(Accelerator, Accelerator.id == ServingGpuLease.accelerator_id)
+ .outerjoin(GatewayRequest, GatewayRequest.request_id == ServingGpuLease.request_id)
+ .outerjoin(ServiceClient, ServiceClient.id == GatewayRequest.service_client_id)
+ .outerjoin(ProjectBinding, ProjectBinding.id == ServiceClient.project_binding_id)
+ .where(
+ ServingGpuLease.state.in_(("pending", "granted", "active", "releasing")),
+ ServingGpuLease.expires_at > now,
+ )
+ .order_by(primary_order, ServingGpuLease.id.asc())
+ .limit(limit + 1)
+ ).all()
+ truncated = len(rows) > limit
+ selected = rows[:limit]
+ lease_states: dict[str, Literal["PENDING", "GRANTED", "ACTIVE", "RELEASING"]] = {
+ "pending": "PENDING",
+ "granted": "GRANTED",
+ "active": "ACTIVE",
+ "releasing": "RELEASING",
+ }
+ lease_types: dict[
+ str, Literal["RESIDENCY_RESERVATION", "REQUEST_EXECUTION", "LOAD_TRANSITION"]
+ ] = {
+ "residency_reservation": "RESIDENCY_RESERVATION",
+ "request_execution": "REQUEST_EXECUTION",
+ "load_transition": "LOAD_TRANSITION",
+ }
+ known_priorities = {
+ "production",
+ "interactive",
+ "background",
+ "benchmark",
+ "lab",
+ "maintenance",
+ }
+ def linked_workload_status(
+ lease: ServingGpuLease,
+ request_status: str | None,
+ request_deployment_id: uuid.UUID | None,
+ request_node_id: uuid.UUID | None,
+ node_id: uuid.UUID | None,
+ ) -> Literal["QUEUED", "RUNNING", "UNKNOWN"]:
+ if (
+ request_status is None
+ or request_deployment_id != lease.capability_deployment_id
+ or request_node_id is None
+ or request_node_id != node_id
+ or node_id is None
+ ):
+ return "UNKNOWN"
+ status = request_status.casefold()
+ if status == "queued" and lease.state in {"pending", "granted"}:
+ return "QUEUED"
+ if status == "running" and lease.state == "active":
+ return "RUNNING"
+ return "UNKNOWN"
+
+ inventory = [
+ ActiveSchedulerLeaseResponse(
+ lease_id=lease.id,
+ project_id=project_id,
+ node_id=node_id,
+ accelerator_id=lease.accelerator_id,
+ deployment_id=lease.capability_deployment_id,
+ lease_state=lease_states[lease.state],
+ workload_status=linked_workload_status(
+ lease,
+ request_status,
+ request_deployment_id,
+ request_node_id,
+ node_id,
+ ),
+ priority=(
+ lease.priority.upper()
+ if lease.priority in known_priorities
+ else "UNKNOWN"
+ ),
+ lease_type=lease_types.get(lease.lease_type, "UNKNOWN"),
+ reserved_vram_bytes=lease.reserved_vram_bytes,
+ materialized_vram_bytes=lease.materialized_vram_bytes,
+ created_at=_aware(lease.created_at),
+ acquired_at=_aware(lease.acquired_at) if lease.acquired_at else None,
+ heartbeat_at=_aware(lease.heartbeat_at) if lease.heartbeat_at else None,
+ expires_at=_aware(lease.expires_at),
+ )
+ for (
+ lease,
+ node_id,
+ request_status,
+ request_deployment_id,
+ request_node_id,
+ project_id,
+ ) in selected
+ ]
+ return ActiveSchedulerLeaseInventoryResponse(
+ observed_at=now,
+ completeness="TRUNCATED" if truncated else "COMPLETE",
+ limit=limit,
+ returned_count=len(inventory),
+ leases=inventory,
+ )
+
def capacity_trend(self, node_id: uuid.UUID, hours: int = 24) -> TrendResponse:
start = _now() - timedelta(hours=hours)
rows = list(
@@ -1732,18 +1862,19 @@ class ObservabilityService:
)
self.session.add(alert)
self.session.flush()
- self.session.add(
- AlertHistoryEvent(
- alert_id=alert.id,
- from_state=None,
- to_state=initial.value,
- actor="observability-engine",
- reason="Signal condition detected",
- evidence=signal["details"],
- occurred_at=now,
- )
+ event = AlertHistoryEvent(
+ alert_id=alert.id,
+ from_state=None,
+ to_state=initial.value,
+ actor="observability-engine",
+ reason="Signal condition detected",
+ evidence=signal["details"],
+ occurred_at=now,
)
+ self.session.add(event)
+ self.session.flush()
if initial is AlertState.FIRING:
+ enqueue_alert_transition(self.session, self.settings, alert, event)
self._correlate_incident(alert, signal, now)
return
alert.last_seen_at = now
@@ -1817,17 +1948,19 @@ class ObservabilityService:
alert.resolved_at = now
rule = self.session.get(AlertRuleRevision, alert.rule_id)
alert.cooldown_until = now + timedelta(seconds=rule.cooldown_seconds if rule else 0)
- self.session.add(
- AlertHistoryEvent(
- alert_id=alert.id,
- from_state=previous,
- to_state=state.value,
- actor=actor,
- reason=reason,
- evidence=evidence,
- occurred_at=now,
- )
+ event = AlertHistoryEvent(
+ alert_id=alert.id,
+ from_state=previous,
+ to_state=state.value,
+ actor=actor,
+ reason=reason,
+ evidence=evidence,
+ occurred_at=now,
)
+ self.session.add(event)
+ if state in {AlertState.FIRING, AlertState.RESOLVED}:
+ self.session.flush()
+ enqueue_alert_transition(self.session, self.settings, alert, event)
def _suppressed_until(
self, rule: AlertRuleRevision, signal: dict[str, Any], now: datetime
diff --git a/backend/src/modelforge_api/services/observability_polling.py b/backend/src/modelforge_api/services/observability_polling.py
index c038377..3e80d29 100644
--- a/backend/src/modelforge_api/services/observability_polling.py
+++ b/backend/src/modelforge_api/services/observability_polling.py
@@ -9,6 +9,7 @@ from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from modelforge_api.domain.observability import metrics
+from modelforge_api.services.alert_notifications import dispatch_pending
from modelforge_api.services.observability import ObservabilityService
from modelforge_api.settings import Settings
@@ -30,11 +31,20 @@ class ObservabilityPollingService:
try:
with Session(self.engine) as session:
service = ObservabilityService(session, self.settings)
- service.ensure_defaults()
- service.collect_capacity()
- service.evaluate_slos()
- service.evaluate_alerts()
- service.apply_retention()
+ evaluation_enabled = (
+ self.settings.observability_monitor_enabled
+ or self.settings.alert_notifications_enabled
+ )
+ if evaluation_enabled:
+ service.ensure_defaults()
+ if self.settings.observability_monitor_enabled:
+ service.collect_capacity()
+ service.evaluate_slos()
+ if evaluation_enabled:
+ service.evaluate_alerts()
+ if self.settings.observability_monitor_enabled:
+ service.apply_retention()
+ dispatch_pending(session, self.settings)
metrics.gauge("modelforge_observability_degraded", {}, 0)
except (SQLAlchemyError, OSError, RuntimeError, ValueError) as error:
metrics.gauge("modelforge_observability_degraded", {}, 1)
diff --git a/backend/src/modelforge_api/services/operator_identities.py b/backend/src/modelforge_api/services/operator_identities.py
new file mode 100644
index 0000000..3670825
--- /dev/null
+++ b/backend/src/modelforge_api/services/operator_identities.py
@@ -0,0 +1,254 @@
+"""Online personal-key lifecycle, isolated from deployment and machine identities."""
+from __future__ import annotations
+
+import hashlib
+import re
+import secrets
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
+from typing import cast
+
+from sqlalchemy import func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.db import rows_affected
+from modelforge_api.domain.managed_identity import (
+ DeploymentIdentityMetadata,
+ ManagedCredentialMetadata,
+ ManagedIdentityChange,
+ ManagedIdentityCreate,
+ ManagedIdentityIssued,
+ ManagedIdentityList,
+ ManagedIdentityMetadata,
+ ManagedIdentityResult,
+ ManagedIdentityRevoke,
+ ManagedIdentityRotate,
+ Role,
+)
+from modelforge_api.domain.operator_identity import parse_operator_identities
+from modelforge_api.persistence.models import (
+ ManagedOperatorCredential,
+ ManagedOperatorCredentialActivation,
+ ManagedOperatorIdentity,
+ ManagedOperatorState,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings
+
+
+class ManagedIdentityError(RuntimeError):
+ def __init__(self, status_code: int, code: str, message: str) -> None:
+ super().__init__(message)
+ self.status_code, self.code, self.message = status_code, code, message
+
+
+def _aware(value: datetime) -> datetime:
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value
+
+
+def _usable(credential: ManagedOperatorCredential, identity: ManagedOperatorIdentity,
+ activated: bool) -> bool:
+ return (identity.enabled and activated and credential.revoked_at is None
+ and _aware(credential.expires_at) > datetime.now(UTC))
+
+
+def authenticate_managed_operator(session: Session, token: str) -> Principal:
+ if not re.fullmatch(r"mfopm_[A-Za-z0-9_-]{43}", token):
+ raise ManagedIdentityError(401, "MANAGED_CREDENTIAL_INVALID", "invalid operator credential")
+ credential = session.scalar(select(ManagedOperatorCredential).where(
+ ManagedOperatorCredential.token_sha256 == hashlib.sha256(token.encode()).hexdigest(),
+ ))
+ identity = session.get(ManagedOperatorIdentity, credential.identity_slug) if credential else None
+ activation = session.get(ManagedOperatorCredentialActivation, credential.id) if credential else None
+ if credential is None or identity is None or not _usable(credential, identity, activation is not None):
+ raise ManagedIdentityError(401, "MANAGED_CREDENTIAL_INVALID", "invalid operator credential")
+ credential.last_used_at = datetime.now(UTC)
+ return Principal(
+ subject=f"managed:{identity.slug}", role=PrincipalRole(identity.role),
+ authentication_method="managed_operator_key", credential_id=str(credential.id),
+ expires_at=_aware(credential.expires_at),
+ )
+
+
+class ManagedIdentityService:
+ def __init__(self, session: Session, principal: Principal, correlation_id: str) -> None:
+ self.session, self.principal = session, principal
+ self.audit = AuditWriter(session, context=AuditContext(
+ "operator", principal.subject, correlation_id,
+ ))
+
+ def _admin(self) -> None:
+ if self.principal.role != PrincipalRole.ADMIN:
+ raise ManagedIdentityError(403, "MANAGED_ADMIN_REQUIRED", "administrator role required")
+
+ @contextmanager
+ def _mutation(self) -> Iterator[ManagedOperatorState]:
+ self._admin()
+ try:
+ # Updating the singleton serializes every administration transaction, including two
+ # administrators removing one another. It also locks SQLite's test write transaction.
+ claimed = self.session.execute(update(ManagedOperatorState).where(
+ ManagedOperatorState.singleton_id == 1,
+ ).values(revision=ManagedOperatorState.revision))
+ if rows_affected(claimed) != 1:
+ raise ManagedIdentityError(503, "MANAGED_IDENTITY_STATE_MISSING", "identity state unavailable")
+ self.session.expire_all()
+ state = self.session.get(ManagedOperatorState, 1)
+ if state is None:
+ raise ManagedIdentityError(503, "MANAGED_IDENTITY_STATE_MISSING", "identity state unavailable")
+ if self.principal.authentication_method == "managed_operator_key":
+ try:
+ credential_id = uuid.UUID(self.principal.credential_id or "")
+ except ValueError as exc:
+ raise ManagedIdentityError(401, "MANAGED_CREDENTIAL_INVALID", "invalid operator credential") from exc
+ credential = self.session.get(ManagedOperatorCredential, credential_id)
+ identity = self.session.get(ManagedOperatorIdentity, credential.identity_slug) if credential else None
+ active = self.session.get(ManagedOperatorCredentialActivation, credential_id)
+ if (credential is None or identity is None or identity.role != "admin"
+ or self.principal.subject != f"managed:{identity.slug}"
+ or not _usable(credential, identity, active is not None)):
+ raise ManagedIdentityError(403, "MANAGED_ADMIN_NO_LONGER_AUTHORIZED", "administrator authorization changed")
+ yield state
+ self.session.flush()
+ usable_admin = self.session.scalar(select(ManagedOperatorIdentity.slug).join(
+ ManagedOperatorCredential,
+ ManagedOperatorCredential.identity_slug == ManagedOperatorIdentity.slug,
+ ).join(ManagedOperatorCredentialActivation,
+ ManagedOperatorCredentialActivation.credential_id == ManagedOperatorCredential.id).where(
+ ManagedOperatorIdentity.role == "admin", ManagedOperatorIdentity.enabled.is_(True),
+ ManagedOperatorCredential.revoked_at.is_(None),
+ ManagedOperatorCredential.expires_at > datetime.now(UTC),
+ ).limit(1))
+ if usable_admin is None:
+ raise ManagedIdentityError(409, "LAST_MANAGED_ADMIN", "retain a usable managed administrator; recovery requires an external administrator to issue a new admin key")
+ state.revision += 1
+ self.session.commit()
+ except Exception:
+ self.session.rollback()
+ raise
+
+ def _identity(self, slug: str, version: int) -> ManagedOperatorIdentity:
+ identity = self.session.get(ManagedOperatorIdentity, slug)
+ if identity is None:
+ raise ManagedIdentityError(404, "MANAGED_IDENTITY_NOT_FOUND", "managed identity not found")
+ if identity.version != version:
+ raise ManagedIdentityError(409, "MANAGED_IDENTITY_VERSION_CONFLICT", "identity changed; reload before retrying")
+ return identity
+
+ def _metadata(self, identity: ManagedOperatorIdentity) -> ManagedIdentityMetadata:
+ credentials = list(self.session.scalars(select(ManagedOperatorCredential).where(
+ ManagedOperatorCredential.identity_slug == identity.slug,
+ ).order_by(ManagedOperatorCredential.created_at.desc(), ManagedOperatorCredential.id.desc()).limit(10)))
+ active = set(self.session.scalars(select(ManagedOperatorCredentialActivation.credential_id).where(
+ ManagedOperatorCredentialActivation.credential_id.in_([item.id for item in credentials]),
+ )))
+ return ManagedIdentityMetadata(
+ subject=f"managed:{identity.slug}", slug=identity.slug, role=cast(Role, identity.role),
+ enabled=identity.enabled, version=identity.version, created_at=_aware(identity.created_at),
+ credentials=[ManagedCredentialMetadata(
+ id=str(item.id), expires_at=_aware(item.expires_at),
+ revoked_at=_aware(item.revoked_at) if item.revoked_at else None,
+ superseded_at=_aware(item.superseded_at) if item.superseded_at else None,
+ last_used_at=_aware(item.last_used_at) if item.last_used_at else None,
+ activated=item.id in active, usable=_usable(item, identity, item.id in active),
+ ) for item in credentials],
+ )
+
+ def list(self, settings: Settings | None = None) -> ManagedIdentityList:
+ self._admin()
+ state = self.session.get(ManagedOperatorState, 1)
+ if state is None:
+ raise ManagedIdentityError(503, "MANAGED_IDENTITY_STATE_MISSING", "identity state unavailable")
+ identities = list(self.session.scalars(select(ManagedOperatorIdentity).order_by(ManagedOperatorIdentity.slug).limit(100)))
+ document = settings.operator_identities if settings else None
+ deployed = parse_operator_identities(document.get_secret_value()) if document else ()
+ return ManagedIdentityList(revision=state.revision, initialized=state.revision > 0,
+ identities=[self._metadata(item) for item in identities],
+ deployment_identities=[DeploymentIdentityMetadata(
+ subject=item.subject, role=item.role, credential_id=item.credential_id,
+ expires_at=item.expires_at,
+ ) for item in deployed])
+
+ def _issue(self, identity: ManagedOperatorIdentity, days: int) -> str:
+ token = "mfopm_" + secrets.token_urlsafe(32)
+ now = datetime.now(UTC)
+ credential = ManagedOperatorCredential(
+ id=uuid.uuid4(), identity_slug=identity.slug,
+ token_sha256=hashlib.sha256(token.encode()).hexdigest(),
+ expires_at=now + timedelta(days=days), created_at=now,
+ )
+ self.session.add(credential)
+ self.session.flush()
+ self.session.add(ManagedOperatorCredentialActivation(credential_id=credential.id, activated_at=now))
+ return token
+
+ def _audit(self, action: str, identity: ManagedOperatorIdentity, reason: str) -> None:
+ self.audit.write(action, "managed_operator_identity", f"managed:{identity.slug}", {
+ "role": identity.role, "enabled": identity.enabled, "version": identity.version,
+ "initiating_credential_id": self.principal.credential_id, "reason": reason,
+ })
+
+ def create(self, request: ManagedIdentityCreate) -> ManagedIdentityIssued:
+ with self._mutation() as state:
+ if state.revision != request.expected_revision:
+ raise ManagedIdentityError(409, "MANAGED_IDENTITY_VERSION_CONFLICT", "registry changed; reload before retrying")
+ if self.session.get(ManagedOperatorIdentity, request.slug):
+ raise ManagedIdentityError(409, "MANAGED_IDENTITY_EXISTS", "managed subject already exists")
+ count = self.session.scalar(select(func.count()).select_from(ManagedOperatorIdentity)) or 0
+ if count >= 100:
+ raise ManagedIdentityError(409, "MANAGED_IDENTITY_LIMIT", "maximum 100 managed identities")
+ if count == 0 and request.role != "admin":
+ raise ManagedIdentityError(409, "FIRST_MANAGED_ADMIN_REQUIRED", "first managed identity must be an administrator")
+ identity = ManagedOperatorIdentity(slug=request.slug, role=request.role,
+ enabled=True, version=1, created_at=datetime.now(UTC))
+ self.session.add(identity)
+ self.session.flush()
+ token = self._issue(identity, request.expires_in_days)
+ self._audit("managed_identity.created", identity, request.reason)
+ result = ManagedIdentityIssued(revision=state.revision + 1, identity=self._metadata(identity), one_time_key=token)
+ return result
+
+ def rotate(self, slug: str, request: ManagedIdentityRotate) -> ManagedIdentityIssued:
+ with self._mutation() as state:
+ identity = self._identity(slug, request.expected_version)
+ now = datetime.now(UTC)
+ credentials = list(self.session.scalars(select(ManagedOperatorCredential).where(
+ ManagedOperatorCredential.identity_slug == slug, ManagedOperatorCredential.revoked_at.is_(None),
+ )))
+ for credential in credentials:
+ if credential.superseded_at is not None or request.overlap_seconds == 0:
+ credential.revoked_at = now
+ else:
+ credential.superseded_at = now
+ credential.expires_at = min(_aware(credential.expires_at), now + timedelta(seconds=request.overlap_seconds))
+ self.session.flush()
+ token = self._issue(identity, request.expires_in_days)
+ identity.version += 1
+ self._audit("managed_identity.rotated", identity, request.reason)
+ result = ManagedIdentityIssued(revision=state.revision + 1, identity=self._metadata(identity), one_time_key=token)
+ return result
+
+ def change(self, slug: str, request: ManagedIdentityChange) -> ManagedIdentityResult:
+ with self._mutation() as state:
+ identity = self._identity(slug, request.expected_version)
+ identity.role, identity.enabled = request.role, request.enabled
+ identity.version += 1
+ self._audit("managed_identity.changed", identity, request.reason)
+ result = ManagedIdentityResult(revision=state.revision + 1, identity=self._metadata(identity))
+ return result
+
+ def revoke(self, slug: str, request: ManagedIdentityRevoke) -> ManagedIdentityResult:
+ with self._mutation() as state:
+ identity = self._identity(slug, request.expected_version)
+ if request.confirm_subject != f"managed:{slug}":
+ raise ManagedIdentityError(422, "MANAGED_IDENTITY_CONFIRMATION_REQUIRED", "type the exact managed subject to revoke all keys")
+ self.session.execute(update(ManagedOperatorCredential).where(
+ ManagedOperatorCredential.identity_slug == slug, ManagedOperatorCredential.revoked_at.is_(None),
+ ).values(revoked_at=datetime.now(UTC)))
+ identity.version += 1
+ self._audit("managed_identity.revoked", identity, request.reason)
+ result = ManagedIdentityResult(revision=state.revision + 1, identity=self._metadata(identity))
+ return result
diff --git a/backend/src/modelforge_api/services/operator_password.py b/backend/src/modelforge_api/services/operator_password.py
new file mode 100644
index 0000000..7db7cb1
--- /dev/null
+++ b/backend/src/modelforge_api/services/operator_password.py
@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import re
+import secrets
+import time
+
+HASH_PREFIX = "scrypt$32768$8$1$"
+SESSION_SECONDS = 900
+
+
+def _encode(value: bytes) -> str:
+ return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
+
+
+def _decode(value: str, length: int) -> bytes:
+ if not re.fullmatch(r"[A-Za-z0-9_-]+", value):
+ raise ValueError("Invalid password credential encoding")
+ result = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
+ if len(result) != length or _encode(result) != value:
+ raise ValueError("Invalid password credential encoding")
+ return result
+
+
+def parse_password_hash(encoded: str) -> tuple[bytes, bytes]:
+ # Accept only one fixed, bounded work factor; never execute attacker-selected KDF parameters.
+ if len(encoded) != 83 or not encoded.startswith(HASH_PREFIX):
+ raise ValueError("Invalid operator password hash")
+ fields = encoded[len(HASH_PREFIX) :].split("$")
+ if len(fields) != 2:
+ raise ValueError("Invalid operator password hash")
+ return _decode(fields[0], 16), _decode(fields[1], 32)
+
+
+def hash_password(password: str) -> str:
+ if not 1 <= len(password) <= 1024:
+ raise ValueError("Password must contain 1 to 1024 characters")
+ salt = secrets.token_bytes(16)
+ digest = hashlib.scrypt(
+ password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, maxmem=64 * 1024 * 1024, dklen=32
+ )
+ return HASH_PREFIX + _encode(salt) + "$" + _encode(digest)
+
+
+def verify_password(password: str, encoded: str) -> bool:
+ salt, expected = parse_password_hash(encoded)
+ if not 1 <= len(password) <= 1024:
+ return False
+ actual = hashlib.scrypt(
+ password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, maxmem=64 * 1024 * 1024, dklen=32
+ )
+ return hmac.compare_digest(actual, expected)
+
+
+def _session_key(bootstrap: str, password_hash: str) -> bytes:
+ return hmac.digest(
+ bootstrap.encode(), b"modelforge-password-session-v1\0" + password_hash.encode(), "sha256"
+ )
+
+
+def issue_session(bootstrap: str, password_hash: str, *, now: int | None = None) -> tuple[str, int]:
+ issued = int(time.time()) if now is None else now
+ expires = issued + SESSION_SECONDS
+ body = f"{issued}.{expires}.{_encode(secrets.token_bytes(24))}"
+ signature = _encode(
+ hmac.digest(_session_key(bootstrap, password_hash), body.encode(), "sha256")
+ )
+ return f"mfpw_{body}.{signature}", expires
+
+
+def verify_session(
+ token: str, bootstrap: str, password_hash: str, *, now: int | None = None
+) -> bool:
+ if len(token) > 160 or not token.startswith("mfpw_"):
+ return False
+ try:
+ issued_text, expires_text, nonce, signature = token[5:].split(".")
+ if not re.fullmatch(r"[0-9]{1,12}", issued_text) or not re.fullmatch(
+ r"[0-9]{1,12}", expires_text
+ ):
+ return False
+ issued, expires = int(issued_text), int(expires_text)
+ current = int(time.time()) if now is None else now
+ if not issued <= current < expires or expires - issued != SESSION_SECONDS:
+ return False
+ _decode(nonce, 24)
+ provided = _decode(signature, 32)
+ body = f"{issued_text}.{expires_text}.{nonce}"
+ expected = hmac.digest(_session_key(bootstrap, password_hash), body.encode(), "sha256")
+ return hmac.compare_digest(provided, expected)
+ except (ValueError, UnicodeError):
+ return False
diff --git a/backend/src/modelforge_api/services/portfolio_registry.py b/backend/src/modelforge_api/services/portfolio_registry.py
new file mode 100644
index 0000000..a3f6f9a
--- /dev/null
+++ b/backend/src/modelforge_api/services/portfolio_registry.py
@@ -0,0 +1,545 @@
+"""Idempotent, secret-free reconciliation of repository inventory into project identities."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime
+from typing import Any, cast
+
+from sqlalchemy import func, select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.portfolio import (
+ PROJECT_LINK_CLASSIFICATIONS,
+ PortfolioCodeEvidence,
+ PortfolioInventoryResponse,
+ PortfolioReconcileRequest,
+ PortfolioReconcileResponse,
+ PortfolioRepositoryInput,
+ PortfolioRepositoryView,
+ PortfolioRuntimeEvidence,
+)
+from modelforge_api.persistence.models import (
+ PortfolioReconciliationRun,
+ PortfolioRepositoryObservation,
+ Project,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+
+
+class PortfolioRegistryError(Exception):
+ def __init__(self, status_code: int, code: str, message: str) -> None:
+ super().__init__(message)
+ self.status_code = status_code
+ self.code = code
+ self.message = message
+
+
+def _utc(value: datetime) -> datetime:
+ if value.tzinfo is None or value.utcoffset() is None:
+ return value.replace(tzinfo=UTC)
+ return value.astimezone(UTC)
+
+
+def snapshot_digest(request: PortfolioReconcileRequest) -> str:
+ """Hash normalized, order-independent content so replay does not depend on JSON formatting."""
+
+ payload = request.model_dump(mode="json")
+ repositories = cast(list[dict[str, Any]], payload["repositories"])
+ for repository in repositories:
+ repository["code_evidence"] = sorted(
+ repository["code_evidence"],
+ key=lambda item: (item["path"], item["signal"], tuple(item["capabilities"])),
+ )
+ repository["runtime_evidence"] = sorted(
+ repository["runtime_evidence"], key=lambda item: item["component"]
+ )
+ for evidence in repository["code_evidence"]:
+ evidence["capabilities"] = sorted(evidence["capabilities"])
+ payload["repositories"] = sorted(repositories, key=lambda item: item["external_repository_id"])
+ encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+class PortfolioRegistryService:
+ def __init__(self, session: Session) -> None:
+ self.session = session
+
+ def inventory(self) -> PortfolioInventoryResponse:
+ rows = self.session.execute(
+ select(PortfolioRepositoryObservation, Project)
+ .outerjoin(Project, Project.id == PortfolioRepositoryObservation.project_id)
+ .order_by(
+ PortfolioRepositoryObservation.slug,
+ PortfolioRepositoryObservation.external_repository_id,
+ )
+ ).all()
+ return PortfolioInventoryResponse(
+ repositories=[
+ PortfolioRepositoryView(
+ id=str(repository.id),
+ project_key=repository.project_key,
+ project_name=project.name if project is not None else None,
+ project_id=str(project.id) if project is not None else None,
+ source_system=repository.source_system,
+ external_repository_id=repository.external_repository_id,
+ slug=repository.slug,
+ default_branch=repository.default_branch,
+ head_sha=repository.head_sha,
+ classification=repository.classification,
+ code_evidence=[
+ PortfolioCodeEvidence.model_validate(item)
+ for item in repository.code_evidence
+ ],
+ runtime_evidence=[
+ PortfolioRuntimeEvidence.model_validate(item)
+ for item in repository.runtime_evidence
+ ],
+ observed_at=_utc(repository.source_observed_at),
+ snapshot_id=repository.source_snapshot_id,
+ snapshot_digest=repository.source_snapshot_digest,
+ pending_head_sha=repository.pending_head_sha,
+ pending_default_branch=repository.pending_default_branch,
+ pending_project_key=repository.pending_project_key,
+ pending_classification=repository.pending_classification,
+ pending_code_evidence=[
+ PortfolioCodeEvidence.model_validate(item)
+ for item in (repository.pending_code_evidence or [])
+ ],
+ pending_observed_at=(
+ _utc(repository.pending_observed_at)
+ if repository.pending_observed_at
+ else None
+ ),
+ pending_snapshot_id=repository.pending_snapshot_id,
+ pending_snapshot_digest=repository.pending_snapshot_digest,
+ last_seen_at=_utc(repository.last_seen_at),
+ stale_at=_utc(repository.stale_at) if repository.stale_at else None,
+ )
+ for repository, project in rows
+ ]
+ )
+
+ def reconcile(
+ self, request: PortfolioReconcileRequest, audit_context: AuditContext
+ ) -> PortfolioReconcileResponse:
+ digest = snapshot_digest(request)
+ try:
+ existing_run = self.session.scalar(
+ select(PortfolioReconciliationRun).where(
+ PortfolioReconciliationRun.source_system == request.source_system,
+ PortfolioReconciliationRun.snapshot_id == request.snapshot_id,
+ )
+ )
+ if existing_run is not None:
+ if existing_run.payload_digest != digest:
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_SNAPSHOT_CONFLICT",
+ "snapshot ID was already used with different content",
+ )
+ return self._run_response(existing_run, replayed=True)
+
+ latest_complete = self.session.scalar(
+ select(func.max(PortfolioReconciliationRun.observed_at)).where(
+ PortfolioReconciliationRun.source_system == request.source_system,
+ PortfolioReconciliationRun.complete.is_(True),
+ PortfolioReconciliationRun.stale_snapshot.is_(False),
+ )
+ )
+ observation_time = _utc(request.observed_at)
+ latest_complete_time = _utc(latest_complete) if latest_complete else None
+ stale_snapshot = bool(
+ latest_complete_time
+ and (
+ observation_time < latest_complete_time
+ or (request.complete and observation_time <= latest_complete_time)
+ )
+ )
+ counts = {
+ "projects_created": 0,
+ "repositories_created": 0,
+ "repositories_updated": 0,
+ "stale_observations": len(request.repositories) if stale_snapshot else 0,
+ "repositories_marked_stale": 0,
+ "runtime_observations_updated": 0,
+ "head_changes_review_required": 0,
+ "head_changes_accepted": 0,
+ }
+ seen_repository_ids = {item.external_repository_id for item in request.repositories}
+
+ for item in request.repositories:
+ repository = self.session.scalar(
+ select(PortfolioRepositoryObservation).where(
+ PortfolioRepositoryObservation.source_system == request.source_system,
+ PortfolioRepositoryObservation.external_repository_id
+ == item.external_repository_id,
+ )
+ )
+ if item.project_key is not None:
+ key_owner = self.session.scalar(
+ select(PortfolioRepositoryObservation).where(
+ PortfolioRepositoryObservation.source_system == request.source_system,
+ PortfolioRepositoryObservation.project_key == item.project_key,
+ PortfolioRepositoryObservation.external_repository_id
+ != item.external_repository_id,
+ )
+ )
+ if key_owner is not None:
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_PROJECT_KEY_CONFLICT",
+ "project key is already proposed for another repository",
+ )
+ project: Project | None = None
+ if repository is not None and repository.project_id is not None:
+ project = self.session.get(Project, repository.project_id)
+ if project is None:
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_PROJECT_MISSING",
+ "repository is linked to a missing project identity",
+ )
+ if item.project_key is not None and project.key != item.project_key:
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_IDENTITY_CONFLICT",
+ "repository identity cannot be reassigned to another project",
+ )
+
+ source_is_newer = repository is None or observation_time > _utc(
+ repository.last_seen_at
+ )
+ if not stale_snapshot and source_is_newer:
+ if repository is None:
+ project = (
+ self._project_for_observation(item, counts)
+ if item.classification in PROJECT_LINK_CLASSIFICATIONS
+ else None
+ )
+ repository = PortfolioRepositoryObservation(
+ project_id=project.id if project is not None else None,
+ project_key=item.project_key,
+ source_system=request.source_system,
+ external_repository_id=item.external_repository_id,
+ slug=item.slug,
+ default_branch=item.default_branch,
+ head_sha=item.head_sha,
+ classification=item.classification,
+ code_evidence=self._code_evidence(item),
+ runtime_evidence=[],
+ source_observed_at=observation_time,
+ source_snapshot_id=request.snapshot_id,
+ source_snapshot_digest=digest,
+ pending_head_sha=None,
+ pending_default_branch=None,
+ pending_project_key=None,
+ pending_classification=None,
+ pending_code_evidence=None,
+ pending_observed_at=None,
+ pending_snapshot_id=None,
+ pending_snapshot_digest=None,
+ last_seen_at=observation_time,
+ stale_at=None,
+ )
+ self.session.add(repository)
+ self.session.flush()
+ counts["repositories_created"] += 1
+ else:
+ repository.last_seen_at = observation_time
+ repository.stale_at = None
+ head_changed = (
+ item.head_sha != repository.head_sha
+ or item.default_branch != repository.default_branch
+ )
+ pending_review = repository.pending_head_sha is not None
+ if pending_review and item.accept_head_update and not self._pending_matches(
+ repository, item
+ ):
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_PENDING_HEAD_CONFLICT",
+ "head acceptance must match the pending repository observation",
+ )
+ if pending_review and not item.accept_head_update:
+ if head_changed:
+ self._set_pending_head(
+ repository, item, request, observation_time, digest
+ )
+ counts["head_changes_review_required"] += 1
+ elif head_changed and not item.accept_head_update:
+ self._set_pending_head(
+ repository, item, request, observation_time, digest
+ )
+ counts["head_changes_review_required"] += 1
+ else:
+ if (
+ repository.project_id is None
+ and item.classification in PROJECT_LINK_CLASSIFICATIONS
+ ):
+ project = self._project_for_observation(item, counts)
+ repository.project_id = project.id
+ repository.project_key = item.project_key
+ if repository.project_id is not None and item.project_key is not None:
+ linked_project = project or self.session.get(
+ Project, repository.project_id
+ )
+ if linked_project is None or linked_project.key != item.project_key:
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_IDENTITY_CONFLICT",
+ "repository identity cannot be reassigned to another project",
+ )
+ elif repository.project_id is None:
+ repository.project_key = item.project_key
+ if head_changed:
+ counts["head_changes_accepted"] += 1
+ repository.slug = item.slug
+ repository.default_branch = item.default_branch
+ repository.head_sha = item.head_sha
+ repository.classification = item.classification
+ repository.code_evidence = self._code_evidence(item)
+ repository.source_observed_at = observation_time
+ repository.source_snapshot_id = request.snapshot_id
+ repository.source_snapshot_digest = digest
+ self._clear_pending_head(repository)
+ counts["repositories_updated"] += 1
+ elif (
+ not stale_snapshot
+ and repository is not None
+ and (
+ observation_time < _utc(repository.last_seen_at)
+ or not self._source_matches(repository, item)
+ )
+ ):
+ counts["stale_observations"] += 1
+
+ if repository is not None:
+ counts["runtime_observations_updated"] += self._merge_runtime_evidence(
+ repository, item.runtime_evidence
+ )
+
+ if request.complete and not stale_snapshot:
+ for repository in self.session.scalars(
+ select(PortfolioRepositoryObservation).where(
+ PortfolioRepositoryObservation.source_system == request.source_system
+ )
+ ):
+ if repository.external_repository_id in seen_repository_ids:
+ continue
+ repository_time = _utc(repository.last_seen_at)
+ if observation_time <= repository_time:
+ if observation_time < repository_time:
+ counts["stale_observations"] += 1
+ continue
+ if repository.stale_at is None or observation_time > _utc(repository.stale_at):
+ repository.stale_at = observation_time
+ counts["repositories_marked_stale"] += 1
+
+ run = PortfolioReconciliationRun(
+ source_system=request.source_system,
+ snapshot_id=request.snapshot_id,
+ payload_digest=digest,
+ observed_at=observation_time,
+ complete=request.complete,
+ stale_snapshot=stale_snapshot,
+ **counts,
+ )
+ self.session.add(run)
+ self.session.flush()
+ AuditWriter(self.session, context=audit_context).write(
+ "PORTFOLIO_RECONCILED",
+ "portfolio_snapshot",
+ str(run.id),
+ {
+ "source_system": run.source_system,
+ "snapshot_id": run.snapshot_id,
+ "payload_digest": run.payload_digest,
+ "complete": run.complete,
+ "stale_snapshot": run.stale_snapshot,
+ **counts,
+ },
+ )
+ self.session.commit()
+ return self._run_response(run, replayed=False)
+ except PortfolioRegistryError:
+ self.session.rollback()
+ raise
+ except IntegrityError as exc:
+ self.session.rollback()
+ persisted_run = self.session.scalar(
+ select(PortfolioReconciliationRun).where(
+ PortfolioReconciliationRun.source_system == request.source_system,
+ PortfolioReconciliationRun.snapshot_id == request.snapshot_id,
+ )
+ )
+ if persisted_run is not None:
+ if persisted_run.payload_digest == digest:
+ return self._run_response(persisted_run, replayed=True)
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_SNAPSHOT_CONFLICT",
+ "snapshot ID was already used with different content",
+ ) from exc
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_IDENTITY_CONFLICT",
+ "repository or project identity conflicts with existing inventory",
+ ) from exc
+ except Exception:
+ self.session.rollback()
+ raise
+
+ @staticmethod
+ def _project_name(item: PortfolioRepositoryInput) -> str:
+ return item.slug.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ").title()
+
+ def _project_for_observation(
+ self, item: PortfolioRepositoryInput, counts: dict[str, int]
+ ) -> Project:
+ if item.project_key is None:
+ raise PortfolioRegistryError(
+ 422,
+ "PORTFOLIO_PROJECT_KEY_REQUIRED",
+ "project key is required before linking a local consumer or provider",
+ )
+ project = self.session.scalar(select(Project).where(Project.key == item.project_key))
+ if project is None:
+ project = Project(
+ key=item.project_key,
+ name=self._project_name(item),
+ description=f"Repository inventory for {item.slug}.",
+ active=True,
+ )
+ self.session.add(project)
+ self.session.flush()
+ counts["projects_created"] += 1
+ linked_repository = self.session.scalar(
+ select(PortfolioRepositoryObservation).where(
+ PortfolioRepositoryObservation.project_id == project.id,
+ PortfolioRepositoryObservation.external_repository_id
+ != item.external_repository_id,
+ )
+ )
+ if linked_repository is not None:
+ raise PortfolioRegistryError(
+ 409,
+ "PORTFOLIO_PROJECT_ALREADY_LINKED",
+ "project identity is already linked to another repository",
+ )
+ return project
+
+ @staticmethod
+ def _code_evidence(item: PortfolioRepositoryInput) -> list[dict[str, Any]]:
+ payloads = [evidence.model_dump(mode="json") for evidence in item.code_evidence]
+ for evidence in payloads:
+ evidence["capabilities"] = sorted(evidence["capabilities"])
+ return sorted(
+ payloads,
+ key=lambda evidence: (
+ evidence["path"], evidence["signal"], tuple(evidence["capabilities"])
+ ),
+ )
+
+ @staticmethod
+ def _source_matches(
+ repository: PortfolioRepositoryObservation, item: PortfolioRepositoryInput
+ ) -> bool:
+ return (
+ repository.slug == item.slug
+ and repository.default_branch == item.default_branch
+ and repository.head_sha == item.head_sha
+ and repository.classification == item.classification
+ and repository.code_evidence == PortfolioRegistryService._code_evidence(item)
+ )
+
+ @staticmethod
+ def _clear_pending_head(repository: PortfolioRepositoryObservation) -> None:
+ repository.pending_head_sha = None
+ repository.pending_default_branch = None
+ repository.pending_project_key = None
+ repository.pending_classification = None
+ repository.pending_code_evidence = None
+ repository.pending_observed_at = None
+ repository.pending_snapshot_id = None
+ repository.pending_snapshot_digest = None
+
+ @staticmethod
+ def _set_pending_head(
+ repository: PortfolioRepositoryObservation,
+ item: PortfolioRepositoryInput,
+ request: PortfolioReconcileRequest,
+ observed_at: datetime,
+ digest: str,
+ ) -> None:
+ repository.pending_head_sha = item.head_sha
+ repository.pending_default_branch = item.default_branch
+ repository.pending_project_key = item.project_key
+ repository.pending_classification = item.classification
+ repository.pending_code_evidence = PortfolioRegistryService._code_evidence(item)
+ repository.pending_observed_at = observed_at
+ repository.pending_snapshot_id = request.snapshot_id
+ repository.pending_snapshot_digest = digest
+
+ @staticmethod
+ def _pending_matches(
+ repository: PortfolioRepositoryObservation, item: PortfolioRepositoryInput
+ ) -> bool:
+ return (
+ repository.pending_head_sha == item.head_sha
+ and repository.pending_default_branch == item.default_branch
+ and repository.pending_project_key == item.project_key
+ and repository.pending_classification == item.classification
+ and repository.pending_code_evidence == PortfolioRegistryService._code_evidence(item)
+ )
+
+ @staticmethod
+ def _merge_runtime_evidence(
+ repository: PortfolioRepositoryObservation,
+ incoming: list[PortfolioRuntimeEvidence],
+ ) -> int:
+ if not incoming:
+ return 0
+ current = {
+ evidence.component: evidence
+ for evidence in (
+ PortfolioRuntimeEvidence.model_validate(item)
+ for item in repository.runtime_evidence
+ )
+ }
+ updated = 0
+ for evidence in incoming:
+ previous = current.get(evidence.component)
+ if previous is not None and _utc(evidence.observed_at) <= _utc(previous.observed_at):
+ continue
+ current[evidence.component] = evidence
+ updated += 1
+ if updated:
+ repository.runtime_evidence = [
+ item.model_dump(mode="json") for item in sorted(current.values(), key=lambda x: x.component)
+ ]
+ return updated
+
+ @staticmethod
+ def _run_response(
+ run: PortfolioReconciliationRun, *, replayed: bool
+ ) -> PortfolioReconcileResponse:
+ return PortfolioReconcileResponse(
+ source_system=run.source_system,
+ snapshot_id=run.snapshot_id,
+ snapshot_digest=run.payload_digest,
+ observed_at=_utc(run.observed_at),
+ complete=run.complete,
+ replayed=replayed,
+ stale_snapshot=run.stale_snapshot,
+ projects_created=run.projects_created,
+ repositories_created=run.repositories_created,
+ repositories_updated=run.repositories_updated,
+ stale_observations=run.stale_observations,
+ repositories_marked_stale=run.repositories_marked_stale,
+ runtime_observations_updated=run.runtime_observations_updated,
+ head_changes_review_required=run.head_changes_review_required,
+ head_changes_accepted=run.head_changes_accepted,
+ )
diff --git a/backend/src/modelforge_api/services/project_registry.py b/backend/src/modelforge_api/services/project_registry.py
index f1488ee..e7cc607 100644
--- a/backend/src/modelforge_api/services/project_registry.py
+++ b/backend/src/modelforge_api/services/project_registry.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass
from sqlalchemy import select
from sqlalchemy.orm import Session
+from modelforge_api.domain.contracts import CapabilityStability
from modelforge_api.persistence.models import (
Capability,
CapabilityContract,
@@ -15,11 +16,20 @@ from modelforge_api.services.audit import AuditWriter
from modelforge_api.services.manifest_registry import ManifestRegistry
+@dataclass(frozen=True)
+class UnavailableProjectBinding:
+ project_key: str
+ capability: str
+ contract_version: int
+ declared_stability: CapabilityStability
+
+
@dataclass(frozen=True)
class ProjectRegistrySyncResult:
projects_created: int
bindings_created: int
unavailable_contracts: tuple[str, ...]
+ unavailable_bindings: tuple[UnavailableProjectBinding, ...]
def sync_project_registry(
@@ -28,13 +38,19 @@ def sync_project_registry(
"""Materialize generic manifest projects against contracts that exist in the DB.
A capability contract is lifecycle-owned and may not exist yet on a fresh control plane.
- Such bindings remain in the validated manifest and are reported, never fabricated.
- Existing bindings are operator-owned and are not overwritten by startup reconciliation.
+ Missing contracts are reported, never fabricated. Contracts declared planned or blocked also
+ remain unbound and are reported with their declared stability. Existing bindings are
+ operator-owned and are not overwritten or removed by startup reconciliation.
"""
audit = AuditWriter(session, "system", "project-manifest-sync")
projects_created = 0
bindings_created = 0
unavailable: list[str] = []
+ unavailable_bindings: list[UnavailableProjectBinding] = []
+ capability_stability = {
+ (item.capability, item.version): item.estate.stability
+ for item in manifests.capabilities()
+ }
for item in manifests.projects():
project = session.scalar(select(Project).where(Project.key == item.project.id))
if project is None:
@@ -66,6 +82,17 @@ def sync_project_registry(
if contract is None:
unavailable.append(identity)
continue
+ stability = capability_stability[(capability_key, binding.contract_version)]
+ if stability in {"planned", "blocked"}:
+ unavailable_bindings.append(
+ UnavailableProjectBinding(
+ project_key=item.project.id,
+ capability=capability_key,
+ contract_version=binding.contract_version,
+ declared_stability=stability,
+ )
+ )
+ continue
existing = session.scalar(
select(ProjectBinding).where(
ProjectBinding.project_id == project.id,
@@ -104,4 +131,14 @@ def sync_project_registry(
projects_created=projects_created,
bindings_created=bindings_created,
unavailable_contracts=tuple(sorted(unavailable)),
+ unavailable_bindings=tuple(
+ sorted(
+ unavailable_bindings,
+ key=lambda entry: (
+ entry.project_key,
+ entry.capability,
+ entry.contract_version,
+ ),
+ )
+ ),
)
diff --git a/backend/src/modelforge_api/services/projects.py b/backend/src/modelforge_api/services/projects.py
new file mode 100644
index 0000000..0a1cb72
--- /dev/null
+++ b/backend/src/modelforge_api/services/projects.py
@@ -0,0 +1,237 @@
+"""Persisted, capability-only onboarding; readiness is derived from observed gateway use."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from datetime import timedelta
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import selectinload
+
+from modelforge_api.domain.projects import (
+ ProjectCreate,
+ ProjectRequirementCreate,
+ ProjectRequirementResponse,
+ ProjectServingModelResponse,
+ ProjectWorkspaceResponse,
+)
+from modelforge_api.persistence.models import (
+ ArtifactSet,
+ Capability,
+ CapabilityContract,
+ GatewayRequest,
+ Model,
+ ModelRevision,
+ Project,
+ ProjectBinding,
+ ProjectOnboarding,
+ ServiceClient,
+)
+from modelforge_api.services.serving import ServingError, ServingService, _aware, _now
+
+
+class ProjectService:
+ def __init__(self, serving: ServingService) -> None:
+ self.serving = serving
+ self.session = serving.session
+ self.audit = serving.audit
+
+ def _project(self, key: str) -> Project:
+ project = self.session.scalar(select(Project).where(Project.key == key))
+ if project is None:
+ raise ServingError(404, "PROJECT_NOT_FOUND", "project was not found")
+ return project
+
+ def create(self, request: ProjectCreate) -> ProjectWorkspaceResponse:
+ if any(not label or len(label) > 64 for label in request.labels):
+ raise ServingError(422, "PROJECT_LABEL_INVALID", "labels must contain 1 to 64 characters")
+ try:
+ project = Project(key=request.key, name=request.name, description=request.description)
+ self.session.add(project)
+ self.session.flush()
+ self.session.add(ProjectOnboarding(
+ project_id=project.id, owner=request.owner, environment=request.environment,
+ labels=sorted(set(request.labels)), created_at=_now(),
+ ))
+ self.audit.write("PROJECT_CREATED", "project", str(project.id), {
+ "project_key": project.key, "environment": request.environment,
+ })
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise ServingError(409, "PROJECT_EXISTS", "project slug already exists") from exc
+ return self.detail(project.key)
+
+ def bind(self, key: str, request: ProjectRequirementCreate) -> ProjectWorkspaceResponse:
+ project = self._project(key)
+ if not project.active:
+ raise ServingError(409, "PROJECT_INACTIVE", "project is inactive")
+ capability, version = request.capability.rsplit("@", 1)
+ contract = self.serving.repo.contract(capability, int(version))
+ if contract is None:
+ # Only reviewed versioned manifests can materialize a missing contract.
+ self.serving.ensure_contract(capability, int(version))
+ contract = self.serving.repo.contract(capability, int(version))
+ if contract is None or contract.deprecated_at:
+ raise ServingError(409, "CAPABILITY_UNAVAILABLE", "capability contract is unavailable")
+ if contract.upgrade_class == "requires_reindex" and request.migration_support != "reindex":
+ raise ServingError(422, "REINDEX_SUPPORT_REQUIRED", "this contract requires explicit reindex support")
+ if contract.upgrade_class == "schema_breaking" and request.migration_support != "schema":
+ raise ServingError(422, "SCHEMA_SUPPORT_REQUIRED", "this contract requires explicit schema migration support")
+ try:
+ binding = ProjectBinding(
+ project_id=project.id, capability_contract_id=contract.id, channel="stable",
+ priority=request.priority, migration_support=request.migration_support,
+ )
+ self.session.add(binding)
+ self.session.flush()
+ self.audit.write("PROJECT_REQUIREMENT_BOUND", "project_binding", str(binding.id), {
+ "project_key": key, "capability": request.capability, "channel": "stable",
+ "migration_support": request.migration_support,
+ })
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise ServingError(409, "PROJECT_BINDING_EXISTS", "this capability is already bound") from exc
+ return self.detail(key)
+
+ def list(
+ self, limit: int | None = None, after_key: str | None = None
+ ) -> list[ProjectWorkspaceResponse]:
+ statement = select(Project).options(selectinload(Project.bindings))
+ if after_key is not None:
+ statement = statement.where(Project.key > after_key)
+ if limit is not None:
+ statement = statement.limit(limit)
+ projects = list(self.session.scalars(statement.order_by(Project.key)))
+ if not projects:
+ return []
+
+ onboarding_by_project = {
+ item.project_id: item
+ for item in self.session.scalars(
+ select(ProjectOnboarding).where(
+ ProjectOnboarding.project_id.in_([project.id for project in projects])
+ )
+ )
+ }
+ return [
+ self._workspace_response(
+ project,
+ onboarding_by_project.get(project.id),
+ sorted(project.bindings, key=lambda binding: (binding.created_at, binding.id.hex)),
+ )
+ for project in projects
+ ]
+
+ def detail(self, key: str) -> ProjectWorkspaceResponse:
+ project = self._project(key)
+ metadata = self.session.get(ProjectOnboarding, project.id)
+ bindings = list(self.session.scalars(select(ProjectBinding).where(
+ ProjectBinding.project_id == project.id,
+ ).order_by(ProjectBinding.created_at, ProjectBinding.id)))
+ return self._workspace_response(project, metadata, bindings)
+
+ def _workspace_response(
+ self,
+ project: Project,
+ metadata: ProjectOnboarding | None,
+ bindings: Iterable[ProjectBinding],
+ ) -> ProjectWorkspaceResponse:
+ requirements: list[ProjectRequirementResponse] = []
+ clients = []
+ now = _now()
+ freshness = now - timedelta(hours=24)
+ for binding in bindings:
+ contract = self.session.get(CapabilityContract, binding.capability_contract_id)
+ capability = self.session.get(Capability, contract.capability_id) if contract else None
+ if contract is None or capability is None:
+ raise ServingError(409, "PROJECT_CONTRACT_MISSING", "binding contract is unavailable")
+ deployment = self.serving.repo.stable_deployment(contract.id)
+ residency = self.serving.repo.residency(deployment.id) if deployment else None
+ health = self.serving._health_status(deployment, residency) if deployment and residency else "unavailable"
+
+ artifact_set = self.session.get(ArtifactSet, deployment.artifact_set_id) if deployment else None
+ revision = self.session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
+ model = self.session.get(Model, revision.model_id) if revision else None
+ serving_model = None
+ if deployment and artifact_set and revision and model:
+ serving_model = ProjectServingModelResponse(
+ deployment_id=deployment.id,
+ artifact_set_id=artifact_set.id,
+ model_id=model.id,
+ model_key=model.key,
+ display_name=model.display_name,
+ upstream_source=model.upstream_source,
+ revision_id=revision.id,
+ upstream_revision=revision.upstream_revision,
+ resolved_commit_sha=revision.resolved_commit_sha,
+ runtime_profile_id=deployment.runtime_profile_id,
+ compute_node_id=deployment.compute_node_id,
+ channel=deployment.channel,
+ production=deployment.production,
+ artifact_availability=artifact_set.availability,
+ )
+
+ connection_times = []
+ successes: list[GatewayRequest] = []
+ has_credential = False
+ for client in self.session.scalars(select(ServiceClient).where(
+ ServiceClient.project_binding_id == binding.id,
+ ).order_by(ServiceClient.created_at)):
+ credential = self.serving.repo.active_credential(client.id)
+ clients.append(self.serving.client_response(client, credential))
+ if (credential is None or client.status != "active" or client.disabled_at
+ or credential.expires_at and _aware(credential.expires_at) <= now):
+ continue
+ has_credential = True
+ if credential.last_used_at and _aware(credential.last_used_at) >= freshness:
+ connection_times.append(_aware(credential.last_used_at))
+ if deployment:
+ success = self.session.scalar(select(GatewayRequest).where(
+ GatewayRequest.service_client_id == client.id,
+ GatewayRequest.capability_deployment_id == deployment.id,
+ GatewayRequest.status == "completed",
+ GatewayRequest.decision_evidence["service_credential_id"].as_string() == str(credential.id),
+ GatewayRequest.created_at >= max(freshness, _aware(credential.created_at)),
+ ).order_by(GatewayRequest.created_at.desc()).limit(1))
+ if success:
+ successes.append(success)
+ latest = max(successes, key=lambda item: _aware(item.created_at)) if successes else None
+ blockers = []
+ if not project.active or binding.deprecated_at or contract.deprecated_at:
+ blockers.append("Project, binding or contract is inactive.")
+ if binding.channel != "stable":
+ blockers.append("This planning binding requires its separate experiment workflow.")
+ if not deployment or health not in {"healthy", "ready_on_demand"}:
+ blockers.append("No healthy stable deployment is available.")
+ if deployment and serving_model is None:
+ blockers.append("Stable deployment model provenance is incomplete.")
+ if not has_credential:
+ blockers.append("Issue an active scoped credential.")
+ if not connection_times:
+ blockers.append("Verify the credential through the gateway (evidence expires after 24 hours).")
+ if latest is None:
+ blockers.append("Complete a capability request with the current credential and deployment.")
+ requirements.append(ProjectRequirementResponse(
+ id=binding.id, capability=f"{capability.key}@{contract.version}",
+ channel=binding.channel, priority=binding.priority,
+ migration_support=binding.migration_support, deprecated=bool(binding.deprecated_at),
+ deployment_id=deployment.id if deployment else None, deployment_health=health,
+ serving_model=serving_model, credential_ready=has_credential,
+ connectivity_checked_at=max(connection_times) if connection_times else None,
+ capability_checked_at=latest.created_at if latest else None,
+ capability_check_request_id=str(latest.request_id) if latest else None,
+ state="ready" if not blockers else "needs_attention", blockers=blockers,
+ ))
+ return ProjectWorkspaceResponse(
+ id=project.id, key=project.key, name=project.name, description=project.description,
+ active=project.active, owner=metadata.owner if metadata else None,
+ environment=metadata.environment if metadata else None,
+ labels=metadata.labels if metadata else [], source="operator" if metadata else "manifest",
+ # Engineering readiness cannot grant a production lifecycle approval.
+ production_eligible=None,
+ state="ready" if requirements and all(item.state == "ready" for item in requirements) else "needs_attention",
+ requirements=requirements, clients=clients,
+ )
diff --git a/backend/src/modelforge_api/services/recovery.py b/backend/src/modelforge_api/services/recovery.py
index 5d5fed1..cf3f2b5 100644
--- a/backend/src/modelforge_api/services/recovery.py
+++ b/backend/src/modelforge_api/services/recovery.py
@@ -15,10 +15,11 @@ import shutil
import time
import uuid
from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from functools import lru_cache
from pathlib import Path
-from typing import Any, cast
+from typing import Any, Literal, cast
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session
@@ -49,6 +50,8 @@ from modelforge_api.domain.recovery import (
RecoveryPolicyResponse,
RecoveryReadiness,
RecoveryReadinessEntry,
+ RecoveryRetentionDecision,
+ RecoveryRetentionPreview,
RestoreAdvanceRequest,
RestoreMode,
RestoreOperationEventResponse,
@@ -91,7 +94,10 @@ from modelforge_api.services.recovery_crypto import (
BackupCryptoError,
)
from modelforge_api.services.recovery_fingerprint import (
+ CREATIVE_RESTORE_ERROR,
CURRENT_TRUTH_TABLES,
+ SELECTED_COMPARISON_ACTIVE_STATUSES,
+ SELECTED_COMPARISON_RESTORE_ERROR,
audit_fingerprint_differences_are_compatible,
diff_fingerprints,
fingerprint_compatibility,
@@ -184,6 +190,29 @@ class RecoveryError(Exception):
self.details = details or {}
+@dataclass(frozen=True)
+class _RetentionDecision:
+ record: BackupSet
+ decision: Literal["RETAIN", "EXPIRE"]
+ reason: Literal[
+ "ACTIVE_RESTORE_IN_PROGRESS",
+ "LEGAL_HOLD",
+ "MILESTONE",
+ "MINIMUM_VERIFIED_BACKUPS",
+ "RETENTION_NOT_REACHED",
+ "RETENTION_ELIGIBLE",
+ ]
+
+
+@dataclass(frozen=True)
+class _RetentionPlan:
+ observed_at: datetime
+ policy: RecoveryPolicyRevision | None
+ active_restore_in_progress: bool
+ protected_count: int
+ decisions: tuple[_RetentionDecision, ...]
+
+
class RecoveryService:
def __init__(
self,
@@ -377,6 +406,24 @@ class RecoveryService:
500, RecoveryFailureCode.PATH_NOT_ALLOWED.value, str(error)
) from error
+ def _plaintext_staging_path(self, operation: str) -> Path:
+ """Keep decrypted working files off the immutable backup volume."""
+
+ try:
+ root = normalise_root(self.settings.backup_restore_root)
+ except (RecoveryPathError, OSError) as error:
+ raise RecoveryError(
+ 500, RecoveryFailureCode.PATH_NOT_ALLOWED.value, str(error)
+ ) from error
+ backup_root = self._backup_root()
+ if root == backup_root or backup_root in root.parents:
+ raise RecoveryError(
+ 409,
+ RecoveryFailureCode.PATH_NOT_ALLOWED.value,
+ "plaintext staging must be outside the backup root",
+ )
+ return root / f"database-{operation}-{uuid.uuid4().hex}.dump"
+
def estimate_capacity(self) -> BackupCapacityEstimate:
"""Backups check headroom before writing; they never fill a disk to find out."""
@@ -1021,7 +1068,7 @@ class RecoveryService:
raise RecoveryError(
409, RecoveryFailureCode.ENCRYPTION_KEY_UNAVAILABLE.value, "no key available"
)
- staging = path.with_name(path.name + f".verify-{uuid.uuid4().hex[:8]}")
+ staging = self._plaintext_staging_path("verify")
try:
cipher.decrypt_file(path, staging)
expected = (entry.get("dependency_refs") or {}).get("plaintext_sha256")
@@ -1063,12 +1110,16 @@ class RecoveryService:
self.session.commit()
return len(interrupted)
- def apply_retention(self, observed_at: datetime | None = None) -> dict[str, int]:
- """Expire aged backups, but never drop the last verified recovery point."""
+ def _retention_plan(
+ self,
+ observed_at: datetime | None = None,
+ *,
+ seed_missing_policy: bool,
+ include_active_restore_decisions: bool,
+ ) -> _RetentionPlan:
+ """Evaluate the active retention rules without changing backup state or the filesystem."""
now = observed_at or _now()
- expired = 0
- protected = 0
verified = [
item
for item in self.session.scalars(
@@ -1077,15 +1128,69 @@ class RecoveryService:
.order_by(BackupSet.verified_at.desc())
)
]
- if self.session.scalar(
+ active_restore_in_progress = self.session.scalar(
select(RestoreOperation)
.where(RestoreOperation.state.in_(ACTIVE_RESTORE_STATES))
.limit(1)
- ):
- return {"expired": 0, "protected": len(verified), "skipped_active_restore": 1}
- policy = self._database_policy()
+ ) is not None
+ if active_restore_in_progress and not include_active_restore_decisions:
+ # Preserve apply_retention's original short circuit: no policy lookup, default seeding,
+ # candidate scan, state change or audit event occurs during a restore.
+ return _RetentionPlan(
+ observed_at=now,
+ policy=None,
+ active_restore_in_progress=True,
+ protected_count=len(verified),
+ decisions=(),
+ )
+ if active_restore_in_progress:
+ policy = self.session.scalar(
+ select(RecoveryPolicyRevision).where(
+ RecoveryPolicyRevision.key == "control-plane.database",
+ RecoveryPolicyRevision.active.is_(True),
+ )
+ )
+ if policy is None:
+ raise RecoveryError(
+ 500, "recovery_policy_missing", "no active database backup policy"
+ )
+ active_decisions = tuple(
+ _RetentionDecision(record, "RETAIN", "ACTIVE_RESTORE_IN_PROGRESS")
+ for record in self.session.scalars(
+ select(BackupSet).where(
+ BackupSet.state.in_(
+ (
+ BackupState.VERIFIED.value,
+ BackupState.CREATED.value,
+ BackupState.FAILED.value,
+ )
+ )
+ )
+ )
+ )
+ return _RetentionPlan(
+ observed_at=now,
+ policy=policy,
+ active_restore_in_progress=True,
+ protected_count=len(verified),
+ decisions=active_decisions,
+ )
+ policy = self.session.scalar(
+ select(RecoveryPolicyRevision).where(
+ RecoveryPolicyRevision.key == "control-plane.database",
+ RecoveryPolicyRevision.active.is_(True),
+ )
+ )
+ if policy is None and seed_missing_policy:
+ # Keep the established run behavior. A preview passes False so its GET path can never
+ # seed policy rows as a side effect.
+ policy = self._database_policy()
+ if policy is None:
+ raise RecoveryError(500, "recovery_policy_missing", "no active database backup policy")
keep = max(policy.minimum_verified_backups, 1)
protected_ids = {item.id for item in verified[:keep]}
+ protected = 0
+ decisions: list[_RetentionDecision] = []
for record in self.session.scalars(
select(BackupSet).where(
BackupSet.state.in_(
@@ -1093,25 +1198,103 @@ class RecoveryService:
)
)
):
- expires_at = _aware(record.expires_at)
- if record.legal_hold or record.milestone:
+ if active_restore_in_progress:
+ decisions.append(
+ _RetentionDecision(record, "RETAIN", "ACTIVE_RESTORE_IN_PROGRESS")
+ )
+ elif record.legal_hold:
protected += 1
- continue
- if record.id in protected_ids:
+ decisions.append(_RetentionDecision(record, "RETAIN", "LEGAL_HOLD"))
+ elif record.milestone:
protected += 1
+ decisions.append(_RetentionDecision(record, "RETAIN", "MILESTONE"))
+ elif record.id in protected_ids:
+ protected += 1
+ decisions.append(_RetentionDecision(record, "RETAIN", "MINIMUM_VERIFIED_BACKUPS"))
+ else:
+ expires_at = _aware(record.expires_at)
+ if expires_at is None or expires_at > now:
+ decisions.append(
+ _RetentionDecision(record, "RETAIN", "RETENTION_NOT_REACHED")
+ )
+ else:
+ decisions.append(_RetentionDecision(record, "EXPIRE", "RETENTION_ELIGIBLE"))
+ return _RetentionPlan(
+ observed_at=now,
+ policy=policy,
+ active_restore_in_progress=active_restore_in_progress,
+ protected_count=protected,
+ decisions=tuple(decisions),
+ )
+
+ def preview_retention(self, observed_at: datetime | None = None) -> RecoveryRetentionPreview:
+ """Return the exact retention decisions without writing rows, audit events or files."""
+
+ # GET preview requests must not flush unrelated pending ORM changes in their session.
+ with self.session.no_autoflush:
+ plan = self._retention_plan(
+ observed_at,
+ seed_missing_policy=False,
+ include_active_restore_decisions=True,
+ )
+ if plan.policy is None: # pragma: no cover - read-only previews fail before this point
+ raise RecoveryError(500, "recovery_policy_missing", "no active database backup policy")
+ decisions = sorted(plan.decisions, key=lambda item: item.record.backup_id)
+ public_decisions = [
+ RecoveryRetentionDecision(
+ backup_id=item.record.backup_id,
+ current_state=BackupState(item.record.state),
+ decision=item.decision,
+ reason=item.reason,
+ expires_at=_aware(item.record.expires_at),
+ )
+ for item in decisions
+ ]
+ return RecoveryRetentionPreview(
+ observed_at=plan.observed_at,
+ policy_key=plan.policy.key,
+ policy_revision=plan.policy.revision,
+ minimum_verified_backups=max(plan.policy.minimum_verified_backups, 1),
+ active_restore_in_progress=plan.active_restore_in_progress,
+ retained_backup_ids=[
+ item.backup_id for item in public_decisions if item.decision == "RETAIN"
+ ],
+ expiring_backup_ids=[
+ item.backup_id for item in public_decisions if item.decision == "EXPIRE"
+ ],
+ decisions=public_decisions,
+ )
+
+ def apply_retention(self, observed_at: datetime | None = None) -> dict[str, int]:
+ """Expire aged backups, but never drop the last verified recovery point."""
+
+ plan = self._retention_plan(
+ observed_at,
+ seed_missing_policy=True,
+ include_active_restore_decisions=False,
+ )
+ if plan.active_restore_in_progress:
+ return {
+ "expired": 0,
+ "protected": plan.protected_count,
+ "skipped_active_restore": 1,
+ }
+ if plan.policy is None: # pragma: no cover - non-active runs seed the default policy
+ raise RecoveryError(500, "recovery_policy_missing", "no active database backup policy")
+ expired = 0
+ for item in plan.decisions:
+ if item.decision != "EXPIRE":
continue
- if expires_at is None or expires_at > now:
- continue
- record.state = BackupState.EXPIRED.value
+ item.record.state = BackupState.EXPIRED.value
expired += 1
self.audit.write(
"BACKUP_EXPIRED",
AUDIT_RESOURCE,
- str(record.id),
- {"backup_id": record.backup_id, "expired_at": now.isoformat()},
+ str(item.record.id),
+ {"backup_id": item.record.backup_id, "expired_at": plan.observed_at.isoformat()},
)
self.session.commit()
- return {"expired": expired, "protected": protected, "skipped_active_restore": 0}
+ return {"expired": expired, "protected": plan.protected_count, "skipped_active_restore": 0}
def backups(self, limit: int = 100) -> list[BackupSetResponse]:
rows = self.session.scalars(
@@ -1654,7 +1837,7 @@ class RecoveryService:
RecoveryFailureCode.ENCRYPTION_KEY_UNAVAILABLE.value,
"no decryption key",
)
- staging = stored.with_name(stored.name + f".restore-{uuid.uuid4().hex[:8]}")
+ staging = self._plaintext_staging_path("restore")
cipher.decrypt_file(stored, staging)
expected = (entry.dependency_refs or {}).get("plaintext_sha256")
if expected and sha256_file(staging) != expected:
@@ -1851,7 +2034,7 @@ class RecoveryService:
"coalesce(canonical_payload, '') as canonical_payload, "
"(canonical_payload is null)::text as canonical_payload_is_null "
"from audit_events "
- f"{cursor_sql}order by sequence, id limit {page_size}"
+ f"{cursor_sql}order by audit_events.sequence, audit_events.id limit {page_size}"
)
rows = self.engine.query_rows(
target,
@@ -1863,7 +2046,8 @@ class RecoveryService:
for row in rows:
details = json.loads(row["details"])
if not isinstance(details, dict):
- raise ValueError("restored audit details are not an object")
+ # This is corrupt restored content, not a caller type contract violation.
+ raise ValueError("restored audit details are not an object") # noqa: TRY004
sequence = int(row["sequence"])
event_id = str(uuid.UUID(row["id"]))
yield AuditEventRecord(
@@ -2003,8 +2187,65 @@ class RecoveryService:
)
return event_hash
+ def _reconcile_creative_allocations(self, target: PostgresTarget) -> int:
+ exists = self.engine.scalar(target,
+ "select count(*) from information_schema.tables where table_schema='public' "
+ "and table_name='external_creative_allocations'")
+ if exists.strip() in {"0", ""}:
+ return 0
+ affected = int(self.engine.scalar(target,
+ "select count(*) from external_creative_allocations where state in ('queued', 'granted')") or 0)
+ self.engine.scalar(target,
+ "update external_creative_allocations set state='uncertain', " # noqa: S608 - fixed module constant and literal status allowlist
+ f"decision_code='{CREATIVE_RESTORE_ERROR}' where state in ('queued', 'granted')")
+ return affected
+
+ def _creative_allocation_residuals(self, target: PostgresTarget) -> int:
+ exists = self.engine.scalar(target,
+ "select count(*) from information_schema.tables where table_schema='public' "
+ "and table_name='external_creative_allocations'")
+ if exists.strip() in {"0", ""}:
+ return 0
+ return int(self.engine.scalar(target,
+ "select count(*) from external_creative_allocations where state in ('queued', 'granted')") or 0)
+
+ def _reconcile_selected_comparisons(self, target: PostgresTarget) -> dict[str, int]:
+ exists = self.engine.scalar(target,
+ "select count(*) from information_schema.tables where table_schema='public' "
+ "and table_name='selected_model_comparisons'")
+ if exists.strip() in {"0", ""}:
+ return {"interrupted": 0, "claims_cleared": 0}
+ active = _quoted(tuple(sorted(SELECTED_COMPARISON_ACTIVE_STATUSES)))
+ interrupted = int(self.engine.scalar(target,
+ f"select count(*) from selected_model_comparisons where status in ({active})") or 0) # noqa: S608
+ claims = int(self.engine.scalar(target,
+ "select count(*) from selected_model_comparisons "
+ "where claim_token_hash is not null or claim_expires_at is not null") or 0)
+ # Preserve all timestamps/pins/results. This does not assert that an external worker stopped.
+ # The same deterministic transformation is committed in each source restore_projection.
+ self.engine.scalar(target,
+ f"update selected_model_comparisons set " # noqa: S608 - module-level literal status/code allowlists
+ f"version=case when status in ({active}) then version+1 else version end, "
+ f"error_code=case when status in ({active}) then '{SELECTED_COMPARISON_RESTORE_ERROR}' else error_code end, "
+ f"status=case when status in ({active}) then 'interrupted' else status end, "
+ "claim_token_hash=NULL, claim_expires_at=NULL "
+ f"where status in ({active}) or claim_token_hash is not null or claim_expires_at is not null")
+ return {"interrupted": interrupted, "claims_cleared": claims}
+
+ def _selected_comparison_residuals(self, target: PostgresTarget) -> int:
+ exists = self.engine.scalar(target,
+ "select count(*) from information_schema.tables where table_schema='public' "
+ "and table_name='selected_model_comparisons'")
+ if exists.strip() in {"0", ""}:
+ return 0
+ active = _quoted(tuple(sorted(SELECTED_COMPARISON_ACTIVE_STATUSES)))
+ return int(self.engine.scalar(target,
+ "select count(*) from selected_model_comparisons " # noqa: S608 - literal status allowlist
+ f"where status in ({active}) or claim_token_hash is not null "
+ "or claim_expires_at is not null") or 0)
+
def _reconcile_restored(self, record: RestoreOperation, plan: RestorePlan) -> dict[str, Any]:
- """Clear stale current-truth rows exactly once; history is never rewritten."""
+ """Clear stale truth and apply the exact comparison interruption projection once."""
target = parse_target(plan.database_destination)
# Fail before the idempotency marker lookup and before any target mutation. A valid marker
@@ -2045,6 +2286,21 @@ class RecoveryService:
self.engine.scalar(target, f"delete from {table}") # noqa: S608
cleared[table] = int(before or 0)
+ # Watch history is authoritative, but an in-flight metadata check is not restored truth.
+ watch_table = self.engine.scalar(target,
+ "select count(*) from information_schema.tables where table_schema='public' "
+ "and table_name='model_update_watches'")
+ if watch_table.strip() not in {"0", ""}:
+ before = self.engine.scalar(target,
+ "select count(*) from model_update_watches where check_token is not null or check_until is not null")
+ self.engine.scalar(target,
+ "update model_update_watches set check_token=NULL, check_until=NULL")
+ cleared["model_update_watch_claims"] = int(before or 0)
+
+ comparisons = self._reconcile_selected_comparisons(target)
+ cleared["selected_comparison_claims"] = comparisons["claims_cleared"]
+ cleared["creative_allocations_interrupted"] = self._reconcile_creative_allocations(target)
+
# Lifecycle and migration operations use a `stage` vocabulary; anything that is not a
# terminal stage was in flight when the backup was taken and needs reconciliation.
pending_lifecycle = int(
@@ -2090,12 +2346,28 @@ class RecoveryService:
)
or 0
)
+ pending_notifications = int(
+ self.engine.scalar(
+ target,
+ "select count(*) from alert_notification_deliveries "
+ "where status in ('PENDING','DISPATCHING','RETRY_WAIT')",
+ )
+ or 0
+ )
# A restored alert is history, not current truth: it must be re-derived after startup.
self.engine.scalar(
target,
"update operational_alerts set state='SUPPRESSED', "
"suppressed_until = now() + interval '1 hour' where state in ('PENDING','FIRING')",
)
+ # A backup's in-flight webhooks cannot safely be replayed after restore: the remote side
+ # may already have accepted them. Preserve their evidence but require operator review.
+ self.engine.scalar(
+ target,
+ "update alert_notification_deliveries set status='HELD_FOR_REVIEW', "
+ "next_attempt_at=null, lease_expires_at=null, claim_token_hash=null "
+ "where status in ('PENDING','DISPATCHING','RETRY_WAIT')",
+ )
# Re-read immediately before the marker so concurrent target activity cannot make the
# earlier proof stale. The marker CAS binds to this exact verified head.
checkpoint = self._verify_restored_audit_chain(target)
@@ -2113,6 +2385,8 @@ class RecoveryService:
"pending_lifecycle_operations": pending_lifecycle,
"pending_migration_cutovers": pending_migrations,
"suppressed_alerts": firing_alerts,
+ "notification_deliveries_held_for_review": pending_notifications,
+ "interrupted_selected_comparisons": comparisons["interrupted"],
},
)
return {
@@ -2124,8 +2398,10 @@ class RecoveryService:
"in_flight_lifecycle_operations": in_flight_lifecycle,
"in_flight_migration_cutovers": in_flight_migrations,
"restored_firing_alerts_suppressed": firing_alerts,
+ "notification_deliveries_held_for_review": pending_notifications,
+ "interrupted_selected_comparisons": comparisons["interrupted"],
"outcome": "RECOVERY_RECONCILIATION_REQUIRED"
- if pending_lifecycle or pending_migrations
+ if pending_lifecycle or pending_migrations or comparisons["interrupted"] or pending_notifications
else "RECONCILED",
}
@@ -2213,6 +2489,13 @@ class RecoveryService:
).strip()
not in {"0", ""}
}
+ if self.engine.scalar(target,
+ "select count(*) from information_schema.tables where table_schema='public' "
+ "and table_name='model_update_watches'").strip() not in {"0", ""}:
+ stale_current_truth["model_update_watch_claims"] = int(self.engine.scalar(target,
+ "select count(*) from model_update_watches where check_token is not null or check_until is not null") or 0)
+ stale_current_truth["selected_comparison_active_or_claimed"] = self._selected_comparison_residuals(target)
+ stale_current_truth["creative_queued_or_granted"] = self._creative_allocation_residuals(target)
gate = {
"database_validated": table_count >= minimum_tables and constraint_count > 0,
# A restore is not READY while a stale lease or NVML snapshot is still presented as
diff --git a/backend/src/modelforge_api/services/recovery_crypto.py b/backend/src/modelforge_api/services/recovery_crypto.py
index 9ca429c..f28b8ee 100644
--- a/backend/src/modelforge_api/services/recovery_crypto.py
+++ b/backend/src/modelforge_api/services/recovery_crypto.py
@@ -100,7 +100,9 @@ class AesGcmBackupCipher:
written = 0
temporary = destination.with_name(destination.name + ".partial")
try:
- with source.open("rb") as reader, temporary.open("wb") as writer:
+ with source.open("rb") as reader, os.fdopen(
+ os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb"
+ ) as writer:
if reader.read(len(MAGIC)) != MAGIC:
raise BackupCryptoError(
"DECRYPTION_FAILED", "payload is not a ModelForge encrypted backup"
diff --git a/backend/src/modelforge_api/services/recovery_fingerprint.py b/backend/src/modelforge_api/services/recovery_fingerprint.py
index afc6075..12eff1c 100644
--- a/backend/src/modelforge_api/services/recovery_fingerprint.py
+++ b/backend/src/modelforge_api/services/recovery_fingerprint.py
@@ -19,7 +19,7 @@ from datetime import UTC, date, datetime
from decimal import Decimal
from typing import Any
-from sqlalchemy import Table, func, select
+from sqlalchemy import Column, MetaData, Table, func, inspect, or_, select, text
from sqlalchemy.engine import Connection
from sqlalchemy.orm import Session
@@ -31,6 +31,59 @@ LEGACY_FINGERPRINT_VERSION = "m15.1"
AUDIT_CHECKPOINT_SOURCE_SCHEMA = "20260828_0022"
AUDIT_CHECKPOINT_CURRENT_SCHEMA = "20260830_0024"
NON_FINGERPRINT_INFRASTRUCTURE_TABLES = frozenset({"alembic_version"})
+CONFIGURATION_TABLES = frozenset({"configuration_state", "configuration_revisions"})
+CONFIGURATION_SCHEMA = "20260907_0025"
+PROJECT_SCHEMA = "20260908_0026"
+PROJECT_TABLES = frozenset({"project_onboarding"})
+MANAGED_IDENTITY_SCHEMA = "20260908_0027"
+MANAGED_IDENTITY_TABLES = frozenset({
+ "managed_operator_state", "managed_operator_identities", "managed_operator_credentials",
+})
+SECRET_SCHEMA = "20260908_0028" # noqa: S105 - schema identifier
+SECRET_TABLES = frozenset({"platform_secret_state", "platform_secret_versions"})
+SECRET_CURRENT_TABLES = frozenset({"platform_secret_tests", "platform_secret_activations"})
+MODEL_UPDATE_SCHEMA = "20260909_0030"
+MODEL_UPDATE_TABLES = frozenset({"model_update_watches", "model_update_notices"})
+MODEL_UPDATE_TRANSIENT_COLUMNS = frozenset({"check_token", "check_until"})
+SELECTED_COMPARISON_SCHEMA = "20260909_0033"
+SERVING_GENERATION_SCHEMA = "20260910_0034"
+MODEL_CHANGE_SCHEMA = "20260911_0035"
+MODEL_CHANGE_TABLES = frozenset({"model_change_cases"})
+PORTFOLIO_SCHEMA = "20260923_0036"
+PORTFOLIO_TABLES = frozenset({"portfolio_reconciliation_runs", "portfolio_repository_observations"})
+ALERT_NOTIFICATION_SCHEMA = "20260923_0037"
+ALERT_NOTIFICATION_TABLES = frozenset({"alert_notification_deliveries"})
+ALERT_NOTIFICATION_RESTORABLE_STATES = frozenset({"PENDING", "DISPATCHING", "RETRY_WAIT"})
+LAB_EXECUTION_SCHEMA = "20260924_0038"
+LAB_EXECUTION_TABLES = frozenset({"lab_execution_runs"})
+BENCHMARK_RUN_SCHEMA = "20260924_0039"
+BENCHMARK_RUN_MIGRATION_COLUMNS = frozenset(
+ {
+ "capability_deployment_id",
+ "target_snapshot",
+ "actor_subject",
+ "idempotency_key",
+ "request_fingerprint",
+ "failure_code",
+ "cancellation_requested_at",
+ "created_at",
+ "generation",
+ }
+)
+BENCHMARK_RUN_SCHEMA_PROJECTION = "benchmark_run_schema_projection"
+BENCHMARK_RUN_SCHEMA_OBSERVED = "benchmark_runs_schema_observed"
+SELECTED_COMPARISON_TABLES = frozenset({"selected_model_comparisons"})
+SELECTED_COMPARISON_TRANSIENT_COLUMNS = frozenset({"claim_token_hash", "claim_expires_at"})
+SELECTED_COMPARISON_ACTIVE_STATUSES = frozenset({"pending", "running", "cancel_requested"})
+SELECTED_COMPARISON_RESTORE_ERROR = "COMPARISON_RESTORE_INTERRUPTED"
+CREATIVE_SCHEMA = "20260909_0029"
+CREATIVE_GOVERNANCE_SCHEMA = "20260909_0031"
+CREATIVE_ALLOCATION_TABLE = "external_creative_allocations"
+CREATIVE_TABLES = frozenset({CREATIVE_ALLOCATION_TABLE, "external_creative_resources",
+ "external_creative_action_attestations", "external_creative_profile_observations"})
+CREATIVE_RESTORE_ERROR = "CREATIVE_RESTORE_INTERRUPTED"
+DEPLOYED_SOURCE_SCHEMAS = frozenset({"20260909_0029", "20260909_0030", "20260909_0031", "20260909_0032", "20260909_0033"})
+MANAGED_ACTIVATION_TABLE = "managed_operator_credential_activations"
# Column names carrying secret material never enter a fingerprint, even as a hash.
SENSITIVE_COLUMN_TOKENS = (
@@ -43,6 +96,15 @@ SENSITIVE_COLUMN_TOKENS = (
)
FINGERPRINT_GROUPS: dict[str, tuple[str, ...]] = {
+ "alert_notifications": tuple(sorted(ALERT_NOTIFICATION_TABLES)),
+ "portfolio": tuple(sorted(PORTFOLIO_TABLES)),
+ "model_changes": tuple(sorted(MODEL_CHANGE_TABLES)),
+ "external_creative": tuple(sorted(CREATIVE_TABLES)),
+ "selected_comparisons": tuple(sorted(SELECTED_COMPARISON_TABLES)),
+ "model_updates": tuple(sorted(MODEL_UPDATE_TABLES)),
+ "platform_secrets": tuple(sorted(SECRET_TABLES)),
+ "managed_identities": tuple(sorted(MANAGED_IDENTITY_TABLES)),
+ "configuration": ("configuration_state", "configuration_revisions"),
"provenance": (
"models",
"model_revisions",
@@ -68,6 +130,7 @@ FINGERPRINT_GROUPS: dict[str, tuple[str, ...]] = {
"discovery_candidate_assessments",
),
"registry": (
+ "project_onboarding",
"capabilities",
"capability_contracts",
"projects",
@@ -173,6 +236,7 @@ FINGERPRINT_GROUPS: dict[str, tuple[str, ...]] = {
"runtime_probes",
"runtime_probe_metrics",
),
+ "lab_execution": tuple(sorted(LAB_EXECUTION_TABLES)),
"audit": ("audit_events", "audit_chain_heads"),
"recovery": (
"recovery_policy_revisions",
@@ -198,6 +262,8 @@ CURRENT_TRUTH_TABLES: frozenset[str] = frozenset(
"residency_allocations",
"serving_gpu_leases",
"gpu_leases",
+ MANAGED_ACTIVATION_TABLE,
+ *SECRET_CURRENT_TABLES,
}
)
@@ -224,9 +290,14 @@ def _fingerprint_table_contract(
physical_tables = fingerprint.get("physical_tables")
if allow_legacy_unexpected_field_absent and "unexpected_tables" not in fingerprint:
unexpected_tables = []
+ current_truth = CURRENT_TRUTH_TABLES
+ if not MANAGED_IDENTITY_TABLES & expected_tables:
+ current_truth = current_truth - {MANAGED_ACTIVATION_TABLE}
+ if not SECRET_TABLES & expected_tables:
+ current_truth = current_truth - SECRET_CURRENT_TABLES
+ expected_physical = expected_tables | current_truth
if allow_legacy_unexpected_field_absent and "physical_tables" not in fingerprint:
- physical_tables = sorted(expected_tables | CURRENT_TRUTH_TABLES)
- expected_physical = expected_tables | CURRENT_TRUTH_TABLES
+ physical_tables = sorted(expected_physical)
if (
not isinstance(tables, dict)
or set(tables) != set(expected_tables)
@@ -234,8 +305,7 @@ def _fingerprint_table_contract(
or not isinstance(physical_tables, list)
or any(not isinstance(name, str) for name in physical_tables)
or physical_tables != sorted(set(physical_tables))
- or set(physical_tables) - NON_FINGERPRINT_INFRASTRUCTURE_TABLES
- != set(expected_physical)
+ or set(physical_tables) - NON_FINGERPRINT_INFRASTRUCTURE_TABLES != set(expected_physical)
or not isinstance(digest, str)
or len(digest) != 64
or any(character not in "0123456789abcdef" for character in digest)
@@ -266,8 +336,6 @@ def _fingerprint_table_contract(
):
return False
else:
- # A capped or content-redacted digest is useful operational evidence, but it is not
- # a complete count+digest proof and can never authorize READY.
return False
return True
@@ -283,8 +351,6 @@ def _physical_table_transition_is_exact(
if not isinstance(restored_physical, list):
return False
if source_physical is None and introduce_checkpoint:
- # Genuine m15.1 fingerprints predate this field. Their exact schema-0022 presence is
- # inferred from the versioned table contract and checked on the migrated target.
return True
if not isinstance(source_physical, list):
return False
@@ -303,23 +369,1495 @@ def _has_single_complete_checkpoint(fingerprint: dict[str, Any]) -> bool:
)
+def _without_configuration(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {
+ **fingerprint,
+ "tables": {
+ key: value
+ for key, value in fingerprint["tables"].items()
+ if key not in CONFIGURATION_TABLES
+ },
+ "physical_tables": [
+ name
+ for name in fingerprint.get("physical_tables", [])
+ if name not in CONFIGURATION_TABLES
+ ],
+ }
+
+
+def _configuration_is_empty(fingerprint: dict[str, Any]) -> bool:
+ empty = {
+ "row_count": 0,
+ "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": [],
+ }
+ return all(fingerprint.get("tables", {}).get(name) == empty for name in CONFIGURATION_TABLES)
+
+
+def _without_projects(fingerprint: dict[str, Any], *, legacy: bool) -> dict[str, Any]:
+ tables = {key: value for key, value in fingerprint["tables"].items() if key not in PROJECT_TABLES}
+ credential = tables.get("service_credentials", {})
+ if legacy and "pre_onboarding" in credential:
+ tables["service_credentials"] = credential["pre_onboarding"]
+ return {**fingerprint, "tables": tables, "physical_tables": [
+ name for name in fingerprint.get("physical_tables", []) if name not in PROJECT_TABLES
+ ]}
+
+
+def _empty_project_upgrade(fingerprint: dict[str, Any]) -> bool:
+ empty = {"row_count": 0, "digest": hashlib.sha256(b"").hexdigest(), "status": "COMPLETE", "redacted_columns": []}
+ credential = fingerprint.get("tables", {}).get("service_credentials", {})
+ legacy = credential.get("pre_onboarding")
+ return (
+ all(fingerprint.get("tables", {}).get(name) == empty for name in PROJECT_TABLES)
+ and credential.get("superseded_count") == 0
+ and isinstance(legacy, dict)
+ and legacy.get("status") == "COMPLETE"
+ and legacy.get("row_count") == credential.get("row_count")
+ and legacy.get("redacted_columns") == credential.get("redacted_columns")
+ )
+
+
+def _without_managed_identities(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {**fingerprint,
+ "tables": {key: value for key, value in fingerprint["tables"].items() if key not in MANAGED_IDENTITY_TABLES},
+ "physical_tables": [name for name in fingerprint.get("physical_tables", [])
+ if name not in MANAGED_IDENTITY_TABLES and name != MANAGED_ACTIVATION_TABLE]}
+
+
+def _empty_managed_identity_upgrade(fingerprint: dict[str, Any]) -> bool:
+ row_hash = hashlib.sha256(_canonical_row((0, 1))).digest()
+ state_digest = hashlib.sha256(len(row_hash).to_bytes(4, "big") + row_hash).hexdigest()
+ expected = {
+ name: {"row_count": 1 if name == "managed_operator_state" else 0,
+ "digest": state_digest if name == "managed_operator_state" else hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE", "redacted_columns": ["token_sha256"] if name == "managed_operator_credentials" else []}
+ for name in MANAGED_IDENTITY_TABLES
+ }
+ return all(fingerprint.get("tables", {}).get(name) == value for name, value in expected.items())
+
+
+def _without_secrets(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {**fingerprint,
+ "tables": {name: entry for name, entry in fingerprint["tables"].items() if name not in SECRET_TABLES},
+ "physical_tables": [name for name in fingerprint.get("physical_tables", [])
+ if name not in SECRET_TABLES | SECRET_CURRENT_TABLES]}
+
+
+def _empty_secret_upgrade(fingerprint: dict[str, Any]) -> bool:
+ row_hash = hashlib.sha256(_canonical_row((False, 0, 1))).digest()
+ state_digest = hashlib.sha256(len(row_hash).to_bytes(4, "big") + row_hash).hexdigest()
+ return all(fingerprint.get("tables", {}).get(name) == {
+ "row_count": 1 if name == "platform_secret_state" else 0,
+ "digest": state_digest if name == "platform_secret_state" else hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": ["secret_ciphertext", "secret_nonce"] if name == "platform_secret_versions" else [],
+ } for name in SECRET_TABLES)
+
+
+def _pre_updates_tables() -> frozenset[str]:
+ return _pre_comparison_tables() - MODEL_UPDATE_TABLES
+
+
+def _pre_comparison_tables() -> frozenset[str]:
+ return _core_tables() - SELECTED_COMPARISON_TABLES
+
+
+def _without_selected_comparisons(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {**fingerprint,
+ "tables": {name: entry for name, entry in fingerprint["tables"].items()
+ if name not in SELECTED_COMPARISON_TABLES},
+ "physical_tables": [name for name in fingerprint.get("physical_tables", [])
+ if name not in SELECTED_COMPARISON_TABLES]}
+
+
+def _empty_selected_comparison_upgrade(fingerprint: dict[str, Any]) -> bool:
+ empty = {"row_count": 0, "digest": hashlib.sha256(b"").hexdigest(), "status": "COMPLETE",
+ "redacted_columns": ["claim_token_hash"]}
+ return bool(fingerprint.get("tables", {}).get("selected_model_comparisons") == {
+ **empty, "restore_projection": empty,
+ })
+
+
+def _selected_comparison_recovery_matches(source: dict[str, Any], restored: dict[str, Any]) -> bool:
+ before = source.get("tables", {}).get("selected_model_comparisons", {})
+ after = restored.get("tables", {}).get("selected_model_comparisons", {})
+ projection = before.get("restore_projection")
+ actual = {name: value for name, value in after.items() if name != "restore_projection"}
+ return (isinstance(projection, dict) and projection == actual
+ and all(projection.get(name) == before.get(name) for name in ("row_count", "status", "redacted_columns"))
+ and after.get("restore_projection") == actual)
+
+
+def _without_model_updates(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {**fingerprint,
+ "tables": {name: entry for name, entry in fingerprint["tables"].items()
+ if name not in MODEL_UPDATE_TABLES},
+ "physical_tables": [name for name in fingerprint.get("physical_tables", [])
+ if name not in MODEL_UPDATE_TABLES]}
+
+
+def _empty_model_update_upgrade(fingerprint: dict[str, Any]) -> bool:
+ return all(fingerprint.get("tables", {}).get(name) == {
+ "row_count": 0, "digest": hashlib.sha256(b"").hexdigest(), "status": "COMPLETE",
+ "redacted_columns": ["check_token"] if name == "model_update_watches" else [],
+ } for name in MODEL_UPDATE_TABLES)
+
+
+def _without_model_changes(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {
+ **fingerprint,
+ "tables": {
+ name: entry
+ for name, entry in fingerprint["tables"].items()
+ if name not in MODEL_CHANGE_TABLES
+ },
+ "physical_tables": [
+ name
+ for name in fingerprint.get("physical_tables", [])
+ if name not in MODEL_CHANGE_TABLES
+ ],
+ }
+
+
+def _without_portfolio(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {
+ **fingerprint,
+ "tables": {
+ name: entry
+ for name, entry in fingerprint["tables"].items()
+ if name not in PORTFOLIO_TABLES
+ },
+ "physical_tables": [
+ name
+ for name in fingerprint.get("physical_tables", [])
+ if name not in PORTFOLIO_TABLES
+ ],
+ }
+
+
+def _portfolio_is_empty(fingerprint: dict[str, Any]) -> bool:
+ empty = {
+ "row_count": 0,
+ "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": [],
+ }
+ return all(fingerprint.get("tables", {}).get(name) == empty for name in PORTFOLIO_TABLES)
+
+
+def _portfolio_has_rows(fingerprint: dict[str, Any]) -> bool:
+ tables = fingerprint.get("tables", {})
+ return any(
+ isinstance(tables.get(name), dict)
+ and isinstance(tables[name].get("row_count"), int)
+ and tables[name].get("row_count", 0) > 0
+ for name in PORTFOLIO_TABLES
+ )
+
+
+def _empty_model_change_upgrade(fingerprint: dict[str, Any]) -> bool:
+ empty = {
+ "row_count": 0,
+ "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": [],
+ }
+ return all(fingerprint.get("tables", {}).get(name) == empty for name in MODEL_CHANGE_TABLES)
+
+
+def _pre_model_change_tables() -> frozenset[str]:
+ return classified_tables() - MODEL_CHANGE_TABLES
+
+
+def _core_tables() -> frozenset[str]:
+ return (
+ _pre_model_change_tables()
+ - CREATIVE_TABLES
+ - PORTFOLIO_TABLES
+ - ALERT_NOTIFICATION_TABLES
+ - LAB_EXECUTION_TABLES
+ )
+
+
+def _without_creative(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ """Projection seam; callers must validate the removed authoritative state first."""
+ projected = {**fingerprint,
+ "tables": {name: entry for name, entry in fingerprint["tables"].items() if name not in CREATIVE_TABLES}}
+ if "physical_tables" in fingerprint:
+ projected["physical_tables"] = [name for name in fingerprint["physical_tables"] if name not in CREATIVE_TABLES]
+ return projected
+
+
+def _deployed_tables(schema: str) -> frozenset[str]:
+ expected = _core_tables()
+ if schema < ALERT_NOTIFICATION_SCHEMA:
+ expected -= ALERT_NOTIFICATION_TABLES
+ if schema < SELECTED_COMPARISON_SCHEMA:
+ expected -= SELECTED_COMPARISON_TABLES
+ if schema < MODEL_UPDATE_SCHEMA:
+ expected -= MODEL_UPDATE_TABLES
+ if schema >= ALERT_NOTIFICATION_SCHEMA:
+ expected |= ALERT_NOTIFICATION_TABLES
+ if schema >= CREATIVE_GOVERNANCE_SCHEMA:
+ return expected | CREATIVE_TABLES
+ return expected | {CREATIVE_ALLOCATION_TABLE}
+
+
+def _watch_controls_for_core(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ entry = fingerprint.get("tables", {}).get("model_update_watches")
+ if not isinstance(entry, dict) or "hub_controls_projection" not in entry:
+ return fingerprint
+ return {**fingerprint, "tables": {**fingerprint["tables"],
+ "model_update_watches": entry["hub_controls_projection"]}}
+
+
+def _valid_watch_projection(fingerprint: dict[str, Any]) -> bool:
+ entry = fingerprint.get("tables", {}).get("model_update_watches")
+ if not isinstance(entry, dict):
+ return False
+ projection = entry.get("hub_controls_projection")
+ if not isinstance(projection, dict):
+ return entry == {"row_count": 0, "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE", "redacted_columns": ["check_token"]}
+ return all(projection.get(key) == entry.get(key) for key in ("row_count", "status", "redacted_columns"))
+
+
+def _creative_recovery_matches(source: dict[str, Any], restored: dict[str, Any]) -> bool:
+ empty = {"row_count": 0, "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE", "redacted_columns": []}
+ for name in CREATIVE_TABLES:
+ before, after = source.get("tables", {}).get(name), restored.get("tables", {}).get(name)
+ if before is None:
+ if after is not None and after != (dict(empty, restore_projection=empty) if name == CREATIVE_ALLOCATION_TABLE else empty):
+ return False
+ elif after is None:
+ return False
+ elif name == CREATIVE_ALLOCATION_TABLE:
+ projection = before.get("restore_projection", before)
+ actual = {key: value for key, value in after.items() if key != "restore_projection"}
+ if (projection != actual or after.get("restore_projection", actual) != actual
+ or any(projection.get(key) != before.get(key) for key in ("row_count", "status", "redacted_columns"))):
+ return False
+ elif before != after:
+ return False
+ return True
+
+
+def _without_alert_notifications(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ return {
+ **fingerprint,
+ "tables": {
+ name: entry
+ for name, entry in fingerprint.get("tables", {}).items()
+ if name not in ALERT_NOTIFICATION_TABLES
+ },
+ "physical_tables": [
+ name
+ for name in fingerprint.get("physical_tables", [])
+ if name not in ALERT_NOTIFICATION_TABLES
+ ],
+ }
+
+
+def _alert_notifications_empty(fingerprint: dict[str, Any]) -> bool:
+ empty = {
+ "row_count": 0,
+ "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": ["claim_token_hash"],
+ "restore_projection": {
+ "row_count": 0,
+ "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": ["claim_token_hash"],
+ },
+ }
+ return all(fingerprint.get("tables", {}).get(name) == empty for name in ALERT_NOTIFICATION_TABLES)
+
+
+def _alert_notifications_have_rows(fingerprint: dict[str, Any]) -> bool:
+ return any(
+ isinstance(fingerprint.get("tables", {}).get(name), dict)
+ and int(fingerprint["tables"][name].get("row_count") or 0) > 0
+ for name in ALERT_NOTIFICATION_TABLES
+ )
+
+
+def _alert_notification_recovery_matches(
+ source: dict[str, Any], restored: dict[str, Any]
+) -> bool:
+ for name in ALERT_NOTIFICATION_TABLES:
+ before = source.get("tables", {}).get(name)
+ after = restored.get("tables", {}).get(name)
+ if not isinstance(before, dict) or not isinstance(after, dict):
+ return False
+ projection = before.get("restore_projection")
+ actual = {key: value for key, value in after.items() if key != "restore_projection"}
+ if (
+ not isinstance(projection, dict)
+ or projection != actual
+ or after.get("restore_projection") != actual
+ or any(
+ projection.get(key) != before.get(key)
+ for key in ("row_count", "status", "redacted_columns")
+ )
+ ):
+ return False
+ return True
+
+
+def _generation_for_core(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ entry = fingerprint.get("tables", {}).get("capability_contracts", {})
+ projection = entry.get("serving_generation_projection")
+ if not isinstance(projection, dict):
+ return fingerprint
+ return {**fingerprint, "tables": {**fingerprint["tables"], "capability_contracts": projection}}
+
+
+def _without_lab_execution(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ """Project post-0037 schema additions only when their state remains legacy-compatible."""
+ tables = {
+ name: entry
+ for name, entry in fingerprint.get("tables", {}).items()
+ if name not in LAB_EXECUTION_TABLES
+ }
+ serving_jobs = tables.get("serving_jobs")
+ if isinstance(serving_jobs, dict):
+ projection = serving_jobs.get("lab_execution_projection")
+ if isinstance(projection, dict) and isinstance(projection.get("base"), dict):
+ tables["serving_jobs"] = projection["base"]
+ if _benchmark_runs_legacy_upgrade_is_safe(fingerprint):
+ tables["benchmark_runs"] = fingerprint["tables"]["benchmark_runs"][
+ BENCHMARK_RUN_SCHEMA_PROJECTION
+ ]["base"]
+ return {
+ **fingerprint,
+ "tables": tables,
+ "physical_tables": [
+ name
+ for name in fingerprint.get("physical_tables", [])
+ if name not in LAB_EXECUTION_TABLES
+ ],
+ }
+
+
+def _valid_table_digest_entry(entry: Any) -> bool:
+ if not isinstance(entry, dict):
+ return False
+ row_count = entry.get("row_count")
+ digest = entry.get("digest")
+ redacted_columns = entry.get("redacted_columns")
+ return (
+ entry.get("status") == "COMPLETE"
+ and isinstance(row_count, int)
+ and not isinstance(row_count, bool)
+ and 0 <= row_count <= MAX_ROWS_PER_TABLE
+ and isinstance(digest, str)
+ and len(digest) == 64
+ and all(character in "0123456789abcdef" for character in digest)
+ and isinstance(redacted_columns, list)
+ and all(isinstance(column, str) for column in redacted_columns)
+ and redacted_columns == sorted(set(redacted_columns))
+ and set(entry) == {"row_count", "digest", "status", "redacted_columns"}
+ )
+
+
+def _valid_lab_execution_projection(fingerprint: dict[str, Any]) -> bool:
+ tables = fingerprint.get("tables")
+ serving_jobs = tables.get("serving_jobs") if isinstance(tables, dict) else None
+ projection = serving_jobs.get("lab_execution_projection") if isinstance(serving_jobs, dict) else None
+ if not isinstance(serving_jobs, dict) or not isinstance(projection, dict):
+ return False
+ base = projection.get("base")
+ run_link_count = projection.get("run_link_count")
+ run_link_digest = projection.get("run_link_digest")
+ return (
+ isinstance(base, dict)
+ and _valid_table_digest_entry(base)
+ and isinstance(run_link_count, int)
+ and not isinstance(run_link_count, bool)
+ and run_link_count >= 0
+ and isinstance(run_link_digest, str)
+ and len(run_link_digest) == 64
+ and all(character in "0123456789abcdef" for character in run_link_digest)
+ and set(projection) == {"base", "run_link_count", "run_link_digest"}
+ and set(serving_jobs) == {*base, "lab_execution_projection"}
+ and serving_jobs.get("status") == base.get("status")
+ and serving_jobs.get("row_count") == base.get("row_count")
+ and serving_jobs.get("redacted_columns") == base.get("redacted_columns")
+ )
+
+
+def _lab_execution_upgrade_is_empty(fingerprint: dict[str, Any]) -> bool:
+ empty = {
+ "row_count": 0,
+ "digest": hashlib.sha256(b"").hexdigest(),
+ "status": "COMPLETE",
+ "redacted_columns": [],
+ }
+ entry = fingerprint.get("tables", {}).get("lab_execution_runs")
+ return (
+ entry == empty
+ and _valid_lab_execution_projection(fingerprint)
+ and fingerprint["tables"]["serving_jobs"]["lab_execution_projection"][
+ "run_link_count"
+ ]
+ == 0
+ and fingerprint["tables"]["serving_jobs"]["lab_execution_projection"][
+ "run_link_digest"
+ ]
+ == hashlib.sha256(b"").hexdigest()
+ )
+
+
+def _lab_execution_state_matches(
+ source: dict[str, Any], restored: dict[str, Any]
+) -> bool:
+ if source.get("tables", {}).get("lab_execution_runs") != restored.get("tables", {}).get(
+ "lab_execution_runs"
+ ):
+ return False
+ if not _valid_lab_execution_projection(source) or not _valid_lab_execution_projection(restored):
+ return False
+ source_projection = source["tables"]["serving_jobs"]["lab_execution_projection"]
+ restored_projection = restored["tables"]["serving_jobs"]["lab_execution_projection"]
+ return all(
+ source_projection[key] == restored_projection[key]
+ for key in ("run_link_count", "run_link_digest")
+ )
+
+
+def _benchmark_runs_projection_present(fingerprint: dict[str, Any]) -> bool:
+ entry = fingerprint.get("tables", {}).get("benchmark_runs")
+ return isinstance(entry, dict) and BENCHMARK_RUN_SCHEMA_PROJECTION in entry
+
+
+def _benchmark_runs_schema_observed(fingerprint: dict[str, Any]) -> bool:
+ entry = fingerprint.get("tables", {}).get("benchmark_runs")
+ return isinstance(entry, dict) and entry.get(BENCHMARK_RUN_SCHEMA_OBSERVED) is True
+
+
+def _valid_benchmark_runs_projection(fingerprint: dict[str, Any]) -> bool:
+ entry = fingerprint.get("tables", {}).get("benchmark_runs")
+ projection = (
+ entry.get(BENCHMARK_RUN_SCHEMA_PROJECTION) if isinstance(entry, dict) else None
+ )
+ if not isinstance(entry, dict) or not isinstance(projection, dict):
+ return False
+ base = projection.get("base")
+ non_legacy_row_count = projection.get("non_legacy_row_count")
+ if not isinstance(base, dict):
+ return False
+ return (
+ set(entry) == {
+ "row_count",
+ "digest",
+ "status",
+ "redacted_columns",
+ BENCHMARK_RUN_SCHEMA_PROJECTION,
+ }
+ and _valid_table_digest_entry(base)
+ and set(projection) == {"base", "non_legacy_row_count"}
+ and isinstance(non_legacy_row_count, int)
+ and not isinstance(non_legacy_row_count, bool)
+ and 0 <= non_legacy_row_count <= base["row_count"]
+ and all(
+ entry.get(key) == base.get(key)
+ for key in ("row_count", "status", "redacted_columns")
+ )
+ and _valid_table_digest_entry({key: entry[key] for key in (
+ "row_count", "digest", "status", "redacted_columns"
+ )})
+ )
+
+
+def _benchmark_runs_legacy_upgrade_is_safe(fingerprint: dict[str, Any]) -> bool:
+ return (
+ _valid_benchmark_runs_projection(fingerprint)
+ and fingerprint["tables"]["benchmark_runs"][BENCHMARK_RUN_SCHEMA_PROJECTION][
+ "non_legacy_row_count"
+ ]
+ == 0
+ )
+
+
+def _without_benchmark_runs_schema(fingerprint: dict[str, Any]) -> dict[str, Any]:
+ """Project migration-0039 columns away after validating their restore state."""
+ tables = dict(fingerprint.get("tables", {}))
+ entry = tables.get("benchmark_runs")
+ projection = (
+ entry.get(BENCHMARK_RUN_SCHEMA_PROJECTION) if isinstance(entry, dict) else None
+ )
+ if isinstance(projection, dict) and isinstance(projection.get("base"), dict):
+ tables["benchmark_runs"] = projection["base"]
+ return {**fingerprint, "tables": tables}
+
+
+def _lab_execution_fingerprint_present(fingerprint: dict[str, Any]) -> bool:
+ tables = fingerprint.get("tables")
+ physical_tables = fingerprint.get("physical_tables")
+ serving_jobs = tables.get("serving_jobs") if isinstance(tables, dict) else None
+ return bool(
+ (isinstance(tables, dict) and LAB_EXECUTION_TABLES & set(tables))
+ or (
+ isinstance(physical_tables, list)
+ and any(isinstance(name, str) and name in LAB_EXECUTION_TABLES for name in physical_tables)
+ )
+ or (
+ isinstance(serving_jobs, dict)
+ and (
+ "lab_execution_projection" in serving_jobs
+ or serving_jobs.get("lab_execution_schema_observed") is True
+ )
+ )
+ )
+
+
def fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ schema = source_schema_revision
+ source_names = set(source.get("tables", {}))
+ restored_names = set(restored.get("tables", {}))
+ if _benchmark_runs_schema_observed(source) or _benchmark_runs_schema_observed(restored):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "benchmark_runs physical columns do not match a complete declared schema",
+ )
+ source_has_benchmark_projection = _benchmark_runs_projection_present(source)
+ restored_has_benchmark_projection = _benchmark_runs_projection_present(restored)
+ if schema == BENCHMARK_RUN_SCHEMA:
+ if not (
+ _fingerprint_table_contract(source, expected_tables=classified_tables())
+ and _fingerprint_table_contract(restored, expected_tables=classified_tables())
+ and _valid_benchmark_runs_projection(source)
+ and _valid_benchmark_runs_projection(restored)
+ and source["tables"]["benchmark_runs"] == restored["tables"]["benchmark_runs"]
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0039 benchmark operator history requires exact retention",
+ )
+ base = fingerprint_compatibility(
+ _without_benchmark_runs_schema(source),
+ _without_benchmark_runs_schema(restored),
+ source_schema_revision=LAB_EXECUTION_SCHEMA,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"BENCHMARK_RUNS_STRICT|{base.mode}" if base.compatible else "INCOMPATIBLE",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if source_has_benchmark_projection:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "pre-0039 source contains future benchmark operator evidence",
+ )
+ if restored_has_benchmark_projection:
+ source_benchmark_runs = source.get("tables", {}).get("benchmark_runs")
+ if not isinstance(schema, str) or schema >= BENCHMARK_RUN_SCHEMA:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "benchmark schema projection requires a known pre-0039 source revision",
+ )
+ if not (
+ _fingerprint_table_contract(restored, expected_tables=classified_tables())
+ and _valid_table_digest_entry(source_benchmark_runs)
+ and _benchmark_runs_legacy_upgrade_is_safe(restored)
+ and source_benchmark_runs
+ == restored["tables"]["benchmark_runs"][BENCHMARK_RUN_SCHEMA_PROJECTION]["base"]
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0039 upgrade may add only default benchmark metadata while preserving legacy rows",
+ )
+ base = fingerprint_compatibility(
+ source,
+ _without_benchmark_runs_schema(restored),
+ source_schema_revision=schema,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"BENCHMARK_RUNS_UPGRADE|{schema}|{base.mode}" if base.compatible else "INCOMPATIBLE",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if schema == LAB_EXECUTION_SCHEMA:
+ valid_contract = (
+ _fingerprint_table_contract(source, expected_tables=classified_tables())
+ and _fingerprint_table_contract(restored, expected_tables=classified_tables())
+ )
+ if not valid_contract or not _lab_execution_state_matches(source, restored):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0038 LAB run history and serving-job links require exact retention",
+ )
+ base = fingerprint_compatibility(
+ _without_lab_execution(source),
+ _without_lab_execution(restored),
+ source_schema_revision=ALERT_NOTIFICATION_SCHEMA,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"LAB_EXECUTION_STRICT|{base.mode}" if base.compatible else "INCOMPATIBLE",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if isinstance(schema, str) and schema < LAB_EXECUTION_SCHEMA:
+ source_has_lab = _lab_execution_fingerprint_present(source)
+ restored_has_lab = _lab_execution_fingerprint_present(restored)
+ if source_has_lab:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "pre-0038 source contains future LAB schema or execution evidence",
+ )
+ if restored_has_lab:
+ if not (
+ _fingerprint_table_contract(restored, expected_tables=classified_tables())
+ and _lab_execution_upgrade_is_empty(restored)
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0038 upgrade may introduce only an empty LAB table and unlinked serving jobs",
+ )
+ base = fingerprint_compatibility(
+ source,
+ _without_lab_execution(restored),
+ source_schema_revision=schema,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"LAB_EXECUTION_UPGRADE|{schema}|{base.mode}" if base.compatible else "INCOMPATIBLE",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if schema == ALERT_NOTIFICATION_SCHEMA:
+ if not (
+ _fingerprint_table_contract(source, expected_tables=_pre_lab_execution_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_pre_lab_execution_tables())
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "current alert notification history requires the complete schema-0037 table contract",
+ )
+ base = fingerprint_compatibility(
+ _without_alert_notifications(source),
+ _without_alert_notifications(restored),
+ source_schema_revision=PORTFOLIO_SCHEMA,
+ )
+ notification_match = _alert_notification_recovery_matches(source, restored)
+ return FingerprintCompatibility(
+ base.compatible and notification_match,
+ f"ALERT_NOTIFICATION_STRICT|{base.mode}" if base.compatible and notification_match else "INCOMPATIBLE",
+ base.allow_audit_checkpoint_introduction,
+ base.reason if notification_match else "alert deliveries differ beyond the exact pending-to-held restore projection",
+ )
+ if ALERT_NOTIFICATION_TABLES & restored_names:
+ if not isinstance(schema, str) or schema >= ALERT_NOTIFICATION_SCHEMA:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "alert notification history predates its declared migration",
+ )
+ if not _alert_notifications_empty(restored):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0037 upgrade may introduce only empty alert notification history",
+ )
+ if _alert_notifications_have_rows(source):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "alert notification history predates its declared migration",
+ )
+ base = fingerprint_compatibility(
+ _without_alert_notifications(source),
+ _without_alert_notifications(restored),
+ source_schema_revision=schema,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"ALERT_NOTIFICATION_UPGRADE|{base.mode}",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if ALERT_NOTIFICATION_TABLES & source_names:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "alert notification history cannot exist before schema 0037",
+ )
+ if schema == PORTFOLIO_SCHEMA:
+ if not (
+ _fingerprint_table_contract(
+ source, expected_tables=_pre_lab_execution_tables() - ALERT_NOTIFICATION_TABLES
+ )
+ and _fingerprint_table_contract(
+ restored, expected_tables=_pre_lab_execution_tables() - ALERT_NOTIFICATION_TABLES
+ )
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "current portfolio history requires the complete schema-0036 table contract",
+ )
+ base = fingerprint_compatibility(
+ _without_portfolio(source),
+ _without_portfolio(restored),
+ source_schema_revision=MODEL_CHANGE_SCHEMA,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"PORTFOLIO_STRICT|{base.mode}",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if PORTFOLIO_TABLES & restored_names:
+ if _portfolio_has_rows(source):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "portfolio history predates its declared migration",
+ )
+ if (
+ not _portfolio_is_empty(restored)
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0036 upgrade may introduce only empty portfolio history",
+ )
+ base = fingerprint_compatibility(
+ _without_portfolio(source),
+ _without_portfolio(restored),
+ source_schema_revision=schema,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"PORTFOLIO_UPGRADE|{base.mode}",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if _portfolio_has_rows(source):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "portfolio history cannot exist before schema 0036",
+ )
+ if schema == MODEL_CHANGE_SCHEMA:
+ if not (
+ _fingerprint_table_contract(
+ source,
+ expected_tables=_pre_lab_execution_tables() - PORTFOLIO_TABLES - ALERT_NOTIFICATION_TABLES,
+ )
+ and _fingerprint_table_contract(
+ restored,
+ expected_tables=_pre_lab_execution_tables() - PORTFOLIO_TABLES - ALERT_NOTIFICATION_TABLES,
+ )
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "current model change history requires the complete schema-0035 table contract",
+ )
+ base = fingerprint_compatibility(
+ _without_model_changes(source),
+ _without_model_changes(restored),
+ source_schema_revision=SERVING_GENERATION_SCHEMA,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"MODEL_CHANGE_STRICT|{base.mode}",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if MODEL_CHANGE_TABLES & restored_names:
+ if MODEL_CHANGE_TABLES & source_names:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "model change history predates its declared migration",
+ )
+ if (
+ not _fingerprint_table_contract(
+ restored,
+ expected_tables=_pre_lab_execution_tables() - PORTFOLIO_TABLES - ALERT_NOTIFICATION_TABLES,
+ )
+ or not _empty_model_change_upgrade(restored)
+ ):
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "schema-0035 upgrade may introduce only empty model change history",
+ )
+ base = fingerprint_compatibility(
+ source,
+ _without_model_changes(restored),
+ source_schema_revision=schema,
+ )
+ return FingerprintCompatibility(
+ base.compatible,
+ f"MODEL_CHANGE_UPGRADE|{base.mode}",
+ base.allow_audit_checkpoint_introduction,
+ base.reason,
+ )
+ if MODEL_CHANGE_TABLES & source_names:
+ return FingerprintCompatibility(
+ False,
+ "INCOMPATIBLE",
+ False,
+ "model change history cannot exist before schema 0035",
+ )
+
+ before = source.get("tables", {}).get("capability_contracts", {})
+ after = restored.get("tables", {}).get("capability_contracts", {})
+ projection = before.get("serving_generation_projection")
+ current = source_schema_revision == SERVING_GENERATION_SCHEMA
+ if current and (projection is not None or "serving_generation_projection" in after):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "current serving generation cannot be projected away")
+ if projection is not None and (
+ not isinstance(projection, dict) or "serving_generation_projection" in after
+ or not all(projection.get(key) == before.get(key)
+ for key in ("row_count", "status", "redacted_columns"))
+ or projection.get("status") != "COMPLETE"
+ ):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "serving generation requires an exact source default-zero projection")
+ mapped_schema = SELECTED_COMPARISON_SCHEMA if current else source_schema_revision
+ contract = _deployed_fingerprint_compatibility(
+ _generation_for_core(source), restored, source_schema_revision=mapped_schema)
+ if current or projection is not None:
+ return FingerprintCompatibility(contract.compatible,
+ f"SERVING|{source_schema_revision or 'none'}|{contract.mode}",
+ contract.allow_audit_checkpoint_introduction, contract.reason)
+ return contract
+
+
+def audit_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode.startswith("BENCHMARK_RUNS_STRICT|"):
+ base_mode = contract.mode.removeprefix("BENCHMARK_RUNS_STRICT|")
+ verified = fingerprint_compatibility(
+ source, restored, source_schema_revision=BENCHMARK_RUN_SCHEMA
+ )
+ return (
+ verified.compatible
+ and verified.mode == contract.mode
+ and source.get("tables", {}).get("benchmark_runs")
+ == restored.get("tables", {}).get("benchmark_runs")
+ and audit_fingerprint_differences_are_compatible(
+ _without_benchmark_runs_schema(source),
+ _without_benchmark_runs_schema(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("BENCHMARK_RUNS_UPGRADE|"):
+ _, schema, base_mode = contract.mode.split("|", 2)
+ verified = fingerprint_compatibility(source, restored, source_schema_revision=schema)
+ source_entry = source.get("tables", {}).get("benchmark_runs")
+ projection = (
+ restored.get("tables", {})
+ .get("benchmark_runs", {})
+ .get(BENCHMARK_RUN_SCHEMA_PROJECTION)
+ )
+ return (
+ verified.compatible
+ and verified.mode == contract.mode
+ and not _benchmark_runs_projection_present(source)
+ and _benchmark_runs_legacy_upgrade_is_safe(restored)
+ and isinstance(projection, dict)
+ and source_entry == projection.get("base")
+ and audit_fingerprint_differences_are_compatible(
+ source,
+ _without_benchmark_runs_schema(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("LAB_EXECUTION_STRICT|"):
+ base_mode = contract.mode.removeprefix("LAB_EXECUTION_STRICT|")
+ verified = fingerprint_compatibility(
+ source, restored, source_schema_revision=LAB_EXECUTION_SCHEMA
+ )
+ return (
+ verified.compatible
+ and verified.mode == contract.mode
+ and _lab_execution_state_matches(source, restored)
+ and audit_fingerprint_differences_are_compatible(
+ _without_lab_execution(source),
+ _without_lab_execution(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("LAB_EXECUTION_UPGRADE|"):
+ _, schema, base_mode = contract.mode.split("|", 2)
+ verified = fingerprint_compatibility(
+ source, restored, source_schema_revision=schema
+ )
+ return (
+ verified.compatible
+ and verified.mode == contract.mode
+ and _lab_execution_upgrade_is_empty(restored)
+ and not _lab_execution_fingerprint_present(source)
+ and audit_fingerprint_differences_are_compatible(
+ _without_lab_execution(source),
+ _without_lab_execution(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("PORTFOLIO_STRICT|"):
+ base_mode = contract.mode.removeprefix("PORTFOLIO_STRICT|")
+ return (
+ _fingerprint_table_contract(
+ source, expected_tables=_pre_lab_execution_tables() - ALERT_NOTIFICATION_TABLES
+ )
+ and _fingerprint_table_contract(
+ restored, expected_tables=_pre_lab_execution_tables() - ALERT_NOTIFICATION_TABLES
+ )
+ and all(
+ source["tables"][name] == restored["tables"][name]
+ for name in PORTFOLIO_TABLES
+ )
+ and audit_fingerprint_differences_are_compatible(
+ _without_portfolio(source),
+ _without_portfolio(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("PORTFOLIO_UPGRADE|"):
+ base_mode = contract.mode.removeprefix("PORTFOLIO_UPGRADE|")
+ return (
+ _portfolio_is_empty(restored)
+ and audit_fingerprint_differences_are_compatible(
+ _without_portfolio(source),
+ _without_portfolio(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("ALERT_NOTIFICATION_STRICT|"):
+ base_mode = contract.mode.removeprefix("ALERT_NOTIFICATION_STRICT|")
+ verified = fingerprint_compatibility(
+ source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ return (
+ verified.compatible
+ and verified.mode == contract.mode
+ and _fingerprint_table_contract(source, expected_tables=_pre_lab_execution_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_pre_lab_execution_tables())
+ and _alert_notification_recovery_matches(source, restored)
+ and audit_fingerprint_differences_are_compatible(
+ _without_alert_notifications(source),
+ _without_alert_notifications(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|"):
+ base_mode = contract.mode.removeprefix("ALERT_NOTIFICATION_UPGRADE|")
+ return (
+ _alert_notifications_empty(restored)
+ and not _alert_notifications_have_rows(source)
+ and audit_fingerprint_differences_are_compatible(
+ _without_alert_notifications(source),
+ _without_alert_notifications(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("MODEL_CHANGE_STRICT|"):
+ base_mode = contract.mode.removeprefix("MODEL_CHANGE_STRICT|")
+ return (
+ _fingerprint_table_contract(
+ source,
+ expected_tables=_pre_lab_execution_tables() - PORTFOLIO_TABLES - ALERT_NOTIFICATION_TABLES,
+ )
+ and _fingerprint_table_contract(
+ restored,
+ expected_tables=_pre_lab_execution_tables() - PORTFOLIO_TABLES - ALERT_NOTIFICATION_TABLES,
+ )
+ and all(
+ source["tables"][name] == restored["tables"][name]
+ for name in MODEL_CHANGE_TABLES
+ )
+ and audit_fingerprint_differences_are_compatible(
+ _without_model_changes(source),
+ _without_model_changes(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if contract.mode.startswith("MODEL_CHANGE_UPGRADE|"):
+ base_mode = contract.mode.removeprefix("MODEL_CHANGE_UPGRADE|")
+ return (
+ _fingerprint_table_contract(
+ restored,
+ expected_tables=_pre_lab_execution_tables() - PORTFOLIO_TABLES - ALERT_NOTIFICATION_TABLES,
+ )
+ and _empty_model_change_upgrade(restored)
+ and audit_fingerprint_differences_are_compatible(
+ source,
+ _without_model_changes(restored),
+ FingerprintCompatibility(
+ True,
+ base_mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ )
+ if not contract.mode.startswith("SERVING|"):
+ if "serving_generation_projection" in source.get("tables", {}).get("capability_contracts", {}):
+ return False
+ return _deployed_fingerprint_differences_are_compatible(source, restored, contract)
+ _, schema, core_mode = contract.mode.split("|", 2)
+ verified = fingerprint_compatibility(source, restored, source_schema_revision=None if schema == "none" else schema)
+ return (verified.compatible and verified.mode == contract.mode
+ and _deployed_fingerprint_differences_are_compatible(_generation_for_core(source), restored,
+ FingerprintCompatibility(True, core_mode, contract.allow_audit_checkpoint_introduction, contract.reason)))
+
+
+def _deployed_fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ schema = source_schema_revision
+ source_names = set(source.get("tables", {}))
+ restored_names = set(restored.get("tables", {}))
+ if CREATIVE_TABLES & set(restored.get("physical_tables", [])) != CREATIVE_TABLES & restored_names:
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "creative physical state lacks its exact authoritative table proof")
+ if isinstance(schema, str) and schema >= CREATIVE_SCHEMA:
+ if schema not in DEPLOYED_SOURCE_SCHEMAS or not _fingerprint_table_contract(source, expected_tables=_deployed_tables(schema)):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "deployed source table contract is incomplete")
+ if MODEL_UPDATE_SCHEMA <= schema < SELECTED_COMPARISON_SCHEMA and not _valid_watch_projection(source):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False,
+ "legacy watch history requires a freshly qualified hub_controls_projection; opaque old hashes cannot be upgraded")
+ if schema == SELECTED_COMPARISON_SCHEMA and "hub_controls_projection" in source.get("tables", {}).get("model_update_watches", {}):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "current watch controls cannot be projected away")
+ elif CREATIVE_TABLES & (source_names | set(source.get("physical_tables", []))):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "creative tables predate their declared migration")
+ if CREATIVE_TABLES & restored_names:
+ restored_schema = (SELECTED_COMPARISON_SCHEMA if SELECTED_COMPARISON_TABLES & restored_names
+ else CREATIVE_GOVERNANCE_SCHEMA if (CREATIVE_TABLES - {CREATIVE_ALLOCATION_TABLE}) & restored_names
+ else MODEL_UPDATE_SCHEMA if MODEL_UPDATE_TABLES & restored_names else CREATIVE_SCHEMA)
+ if not _fingerprint_table_contract(restored, expected_tables=_deployed_tables(restored_schema)):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "deployed restore table contract is incomplete")
+ if restored_schema == SELECTED_COMPARISON_SCHEMA and "hub_controls_projection" in restored.get("tables", {}).get("model_update_watches", {}):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "current restored watch controls cannot be projected away")
+ if any(name not in restored_names for name in CREATIVE_TABLES & source_names):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "creative history disappeared")
+ introduced = {name: restored["tables"][name] for name in (CREATIVE_TABLES & restored_names) - source_names}
+ if not _creative_recovery_matches({"tables": {}}, {"tables": introduced}):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "new creative tables must be exactly empty")
+ core_schema = (SECRET_SCHEMA if schema == CREATIVE_SCHEMA
+ else MODEL_UPDATE_SCHEMA if isinstance(schema, str) and MODEL_UPDATE_SCHEMA <= schema < SELECTED_COMPARISON_SCHEMA else schema)
+ core = _core_fingerprint_compatibility(_watch_controls_for_core(_without_creative(source)),
+ _watch_controls_for_core(_without_creative(restored)), source_schema_revision=core_schema)
+ if not CREATIVE_TABLES & (source_names | restored_names):
+ return core
+ return FingerprintCompatibility(core.compatible, f"DEPLOYED|{schema or 'none'}|{core.mode}",
+ core.allow_audit_checkpoint_introduction, core.reason)
+
+
+def _deployed_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if not contract.mode.startswith("DEPLOYED|"):
+ if CREATIVE_TABLES & (set(source.get("tables", {})) | set(restored.get("tables", {}))
+ | set(source.get("physical_tables", [])) | set(restored.get("physical_tables", []))):
+ return False
+ return _core_fingerprint_differences_are_compatible(source, restored, contract)
+ _, schema, core_mode = contract.mode.split("|", 2)
+ verified = _deployed_fingerprint_compatibility(source, restored, source_schema_revision=None if schema == "none" else schema)
+ return (verified.compatible and verified.mode == contract.mode
+ and _creative_recovery_matches(source, restored)
+ and _core_fingerprint_differences_are_compatible(
+ _watch_controls_for_core(_without_creative(source)), _watch_controls_for_core(_without_creative(restored)),
+ FingerprintCompatibility(True, core_mode, contract.allow_audit_checkpoint_introduction, contract.reason)))
+
+
+def _core_fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ if source_schema_revision == SELECTED_COMPARISON_SCHEMA:
+ valid = (_fingerprint_table_contract(source, expected_tables=_core_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_core_tables()))
+ return FingerprintCompatibility(valid, "SELECTED_COMPARISON_STRICT" if valid else "INCOMPATIBLE",
+ False, "comparison history requires the exact interrupted restore projection")
+ if SELECTED_COMPARISON_TABLES & set(restored.get("tables", {})):
+ if (not _fingerprint_table_contract(restored, expected_tables=_core_tables())
+ or not _empty_selected_comparison_upgrade(restored)):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False,
+ "unexpected selected comparison history during upgrade")
+ prior = _model_update_fingerprint_compatibility(source, _without_selected_comparisons(restored),
+ source_schema_revision=source_schema_revision)
+ return FingerprintCompatibility(prior.compatible, "SELECTED_COMPARISON_UPGRADE_" + prior.mode,
+ prior.allow_audit_checkpoint_introduction, prior.reason)
+ return _model_update_fingerprint_compatibility(source, restored, source_schema_revision=source_schema_revision)
+
+
+def _core_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode == "SELECTED_COMPARISON_STRICT":
+ return (not contract.allow_audit_checkpoint_introduction
+ and _fingerprint_table_contract(source, expected_tables=_core_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_core_tables())
+ and _selected_comparison_recovery_matches(source, restored)
+ and _model_update_fingerprint_differences_are_compatible(
+ _without_selected_comparisons(source), _without_selected_comparisons(restored),
+ FingerprintCompatibility(True, "MODEL_UPDATE_STRICT", False, contract.reason)))
+ if contract.mode.startswith("SELECTED_COMPARISON_UPGRADE_"):
+ return (_fingerprint_table_contract(restored, expected_tables=_core_tables())
+ and _empty_selected_comparison_upgrade(restored)
+ and _model_update_fingerprint_differences_are_compatible(
+ source, _without_selected_comparisons(restored),
+ FingerprintCompatibility(True, contract.mode.removeprefix("SELECTED_COMPARISON_UPGRADE_"),
+ contract.allow_audit_checkpoint_introduction, contract.reason)))
+ return _model_update_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def _model_update_fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ if source_schema_revision == MODEL_UPDATE_SCHEMA:
+ valid = (_fingerprint_table_contract(source, expected_tables=_pre_comparison_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_pre_comparison_tables()))
+ return FingerprintCompatibility(valid, "MODEL_UPDATE_STRICT" if valid else "INCOMPATIBLE",
+ False, "selected-model watch and notice history must be retained exactly")
+ if MODEL_UPDATE_TABLES & set(restored.get("tables", {})):
+ if (not _fingerprint_table_contract(restored, expected_tables=_pre_comparison_tables())
+ or not _empty_model_update_upgrade(restored)):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False,
+ "unexpected model update state during upgrade")
+ prior = _secret_fingerprint_compatibility(source, _without_model_updates(restored),
+ source_schema_revision=source_schema_revision)
+ return FingerprintCompatibility(prior.compatible, "MODEL_UPDATE_UPGRADE_" + prior.mode,
+ prior.allow_audit_checkpoint_introduction, prior.reason)
+ return _secret_fingerprint_compatibility(source, restored, source_schema_revision=source_schema_revision)
+
+
+def _model_update_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode == "MODEL_UPDATE_STRICT":
+ return (not contract.allow_audit_checkpoint_introduction
+ and _fingerprint_table_contract(source, expected_tables=_pre_comparison_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_pre_comparison_tables())
+ and all(source["tables"][name] == restored["tables"][name] for name in MODEL_UPDATE_TABLES)
+ and _secret_fingerprint_differences_are_compatible(
+ _without_model_updates(source), _without_model_updates(restored),
+ FingerprintCompatibility(True, "SECRET_STRICT", False, contract.reason)))
+ if contract.mode.startswith("MODEL_UPDATE_UPGRADE_"):
+ return (_fingerprint_table_contract(restored, expected_tables=_pre_comparison_tables())
+ and _empty_model_update_upgrade(restored)
+ and _secret_fingerprint_differences_are_compatible(
+ source, _without_model_updates(restored),
+ FingerprintCompatibility(True, contract.mode.removeprefix("MODEL_UPDATE_UPGRADE_"),
+ contract.allow_audit_checkpoint_introduction, contract.reason)))
+ return _secret_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def _secret_fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ if source_schema_revision == SECRET_SCHEMA:
+ valid = (_fingerprint_table_contract(source, expected_tables=_pre_updates_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_pre_updates_tables()))
+ return FingerprintCompatibility(valid, "SECRET_STRICT" if valid else "INCOMPATIBLE", False,
+ "secret version metadata must be retained exactly")
+ if SECRET_TABLES & set(restored.get("tables", {})):
+ if (not _fingerprint_table_contract(restored, expected_tables=_pre_updates_tables())
+ or not _empty_secret_upgrade(restored)):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "unexpected secret state during upgrade")
+ old = _managed_fingerprint_compatibility(source, _without_secrets(restored),
+ source_schema_revision=source_schema_revision)
+ return FingerprintCompatibility(old.compatible, "SECRET_UPGRADE_" + old.mode,
+ old.allow_audit_checkpoint_introduction, old.reason)
+ return _managed_fingerprint_compatibility(source, restored, source_schema_revision=source_schema_revision)
+
+
+def _secret_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode == "SECRET_STRICT":
+ return (not contract.allow_audit_checkpoint_introduction
+ and _fingerprint_table_contract(source, expected_tables=_pre_updates_tables())
+ and _fingerprint_table_contract(restored, expected_tables=_pre_updates_tables())
+ and all(source["tables"][name] == restored["tables"][name] for name in SECRET_TABLES)
+ and _managed_fingerprint_differences_are_compatible(
+ _without_secrets(source), _without_secrets(restored),
+ FingerprintCompatibility(True, "MANAGED_STRICT", False, contract.reason)))
+ if contract.mode.startswith("SECRET_UPGRADE_"):
+ return (_fingerprint_table_contract(restored, expected_tables=_pre_updates_tables())
+ and _empty_secret_upgrade(restored)
+ and _managed_fingerprint_differences_are_compatible(
+ source, _without_secrets(restored),
+ FingerprintCompatibility(True, contract.mode.removeprefix("SECRET_UPGRADE_"),
+ contract.allow_audit_checkpoint_introduction, contract.reason)))
+ return _managed_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def _managed_fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ if source_schema_revision == MANAGED_IDENTITY_SCHEMA:
+ valid = (_fingerprint_table_contract(source, expected_tables=(_pre_updates_tables() - SECRET_TABLES))
+ and _fingerprint_table_contract(restored, expected_tables=(_pre_updates_tables() - SECRET_TABLES)))
+ return FingerprintCompatibility(valid, "MANAGED_STRICT" if valid else "INCOMPATIBLE", False,
+ "managed identity history must be retained exactly")
+ if MANAGED_IDENTITY_TABLES & set(restored.get("tables", {})):
+ if (not _fingerprint_table_contract(restored, expected_tables=(_pre_updates_tables() - SECRET_TABLES))
+ or not _empty_managed_identity_upgrade(restored)):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "unexpected managed identity state during upgrade")
+ old = _project_fingerprint_compatibility(source, _without_managed_identities(restored),
+ source_schema_revision=source_schema_revision)
+ return FingerprintCompatibility(old.compatible, "MANAGED_UPGRADE_" + old.mode,
+ old.allow_audit_checkpoint_introduction, old.reason)
+ return _project_fingerprint_compatibility(source, restored, source_schema_revision=source_schema_revision)
+
+
+def _managed_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode == "MANAGED_STRICT":
+ return (not contract.allow_audit_checkpoint_introduction
+ and _fingerprint_table_contract(source, expected_tables=(_pre_updates_tables() - SECRET_TABLES))
+ and _fingerprint_table_contract(restored, expected_tables=(_pre_updates_tables() - SECRET_TABLES))
+ and all(source["tables"][name] == restored["tables"][name]
+ for name in MANAGED_IDENTITY_TABLES)
+ and _project_fingerprint_differences_are_compatible(
+ _without_managed_identities(source), _without_managed_identities(restored),
+ FingerprintCompatibility(True, "PROJECT_STRICT", False, contract.reason)))
+ if contract.mode.startswith("MANAGED_UPGRADE_"):
+ return (_fingerprint_table_contract(restored, expected_tables=(_pre_updates_tables() - SECRET_TABLES))
+ and _empty_managed_identity_upgrade(restored)
+ and _project_fingerprint_differences_are_compatible(
+ source, _without_managed_identities(restored),
+ FingerprintCompatibility(True, contract.mode.removeprefix("MANAGED_UPGRADE_"),
+ contract.allow_audit_checkpoint_introduction, contract.reason)))
+ return _project_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def _project_fingerprint_compatibility(
+ source: dict[str, Any], restored: dict[str, Any], *, source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+ if source_schema_revision == PROJECT_SCHEMA:
+ valid = _fingerprint_table_contract(source, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES)) and _fingerprint_table_contract(restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES))
+ return FingerprintCompatibility(valid, "PROJECT_STRICT" if valid else "INCOMPATIBLE", False, "schema 0026 retains onboarding and credential history exactly")
+ if PROJECT_TABLES & set(restored.get("tables", {})):
+ if not _fingerprint_table_contract(restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES)) or not _empty_project_upgrade(restored):
+ return FingerprintCompatibility(False, "INCOMPATIBLE", False, "unexpected project or credential state during upgrade")
+ old = _configuration_fingerprint_compatibility(source, _without_projects(restored, legacy=True), source_schema_revision=source_schema_revision)
+ return FingerprintCompatibility(old.compatible, "PROJECT_UPGRADE_" + old.mode, old.allow_audit_checkpoint_introduction, old.reason)
+ return _configuration_fingerprint_compatibility(source, restored, source_schema_revision=source_schema_revision)
+
+
+def _project_fingerprint_differences_are_compatible(
+ source: dict[str, Any], restored: dict[str, Any], contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode == "PROJECT_STRICT":
+ return (not contract.allow_audit_checkpoint_introduction
+ and _fingerprint_table_contract(source, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES))
+ and _fingerprint_table_contract(restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES))
+ and all(source["tables"][name] == restored["tables"][name]
+ for name in PROJECT_TABLES | {"service_credentials"})
+ and _configuration_fingerprint_differences_are_compatible(
+ _without_projects(source, legacy=False), _without_projects(restored, legacy=False),
+ FingerprintCompatibility(True, "CONFIGURATION_STRICT", False, contract.reason)))
+ if contract.mode.startswith("PROJECT_UPGRADE_"):
+ return (_fingerprint_table_contract(restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES))
+ and _empty_project_upgrade(restored)
+ and _configuration_fingerprint_differences_are_compatible(
+ source, _without_projects(restored, legacy=True),
+ FingerprintCompatibility(True, contract.mode.removeprefix("PROJECT_UPGRADE_"), contract.allow_audit_checkpoint_introduction, contract.reason)))
+ return _configuration_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def _configuration_fingerprint_compatibility(
source: dict[str, Any],
restored: dict[str, Any],
*,
source_schema_revision: str | None,
) -> FingerprintCompatibility:
- """Return the only supported semantic-fingerprint comparison modes.
+ if source_schema_revision == CONFIGURATION_SCHEMA:
+ if _fingerprint_table_contract(
+ source, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - PROJECT_TABLES
+ ) and _fingerprint_table_contract(restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - PROJECT_TABLES):
+ old = _audit_fingerprint_compatibility(
+ _without_configuration(source),
+ _without_configuration(restored),
+ source_schema_revision=AUDIT_CHECKPOINT_CURRENT_SCHEMA,
+ )
+ if old.compatible:
+ return FingerprintCompatibility(
+ True,
+ "CONFIGURATION_STRICT",
+ False,
+ "schema 0025 retains configuration history exactly",
+ )
+ return FingerprintCompatibility(
+ False, "INCOMPATIBLE", False, "schema 0025 requires complete configuration tables"
+ )
+ if CONFIGURATION_TABLES & set(restored.get("tables", {})):
+ if _fingerprint_table_contract(
+ restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - PROJECT_TABLES
+ ) and _configuration_is_empty(restored):
+ old = _audit_fingerprint_compatibility(
+ source,
+ _without_configuration(restored),
+ source_schema_revision=source_schema_revision,
+ )
+ if old.compatible:
+ return FingerprintCompatibility(
+ True,
+ "CONFIGURATION_UPGRADE_" + old.mode,
+ old.allow_audit_checkpoint_introduction,
+ "migration introduces exactly two empty configuration tables",
+ )
+ return FingerprintCompatibility(
+ False, "INCOMPATIBLE", False, "unexpected configuration state during upgrade"
+ )
+ return _audit_fingerprint_compatibility(
+ source, restored, source_schema_revision=source_schema_revision
+ )
- ``m15.2`` is strict and requires the singleton checkpoint on both sides. The one cross-version
- exception is a genuine schema-0022 ``m15.1`` backup: 0022 had no checkpoint table, while the
- 0024 migration must introduce exactly one verified head. No other missing/new table or version
- pairing is interpreted as a migration side effect.
- """
+def _configuration_fingerprint_differences_are_compatible(
+ source: dict[str, Any],
+ restored: dict[str, Any],
+ contract: FingerprintCompatibility,
+) -> bool:
+ if not contract.compatible:
+ return False
+ if contract.mode == "CONFIGURATION_STRICT":
+ if (
+ contract.allow_audit_checkpoint_introduction
+ or not _fingerprint_table_contract(source, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - PROJECT_TABLES)
+ or not _fingerprint_table_contract(restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - PROJECT_TABLES)
+ or any(
+ source["tables"][name] != restored["tables"][name] for name in CONFIGURATION_TABLES
+ )
+ ):
+ return False
+ return _audit_fingerprint_differences_are_compatible(
+ _without_configuration(source),
+ _without_configuration(restored),
+ FingerprintCompatibility(True, "CURRENT_STRICT", False, contract.reason),
+ )
+ if contract.mode.startswith("CONFIGURATION_UPGRADE_"):
+ if not _fingerprint_table_contract(
+ restored, expected_tables=((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - PROJECT_TABLES
+ ) or not _configuration_is_empty(restored):
+ return False
+ return _audit_fingerprint_differences_are_compatible(
+ source,
+ _without_configuration(restored),
+ FingerprintCompatibility(
+ True,
+ contract.mode.removeprefix("CONFIGURATION_UPGRADE_"),
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ ),
+ )
+ return _audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def _audit_fingerprint_compatibility(
+ source: dict[str, Any],
+ restored: dict[str, Any],
+ *,
+ source_schema_revision: str | None,
+) -> FingerprintCompatibility:
source_version = source.get("version")
restored_version = restored.get("version")
- current_tables = classified_tables()
+ current_tables = ((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - CONFIGURATION_TABLES - PROJECT_TABLES
legacy_tables = current_tables - {"audit_chain_heads"}
if (
source_schema_revision == AUDIT_CHECKPOINT_CURRENT_SCHEMA
@@ -329,9 +1867,7 @@ def fingerprint_compatibility(
and _fingerprint_table_contract(restored, expected_tables=current_tables)
and _has_single_complete_checkpoint(source)
and _has_single_complete_checkpoint(restored)
- and _physical_table_transition_is_exact(
- source, restored, introduce_checkpoint=False
- )
+ and _physical_table_transition_is_exact(source, restored, introduce_checkpoint=False)
):
return FingerprintCompatibility(
compatible=True,
@@ -350,9 +1886,7 @@ def fingerprint_compatibility(
)
and _fingerprint_table_contract(restored, expected_tables=current_tables)
and _has_single_complete_checkpoint(restored)
- and _physical_table_transition_is_exact(
- source, restored, introduce_checkpoint=True
- )
+ and _physical_table_transition_is_exact(source, restored, introduce_checkpoint=True)
):
return FingerprintCompatibility(
compatible=True,
@@ -372,16 +1906,14 @@ def fingerprint_compatibility(
)
-def audit_fingerprint_differences_are_compatible(
+def _audit_fingerprint_differences_are_compatible(
source: dict[str, Any],
restored: dict[str, Any],
contract: FingerprintCompatibility,
) -> bool:
- """Require exact authoritative tables except the enumerated reconciliation mutations."""
-
if not contract.compatible:
return False
- current_tables = classified_tables()
+ current_tables = ((_pre_updates_tables() - SECRET_TABLES) - MANAGED_IDENTITY_TABLES) - CONFIGURATION_TABLES - PROJECT_TABLES
if contract.mode == "CURRENT_STRICT":
if (
contract.allow_audit_checkpoint_introduction
@@ -391,9 +1923,7 @@ def audit_fingerprint_differences_are_compatible(
or not _fingerprint_table_contract(restored, expected_tables=current_tables)
or not _has_single_complete_checkpoint(source)
or not _has_single_complete_checkpoint(restored)
- or not _physical_table_transition_is_exact(
- source, restored, introduce_checkpoint=False
- )
+ or not _physical_table_transition_is_exact(source, restored, introduce_checkpoint=False)
):
return False
elif contract.mode == "SCHEMA_0022_TO_0024":
@@ -408,9 +1938,7 @@ def audit_fingerprint_differences_are_compatible(
)
or not _fingerprint_table_contract(restored, expected_tables=current_tables)
or not _has_single_complete_checkpoint(restored)
- or not _physical_table_transition_is_exact(
- source, restored, introduce_checkpoint=True
- )
+ or not _physical_table_transition_is_exact(source, restored, introduce_checkpoint=True)
):
return False
else:
@@ -422,14 +1950,8 @@ def audit_fingerprint_differences_are_compatible(
def same_static_fields(left: dict[str, Any], right: dict[str, Any]) -> bool:
return {
- key: value
- for key, value in left.items()
- if key not in {"row_count", "digest"}
- } == {
- key: value
- for key, value in right.items()
- if key not in {"row_count", "digest"}
- }
+ key: value for key, value in left.items() if key not in {"row_count", "digest"}
+ } == {key: value for key, value in right.items() if key not in {"row_count", "digest"}}
changed: set[str] = set()
for table in sorted(set(source_tables) | set(restored_tables)):
@@ -481,9 +2003,11 @@ def classified_tables() -> frozenset[str]:
return frozenset(name for tables in FINGERPRINT_GROUPS.values() for name in tables)
-def unclassified_tables() -> frozenset[str]:
- """Any persisted table must be either fingerprinted or explicitly current-truth."""
+def _pre_lab_execution_tables() -> frozenset[str]:
+ return classified_tables() - LAB_EXECUTION_TABLES
+
+def unclassified_tables() -> frozenset[str]:
return frozenset(Base.metadata.tables) - classified_tables() - CURRENT_TRUTH_TABLES
@@ -514,9 +2038,9 @@ def _canonical_cell_payload(value: Any) -> tuple[bytes, bytes]:
if isinstance(value, bytes | bytearray | memoryview):
return b"y", bytes(value)
if isinstance(value, dict | list | tuple):
- payload = json.dumps(
- value, sort_keys=True, separators=(",", ":"), default=str
- ).encode("utf-8")
+ payload = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode(
+ "utf-8"
+ )
return b"j", payload
if isinstance(value, str):
return b"s", value.encode("utf-8")
@@ -534,17 +2058,18 @@ def _canonical_row(values: Any) -> bytes:
return b"r" + len(payload).to_bytes(8, byteorder="big") + payload
-def _table_digest(connection: Connection, table: Table) -> dict[str, Any]:
+def _table_digest(connection: Connection, table: Table, *, exclude: frozenset[str] = frozenset(),
+ include_comparison_projection: bool = False, include_creative_projection: bool = False,
+ include_watch_controls: bool = False, include_serving_generation: bool = False,
+ include_alert_notification_projection: bool = False) -> dict[str, Any]:
columns = [
column
for column in sorted(table.columns, key=lambda item: item.name)
- if not _sensitive(column.name)
+ if not _sensitive(column.name) and column.name not in exclude
]
redacted = sorted(column.name for column in table.columns if _sensitive(column.name))
if not columns:
- row_count = int(
- connection.scalar(select(func.count()).select_from(table)) or 0
- )
+ row_count = int(connection.scalar(select(func.count()).select_from(table)) or 0)
return {
"row_count": row_count,
"digest": hashlib.sha256(_canonical_row((row_count,))).hexdigest(),
@@ -553,31 +2078,101 @@ def _table_digest(connection: Connection, table: Table) -> dict[str, Any]:
}
statement = select(*columns).select_from(table)
hashes: list[bytes] = []
+ projected_hashes: list[bytes] = []
+ names = [column.name for column in columns]
truncated = False
for row in connection.execute(statement).yield_per(2_000):
if len(hashes) >= MAX_ROWS_PER_TABLE:
truncated = True
break
- hashes.append(hashlib.sha256(_canonical_row(row)).digest())
+ values = list(row)
+ hashes.append(hashlib.sha256(_canonical_row(values)).digest())
+ if include_comparison_projection:
+ status_index = names.index("status")
+ if values[status_index] in SELECTED_COMPARISON_ACTIVE_STATUSES:
+ values[status_index] = "interrupted"
+ values[names.index("version")] += 1
+ values[names.index("error_code")] = SELECTED_COMPARISON_RESTORE_ERROR
+ projected_hashes.append(hashlib.sha256(_canonical_row(values)).digest())
+ elif include_creative_projection:
+ if values[names.index("state")] in {"queued", "granted"}:
+ values[names.index("state")] = "uncertain"
+ values[names.index("decision_code")] = CREATIVE_RESTORE_ERROR
+ projected_hashes.append(hashlib.sha256(_canonical_row(values)).digest())
+ elif include_serving_generation:
+ projected_values = dict(zip(names, values, strict=True))
+ projected_values["serving_generation"] = 0
+ projected_hashes.append(hashlib.sha256(_canonical_row(
+ [projected_values[name] for name in sorted(projected_values)])).digest())
+ elif include_alert_notification_projection:
+ projected_values = dict(zip(names, values, strict=True))
+ if projected_values.get("status") in ALERT_NOTIFICATION_RESTORABLE_STATES:
+ projected_values["status"] = "HELD_FOR_REVIEW"
+ projected_values["next_attempt_at"] = None
+ projected_values["lease_expires_at"] = None
+ projected_hashes.append(hashlib.sha256(_canonical_row(
+ [projected_values[name] for name in names])).digest())
+ elif include_watch_controls:
+ projected_values = {name: value for name, value in zip(names, values, strict=True)
+ if name not in MODEL_UPDATE_TRANSIENT_COLUMNS}
+ projected_values.update(enabled=True, version=1)
+ projected_hashes.append(hashlib.sha256(_canonical_row(
+ [projected_values[name] for name in sorted(projected_values)])).digest())
hashes.sort()
aggregate = hashlib.sha256()
for item in hashes:
aggregate.update(len(item).to_bytes(4, byteorder="big"))
aggregate.update(item)
- return {
+ entry = {
"row_count": len(hashes),
"digest": aggregate.hexdigest(),
"status": "BOUNDED" if truncated else "COMPLETE",
"redacted_columns": redacted,
}
+ if (include_comparison_projection or include_creative_projection or include_watch_controls
+ or include_serving_generation or include_alert_notification_projection):
+ projected_hashes.sort()
+ projected = hashlib.sha256()
+ for item in projected_hashes:
+ projected.update(len(item).to_bytes(4, byteorder="big"))
+ projected.update(item)
+ projection_key = ("restore_projection" if include_alert_notification_projection else
+ "serving_generation_projection" if include_serving_generation else
+ "hub_controls_projection" if include_watch_controls else "restore_projection")
+ entry[projection_key] = {**entry, "digest": projected.hexdigest()}
+ return entry
+
+
+def _actual_table(connection: Connection, name: str) -> Table:
+ declared = Base.metadata.tables[name]
+ return Table(name, MetaData(), *(Column(column["name"],
+ declared.columns[column["name"]].type if column["name"] in declared.columns else column["type"])
+ for column in inspect(connection).get_columns(name)))
def semantic_fingerprint(connection: Connection) -> dict[str, Any]:
- """Compute the bounded semantic fingerprint of one control-plane database."""
-
tables: dict[str, Any] = {}
groups: dict[str, Any] = {}
inspector_tables = set(connection.dialect.get_table_names(connection))
+ benchmark_run_columns = (
+ {column["name"] for column in inspect(connection).get_columns("benchmark_runs")}
+ if "benchmark_runs" in inspector_tables
+ else set()
+ )
+ if "alembic_version" in inspector_tables:
+ schema = connection.scalar(text("SELECT version_num FROM alembic_version"))
+ else:
+ # Metadata-created databases can already carry the current benchmark shape without an
+ # Alembic marker. Infer that shape from physical columns so operator state cannot be
+ # silently treated as a legacy 0038 row. Partial additions are marked below and reject
+ # compatibility rather than being projected.
+ benchmark_columns_present = benchmark_run_columns & BENCHMARK_RUN_MIGRATION_COLUMNS
+ schema = (
+ BENCHMARK_RUN_SCHEMA
+ if benchmark_columns_present
+ else LAB_EXECUTION_SCHEMA
+ )
+ legacy_projects = isinstance(schema, str) and schema < PROJECT_SCHEMA
unexpected_tables = sorted(
inspector_tables
- classified_tables()
@@ -585,6 +2180,30 @@ def semantic_fingerprint(connection: Connection) -> dict[str, Any]:
- NON_FINGERPRINT_INFRASTRUCTURE_TABLES
)
for group, names in FINGERPRINT_GROUPS.items():
+ if isinstance(schema, str) and schema < LAB_EXECUTION_SCHEMA:
+ names = tuple(name for name in names if name not in LAB_EXECUTION_TABLES)
+ if isinstance(schema, str) and schema < ALERT_NOTIFICATION_SCHEMA:
+ names = tuple(name for name in names if name not in ALERT_NOTIFICATION_TABLES)
+ if isinstance(schema, str) and schema < PORTFOLIO_SCHEMA:
+ names = tuple(name for name in names if name not in PORTFOLIO_TABLES)
+ if isinstance(schema, str) and schema < MODEL_CHANGE_SCHEMA:
+ names = tuple(name for name in names if name not in MODEL_CHANGE_TABLES)
+ if isinstance(schema, str) and schema < CREATIVE_SCHEMA:
+ names = tuple(name for name in names if name not in CREATIVE_TABLES)
+ elif isinstance(schema, str) and schema < CREATIVE_GOVERNANCE_SCHEMA:
+ names = tuple(name for name in names if name not in CREATIVE_TABLES - {CREATIVE_ALLOCATION_TABLE})
+ if isinstance(schema, str) and schema < SELECTED_COMPARISON_SCHEMA:
+ names = tuple(name for name in names if name not in SELECTED_COMPARISON_TABLES)
+ if isinstance(schema, str) and schema < MODEL_UPDATE_SCHEMA:
+ names = tuple(name for name in names if name not in MODEL_UPDATE_TABLES)
+ if isinstance(schema, str) and schema < SECRET_SCHEMA:
+ names = tuple(name for name in names if name not in SECRET_TABLES)
+ if isinstance(schema, str) and schema < MANAGED_IDENTITY_SCHEMA:
+ names = tuple(name for name in names if name not in MANAGED_IDENTITY_TABLES)
+ if legacy_projects:
+ names = tuple(name for name in names if name not in PROJECT_TABLES)
+ if not names:
+ continue
group_hash = hashlib.sha256()
group_rows = 0
for name in sorted(names):
@@ -593,7 +2212,118 @@ def semantic_fingerprint(connection: Connection) -> dict[str, Any]:
tables[name] = {"row_count": None, "digest": None, "status": "ABSENT"}
group_hash.update(_canonical_row(("table", name, "ABSENT", None, None)))
continue
- entry = _table_digest(connection, table)
+ old_watch = name == "model_update_watches" and isinstance(schema, str) and schema < SELECTED_COMPARISON_SCHEMA
+ old_lab = name == "serving_jobs" and isinstance(schema, str) and schema < LAB_EXECUTION_SCHEMA
+ old_benchmark_runs = (
+ name == "benchmark_runs"
+ and isinstance(schema, str)
+ and schema < BENCHMARK_RUN_SCHEMA
+ )
+ stale_benchmark_runs_schema = (
+ old_benchmark_runs
+ and bool(benchmark_run_columns & BENCHMARK_RUN_MIGRATION_COLUMNS)
+ )
+ incomplete_benchmark_runs = (
+ name == "benchmark_runs"
+ and isinstance(schema, str)
+ and schema >= BENCHMARK_RUN_SCHEMA
+ and not benchmark_run_columns >= BENCHMARK_RUN_MIGRATION_COLUMNS
+ )
+ if (
+ old_watch
+ or old_lab
+ or old_benchmark_runs
+ or incomplete_benchmark_runs
+ or name == "capability_contracts"
+ ):
+ table = _actual_table(connection, name)
+ old_generation = name == "capability_contracts" and "serving_generation" not in table.c
+ exclude = (frozenset({"superseded_at"}) if legacy_projects and name == "service_credentials"
+ else MODEL_UPDATE_TRANSIENT_COLUMNS if name == "model_update_watches" and not old_watch
+ else SELECTED_COMPARISON_TRANSIENT_COLUMNS if name in SELECTED_COMPARISON_TABLES
+ else BENCHMARK_RUN_MIGRATION_COLUMNS
+ if old_benchmark_runs and not stale_benchmark_runs_schema
+ else frozenset())
+ entry = _table_digest(connection, table, exclude=exclude,
+ include_comparison_projection=name in SELECTED_COMPARISON_TABLES,
+ include_creative_projection=name == CREATIVE_ALLOCATION_TABLE,
+ include_watch_controls=old_watch,
+ include_serving_generation=old_generation,
+ include_alert_notification_projection=name in ALERT_NOTIFICATION_TABLES)
+ if (
+ name == "benchmark_runs"
+ and isinstance(schema, str)
+ and schema >= BENCHMARK_RUN_SCHEMA
+ and not incomplete_benchmark_runs
+ ):
+ non_legacy_columns = (
+ table.c[column_name].is_not(None)
+ for column_name in (
+ "capability_deployment_id",
+ "target_snapshot",
+ "actor_subject",
+ "idempotency_key",
+ "request_fingerprint",
+ "failure_code",
+ "cancellation_requested_at",
+ )
+ )
+ non_legacy_predicate = or_(
+ *non_legacy_columns,
+ table.c.created_at.is_(None),
+ table.c.generation.is_(None),
+ table.c.generation != 1,
+ )
+ entry[BENCHMARK_RUN_SCHEMA_PROJECTION] = {
+ "base": _table_digest(
+ connection,
+ table,
+ exclude=BENCHMARK_RUN_MIGRATION_COLUMNS,
+ ),
+ "non_legacy_row_count": int(
+ connection.scalar(
+ select(func.count()).select_from(table).where(non_legacy_predicate)
+ )
+ or 0
+ ),
+ }
+ elif incomplete_benchmark_runs or stale_benchmark_runs_schema:
+ entry[BENCHMARK_RUN_SCHEMA_OBSERVED] = True
+ if name == "service_credentials" and not legacy_projects:
+ entry["pre_onboarding"] = _table_digest(connection, table, exclude=frozenset({"superseded_at"}))
+ entry["superseded_count"] = int(connection.scalar(select(func.count()).select_from(table).where(table.c.superseded_at.is_not(None))) or 0)
+ if (
+ name == "serving_jobs"
+ and isinstance(schema, str)
+ and schema >= LAB_EXECUTION_SCHEMA
+ and "lab_execution_run_id" in table.c
+ ):
+ linked_jobs = list(
+ connection.execute(
+ select(table.c.id, table.c.lab_execution_run_id)
+ .where(table.c.lab_execution_run_id.is_not(None))
+ .order_by(table.c.id)
+ )
+ )
+ links = hashlib.sha256()
+ for job_id, run_id in linked_jobs:
+ links.update(_canonical_row((job_id, run_id)))
+ entry["lab_execution_projection"] = {
+ "base": _table_digest(
+ connection,
+ table,
+ exclude=frozenset({"lab_execution_run_id"}),
+ ),
+ "run_link_count": len(linked_jobs),
+ "run_link_digest": links.hexdigest(),
+ }
+ elif (
+ name == "serving_jobs"
+ and isinstance(schema, str)
+ and schema < LAB_EXECUTION_SCHEMA
+ and "lab_execution_run_id" in table.c
+ ):
+ entry["lab_execution_schema_observed"] = True
tables[name] = entry
group_rows += int(entry["row_count"] or 0)
group_hash.update(
@@ -608,6 +2338,17 @@ def semantic_fingerprint(connection: Connection) -> dict[str, Any]:
)
)
)
+ for projection_key in (
+ "restore_projection",
+ "hub_controls_projection",
+ "serving_generation_projection",
+ "lab_execution_projection",
+ "lab_execution_schema_observed",
+ BENCHMARK_RUN_SCHEMA_PROJECTION,
+ BENCHMARK_RUN_SCHEMA_OBSERVED,
+ ):
+ if projection_key in entry:
+ group_hash.update(_canonical_row((projection_key, entry[projection_key])))
groups[group] = {
"digest": group_hash.hexdigest(),
"row_count": group_rows,
@@ -647,8 +2388,6 @@ def fingerprint_session(session: Session) -> dict[str, Any]:
def diff_fingerprints(source: dict[str, Any], restored: dict[str, Any]) -> dict[str, Any]:
- """Explain exactly which subjects differ, so a restore is never declared good by hand-wave."""
-
source_tables: dict[str, Any] = source.get("tables", {})
restored_tables: dict[str, Any] = restored.get("tables", {})
identical: list[str] = []
@@ -684,8 +2423,7 @@ def diff_fingerprints(source: dict[str, Any], restored: dict[str, Any]) -> dict[
"restored_digest": restored.get("digest"),
"source_physical_tables": source.get("physical_tables"),
"restored_physical_tables": restored.get("physical_tables"),
- "physical_tables_match": source.get("physical_tables")
- == restored.get("physical_tables"),
+ "physical_tables_match": source.get("physical_tables") == restored.get("physical_tables"),
"identical_table_count": len(identical),
"differing_groups": group_differences,
"differences": differences,
diff --git a/backend/src/modelforge_api/services/secret_slots.py b/backend/src/modelforge_api/services/secret_slots.py
new file mode 100644
index 0000000..f4f2837
--- /dev/null
+++ b/backend/src/modelforge_api/services/secret_slots.py
@@ -0,0 +1,306 @@
+"""A single versioned, write-only HF control-plane secret; no generic vault."""
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import os
+import time
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
+from typing import Any, Literal
+
+import httpx
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from sqlalchemy import delete, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.db import rows_affected
+from modelforge_api.domain.secret_slots import (
+ SLOT,
+ DeploymentSecretMetadata,
+ SecretSlotCatalog,
+ SecretSlotMetadata,
+ SecretSlotRequest,
+ SecretSlotRevoke,
+ SecretSlotSet,
+ SecretSlotVersionRequest,
+ SecretVersionMetadata,
+)
+from modelforge_api.persistence.models import (
+ ManagedOperatorCredential,
+ ManagedOperatorCredentialActivation,
+ ManagedOperatorIdentity,
+ ManagedOperatorState,
+ SecretSlotActivation,
+ SecretSlotState,
+ SecretSlotTest,
+ SecretSlotVersion,
+)
+from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider, ProviderSnapshot
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings
+
+
+class SecretSlotError(RuntimeError):
+ def __init__(self, status_code: int, code: str, message: str) -> None:
+ super().__init__(message)
+ self.status_code, self.code, self.message = status_code, code, message
+
+
+def _aware(value: datetime) -> datetime:
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value
+
+
+def _root_key(settings: Settings) -> bytes:
+ try:
+ key = base64.b64decode(settings.secret_slot_root_key.get_secret_value(), validate=True) if settings.secret_slot_root_key else b""
+ if len(key) != 32:
+ raise ValueError
+ return key
+ except (ValueError, binascii.Error):
+ raise SecretSlotError(503, "SECRET_SLOT_KEY_UNAVAILABLE", "A valid external 32-byte slot root key is required") from None
+
+
+def _key_available(settings: Settings) -> bool:
+ try:
+ _root_key(settings)
+ return True
+ except SecretSlotError:
+ return False
+
+
+def _aad(version: int, key_id: str) -> bytes:
+ return f"modelforge:secret-slot:v1:{SLOT}:{version}:{key_id}".encode()
+
+
+def _decrypt(row: SecretSlotVersion, settings: Settings) -> str:
+ key = _root_key(settings)
+ if row.key_id != settings.secret_slot_root_key_id:
+ raise SecretSlotError(503, "SECRET_SLOT_KEY_UNAVAILABLE", "The external key for this version is unavailable")
+ try:
+ return AESGCM(key).decrypt(base64.b64decode(row.secret_nonce, validate=True),
+ base64.b64decode(row.secret_ciphertext, validate=True), _aad(row.version, row.key_id)).decode()
+ except (InvalidTag, ValueError, UnicodeError, binascii.Error):
+ raise SecretSlotError(503, "SECRET_SLOT_DECRYPTION_FAILED", "The managed credential cannot be decrypted") from None
+
+
+def test_huggingface_token(token: str) -> tuple[bool, str]:
+ """Only provider authentication; never follow a response URL or retain response details."""
+ deadline = time.monotonic() + 10.0
+ try:
+ with httpx.Client(timeout=httpx.Timeout(10.0), follow_redirects=False, trust_env=False) as client: # noqa: SIM117
+ with client.stream("GET", "https://huggingface.co/api/whoami-v2",
+ headers={"Authorization": f"Bearer {token}", "Accept": "application/json",
+ "Accept-Encoding": "identity"}) as response:
+ if response.status_code in (401, 403):
+ return False, "HF_AUTHENTICATION_REJECTED"
+ if response.status_code != 200:
+ return False, "HF_PROVIDER_UNAVAILABLE"
+ if response.headers.get("content-encoding", "identity").lower() != "identity":
+ return False, "HF_RESPONSE_INVALID"
+ body = bytearray()
+ for chunk in response.iter_raw():
+ if time.monotonic() > deadline:
+ return False, "HF_PROVIDER_TIMEOUT"
+ if len(body) + len(chunk) > 65536:
+ return False, "HF_RESPONSE_INVALID"
+ body.extend(chunk)
+ if time.monotonic() > deadline:
+ return False, "HF_PROVIDER_TIMEOUT"
+ payload = json.loads(body)
+ if not isinstance(payload, dict) or not isinstance(payload.get("name"), str) or not payload["name"]:
+ return False, "HF_RESPONSE_INVALID"
+ return True, "HF_AUTHENTICATED"
+ except RecursionError:
+ return False, "HF_RESPONSE_INVALID"
+ except (httpx.HTTPError, ValueError):
+ return False, "HF_PROVIDER_UNAVAILABLE"
+
+
+def resolve_huggingface_token(session: Session, settings: Settings) -> str | None:
+ state = session.get(SecretSlotState, 1)
+ if state is None:
+ raise SecretSlotError(503, "SECRET_SLOT_STATE_MISSING", "Secret slot state is unavailable")
+ if not state.initialized:
+ return settings.hf_token.get_secret_value() if settings.hf_token else None
+ activation = session.get(SecretSlotActivation, 1)
+ row = session.get(SecretSlotVersion, activation.version) if activation else None
+ if row is None or row.revoked_at is not None:
+ raise SecretSlotError(409, "SECRET_SLOT_INACTIVE", "Managed HF authentication requires a tested, explicitly activated version")
+ return _decrypt(row, settings)
+
+
+class SecretAwareHuggingFaceProvider:
+ """Resolve once when upstream access starts, without blocking local job bookkeeping."""
+
+ def __init__(self, session: Session, settings: Settings) -> None:
+ self.session, self.settings = session, settings
+ self._resolved: OfficialHuggingFaceProvider | None = None
+
+ def resolve(self) -> OfficialHuggingFaceProvider:
+ if self._resolved is None:
+ self._resolved = OfficialHuggingFaceProvider(
+ token=resolve_huggingface_token(self.session, self.settings),
+ timeout=self.settings.hf_timeout_seconds,
+ )
+ return self._resolved
+
+ def credential_configured(self) -> bool:
+ return bool(self.resolve().token)
+
+ def search(self, query: str, *, limit: int, sort: str,
+ pipeline_tag: str | None) -> list[dict[str, Any]]:
+ return self.resolve().search(query, limit=limit, sort=sort, pipeline_tag=pipeline_tag)
+
+ def snapshot(self, repository_id: str, revision: str) -> ProviderSnapshot:
+ return self.resolve().snapshot(repository_id, revision)
+
+
+class SecretSlotService:
+ def __init__(self, session: Session, settings: Settings, principal: Principal, correlation_id: str) -> None:
+ self.session, self.settings, self.principal = session, settings, principal
+ self.audit = AuditWriter(session, context=AuditContext("operator", principal.subject, correlation_id))
+
+ def _state(self) -> SecretSlotState:
+ state = self.session.get(SecretSlotState, 1)
+ if state is None:
+ raise SecretSlotError(503, "SECRET_SLOT_STATE_MISSING", "Secret slot state is unavailable")
+ return state
+
+ @contextmanager
+ def _mutation(self, request: SecretSlotRequest) -> Iterator[SecretSlotState]:
+ if self.principal.role != PrincipalRole.ADMIN:
+ raise SecretSlotError(403, "SECRET_SLOT_ADMIN_REQUIRED", "Administrator role required")
+ try:
+ # Match identity administration lock order; revocation cannot race authorization.
+ if self.principal.authentication_method == "managed_operator_key":
+ self.session.execute(update(ManagedOperatorState).where(ManagedOperatorState.singleton_id == 1).values(revision=ManagedOperatorState.revision))
+ claimed = self.session.execute(update(SecretSlotState).where(SecretSlotState.singleton_id == 1).values(revision=SecretSlotState.revision))
+ if rows_affected(claimed) != 1:
+ raise SecretSlotError(503, "SECRET_SLOT_STATE_MISSING", "Secret slot state is unavailable")
+ self.session.expire_all()
+ if self.principal.authentication_method == "managed_operator_key":
+ try:
+ identifier = uuid.UUID(self.principal.credential_id or "")
+ except ValueError:
+ raise SecretSlotError(403, "SECRET_SLOT_ADMIN_REQUIRED", "Administrator authorization changed") from None
+ credential = self.session.get(ManagedOperatorCredential, identifier)
+ identity = self.session.get(ManagedOperatorIdentity, credential.identity_slug) if credential else None
+ active = self.session.get(ManagedOperatorCredentialActivation, identifier)
+ if (credential is None or identity is None or active is None or not identity.enabled or identity.role != "admin"
+ or credential.revoked_at is not None or _aware(credential.expires_at) <= datetime.now(UTC)
+ or self.principal.subject != f"managed:{identity.slug}"):
+ raise SecretSlotError(403, "SECRET_SLOT_ADMIN_REQUIRED", "Administrator authorization changed")
+ state = self._state()
+ if state.revision != request.expected_revision:
+ raise SecretSlotError(409, "SECRET_SLOT_VERSION_CONFLICT", "Secret slot changed; refresh before retrying")
+ yield state
+ self.session.commit()
+ except Exception:
+ self.session.rollback()
+ raise
+
+ def metadata(self) -> SecretSlotMetadata:
+ state = self._state()
+ rows = list(self.session.scalars(select(SecretSlotVersion).order_by(SecretSlotVersion.version.desc()).limit(10)))
+ tests = {item.version: item for item in self.session.scalars(select(SecretSlotTest).where(SecretSlotTest.version.in_([row.version for row in rows])))}
+ active = self.session.get(SecretSlotActivation, 1)
+ candidate = rows[0] if rows else None
+ status: Literal["deployment", "pending", "active", "revoked", "quarantined"] = "deployment"
+ if state.initialized:
+ status = "active" if active else "revoked" if candidate and candidate.revoked_at else "pending" if candidate and candidate.version in tests else "quarantined"
+ if candidate and not active and candidate.revoked_at is None and state.revision == 1:
+ status = "pending"
+ return SecretSlotMetadata(revision=state.revision, source="managed" if state.initialized else "deployment",
+ state=status, configured=bool(candidate and candidate.revoked_at is None) if state.initialized else bool(self.settings.hf_token),
+ root_key_available=_key_available(self.settings), candidate_version=candidate.version if candidate else None,
+ active_version=active.version if active else None,
+ versions=[SecretVersionMetadata(version=row.version, key_id=row.key_id, created_at=_aware(row.created_at),
+ revoked_at=_aware(row.revoked_at) if row.revoked_at else None,
+ test_status="passed" if row.version in tests and tests[row.version].passed else "failed" if row.version in tests and tests[row.version].code != "SECRET_SLOT_UNTESTED" else "untested",
+ tested_at=_aware(tests[row.version].tested_at) if row.version in tests and tests[row.version].code != "SECRET_SLOT_UNTESTED" else None,
+ test_code=tests[row.version].code if row.version in tests and tests[row.version].code != "SECRET_SLOT_UNTESTED" else None) for row in rows])
+
+ def catalog(self) -> SecretSlotCatalog:
+ consumers = {"database_url": "API database bootstrap", "migration_database_url": "one-shot schema migration",
+ "operator_api_key": "deployment recovery administrator", "operator_identities": "deployment operator identities",
+ "backup_encryption_key": "backup encryption and historical restore", "secret_slot_root_key": "HF slot decryption",
+ "hf_token": "deployment HF metadata and separately provisioned node downloads", "redis_url": "Redis deployment connection",
+ "alert_webhook_url": "deployment-managed operational alert notifications"}
+ return SecretSlotCatalog(slot=self.metadata(), deployment_secrets=[DeploymentSecretMetadata(
+ key=key, configured=bool(getattr(self.settings, key)), consumer=consumer) for key, consumer in consumers.items()])
+
+ def _finish(self, state: SecretSlotState, action: str, request: SecretSlotRequest, version: int | None) -> SecretSlotMetadata:
+ state.revision += 1
+ self.audit.write(f"secret_slot.{action}", "platform_secret_slot", SLOT,
+ {"version": version, "revision": state.revision, "reason": request.reason,
+ "consumer": "huggingface_control_plane_metadata"})
+ self.session.flush()
+ return self.metadata()
+
+ def set(self, request: SecretSlotSet) -> SecretSlotMetadata:
+ key = _root_key(self.settings)
+ with self._mutation(request) as state:
+ version = (self.session.scalar(select(func.max(SecretSlotVersion.version))) or 0) + 1
+ nonce = os.urandom(12)
+ encrypted = AESGCM(key).encrypt(nonce, request.value.get_secret_value().encode(), _aad(version, self.settings.secret_slot_root_key_id))
+ self.session.add(SecretSlotVersion(version=version, key_id=self.settings.secret_slot_root_key_id,
+ secret_nonce=base64.b64encode(nonce).decode(), secret_ciphertext=base64.b64encode(encrypted).decode(), created_at=datetime.now(UTC)))
+ self.session.flush()
+ self.session.add(SecretSlotTest(version=version, passed=False, code="SECRET_SLOT_UNTESTED", tested_at=datetime.now(UTC)))
+ state.initialized = True
+ result = self._finish(state, "set", request, version)
+ return result
+
+ def _candidate(self, version: int) -> SecretSlotVersion:
+ row = self.session.get(SecretSlotVersion, version)
+ latest = self.session.scalar(select(func.max(SecretSlotVersion.version)))
+ if row is None or row.revoked_at is not None or version != latest:
+ raise SecretSlotError(409, "SECRET_SLOT_VERSION_CONFLICT", "Select the latest unrevoked candidate")
+ return row
+
+ def test(self, request: SecretSlotVersionRequest) -> SecretSlotMetadata:
+ # Check current managed-admin admission before sending a secret, then release all
+ # locks. Recheck both authorization and CAS before recording the external result.
+ with self._mutation(request):
+ token = _decrypt(self._candidate(request.version), self.settings)
+ passed, code = test_huggingface_token(token)
+ with self._mutation(request) as state:
+ self._candidate(request.version)
+ existing = self.session.get(SecretSlotTest, request.version)
+ if existing:
+ existing.passed, existing.code, existing.tested_at = passed, code, datetime.now(UTC)
+ else:
+ self.session.add(SecretSlotTest(version=request.version, passed=passed, code=code, tested_at=datetime.now(UTC)))
+ result = self._finish(state, "tested", request, request.version)
+ return result
+
+ def activate(self, request: SecretSlotVersionRequest) -> SecretSlotMetadata:
+ with self._mutation(request) as state:
+ row = self._candidate(request.version)
+ test = self.session.get(SecretSlotTest, request.version)
+ if test is None or not test.passed or _aware(test.tested_at) < datetime.now(UTC) - timedelta(minutes=5):
+ raise SecretSlotError(409, "SECRET_SLOT_TEST_REQUIRED", "A successful test within five minutes is required")
+ _decrypt(row, self.settings) # Local consumer acceptance, not node acknowledgement.
+ active = self.session.get(SecretSlotActivation, 1)
+ if active:
+ active.version, active.activated_at = request.version, datetime.now(UTC)
+ else:
+ self.session.add(SecretSlotActivation(singleton_id=1, version=request.version, activated_at=datetime.now(UTC)))
+ result = self._finish(state, "activated", request, request.version)
+ return result
+
+ def revoke(self, request: SecretSlotRevoke) -> SecretSlotMetadata:
+ with self._mutation(request) as state:
+ if not state.initialized:
+ raise SecretSlotError(409, "SECRET_SLOT_INACTIVE", "Deployment credentials require deployment revocation")
+ self.session.execute(update(SecretSlotVersion).where(SecretSlotVersion.revoked_at.is_(None)).values(revoked_at=datetime.now(UTC)))
+ self.session.execute(delete(SecretSlotActivation))
+ self.session.execute(delete(SecretSlotTest))
+ result = self._finish(state, "revoked", request, None)
+ return result
diff --git a/backend/src/modelforge_api/services/selected_model_comparison.py b/backend/src/modelforge_api/services/selected_model_comparison.py
new file mode 100644
index 0000000..ebb29b5
--- /dev/null
+++ b/backend/src/modelforge_api/services/selected_model_comparison.py
@@ -0,0 +1,737 @@
+"""Durable, single-pair consumer coordination; never runs retrieval or promotes models."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import secrets
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import UUID
+
+from sqlalchemy import func, select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+ EvaluationCaseDefinitionResponse,
+ EvaluationComparisonCreate,
+ EvaluationRunComplete,
+ EvaluationRunCreate,
+)
+from modelforge_api.domain.selected_model_comparison import (
+ ComparisonCancel,
+ ComparisonClaim,
+ ComparisonCreate,
+ ComparisonEvaluationSummary,
+ ComparisonFinish,
+ ComparisonList,
+ ComparisonProgress,
+ ComparisonResponse,
+ ComparisonResult,
+)
+from modelforge_api.persistence.models import (
+ ArtifactSet,
+ Capability,
+ CapabilityContract,
+ CapabilityDeployment,
+ CapabilityExperimentRoute,
+ DeploymentCandidate,
+ EmbeddingSpace,
+ EvaluationCase,
+ EvaluationComparison,
+ EvaluationSuite,
+ EvaluationSuiteRevision,
+ GatewayRequest,
+ ModelRevision,
+ Project,
+ ProjectBinding,
+ RuntimeProfile,
+ SelectedModelComparison,
+ ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+
+ACTIVE = ("pending", "running", "cancel_requested")
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _aware(value: datetime) -> datetime:
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value
+
+
+def _digest(value: Any) -> str:
+ return hashlib.sha256(
+ json.dumps(
+ value, sort_keys=True, separators=(",", ":"), default=str, allow_nan=False
+ ).encode()
+ ).hexdigest()
+
+
+def _error(code: str, status: int = 409) -> EvaluationError:
+ return EvaluationError(code, code.replace("_", " "), status_code=status)
+
+
+class SelectedModelComparisonService:
+ def __init__(self, session: Session, audit_context: AuditContext) -> None:
+ self.session = session
+ self.audit_context = audit_context
+
+ def _audit(self, row: SelectedModelComparison, action: str) -> None:
+ AuditWriter(self.session, context=self.audit_context).write(
+ action,
+ "selected_model_comparison",
+ str(row.id),
+ {
+ "status": row.status,
+ "version": row.version,
+ "project_binding_id": str(row.project_binding_id),
+ },
+ )
+
+ def _response(self, row: SelectedModelComparison) -> ComparisonResponse:
+ values = {
+ name: getattr(row, name)
+ for name in ComparisonResponse.model_fields
+ if name not in {"pins_digest", "production_promotion_performed", "evaluation_summary"}
+ }
+ summary = None
+ if row.evaluation_comparison_id is not None:
+ comparison = self.session.get(EvaluationComparison, row.evaluation_comparison_id)
+ if (
+ comparison is None
+ or comparison.baseline_run_id != row.baseline_run_id
+ or comparison.candidate_run_id != row.candidate_run_id
+ ):
+ raise _error("comparison_evidence_missing")
+ summary = ComparisonEvaluationSummary(
+ comparability=comparison.comparability,
+ eligibility=comparison.promotion_eligibility,
+ metric_deltas=comparison.metric_deltas,
+ critical_regressions=comparison.critical_regressions,
+ )
+ elif row.status == "completed":
+ raise _error("comparison_evidence_missing")
+ return ComparisonResponse(
+ **values, pins_digest=_digest(row.pins), evaluation_summary=summary
+ )
+
+ def _row(self, identifier: UUID) -> SelectedModelComparison:
+ row = self.session.scalar(
+ select(SelectedModelComparison)
+ .where(SelectedModelComparison.id == identifier)
+ .with_for_update()
+ .execution_options(populate_existing=True)
+ )
+ if row is None:
+ raise _error("comparison_not_found", 404)
+ return row
+
+ def _consumer(self, row: SelectedModelComparison, client: ServiceClient) -> None:
+ fresh = self.session.get(ServiceClient, client.id, populate_existing=True)
+ binding = self.session.get(ProjectBinding, row.project_binding_id, populate_existing=True)
+ project = self.session.get(Project, binding.project_id) if binding else None
+ if (
+ fresh is None
+ or fresh.id != row.consumer_client_id
+ or fresh.project_binding_id != row.project_binding_id
+ or fresh.status != "active"
+ or fresh.disabled_at is not None
+ or "rag.embedding@1" not in fresh.allowed_capabilities
+ or binding is None
+ or binding.deprecated_at is not None
+ or project is None
+ or not project.active
+ ):
+ raise _error("comparison_consumer_not_authorized", 403)
+
+ def _target(self, identifier: UUID, contract_id: UUID) -> dict[str, Any]:
+ deployment = self.session.get(CapabilityDeployment, identifier, populate_existing=True)
+ if (
+ deployment is None
+ or deployment.capability_contract_id != contract_id
+ or deployment.deprecated_at is not None
+ or deployment.draining_at is not None
+ or deployment.status not in {"stable", "candidate"}
+ ):
+ raise _error("comparison_target_unavailable")
+ candidate = self.session.get(DeploymentCandidate, deployment.deployment_candidate_id)
+ artifact = self.session.get(ArtifactSet, deployment.artifact_set_id)
+ profile = self.session.get(RuntimeProfile, deployment.runtime_profile_id)
+ space = (
+ self.session.get(EmbeddingSpace, deployment.embedding_space_id)
+ if deployment.embedding_space_id
+ else None
+ )
+ revision = self.session.get(ModelRevision, artifact.revision_id) if artifact else None
+ if (
+ candidate is None
+ or artifact is None
+ or profile is None
+ or space is None
+ or revision is None
+ or artifact.immutable_at is None
+ or profile.immutable_at is None
+ or profile.artifact_set_id != artifact.id
+ or not revision.resolved_commit_sha
+ or (candidate.artifact_set_id, candidate.runtime_profile_id, candidate.compute_node_id)
+ != (
+ deployment.artifact_set_id,
+ deployment.runtime_profile_id,
+ deployment.compute_node_id,
+ )
+ or (space.capability_contract_id, space.artifact_set_id, space.runtime_profile_id)
+ != (contract_id, deployment.artifact_set_id, deployment.runtime_profile_id)
+ ):
+ raise _error("comparison_target_identity_mismatch")
+ return {
+ "deployment_id": str(identifier),
+ "config_fingerprint": deployment.config_fingerprint,
+ "artifact_set_id": str(artifact.id),
+ "revision_id": str(revision.id),
+ "upstream_commit_sha": revision.resolved_commit_sha,
+ "runtime_profile_id": str(profile.id),
+ "runtime_fingerprint": profile.fingerprint,
+ "artifact_sha256": profile.artifact_sha256,
+ "embedding_space_id": str(space.id),
+ "embedding_space_digest": space.identity_digest,
+ "dimension": space.dimension,
+ "compute_node_id": str(deployment.compute_node_id),
+ "accelerator_id": str(deployment.accelerator_id),
+ }
+
+ def _pins(self, request: ComparisonCreate) -> tuple[dict[str, Any], int]:
+ binding = self.session.get(ProjectBinding, request.project_binding_id)
+ client = self.session.get(ServiceClient, request.consumer_client_id)
+ project = self.session.get(Project, binding.project_id) if binding else None
+ contract = (
+ self.session.get(CapabilityContract, binding.capability_contract_id)
+ if binding
+ else None
+ )
+ capability = self.session.get(Capability, contract.capability_id) if contract else None
+ revision = self.session.get(EvaluationSuiteRevision, request.suite_revision_id)
+ suite = self.session.get(EvaluationSuite, revision.suite_id) if revision else None
+ if (
+ binding is None
+ or binding.deprecated_at is not None
+ or binding.channel != "stable"
+ or project is None
+ or not project.active
+ or contract is None
+ or contract.deprecated_at is not None
+ or capability is None
+ or capability.key != "rag.embedding"
+ or contract.version != 1
+ or client is None
+ or client.project_binding_id != binding.id
+ or client.status != "active"
+ or client.disabled_at is not None
+ or "rag.embedding@1" not in client.allowed_capabilities
+ or revision is None
+ or suite is None
+ or suite.project_id != binding.project_id
+ ):
+ raise _error("comparison_binding_invalid", 422)
+ cases = list(
+ self.session.scalars(
+ select(EvaluationCase).where(EvaluationCase.suite_revision_id == revision.id)
+ )
+ )
+ if not 1 <= len(cases) <= 100 or any(
+ case.review_status not in {"reviewed", "approved"} for case in cases
+ ):
+ raise _error("comparison_suite_not_bounded_or_reviewed", 422)
+ if any(
+ case.label_provenance.get("corpus_revision") != request.corpus_sha256 for case in cases
+ ):
+ raise _error("comparison_corpus_provenance_mismatch", 422)
+ if request.retrieval_config_sha256 != _digest(revision.retrieval_settings):
+ raise _error("comparison_retrieval_configuration_mismatch", 422)
+ current = self._target(request.current_deployment_id, contract.id)
+ candidate = self._target(request.candidate_deployment_id, contract.id)
+ current_row = self.session.get(CapabilityDeployment, request.current_deployment_id)
+ candidate_row = self.session.get(CapabilityDeployment, request.candidate_deployment_id)
+ route = self.session.get(
+ CapabilityExperimentRoute, request.candidate_route_id, populate_existing=True
+ )
+ if route is None:
+ raise _error("comparison_selection_invalid", 422)
+ common_invalid = (
+ request.current_deployment_id == request.candidate_deployment_id
+ or candidate_row is None
+ or candidate_row.production
+ or candidate_row.channel != "experiment"
+ or candidate_row.status != "candidate"
+ or route.status != "active"
+ or route.capability_contract_id != contract.id
+ or route.capability_deployment_id != candidate_row.id
+ or request.current_index_ref == request.candidate_index_ref
+ )
+ current_route: CapabilityExperimentRoute | None = None
+ if request.current_route_id is None:
+ # Keep the original stable-production comparison contract unchanged.
+ if (
+ common_invalid
+ or current_row is None
+ or not current_row.production
+ or current_row.channel != "stable"
+ or current_row.status != "stable"
+ ):
+ raise _error("comparison_selection_invalid", 422)
+ else:
+ current_route = self.session.get(
+ CapabilityExperimentRoute, request.current_route_id, populate_existing=True
+ )
+ if (
+ common_invalid
+ or current_row is None
+ or current_row.production
+ or current_row.channel != "experiment"
+ or current_row.status != "candidate"
+ or current_route is None
+ or current_route.status != "active"
+ or current_route.capability_contract_id != contract.id
+ or current_route.capability_deployment_id != current_row.id
+ or current_route.id == route.id
+ ):
+ raise _error("comparison_selection_invalid", 422)
+ pins = {
+ "project_id": str(project.id),
+ "current": current,
+ "candidate": candidate,
+ "candidate_route_key": route.route_key,
+ "suite_definition_digest": revision.definition_digest,
+ "dataset_revision": revision.dataset_revision,
+ "corpus_sha256": request.corpus_sha256,
+ "retrieval_config_sha256": request.retrieval_config_sha256,
+ "current_index_ref": request.current_index_ref,
+ "candidate_index_ref": request.candidate_index_ref,
+ }
+ if current_route is not None:
+ pins["current_route_id"] = str(current_route.id)
+ pins["current_route_key"] = current_route.route_key
+ pins["candidate_route_id"] = str(route.id)
+ return pins, len(cases)
+
+ def _revalidate(self, row: SelectedModelComparison) -> None:
+ request = ComparisonCreate(
+ **{
+ name: getattr(row, name)
+ for name in (
+ "project_binding_id",
+ "consumer_client_id",
+ "suite_revision_id",
+ "current_deployment_id",
+ "candidate_deployment_id",
+ "candidate_route_id",
+ "idempotency_key",
+ )
+ },
+ current_route_id=row.pins.get("current_route_id"),
+ **{
+ name: row.pins[name]
+ for name in (
+ "corpus_sha256",
+ "retrieval_config_sha256",
+ "current_index_ref",
+ "candidate_index_ref",
+ )
+ },
+ )
+ pins, count = self._pins(request)
+ if pins != row.pins or count != row.expected_cases:
+ raise _error("comparison_pins_changed")
+
+ def create(self, request: ComparisonCreate) -> ComparisonResponse:
+ # Excluding the optional null route preserves legacy stable-path retry digests.
+ digest = _digest(request.model_dump(mode="json", exclude_none=True))
+ previous = self.session.scalar(
+ select(SelectedModelComparison).where(
+ SelectedModelComparison.project_binding_id == request.project_binding_id,
+ SelectedModelComparison.idempotency_key == request.idempotency_key,
+ )
+ )
+ if previous is not None:
+ if previous.request_digest != digest:
+ raise _error("comparison_idempotency_conflict")
+ return self._response(previous)
+ self.session.scalar(
+ select(ProjectBinding)
+ .where(ProjectBinding.id == request.project_binding_id)
+ .with_for_update()
+ )
+ # The binding lock serializes both idempotency and the one-active-intent decision.
+ previous = self.session.scalar(
+ select(SelectedModelComparison).where(
+ SelectedModelComparison.project_binding_id == request.project_binding_id,
+ SelectedModelComparison.idempotency_key == request.idempotency_key,
+ )
+ )
+ if previous is not None:
+ if previous.request_digest != digest:
+ raise _error("comparison_idempotency_conflict")
+ return self._response(previous)
+ active = self.session.scalar(
+ select(SelectedModelComparison)
+ .where(
+ SelectedModelComparison.project_binding_id == request.project_binding_id,
+ SelectedModelComparison.status.in_(ACTIVE),
+ )
+ .with_for_update()
+ )
+ if active is not None and not self._expire(active):
+ raise _error("comparison_already_active")
+ self.session.flush()
+ pins, count = self._pins(request)
+ row = SelectedModelComparison(
+ **{
+ name: getattr(request, name)
+ for name in (
+ "project_binding_id",
+ "consumer_client_id",
+ "suite_revision_id",
+ "current_deployment_id",
+ "candidate_deployment_id",
+ "candidate_route_id",
+ "idempotency_key",
+ )
+ },
+ request_digest=digest,
+ actor_id=self.audit_context.actor_id,
+ pins=pins,
+ status="pending",
+ version=1,
+ expected_cases=count,
+ current_completed_cases=0,
+ candidate_completed_cases=0,
+ )
+ self.session.add(row)
+ try:
+ self.session.flush()
+ self._audit(row, "SELECTED_MODEL_COMPARISON_CREATED")
+ self.session.commit()
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise _error("comparison_concurrent_create") from exc
+ self.session.refresh(row)
+ return self._response(row)
+
+ def _expire(self, row: SelectedModelComparison) -> bool:
+ if row.status in {"running", "cancel_requested"} and (
+ row.claim_expires_at is None
+ or _aware(row.claim_expires_at) <= _now()
+ or row.deadline_at is None
+ or _aware(row.deadline_at) <= _now()
+ ):
+ row.status, row.error_code, row.finished_at = "interrupted", "CLAIM_EXPIRED", _now()
+ row.claim_token_hash = row.claim_expires_at = None
+ row.version += 1
+ self._audit(row, "SELECTED_MODEL_COMPARISON_INTERRUPTED")
+ return True
+ return False
+
+ def get(self, identifier: UUID) -> ComparisonResponse:
+ row = self._row(identifier)
+ self._expire(row)
+ self.session.commit()
+ return self._response(row)
+
+ def list(
+ self, project_binding_id: UUID | None = None, limit: int = 50, offset: int = 0
+ ) -> ComparisonList:
+ if not 1 <= limit <= 100 or offset < 0:
+ raise _error("comparison_page_invalid", 422)
+ query = select(SelectedModelComparison)
+ if project_binding_id is not None:
+ query = query.where(SelectedModelComparison.project_binding_id == project_binding_id)
+ total = self.session.scalar(select(func.count()).select_from(query.subquery())) or 0
+ rows = self.session.scalars(
+ query.order_by(SelectedModelComparison.created_at.desc(), SelectedModelComparison.id)
+ .limit(limit)
+ .offset(offset)
+ ).all()
+ locked = {
+ identifier: self._row(identifier)
+ for identifier in sorted((item.id for item in rows), key=str)
+ }
+ items = []
+ for item in rows:
+ row = locked[item.id]
+ self._expire(row)
+ items.append(self._response(row))
+ self.session.commit()
+ return ComparisonList(items=items, total=total)
+
+ def cancel(self, identifier: UUID, request: ComparisonCancel) -> ComparisonResponse:
+ row = self._row(identifier)
+ if self._expire(row):
+ self.session.commit()
+ raise _error("comparison_claim_expired")
+ if row.version != request.expected_version:
+ raise _error("comparison_version_conflict")
+ if row.status == "pending":
+ row.status, row.finished_at = "cancelled", _now()
+ elif row.status == "running":
+ row.status = "cancel_requested"
+ elif row.status in {"cancelled", "cancel_requested"}:
+ return self._response(row)
+ else:
+ raise _error("comparison_not_cancellable")
+ row.cancel_requested_at = _now()
+ row.version += 1
+ self._audit(row, "SELECTED_MODEL_COMPARISON_CANCEL_REQUESTED")
+ self.session.commit()
+ return self._response(row)
+
+ def assigned(self, client: ServiceClient, limit: int = 50) -> ComparisonList:
+ if not 1 <= limit <= 100:
+ raise _error("comparison_page_invalid", 422)
+ query = select(SelectedModelComparison).where(
+ SelectedModelComparison.consumer_client_id == client.id,
+ SelectedModelComparison.project_binding_id == client.project_binding_id,
+ SelectedModelComparison.status.in_((*ACTIVE, "interrupted")),
+ )
+ rows = self.session.scalars(
+ query.order_by(SelectedModelComparison.created_at.desc()).limit(limit)
+ ).all()
+ locked = {
+ identifier: self._row(identifier)
+ for identifier in sorted((item.id for item in rows), key=str)
+ }
+ items = []
+ for item in rows:
+ row = locked[item.id]
+ self._consumer(row, client)
+ self._expire(row)
+ items.append(self._response(row))
+ total = self.session.scalar(select(func.count()).select_from(query.subquery())) or 0
+ self.session.commit()
+ return ComparisonList(items=items, total=total)
+
+ def claim(self, identifier: UUID, client: ServiceClient) -> ComparisonClaim:
+ row = self._row(identifier)
+ self._consumer(row, client)
+ if self._expire(row):
+ self.session.commit()
+ raise _error("comparison_claim_expired")
+ if row.status != "pending":
+ raise _error("comparison_not_claimable")
+ self._revalidate(row)
+ token, now = secrets.token_urlsafe(32), _now()
+ result = self.session.execute(
+ update(SelectedModelComparison)
+ .where(
+ SelectedModelComparison.id == row.id,
+ SelectedModelComparison.status == "pending",
+ SelectedModelComparison.version == row.version,
+ )
+ .values(
+ status="running",
+ version=row.version + 1,
+ claim_token_hash=hashlib.sha256(token.encode()).hexdigest(),
+ claimed_at=now,
+ claim_expires_at=now + timedelta(seconds=120),
+ deadline_at=now + timedelta(seconds=3600),
+ )
+ .returning(SelectedModelComparison.id)
+ )
+ if result.scalar_one_or_none() != row.id:
+ self.session.rollback()
+ raise _error("comparison_claim_conflict")
+ self.session.refresh(row)
+ cases = [
+ EvaluationCaseDefinitionResponse.model_validate(case)
+ for case in self.session.scalars(
+ select(EvaluationCase)
+ .where(EvaluationCase.suite_revision_id == row.suite_revision_id)
+ .order_by(EvaluationCase.case_key)
+ )
+ ]
+ self._audit(row, "SELECTED_MODEL_COMPARISON_CLAIMED")
+ self.session.commit()
+ if row.claim_expires_at is None:
+ raise _error("comparison_claim_invalid")
+ return ComparisonClaim(
+ comparison=self._response(row),
+ claim_token=token,
+ claim_expires_at=row.claim_expires_at,
+ cases=cases,
+ )
+
+ def _claim(self, row: SelectedModelComparison, token: str, client: ServiceClient) -> None:
+ self._consumer(row, client)
+ if self._expire(row):
+ self.session.commit()
+ raise _error("comparison_claim_expired")
+ if (
+ row.status not in {"running", "cancel_requested"}
+ or row.claim_token_hash is None
+ or not hmac.compare_digest(
+ row.claim_token_hash, hashlib.sha256(token.encode()).hexdigest()
+ )
+ ):
+ raise _error("comparison_claim_invalid")
+
+ def progress(
+ self, identifier: UUID, request: ComparisonProgress, client: ServiceClient
+ ) -> ComparisonResponse:
+ row = self._row(identifier)
+ self._claim(row, request.claim_token.get_secret_value(), client)
+ if not (
+ row.current_completed_cases <= request.current_completed_cases <= row.expected_cases
+ and row.candidate_completed_cases
+ <= request.candidate_completed_cases
+ <= row.expected_cases
+ ):
+ raise _error("comparison_progress_invalid", 422)
+ row.current_completed_cases, row.candidate_completed_cases = (
+ request.current_completed_cases,
+ request.candidate_completed_cases,
+ )
+ if row.deadline_at is None:
+ raise _error("comparison_claim_invalid")
+ row.claim_expires_at = min(_now() + timedelta(seconds=120), _aware(row.deadline_at))
+ row.version += 1
+ self.session.commit()
+ return self._response(row)
+
+ def finish(
+ self, identifier: UUID, request: ComparisonFinish, client: ServiceClient
+ ) -> ComparisonResponse:
+ row = self._row(identifier)
+ self._claim(row, request.claim_token.get_secret_value(), client)
+ if request.outcome == "cancelled" and row.status != "cancel_requested":
+ raise _error("comparison_cancellation_not_requested")
+ row.status, row.error_code, row.finished_at = request.outcome, request.error_code, _now()
+ row.claim_token_hash = row.claim_expires_at = None
+ row.version += 1
+ self._audit(row, "SELECTED_MODEL_COMPARISON_FINISHED")
+ self.session.commit()
+ return self._response(row)
+
+ def _gateway_evidence(
+ self,
+ row: SelectedModelComparison,
+ leg: EvaluationRunComplete,
+ mapping: dict[UUID, UUID],
+ deployment_id: UUID,
+ seen: set[UUID],
+ ) -> None:
+ successful = {case.case_id for case in leg.results if case.error_code is None}
+ case_ids = {case.case_id for case in leg.results}
+ if not successful <= set(mapping) or not set(mapping) <= case_ids:
+ raise _error("comparison_gateway_evidence_missing", 422)
+ for case_id, identifier in mapping.items():
+ case = self.session.get(EvaluationCase, case_id)
+ gateway = self.session.scalar(
+ select(GatewayRequest).where(GatewayRequest.request_id == identifier)
+ )
+ if (
+ identifier in seen
+ or case is None
+ or case.suite_revision_id != row.suite_revision_id
+ or gateway is None
+ or gateway.service_client_id != row.consumer_client_id
+ or gateway.capability_deployment_id != deployment_id
+ or gateway.capability_key != "rag.embedding"
+ or gateway.input_count != 1
+ or gateway.input_sha256 != _digest([case.query])
+ or gateway.capability_version != 1
+ or gateway.status != "completed"
+ or gateway.finished_at is None
+ or row.claimed_at is None
+ or row.deadline_at is None
+ or not (
+ _aware(row.claimed_at)
+ <= _aware(gateway.created_at)
+ <= _aware(gateway.finished_at)
+ <= min(_now(), _aware(row.deadline_at))
+ )
+ ):
+ raise _error("comparison_gateway_evidence_invalid", 422)
+ seen.add(identifier)
+
+ def result(
+ self, identifier: UUID, request: ComparisonResult, client: ServiceClient
+ ) -> ComparisonResponse:
+ row = self._row(identifier)
+ self._consumer(row, client)
+ digest = _digest(request.model_dump(mode="json", exclude={"claim_token"}))
+ if row.status == "completed":
+ if row.result_digest != digest:
+ raise _error("comparison_result_conflict")
+ return self._response(row)
+ self._claim(row, request.claim_token.get_secret_value(), client)
+ if row.status != "running":
+ raise _error("comparison_cancel_pending")
+ self._revalidate(row)
+ if request.pins_digest != _digest(row.pins):
+ raise _error("comparison_pins_mismatch", 422)
+ seen: set[UUID] = set()
+ self._gateway_evidence(
+ row, request.current, request.current_gateway_requests, row.current_deployment_id, seen
+ )
+ self._gateway_evidence(
+ row,
+ request.candidate,
+ request.candidate_gateway_requests,
+ row.candidate_deployment_id,
+ seen,
+ )
+ evaluation = EvaluationService(self.session, commit=False)
+ try:
+ run_ids = []
+ for name, leg, mapping, target in (
+ ("current", request.current, request.current_gateway_requests, "current"),
+ ("candidate", request.candidate, request.candidate_gateway_requests, "shadow"),
+ ):
+ run = evaluation.create_run(
+ EvaluationRunCreate(
+ project_id=UUID(row.pins["project_id"]),
+ suite_revision_id=row.suite_revision_id,
+ target_kind="current" if target == "current" else "shadow",
+ target_index_ref=row.pins[f"{name}_index_ref"],
+ embedding_space_ref=row.pins[name]["embedding_space_id"],
+ capability_deployment_id=UUID(row.pins[name]["deployment_id"]),
+ corpus_revision=row.pins["corpus_sha256"],
+ retrieval_config_digest=row.pins["retrieval_config_sha256"],
+ environment_fingerprint={
+ "source": "designated_consumer",
+ "comparison_id": str(row.id),
+ "consumer_revision_sha256": request.consumer_revision_sha256,
+ "pins_digest": request.pins_digest,
+ "gateway_request_ids": {
+ str(key): str(value) for key, value in mapping.items()
+ },
+ },
+ )
+ )
+ evaluation.complete_run(run.id, leg)
+ run_ids.append(run.id)
+ comparison = evaluation.compare(
+ EvaluationComparisonCreate(baseline_run_id=run_ids[0], candidate_run_id=run_ids[1])
+ )
+ if (
+ row.claim_expires_at is None
+ or row.deadline_at is None
+ or min(_aware(row.claim_expires_at), _aware(row.deadline_at)) <= _now()
+ ):
+ raise _error("comparison_claim_expired")
+ row.baseline_run_id, row.candidate_run_id = run_ids
+ row.evaluation_comparison_id, row.result_digest = comparison.id, digest
+ row.status, row.finished_at = "completed", _now()
+ row.current_completed_cases = row.candidate_completed_cases = row.expected_cases
+ row.claim_token_hash = row.claim_expires_at = None
+ row.version += 1
+ self._audit(row, "SELECTED_MODEL_COMPARISON_COMPLETED")
+ self.session.commit()
+ except Exception:
+ self.session.rollback()
+ raise
+ return self._response(row)
diff --git a/backend/src/modelforge_api/services/selected_replacement.py b/backend/src/modelforge_api/services/selected_replacement.py
new file mode 100644
index 0000000..39a1da8
--- /dev/null
+++ b/backend/src/modelforge_api/services/selected_replacement.py
@@ -0,0 +1,142 @@
+"""Advisory replacement preview; never approves, executes, reconciles or commits."""
+from uuid import UUID
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
+from modelforge_api.domain.selected_replacement import (
+ ReplacementApproval,
+ ReplacementBinding,
+ ReplacementBlocker,
+ ReplacementPreview,
+)
+from modelforge_api.persistence.models import (
+ CapabilityContract,
+ CapabilityDeployment,
+ EvaluationComparison,
+ ProductionExecutionApproval,
+ Project,
+ ProjectBinding,
+ SelectedModelComparison,
+)
+from modelforge_api.services.evaluation import EvaluationError
+from modelforge_api.services.lifecycle import LifecycleError
+from modelforge_api.services.lifecycle_serving import LifecycleServing
+from modelforge_api.services.serving import ServingError
+
+
+class SelectedReplacementService:
+ def __init__(self, session: Session) -> None:
+ self.session = session
+
+ def preview(self, comparison_id: UUID) -> ReplacementPreview:
+ with self.session.no_autoflush:
+ return self._preview(comparison_id)
+
+ def _preview(self, comparison_id: UUID) -> ReplacementPreview:
+ chosen = self.session.get(SelectedModelComparison, comparison_id)
+ if chosen is None:
+ raise LifecycleError(404, "COMPARISON_NOT_FOUND", "Selected comparison was not found")
+ current = self.session.get(CapabilityDeployment, chosen.current_deployment_id)
+ candidate = self.session.get(CapabilityDeployment, chosen.candidate_deployment_id)
+ contract = self.session.get(CapabilityContract, current.capability_contract_id) if current else None
+ blockers = [ReplacementBlocker(code="PRODUCTION_VALIDATION_REVIEW_REQUIRED",
+ message="Review actual project production validation and explicitly approve the lifecycle plan; this preview grants no production authority.")]
+
+ def block(code: str, message: str, binding_id: UUID | None = None) -> None:
+ if not any(item.code == code and item.project_binding_id == binding_id for item in blockers):
+ blockers.append(ReplacementBlocker(code=code, message=message, project_binding_id=binding_id))
+
+ if chosen.status != "completed":
+ block("COMPARISON_NOT_COMPLETED", "Complete the designated consumer comparison before preparing replacement.")
+ same_space = bool(current and candidate and current.embedding_space_id is not None
+ and current.embedding_space_id == candidate.embedding_space_id)
+ if not same_space:
+ block("SERVING_REINDEX_REQUIRED", "Changed embedding identity requires a separate validated consumer index and trusted cutover; equal dimensions do not suffice.")
+ if (current is None or candidate is None or contract is None or contract.deprecated_at is not None
+ or candidate.capability_contract_id != contract.id or current.id == candidate.id
+ or not current.production or current.channel != "stable" or current.status != "stable"
+ or candidate.production or candidate.channel != "experiment" or candidate.status != "candidate"):
+ block("SERVING_SELECTION_INVALID", "Refresh the comparison: its exact current stable or nonproduction candidate is no longer available.")
+ bindings = []
+ comparison_ids = {}
+ if contract is not None:
+ rows = list(self.session.scalars(select(ProjectBinding).where(
+ ProjectBinding.capability_contract_id == contract.id, ProjectBinding.channel == "stable",
+ ProjectBinding.deprecated_at.is_(None)).order_by(ProjectBinding.id).limit(1001)))
+ if not rows or len(rows) > 1000:
+ block("SERVING_BINDINGS_UNBOUNDED", "One to 1000 affected stable bindings are required.")
+ for binding in rows[:1000]:
+ latest = self.session.scalar(select(SelectedModelComparison).where(
+ SelectedModelComparison.project_binding_id == binding.id
+ ).order_by(SelectedModelComparison.created_at.desc(), SelectedModelComparison.id.desc()).limit(1))
+ exact = bool(latest and latest.current_deployment_id == chosen.current_deployment_id
+ and latest.candidate_deployment_id == chosen.candidate_deployment_id)
+ evaluated = self.session.get(EvaluationComparison, latest.evaluation_comparison_id) if latest and latest.evaluation_comparison_id else None
+ eligible = bool(exact and latest and latest.status == "completed" and evaluated
+ and evaluated.comparability == "comparable" and evaluated.promotion_eligibility == "eligible"
+ and not evaluated.critical_regressions)
+ project = self.session.get(Project, binding.project_id)
+ bindings.append(ReplacementBinding(project_binding_id=binding.id, project_id=binding.project_id,
+ project_name=project.name if project else "Unavailable project",
+ comparison_id=latest.id if exact and latest else None,
+ comparison_status=latest.status if exact and latest else None, eligible=eligible))
+ if eligible and latest:
+ comparison_ids[binding.id] = latest.id
+ else:
+ block("BINDING_COMPARISON_REQUIRED", "This affected project needs its latest eligible comparison of the exact same pair.", binding.id)
+ if binding.id == chosen.project_binding_id and (latest is None or latest.id != chosen.id):
+ block("SELECTED_COMPARISON_SUPERSEDED", "A newer comparison supersedes this selection; review that evidence first.", binding.id)
+ approvals = []
+ selection = None
+ if candidate is not None and contract is not None:
+ choices = list(self.session.scalars(select(ProductionExecutionApproval).where(
+ ProductionExecutionApproval.deployment_candidate_id == candidate.deployment_candidate_id,
+ ProductionExecutionApproval.capability_contract_id == contract.id,
+ ProductionExecutionApproval.status == "approved", ProductionExecutionApproval.revoked_at.is_(None)
+ ).order_by(ProductionExecutionApproval.approved_at.desc(), ProductionExecutionApproval.id.desc()).limit(101)))
+ if len(choices) > 100:
+ block("APPROVAL_CHOICES_UNBOUNDED", "Narrow the production approval history before continuing.")
+ adapter = LifecycleServing(self.session)
+ if choices:
+ # Qualification helpers lock their facts. Follow the writer's lock
+ # order rather than retaining an approval lock before a route lock.
+ adapter.repo.lock_contract_route(contract.id)
+ lock_selection = ServingTransitionSelection(current_deployment_id=chosen.current_deployment_id,
+ candidate_deployment_id=chosen.candidate_deployment_id, production_approval_id=choices[0].id,
+ expected_route_generation=contract.serving_generation,
+ comparison_ids={chosen.project_binding_id: chosen.id})
+ adapter._lock_accelerators(lock_selection)
+ if current is not None:
+ adapter._deployment(current.id)
+ adapter._deployment(candidate.id)
+ rejected = []
+ for approval in choices[:100]:
+ try:
+ # Reuse current production-fact validation, without invoking any write path.
+ LifecycleServing(self.session)._production_evidence(candidate, contract, approval.id)
+ except (LifecycleError, ServingError, EvaluationError) as error:
+ rejected.append(error)
+ continue
+ approvals.append(ReplacementApproval(id=approval.id, approved_by=approval.approved_by,
+ approved_at=approval.approved_at, evidence_fingerprint=approval.evidence_fingerprint))
+ if not approvals:
+ for rejection in rejected:
+ block(rejection.code, str(rejection))
+ block("PRODUCTION_APPROVAL_REQUIRED", "Create or refresh the exact candidate's production execution approval.")
+ if len(blockers) == 1 and approvals and comparison_ids:
+ proposed = ServingTransitionSelection(current_deployment_id=chosen.current_deployment_id,
+ candidate_deployment_id=chosen.candidate_deployment_id, production_approval_id=approvals[0].id,
+ expected_route_generation=contract.serving_generation, comparison_ids=comparison_ids)
+ try:
+ LifecycleServing(self.session).prepare(proposed)
+ except (LifecycleError, ServingError, EvaluationError) as error:
+ block(error.code, str(error))
+ else:
+ selection = proposed
+ return ReplacementPreview(comparison_id=chosen.id,
+ capability_contract_id=contract.id if contract else None,
+ current_deployment_id=chosen.current_deployment_id, candidate_deployment_id=chosen.candidate_deployment_id,
+ route_generation=contract.serving_generation if contract else None, same_embedding_space=same_space,
+ affected_bindings=bindings, production_approvals=approvals, blockers=blockers, selection=selection)
diff --git a/backend/src/modelforge_api/services/self_check.py b/backend/src/modelforge_api/services/self_check.py
new file mode 100644
index 0000000..767acca
--- /dev/null
+++ b/backend/src/modelforge_api/services/self_check.py
@@ -0,0 +1,903 @@
+from __future__ import annotations
+
+import shutil
+import stat
+from contextlib import suppress
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import redis
+import yaml # type: ignore[import-untyped]
+from pydantic import SecretStr
+from sqlalchemy import func, select, text
+from sqlalchemy.orm import Session
+
+from modelforge_api import __version__
+from modelforge_api.domain.configuration_reference import SETTING_DOCS
+from modelforge_api.domain.operator_identity import parse_operator_identities
+from modelforge_api.domain.recovery import BackupState
+from modelforge_api.domain.release import Compatibility, build_identity, schema_compatibility
+from modelforge_api.domain.self_check import (
+ SelfCheckActionCode,
+ SelfCheckKey,
+ SelfCheckNextAction,
+ SelfCheckOverall,
+ SelfCheckReport,
+ SelfCheckResult,
+ SelfCheckStatus,
+)
+from modelforge_api.persistence.models import (
+ Accelerator,
+ AcceleratorTelemetryLatest,
+ BackupSet,
+ CapabilityDeployment,
+ ComputeNode,
+ ManagedOperatorCredential,
+ ManagedOperatorCredentialActivation,
+ ManagedOperatorIdentity,
+ SchedulerPolicyRevision,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.settings import Settings
+
+_NEXT_ACTIONS: dict[SelfCheckKey, tuple[SelfCheckActionCode, str]] = {
+ SelfCheckKey.API: (
+ SelfCheckActionCode.REVIEW_API_IDENTITY,
+ "Confirm the API service and version against the intended deployment.",
+ ),
+ SelfCheckKey.DATABASE: (
+ SelfCheckActionCode.CHECK_DATABASE,
+ "Review database reachability and schema compatibility.",
+ ),
+ SelfCheckKey.REDIS: (
+ SelfCheckActionCode.CHECK_REDIS,
+ "Review Redis availability and its configured connection.",
+ ),
+ SelfCheckKey.MANIFESTS: (
+ SelfCheckActionCode.REVIEW_MANIFESTS,
+ "Review the authoritative manifest files and their deployment mount.",
+ ),
+ SelfCheckKey.STORAGE: (
+ SelfCheckActionCode.REVIEW_STORAGE,
+ "Review storage root availability and capacity evidence.",
+ ),
+ SelfCheckKey.NODES: (
+ SelfCheckActionCode.REVIEW_NODE_FRESHNESS,
+ "Review node heartbeat freshness and enrollment state.",
+ ),
+ SelfCheckKey.SCHEDULER: (
+ SelfCheckActionCode.REVIEW_SCHEDULER_INPUTS,
+ "Review scheduler policy and registered accelerator inputs.",
+ ),
+ SelfCheckKey.DEPLOYMENTS: (
+ SelfCheckActionCode.REVIEW_DEPLOYMENTS,
+ "Review stable production deployment records.",
+ ),
+ SelfCheckKey.BACKUP: (
+ SelfCheckActionCode.VERIFY_BACKUP_PAYLOAD,
+ "Review the latest verified backup and its retained payload.",
+ ),
+ SelfCheckKey.CREDENTIALS: (
+ SelfCheckActionCode.REVIEW_CREDENTIALS,
+ "Review credential source and expiry metadata without exposing secrets.",
+ ),
+ SelfCheckKey.CONFIGURATION: (
+ SelfCheckActionCode.COMPLETE_CONFIGURATION,
+ "Complete documented production-required settings.",
+ ),
+ SelfCheckKey.RELEASE: (
+ SelfCheckActionCode.VERIFY_RELEASE_IDENTITY,
+ "Verify the source commit and image digest for this release.",
+ ),
+}
+
+
+def _result(
+ key: str,
+ label: str,
+ status: SelfCheckStatus,
+ summary: str,
+ *,
+ detail: str | None = None,
+ evidence: dict[str, Any] | None = None,
+) -> SelfCheckResult:
+ check_key = SelfCheckKey(key)
+ action_code, action_summary = _NEXT_ACTIONS[check_key]
+ return SelfCheckResult(
+ key=check_key,
+ label=label,
+ status=status,
+ summary=summary,
+ detail=detail,
+ evidence=evidence or {},
+ next_action=SelfCheckNextAction(
+ code=(
+ SelfCheckActionCode.NONE_REQUIRED
+ if status is SelfCheckStatus.PASS
+ else action_code
+ ),
+ summary=(
+ "No operator follow-up is indicated by this check."
+ if status is SelfCheckStatus.PASS
+ else action_summary
+ ),
+ ),
+ )
+
+
+def _aware(value: datetime) -> datetime:
+ return value if value.tzinfo is not None else value.replace(tzinfo=UTC)
+
+
+def _rollback_failed_read(session: Session) -> None:
+ """Clear a failed read transaction so later independent checks can still report truth."""
+
+ with suppress(Exception):
+ session.rollback()
+
+
+def _database_check(session: Session) -> SelfCheckResult:
+ try:
+ session.execute(text("SELECT 1")).scalar_one()
+ revision = session.execute(text("SELECT version_num FROM alembic_version")).scalar_one_or_none()
+ compatibility = schema_compatibility(revision)
+ except Exception as exc: # noqa: BLE001 - self-check turns dependency failures into typed state
+ _rollback_failed_read(session)
+ return _result(
+ "database",
+ "PostgreSQL and schema",
+ SelfCheckStatus.UNKNOWN,
+ "Database or schema identity could not be read",
+ detail=type(exc).__name__,
+ )
+ if compatibility is not Compatibility.COMPATIBLE:
+ return _result(
+ "database",
+ "PostgreSQL and schema",
+ SelfCheckStatus.FAIL,
+ f"Schema {revision or '(none)'} is {compatibility.value}",
+ evidence={"schema_revision": revision, "compatibility": compatibility.value},
+ )
+ return _result(
+ "database",
+ "PostgreSQL and schema",
+ SelfCheckStatus.PASS,
+ f"Schema {revision} is compatible",
+ evidence={"schema_revision": revision, "compatibility": compatibility.value},
+ )
+
+
+def _api_check(settings: Settings) -> SelfCheckResult:
+ service_name = settings.service_name.strip()
+ if not service_name or not __version__:
+ return _result(
+ "api",
+ "API identity",
+ SelfCheckStatus.UNKNOWN,
+ "API service identity could not be established",
+ )
+ return _result(
+ "api",
+ "API identity",
+ SelfCheckStatus.PASS,
+ "API service identity is available",
+ evidence={"service_name": service_name, "version": __version__},
+ )
+
+
+def _redis_check(settings: Settings) -> SelfCheckResult:
+ client: redis.Redis | None = None
+ try:
+ client = redis.Redis.from_url(
+ settings.redis_url,
+ decode_responses=True,
+ socket_connect_timeout=1.0,
+ socket_timeout=1.0,
+ )
+ pong = bool(client.ping())
+ except Exception as exc: # noqa: BLE001 - dependency health must be reported, not raised
+ return _result(
+ "redis",
+ "Redis",
+ SelfCheckStatus.FAIL,
+ "Redis did not answer the read-only health probe",
+ detail=type(exc).__name__,
+ )
+ finally:
+ if client is not None:
+ with suppress(Exception):
+ client.close()
+ return _result(
+ "redis",
+ "Redis",
+ SelfCheckStatus.PASS if pong else SelfCheckStatus.FAIL,
+ "Redis answered PING" if pong else "Redis PING returned a non-success response",
+ )
+
+
+def _manifest_check(registry: ManifestRegistry) -> SelfCheckResult:
+ try:
+ candidates = registry.candidates()
+ policies = registry.policies()
+ counts = {
+ "capabilities": len(registry.capabilities()),
+ "projects": len(registry.projects()),
+ "candidates": len(candidates.candidates),
+ "benchmarks": len(registry.benchmarks()),
+ "policy_sections": len(policies.model_dump()),
+ }
+ except (FileNotFoundError, TypeError, ValueError, yaml.YAMLError) as exc:
+ return _result(
+ "manifests",
+ "Authoritative manifests",
+ SelfCheckStatus.FAIL,
+ "One or more authoritative manifests could not be loaded",
+ detail=type(exc).__name__,
+ )
+ except Exception as exc: # noqa: BLE001 - an unreadable manifest state is not evidence of health
+ return _result(
+ "manifests",
+ "Authoritative manifests",
+ SelfCheckStatus.UNKNOWN,
+ "Authoritative manifest state could not be inspected",
+ detail=type(exc).__name__,
+ )
+ return _result(
+ "manifests",
+ "Authoritative manifests",
+ SelfCheckStatus.PASS,
+ "All manifest classes loaded successfully",
+ evidence=counts,
+ )
+
+
+def _storage_check(settings: Settings) -> SelfCheckResult:
+ roots = {
+ "artifact_root": Path(settings.artifact_root),
+ "quarantine_root": Path(settings.quarantine_root),
+ "backup_root": Path(settings.backup_root),
+ }
+ evidence: dict[str, Any] = {}
+ invalid: list[str] = []
+ path_unknown: list[str] = []
+ capacity_unknown: list[str] = []
+ for key, path in roots.items():
+ entry: dict[str, Any] = {
+ "path": str(path),
+ "writability": "not_probed_read_only",
+ }
+ try:
+ path_stat = path.stat()
+ except (FileNotFoundError, NotADirectoryError):
+ entry.update({"exists": False, "is_directory": False})
+ invalid.append(key)
+ except OSError as exc:
+ entry.update(
+ {"exists": None, "is_directory": None, "stat_error": type(exc).__name__}
+ )
+ path_unknown.append(key)
+ else:
+ is_directory = stat.S_ISDIR(path_stat.st_mode)
+ entry.update({"exists": True, "is_directory": is_directory})
+ if not is_directory:
+ invalid.append(key)
+ else:
+ try:
+ usage = shutil.disk_usage(path)
+ entry.update(
+ {
+ "capacity_bytes": usage.total,
+ "free_bytes": usage.free,
+ "used_bytes": usage.used,
+ }
+ )
+ except OSError as exc:
+ entry["capacity_error"] = type(exc).__name__
+ capacity_unknown.append(key)
+ evidence[key] = entry
+ if invalid:
+ return _result(
+ "storage",
+ "Storage roots",
+ SelfCheckStatus.FAIL,
+ "One or more configured storage roots are unavailable",
+ detail=", ".join(invalid),
+ evidence=evidence,
+ )
+ if path_unknown or capacity_unknown:
+ return _result(
+ "storage",
+ "Storage roots",
+ SelfCheckStatus.UNKNOWN,
+ "Storage root status or capacity could not be read for every root",
+ detail=(
+ "Writability is intentionally not probed by this read-only workflow; "
+ "metadata unavailable for "
+ f"{', '.join(path_unknown + capacity_unknown)}"
+ ),
+ evidence=evidence,
+ )
+ return _result(
+ "storage",
+ "Storage roots",
+ SelfCheckStatus.PASS,
+ "Configured storage roots exist and capacity was inspected without writes",
+ detail="Writability is intentionally not probed by this read-only workflow",
+ evidence=evidence,
+ )
+
+
+def _node_check(session: Session, settings: Settings, now: datetime) -> SelfCheckResult:
+ try:
+ rows = list(
+ session.execute(
+ select(ComputeNode.id, ComputeNode.display_name, ComputeNode.last_heartbeat_at).where(
+ ComputeNode.decommissioned_at.is_(None)
+ )
+ )
+ )
+ except Exception as exc: # noqa: BLE001
+ _rollback_failed_read(session)
+ return _result(
+ "nodes",
+ "Compute node freshness",
+ SelfCheckStatus.UNKNOWN,
+ "Compute node freshness could not be evaluated",
+ detail=type(exc).__name__,
+ )
+ if not rows:
+ return _result(
+ "nodes",
+ "Compute node freshness",
+ SelfCheckStatus.WARN,
+ "No active compute nodes are enrolled",
+ evidence={"active_nodes": 0},
+ )
+
+ stale: list[str] = []
+ offline: list[str] = []
+ unknown: list[str] = []
+ ages: dict[str, float | None] = {}
+ for row in rows:
+ name = str(row.display_name)
+ heartbeat = row.last_heartbeat_at
+ if heartbeat is None:
+ unknown.append(name)
+ ages[name] = None
+ continue
+ age = max(0.0, (now - _aware(heartbeat)).total_seconds())
+ ages[name] = round(age, 3)
+ if age > settings.node_offline_after_seconds:
+ offline.append(name)
+ elif age > settings.node_stale_after_seconds:
+ stale.append(name)
+
+ evidence = {
+ "active_nodes": len(rows),
+ "stale_nodes": stale,
+ "offline_nodes": offline,
+ "heartbeat_unknown": unknown,
+ "heartbeat_age_seconds": ages,
+ "stale_after_seconds": settings.node_stale_after_seconds,
+ "offline_after_seconds": settings.node_offline_after_seconds,
+ }
+ if len(offline) == len(rows):
+ return _result(
+ "nodes",
+ "Compute node freshness",
+ SelfCheckStatus.FAIL,
+ "All active compute nodes are offline",
+ evidence=evidence,
+ )
+ if unknown:
+ return _result(
+ "nodes",
+ "Compute node freshness",
+ SelfCheckStatus.UNKNOWN,
+ "At least one active compute node has no heartbeat evidence",
+ evidence=evidence,
+ )
+ if stale or offline:
+ return _result(
+ "nodes",
+ "Compute node freshness",
+ SelfCheckStatus.WARN,
+ "At least one compute node is stale, offline, or has no heartbeat",
+ evidence=evidence,
+ )
+ return _result(
+ "nodes",
+ "Compute node freshness",
+ SelfCheckStatus.PASS,
+ "All active compute nodes have fresh heartbeats",
+ evidence=evidence,
+ )
+
+
+def _scheduler_check(session: Session, settings: Settings, now: datetime) -> SelfCheckResult:
+ try:
+ active_accelerators = int(
+ session.scalar(
+ select(func.count(Accelerator.id)).where(Accelerator.status == "active")
+ )
+ or 0
+ )
+ telemetry_missing = int(
+ session.scalar(
+ select(func.count(Accelerator.id))
+ .select_from(Accelerator)
+ .outerjoin(AcceleratorTelemetryLatest)
+ .where(
+ Accelerator.status == "active",
+ AcceleratorTelemetryLatest.accelerator_id.is_(None),
+ )
+ )
+ or 0
+ )
+ telemetry_stale = int(
+ session.scalar(
+ select(func.count(Accelerator.id))
+ .join(AcceleratorTelemetryLatest)
+ .where(
+ Accelerator.status == "active",
+ AcceleratorTelemetryLatest.observed_at
+ < now - timedelta(seconds=settings.scheduler_telemetry_stale_seconds),
+ )
+ )
+ or 0
+ )
+ capacity_unknown = int(
+ session.scalar(
+ select(func.count(Accelerator.id)).where(
+ Accelerator.status == "active",
+ Accelerator.total_vram_bytes.is_(None),
+ )
+ )
+ or 0
+ )
+ active_policies = list(
+ session.execute(
+ select(SchedulerPolicyRevision.revision).where(
+ SchedulerPolicyRevision.active.is_(True)
+ ).limit(2)
+ )
+ )
+ except Exception as exc: # noqa: BLE001 - scheduler readiness is unknown when inputs cannot be read
+ _rollback_failed_read(session)
+ return _result(
+ "scheduler",
+ "Scheduler inputs",
+ SelfCheckStatus.UNKNOWN,
+ "Scheduler policy or accelerator inputs could not be read",
+ detail=type(exc).__name__,
+ )
+
+ evidence: dict[str, Any] = {
+ "active_accelerators": active_accelerators,
+ "telemetry_missing": telemetry_missing,
+ "telemetry_stale": telemetry_stale,
+ "capacity_unknown": capacity_unknown,
+ "active_policy_revisions": len(active_policies),
+ "deployment_queue_limit": settings.scheduler_global_queue_limit,
+ "telemetry_stale_after_seconds": settings.scheduler_telemetry_stale_seconds,
+ }
+ if active_accelerators == 0:
+ return _result(
+ "scheduler",
+ "Scheduler inputs",
+ SelfCheckStatus.UNKNOWN,
+ "Scheduler readiness cannot be inferred without active accelerator inputs",
+ evidence=evidence,
+ )
+ if telemetry_missing or telemetry_stale or capacity_unknown:
+ return _result(
+ "scheduler",
+ "Scheduler inputs",
+ SelfCheckStatus.UNKNOWN,
+ "Scheduler inputs lack fresh telemetry or known accelerator capacity",
+ evidence=evidence,
+ )
+ if len(active_policies) > 1:
+ return _result(
+ "scheduler",
+ "Scheduler inputs",
+ SelfCheckStatus.UNKNOWN,
+ "Multiple active scheduler policies make the effective policy ambiguous",
+ evidence=evidence,
+ )
+ if not active_policies:
+ evidence["policy_source"] = "validated_deployment_settings"
+ return _result(
+ "scheduler",
+ "Scheduler inputs",
+ SelfCheckStatus.WARN,
+ "Scheduler has accelerator inputs and is using deployment policy defaults",
+ evidence=evidence,
+ )
+ evidence["policy_revision"] = str(active_policies[0][0])
+ evidence["policy_source"] = "persisted"
+ return _result(
+ "scheduler",
+ "Scheduler inputs",
+ SelfCheckStatus.PASS,
+ "Scheduler has active accelerator inputs and one persisted policy",
+ evidence=evidence,
+ )
+
+
+def _deployment_check(session: Session) -> SelfCheckResult:
+ try:
+ production_count = int(
+ session.scalar(
+ select(func.count(CapabilityDeployment.id)).where(
+ CapabilityDeployment.status == "stable",
+ CapabilityDeployment.production.is_(True),
+ )
+ )
+ or 0
+ )
+ except Exception as exc: # noqa: BLE001
+ _rollback_failed_read(session)
+ return _result(
+ "deployments",
+ "Production capability deployments",
+ SelfCheckStatus.UNKNOWN,
+ "Production deployment state could not be read",
+ detail=type(exc).__name__,
+ )
+ if production_count == 0:
+ return _result(
+ "deployments",
+ "Production capability deployments",
+ SelfCheckStatus.WARN,
+ "No stable production capability deployment is active",
+ evidence={"stable_production_deployments": 0},
+ )
+ return _result(
+ "deployments",
+ "Production capability deployments",
+ SelfCheckStatus.PASS,
+ f"{production_count} stable production deployment(s) are recorded",
+ evidence={"stable_production_deployments": production_count},
+ )
+
+
+def _backup_check(session: Session, settings: Settings, now: datetime) -> SelfCheckResult:
+ try:
+ latest = session.execute(
+ select(BackupSet.backup_id, BackupSet.verified_at)
+ .where(
+ BackupSet.state == BackupState.VERIFIED.value,
+ BackupSet.verified_at.is_not(None),
+ )
+ .order_by(BackupSet.verified_at.desc())
+ .limit(1)
+ ).one_or_none()
+ except Exception as exc: # noqa: BLE001
+ _rollback_failed_read(session)
+ return _result(
+ "backup",
+ "Verified backup freshness",
+ SelfCheckStatus.UNKNOWN,
+ "Backup freshness could not be read",
+ detail=type(exc).__name__,
+ )
+ if latest is None or latest.verified_at is None:
+ return _result(
+ "backup",
+ "Verified backup freshness",
+ SelfCheckStatus.WARN,
+ "No verified backup is recorded",
+ evidence={"verified_backup": False},
+ )
+ age = max(0.0, (now - _aware(latest.verified_at)).total_seconds())
+ stale = age > settings.backup_stale_after_seconds
+ return _result(
+ "backup",
+ "Verified backup freshness",
+ SelfCheckStatus.WARN if stale else SelfCheckStatus.PASS,
+ "Newest verified backup is stale" if stale else "Newest verified backup is within policy age",
+ evidence={
+ "backup_id": latest.backup_id,
+ "age_seconds": round(age, 3),
+ "stale_after_seconds": settings.backup_stale_after_seconds,
+ },
+ )
+
+
+def _credentials_check(
+ session: Session, settings: Settings, now: datetime
+) -> SelfCheckResult:
+ document = settings.operator_identities
+ try:
+ deployment_identities = (
+ parse_operator_identities(document.get_secret_value())
+ if document is not None and document.get_secret_value()
+ else ()
+ )
+ except ValueError as exc:
+ return _result(
+ "credentials",
+ "Operator credential metadata",
+ SelfCheckStatus.UNKNOWN,
+ "Deployment operator credential metadata could not be parsed",
+ detail=type(exc).__name__,
+ )
+
+ deployment_roles = {role: 0 for role in ("viewer", "operator", "admin")}
+ deployment_expired = 0
+ deployment_expiring = 0
+ expiring_before = now + timedelta(days=7)
+ deployment_admins = 0
+ for identity in deployment_identities:
+ deployment_roles[identity.role] += 1
+ expires_at = _aware(identity.expires_at)
+ if expires_at <= now:
+ deployment_expired += 1
+ elif expires_at <= expiring_before:
+ deployment_expiring += 1
+ if identity.role == "admin" and expires_at > now:
+ deployment_admins += 1
+
+ try:
+ enabled_rows = list(
+ session.execute(
+ select(ManagedOperatorIdentity.role, func.count(ManagedOperatorIdentity.slug))
+ .where(ManagedOperatorIdentity.enabled.is_(True))
+ .group_by(ManagedOperatorIdentity.role)
+ )
+ )
+ usable_managed = int(
+ session.scalar(
+ select(func.count(ManagedOperatorCredential.id))
+ .join(ManagedOperatorIdentity)
+ .join(ManagedOperatorCredentialActivation)
+ .where(
+ ManagedOperatorIdentity.enabled.is_(True),
+ ManagedOperatorCredential.revoked_at.is_(None),
+ ManagedOperatorCredential.superseded_at.is_(None),
+ ManagedOperatorCredential.expires_at > now,
+ )
+ )
+ or 0
+ )
+ usable_managed_admins = int(
+ session.scalar(
+ select(func.count(ManagedOperatorCredential.id))
+ .join(ManagedOperatorIdentity)
+ .join(ManagedOperatorCredentialActivation)
+ .where(
+ ManagedOperatorIdentity.enabled.is_(True),
+ ManagedOperatorIdentity.role == "admin",
+ ManagedOperatorCredential.revoked_at.is_(None),
+ ManagedOperatorCredential.superseded_at.is_(None),
+ ManagedOperatorCredential.expires_at > now,
+ )
+ )
+ or 0
+ )
+ managed_expired = int(
+ session.scalar(
+ select(func.count(ManagedOperatorCredential.id))
+ .join(ManagedOperatorIdentity)
+ .where(
+ ManagedOperatorIdentity.enabled.is_(True),
+ ManagedOperatorCredential.revoked_at.is_(None),
+ ManagedOperatorCredential.superseded_at.is_(None),
+ ManagedOperatorCredential.expires_at <= now,
+ )
+ )
+ or 0
+ )
+ managed_expiring = int(
+ session.scalar(
+ select(func.count(ManagedOperatorCredential.id))
+ .join(ManagedOperatorIdentity)
+ .where(
+ ManagedOperatorIdentity.enabled.is_(True),
+ ManagedOperatorCredential.revoked_at.is_(None),
+ ManagedOperatorCredential.superseded_at.is_(None),
+ ManagedOperatorCredential.expires_at > now,
+ ManagedOperatorCredential.expires_at <= expiring_before,
+ )
+ )
+ or 0
+ )
+ except Exception as exc: # noqa: BLE001 - credential store state is unknown when unreadable
+ _rollback_failed_read(session)
+ return _result(
+ "credentials",
+ "Operator credential metadata",
+ SelfCheckStatus.UNKNOWN,
+ "Managed operator credential metadata could not be read",
+ detail=type(exc).__name__,
+ evidence={
+ "deployment_identity_count": len(deployment_identities),
+ "deployment_roles": deployment_roles,
+ "deployment_expired_identities": deployment_expired,
+ "deployment_expiring_within_7_days": deployment_expiring,
+ },
+ )
+
+ managed_roles = {str(role): int(count) for role, count in enabled_rows}
+ legacy_admin_configured = _legacy_admin_token_configured(settings.operator_api_key)
+ password_session_configured = (
+ legacy_admin_configured and _configured(settings.operator_password_hash)
+ )
+ sources: list[str] = []
+ if legacy_admin_configured:
+ sources.append("legacy_operator_api_key")
+ if deployment_identities:
+ sources.append("deployment_operator_identities")
+ if managed_roles:
+ sources.append("managed_operator_identities")
+ if password_session_configured:
+ sources.append("operator_password_session")
+ administrator_available = bool(
+ legacy_admin_configured or deployment_admins or usable_managed_admins
+ )
+ evidence = {
+ "configured_sources": sources,
+ "deployment_identity_count": len(deployment_identities),
+ "deployment_roles": deployment_roles,
+ "deployment_expired_identities": deployment_expired,
+ "deployment_expiring_within_7_days": deployment_expiring,
+ "managed_enabled_identities_by_role": managed_roles,
+ "managed_usable_credentials": usable_managed,
+ "managed_usable_admin_credentials": usable_managed_admins,
+ "managed_expired_credentials": managed_expired,
+ "managed_expiring_within_7_days": managed_expiring,
+ "operator_password_session_configured": password_session_configured,
+ "administrator_metadata_present": administrator_available,
+ }
+ if not sources:
+ return _result(
+ "credentials",
+ "Operator credential metadata",
+ SelfCheckStatus.FAIL,
+ "No operator credential source is configured",
+ evidence=evidence,
+ )
+ if not administrator_available:
+ return _result(
+ "credentials",
+ "Operator credential metadata",
+ SelfCheckStatus.FAIL,
+ "No usable administrator credential is represented in available metadata",
+ evidence=evidence,
+ )
+ if deployment_expired or deployment_expiring or managed_expired or managed_expiring:
+ return _result(
+ "credentials",
+ "Operator credential metadata",
+ SelfCheckStatus.WARN,
+ "One or more deployment operator credentials are expired or nearing expiry",
+ evidence=evidence,
+ )
+ return _result(
+ "credentials",
+ "Operator credential metadata",
+ SelfCheckStatus.PASS,
+ "Credential source and administrator metadata are available",
+ evidence=evidence,
+ )
+
+
+def _configured(value: object) -> bool:
+ if value is None:
+ return False
+ if isinstance(value, SecretStr):
+ return bool(value.get_secret_value())
+ if isinstance(value, str):
+ return bool(value)
+ return True
+
+
+def _legacy_admin_token_configured(value: object) -> bool:
+ if not isinstance(value, SecretStr):
+ return False
+ token = value.get_secret_value()
+ return bool(
+ token
+ and len(token) <= 512
+ and not token.startswith(("mfopm_", "mfop_", "mfsvc_", "mfnode_", "mfenroll_"))
+ )
+
+
+def _configuration_check(settings: Settings) -> SelfCheckResult:
+ if settings.env != "production":
+ return _result(
+ "configuration",
+ "Required production configuration",
+ SelfCheckStatus.PASS,
+ "Production-only required-setting gate is not applicable in this environment",
+ evidence={"environment": settings.env},
+ )
+ missing = [
+ key
+ for key, documentation in SETTING_DOCS.items()
+ if documentation.required_in_production and not _configured(getattr(settings, key))
+ ]
+ if missing:
+ return _result(
+ "configuration",
+ "Required production configuration",
+ SelfCheckStatus.FAIL,
+ "Required production configuration is incomplete",
+ evidence={"missing_settings": sorted(missing), "environment": settings.env},
+ )
+ return _result(
+ "configuration",
+ "Required production configuration",
+ SelfCheckStatus.PASS,
+ "All documented production-required settings are configured",
+ evidence={"environment": settings.env, "missing_settings": []},
+ )
+
+
+def _release_check(settings: Settings) -> SelfCheckResult:
+ identity = build_identity(
+ source_commit=settings.build_commit,
+ built_at=settings.build_timestamp,
+ image_digest=settings.build_image_digest,
+ )
+ missing = [
+ name
+ for name, value in {
+ "source_commit": identity.source_commit,
+ "image_digest": identity.image_digest,
+ }.items()
+ if not value
+ ]
+ status = (
+ SelfCheckStatus.FAIL
+ if settings.env == "production" and missing
+ else SelfCheckStatus.WARN
+ if missing
+ else SelfCheckStatus.PASS
+ )
+ return _result(
+ "release",
+ "Release identity",
+ status,
+ "Release provenance is complete" if not missing else "Release provenance is incomplete",
+ evidence={
+ "version": identity.version,
+ "channel": identity.channel,
+ "source_commit": identity.source_commit,
+ "image_digest": identity.image_digest,
+ "missing_identity_fields": missing,
+ },
+ )
+
+
+def run_self_check(
+ session: Session,
+ settings: Settings,
+ registry: ManifestRegistry,
+ *,
+ now: datetime | None = None,
+) -> SelfCheckReport:
+ observed_at = _aware(now or datetime.now(UTC))
+ checks = [
+ _api_check(settings),
+ _database_check(session),
+ _redis_check(settings),
+ _manifest_check(registry),
+ _storage_check(settings),
+ _node_check(session, settings, observed_at),
+ _scheduler_check(session, settings, observed_at),
+ _deployment_check(session),
+ _backup_check(session, settings, observed_at),
+ _credentials_check(session, settings, observed_at),
+ _configuration_check(settings),
+ _release_check(settings),
+ ]
+ if any(check.status is SelfCheckStatus.FAIL for check in checks):
+ overall = SelfCheckOverall.UNHEALTHY
+ elif any(check.status in {SelfCheckStatus.WARN, SelfCheckStatus.UNKNOWN} for check in checks):
+ overall = SelfCheckOverall.DEGRADED
+ else:
+ overall = SelfCheckOverall.HEALTHY
+ return SelfCheckReport(observed_at=observed_at, overall=overall, checks=checks)
diff --git a/backend/src/modelforge_api/services/serving.py b/backend/src/modelforge_api/services/serving.py
index edfa8be..ab44cf1 100644
--- a/backend/src/modelforge_api/services/serving.py
+++ b/backend/src/modelforge_api/services/serving.py
@@ -6,6 +6,7 @@ import itertools
import json
import math
import posixpath
+import re
import secrets
import time
import uuid
@@ -14,11 +15,18 @@ from datetime import UTC, datetime, timedelta
from pathlib import PurePosixPath
from typing import Any, cast
-from sqlalchemy import select
+from redis.exceptions import RedisError
+from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
+from modelforge_api.db import rows_affected
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.lab import LabEmbeddingInput, LabInput, LabPreflightRequest, LabTarget
from modelforge_api.domain.serving import (
+ AgentServingJobCancellationAck,
+ AgentServingJobCancellationCheck,
+ AgentServingJobCancellationStatus,
AgentServingJobComplete,
AgentServingJobFailure,
AgentServingJobLease,
@@ -60,6 +68,9 @@ from modelforge_api.domain.serving import (
ServiceClientCreate,
ServiceClientCreated,
ServiceClientResponse,
+ ServiceCredentialResponse,
+ ServiceCredentialRevoke,
+ ServiceCredentialRotate,
SupplyChainReview,
)
from modelforge_api.persistence.models import (
@@ -70,8 +81,11 @@ from modelforge_api.persistence.models import (
CapabilityResourceEnvelope,
ComputeNode,
CoResidencyEvidence,
+ DeploymentCandidate,
EmbeddingSpace,
+ ExternalCreativeAllocation,
GatewayRequest,
+ LabExecutionRun,
PlacementPlanRecord,
ProductionExecutionApproval,
Project,
@@ -88,6 +102,7 @@ from modelforge_api.persistence.models import (
)
from modelforge_api.persistence.serving_repository import ServingRepository
from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.configuration_changes import resolve_serving_settings
from modelforge_api.services.gpu_scheduler import (
AccountingSnapshot,
AttributionConfidence,
@@ -101,6 +116,7 @@ from modelforge_api.services.gpu_scheduler import (
plan_placement,
required_envelope,
)
+from modelforge_api.services.lab import LabPreflightService
from modelforge_api.services.manifest_registry import ManifestRegistry
from modelforge_api.services.runtime import RuntimeService, _selected_manifest_rows
from modelforge_api.settings import Settings
@@ -127,6 +143,71 @@ def _canonical_hash(payload: Any) -> str:
return hashlib.sha256(raw.encode()).hexdigest()
+def _bounded_worker_metrics(raw_metrics: Any) -> dict[str, int | float | str]:
+ """Keep only finite, bounded serving-worker observations; never persist arbitrary payloads."""
+ if not isinstance(raw_metrics, dict):
+ return {}
+ metrics: dict[str, int | float | str] = {}
+ gpu_uuid = raw_metrics.get("gpu_uuid")
+ if isinstance(gpu_uuid, str) and 0 < len(gpu_uuid) <= 255:
+ metrics["gpu_uuid"] = gpu_uuid
+ for key in ("used_vram_bytes", "process_used_vram_bytes", "free_vram_bytes"):
+ value = raw_metrics.get(key)
+ if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 2**63 - 1:
+ metrics[key] = value
+ utilization = raw_metrics.get("gpu_utilization_percent")
+ if isinstance(utilization, int) and not isinstance(utilization, bool) and 0 <= utilization <= 100:
+ metrics["gpu_utilization_percent"] = utilization
+ temperature = raw_metrics.get("temperature_c")
+ if (
+ isinstance(temperature, int | float)
+ and not isinstance(temperature, bool)
+ and -50 <= temperature <= 200
+ and math.isfinite(float(temperature))
+ ):
+ metrics["temperature_c"] = temperature
+ power_draw = raw_metrics.get("power_draw_w")
+ if (
+ isinstance(power_draw, int | float)
+ and not isinstance(power_draw, bool)
+ and 0 <= power_draw <= 100_000
+ and math.isfinite(float(power_draw))
+ ):
+ metrics["power_draw_w"] = power_draw
+ return metrics
+
+
+def _worker_resource_observation(
+ raw_metrics: Any, *, expected_gpu_uuid: str | None
+) -> dict[str, Any]:
+ metrics = _bounded_worker_metrics(raw_metrics)
+ expected = {
+ "gpu_uuid",
+ "used_vram_bytes",
+ "process_used_vram_bytes",
+ "free_vram_bytes",
+ "gpu_utilization_percent",
+ "temperature_c",
+ "power_draw_w",
+ }
+ observed = set(metrics)
+ gpu_uuid = metrics.get("gpu_uuid")
+ device_matches_target = (
+ gpu_uuid == expected_gpu_uuid if isinstance(gpu_uuid, str) and expected_gpu_uuid else None
+ )
+ return {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": (
+ "complete"
+ if expected.issubset(observed) and device_matches_target is True
+ else "partial" if observed else "unavailable"
+ ),
+ "device_matches_target": device_matches_target,
+ "metrics": metrics,
+ }
+
+
def _percentile(values: list[float], fraction: float) -> float | None:
if not values:
return None
@@ -188,12 +269,22 @@ class ServingService:
actor_id: str = "local-operator",
) -> None:
self.session = session
- self.settings = settings
+ self._deployment_settings = settings
+ self._effective_settings: Settings | None = None
+ self.authenticated_credential_id: uuid.UUID | None = None
self.manifests = manifests
self.payload_store = payload_store
self.repo = ServingRepository(session)
self.audit = AuditWriter(session, actor_type, actor_id)
+ @property
+ def settings(self) -> Settings:
+ # Authentication-only service instances must reject absent/invalid credentials before
+ # touching configuration storage. The first policy consumer pins the request snapshot.
+ if self._effective_settings is None:
+ self._effective_settings = resolve_serving_settings(self.session, self._deployment_settings)
+ return self._effective_settings
+
def _commit(self) -> None:
try:
self.session.commit()
@@ -435,7 +526,7 @@ class ServingService:
raise ServingError(
409, "PRODUCTION_APPROVAL_STALE", "production approval is not current"
)
- contract = self.session.get(CapabilityContract, approval.capability_contract_id)
+ contract = self.repo.lock_contract_route(approval.capability_contract_id)
if not contract:
raise ServingError(
409, "CAPABILITY_NOT_FOUND", "approved capability contract disappeared"
@@ -511,6 +602,7 @@ class ServingService:
raise ServingError(
409, "STABLE_DEPLOYMENT_EXISTS", "drain the current stable deployment first"
)
+ self._advance_serving_generation(contract)
deployment = self.repo.add(
CapabilityDeployment(
capability_contract_id=contract.id,
@@ -625,6 +717,7 @@ class ServingService:
)
return CapabilityDeploymentResponse(
id=item.id,
+ route_generation=contract.serving_generation,
capability=capability.key,
contract_version=contract.version,
deployment_candidate_id=item.deployment_candidate_id,
@@ -940,6 +1033,15 @@ class ServingService:
return "unavailable"
def create_client(self, request: ServiceClientCreate) -> ServiceClientCreated:
+ try:
+ return self._create_client(request)
+ except IntegrityError as exc:
+ self.session.rollback()
+ raise ServingError(409, "CLIENT_EXISTS", "service client name already exists") from exc
+
+ def _create_client(self, request: ServiceClientCreate) -> ServiceClientCreated:
+ if request.credential_expires_at is not None and request.credential_expires_at <= _now():
+ raise ServingError(422, "CREDENTIAL_EXPIRY_INVALID", "credential expiry must be in the future")
if self.repo.client_by_name(request.name):
raise ServingError(409, "CLIENT_EXISTS", "service client name already exists")
project: Project | None = None
@@ -976,6 +1078,10 @@ class ServingService:
(item for item in bindings if item.channel in preferred_channels),
bindings[0] if bindings else None,
)
+ if request.project_binding_id is not None:
+ project_binding = next(
+ (item for item in bindings if item.id == request.project_binding_id), None
+ )
if project_binding is None:
raise ServingError(
409,
@@ -1043,6 +1149,7 @@ class ServingService:
integration_environment=client.integration_environment,
purpose=client.purpose,
credential_prefix=credential.secret_prefix if credential else None,
+ credential_id=credential.id if credential else None,
credential_created_at=credential.created_at if credential else None,
credential_expires_at=credential.expires_at if credential else None,
credential_revoked_at=credential.revoked_at if credential else None,
@@ -1053,27 +1160,79 @@ class ServingService:
def clients(self) -> list[ServiceClientResponse]:
return [self.client_response(item) for item in self.repo.clients()]
- def revoke_client_credential(self, client_id: uuid.UUID) -> ServiceClientResponse:
+ def credential_history(self, client_id: uuid.UUID) -> list[ServiceCredentialResponse]:
+ if self.repo.client(client_id) is None:
+ raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
+ return [ServiceCredentialResponse.model_validate(item) for item in self.session.scalars(
+ select(ServiceCredential).where(ServiceCredential.service_client_id == client_id)
+ .order_by(ServiceCredential.created_at.desc()).limit(100)
+ )]
+
+ def _claim_credential(self, credential: ServiceCredential) -> None:
+ # Conditional UPDATE serializes rotation/revocation on the same current credential.
+ # An expired SQLAlchemy identity-map entry can never make a stale browser action win.
+ claimed = self.session.execute(update(ServiceCredential).where(
+ ServiceCredential.id == credential.id,
+ ServiceCredential.revoked_at.is_(None),
+ ServiceCredential.superseded_at.is_(None),
+ ).values(superseded_at=_now()).execution_options(synchronize_session=False))
+ if rows_affected(claimed) != 1:
+ self.session.rollback()
+ raise ServingError(409, "CREDENTIAL_CHANGED", "credential changed; refresh before trying again")
+ self.session.refresh(credential)
+
+ def revoke_client_credential(
+ self, client_id: uuid.UUID, request: ServiceCredentialRevoke | None = None
+ ) -> ServiceClientResponse:
client = self.repo.client(client_id)
credential = self.repo.active_credential(client_id) if client else None
if not client:
raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
- if credential and not credential.revoked_at:
- credential.revoked_at = _now()
+ if request and (credential is None or credential.id != request.expected_credential_id):
+ raise ServingError(409, "CREDENTIAL_CHANGED", "credential changed; refresh before trying again")
+ if request and request.confirmation != client.name:
+ raise ServingError(422, "CONFIRMATION_REQUIRED", "type the exact service client name")
+ if credential:
+ self._claim_credential(credential)
+ self.session.execute(update(ServiceCredential).where(
+ ServiceCredential.service_client_id == client_id,
+ ServiceCredential.revoked_at.is_(None),
+ ).values(revoked_at=_now()))
self.audit.write(
"CLIENT_CREDENTIAL_REVOKED", "service_client", str(client.id), {"name": client.name}
)
self._commit()
return self.client_response(client, credential)
- def rotate_client_credential(self, client_id: uuid.UUID) -> ServiceClientCreated:
+ def rotate_client_credential(
+ self, client_id: uuid.UUID, request: ServiceCredentialRotate | None = None
+ ) -> ServiceClientCreated:
client = self.repo.client(client_id)
if client is None:
raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
current = self.repo.active_credential(client_id)
- if current is not None:
- current.revoked_at = _now()
- self.session.flush()
+ if current is None or (request and current.id != request.expected_credential_id):
+ raise ServingError(409, "CREDENTIAL_CHANGED", "credential changed; refresh before trying again")
+ now = _now()
+ expiry = request.expires_at if request else current.expires_at
+ if expiry is not None and _aware(expiry) <= now:
+ raise ServingError(422, "CREDENTIAL_EXPIRY_INVALID", "credential expiry must be in the future")
+ if client.status != "active" or client.disabled_at:
+ raise ServingError(409, "CLIENT_DISABLED", "disabled clients cannot rotate credentials")
+ self._claim_credential(current)
+ # A second rotation retires any older overlap: at most current + one previous key.
+ self.session.execute(update(ServiceCredential).where(
+ ServiceCredential.service_client_id == client_id,
+ ServiceCredential.id != current.id,
+ ServiceCredential.revoked_at.is_(None),
+ ).values(revoked_at=now))
+ overlap = request.overlap_seconds if request else 0
+ if overlap:
+ deadline = now + timedelta(seconds=overlap)
+ current.expires_at = min(_aware(current.expires_at), deadline) if current.expires_at else deadline
+ else:
+ current.revoked_at = now
+ self.session.flush()
secret = "mf_" + secrets.token_urlsafe(36)
credential = self.repo.add(
ServiceCredential(
@@ -1081,10 +1240,14 @@ class ServingService:
secret_hash=hashlib.sha256(secret.encode()).hexdigest(),
secret_prefix=secret[:12],
created_at=_now(),
+ expires_at=expiry,
)
)
self.audit.write(
- "CLIENT_CREDENTIAL_ROTATED", "service_client", str(client.id), {"name": client.name}
+ "CLIENT_CREDENTIAL_ROTATED", "service_client", str(client.id),
+ {"name": client.name, "previous_credential_id": str(current.id),
+ "credential_id": str(credential.id), "overlap_seconds": overlap,
+ "expires_at": _aware(expiry).isoformat() if expiry else None}
)
self._commit()
return ServiceClientCreated(
@@ -1337,7 +1500,9 @@ class ServingService:
if bound_capability and contract
else None
)
- if binding is None or binding.deprecated_at is not None or bound_key != capability:
+ project = self.session.get(Project, binding.project_id) if binding else None
+ if (binding is None or binding.deprecated_at is not None or bound_key != capability
+ or project is None or not project.active or contract is None or contract.deprecated_at is not None):
raise ServingError(
403, "PROJECT_CAPABILITY_NOT_BOUND", "project binding is inactive"
)
@@ -1352,6 +1517,7 @@ class ServingService:
def authenticate_with_evidence(
self, authorization: str | None, capability: str
) -> tuple[ServiceClient, CapabilityAuthenticationEvidence]:
+ self.authenticated_credential_id = None
secret = self._credential_secret(authorization)
credential_sha256 = hashlib.sha256(secret.encode()).hexdigest()
match = self.repo.credential_by_hash(credential_sha256)
@@ -1368,6 +1534,7 @@ class ServingService:
credential.last_used_at = now
client.last_used_at = now
self._commit_authentication()
+ self.authenticated_credential_id = credential.id
return client, CapabilityAuthenticationEvidence(
credential_id=credential.id,
client_id=client.id,
@@ -1387,6 +1554,7 @@ class ServingService:
) -> ServiceClient:
"""Revalidate current scope/client state while reusing the pre-body credential proof."""
+ self.authenticated_credential_id = None
secret = self._credential_secret(authorization)
credential_sha256 = hashlib.sha256(secret.encode()).hexdigest()
if capability != evidence.capability or not hmac.compare_digest(
@@ -1412,9 +1580,15 @@ class ServingService:
capability,
enforce_request_limits=False,
)
+ self.authenticated_credential_id = credential.id
return client
- def _budget(self, deployment: CapabilityDeployment) -> SchedulerBudgetResponse:
+ def _budget(
+ self,
+ deployment: CapabilityDeployment,
+ *,
+ update_pressure: bool = True,
+ ) -> SchedulerBudgetResponse:
node = self.repo.node(deployment.compute_node_id)
accelerator = self.repo.accelerator(deployment.accelerator_id)
telemetry = self.repo.telemetry(deployment.accelerator_id)
@@ -1437,6 +1611,12 @@ class ServingService:
and item.state != "cold"
)
leases = self.repo.active_leases(accelerator.id, _now())
+ external_reservations = list(self.session.scalars(
+ select(ExternalCreativeAllocation).where(
+ ExternalCreativeAllocation.accelerator_id == accelerator.id,
+ ExternalCreativeAllocation.state.in_(("granted", "uncertain")),
+ )
+ ))
policy = self._scheduler_policy()
pressure_row = self.repo.scheduler_accelerator_state(accelerator.id)
previous_pressure = (
@@ -1449,6 +1629,12 @@ class ServingService:
leases=[
LeaseAllocation(item.reserved_vram_bytes, item.materialized_vram_bytes)
for item in leases
+ ] + [
+ # Unattributed NVML usage is not proof that a specific external
+ # grant materialized. Retain its full reservation until release,
+ # including expired/uncertain grants, in both admission paths.
+ LeaseAllocation(item.required_vram_bytes, 0)
+ for item in external_reservations
],
policy=policy,
telemetry_fresh=(
@@ -1458,10 +1644,12 @@ class ServingService:
),
previous_pressure=previous_pressure,
)
- snapshot = replace(
- snapshot,
- pressure=self._stabilize_pressure(accelerator.id, snapshot.pressure, pressure_row),
+ effective_pressure = (
+ self._stabilize_pressure(accelerator.id, snapshot.pressure, pressure_row)
+ if update_pressure
+ else self._project_pressure(snapshot.pressure, pressure_row)
)
+ snapshot = replace(snapshot, pressure=effective_pressure)
return SchedulerBudgetResponse(
compute_node_id=node.id,
node_name=node.display_name or node.hostname,
@@ -1489,17 +1677,36 @@ class ServingService:
candidate: PressureState,
row: SchedulerAcceleratorState | None,
) -> PressureState:
- now = _now()
+ effective, changes = self._pressure_transition(candidate, row, _now())
if row is None:
- self.repo.add(
- SchedulerAcceleratorState(
- accelerator_id=accelerator_id,
- pressure_state=candidate.value,
- pressure_changed_at=now,
- last_observed_at=now,
- )
- )
- return candidate
+ self.repo.add(SchedulerAcceleratorState(accelerator_id=accelerator_id, **changes))
+ else:
+ for field, value in changes.items():
+ setattr(row, field, value)
+ return effective
+
+ def _project_pressure(
+ self,
+ candidate: PressureState,
+ row: SchedulerAcceleratorState | None,
+ ) -> PressureState:
+ effective, _changes = self._pressure_transition(candidate, row, _now())
+ return effective
+
+ def _pressure_transition(
+ self,
+ candidate: PressureState,
+ row: SchedulerAcceleratorState | None,
+ now: datetime,
+ ) -> tuple[PressureState, dict[str, Any]]:
+ if row is None:
+ return candidate, {
+ "pressure_state": candidate.value,
+ "pressure_changed_at": now,
+ "last_observed_at": now,
+ "recovery_candidate": None,
+ "recovery_candidate_since": None,
+ }
current = PressureState(row.pressure_state)
severity = {
PressureState.NORMAL: 0,
@@ -1507,33 +1714,41 @@ class ServingService:
PressureState.HIGH: 2,
PressureState.CRITICAL: 3,
}
- row.last_observed_at = now
+ changes: dict[str, Any] = {"last_observed_at": now}
if severity[candidate] > severity[current]:
- row.pressure_state = candidate.value
- row.pressure_changed_at = now
- row.recovery_candidate = None
- row.recovery_candidate_since = None
- return candidate
+ changes.update(
+ {
+ "pressure_state": candidate.value,
+ "pressure_changed_at": now,
+ "recovery_candidate": None,
+ "recovery_candidate_since": None,
+ }
+ )
+ return candidate, changes
if candidate == current:
- row.recovery_candidate = None
- row.recovery_candidate_since = None
- return current
+ changes.update({"recovery_candidate": None, "recovery_candidate_since": None})
+ return current, changes
if row.recovery_candidate != candidate.value:
- row.recovery_candidate = candidate.value
- row.recovery_candidate_since = now
- return current
+ changes.update(
+ {"recovery_candidate": candidate.value, "recovery_candidate_since": now}
+ )
+ return current, changes
if (
row.recovery_candidate_since
and _aware(row.recovery_candidate_since)
+ timedelta(seconds=self.settings.scheduler_pressure_stable_seconds)
<= now
):
- row.pressure_state = candidate.value
- row.pressure_changed_at = now
- row.recovery_candidate = None
- row.recovery_candidate_since = None
- return candidate
- return current
+ changes.update(
+ {
+ "pressure_state": candidate.value,
+ "pressure_changed_at": now,
+ "recovery_candidate": None,
+ "recovery_candidate_since": None,
+ }
+ )
+ return candidate, changes
+ return current, changes
def _scheduler_policy(self) -> SchedulerPolicy:
configured = SchedulerPolicy(
@@ -1807,7 +2022,12 @@ class ServingService:
)
return responses
- def scheduler_overview(self) -> list[SchedulerBudgetResponse]:
+ def scheduler_overview(
+ self,
+ *,
+ commit: bool = True,
+ update_pressure: bool = True,
+ ) -> list[SchedulerBudgetResponse]:
representatives: dict[uuid.UUID, CapabilityDeployment] = {}
for deployment in self.repo.deployments():
if deployment.status not in {"stable", "draining"}:
@@ -1815,8 +2035,12 @@ class ServingService:
current = representatives.get(deployment.accelerator_id)
if current is None or (current.status != "stable" and deployment.status == "stable"):
representatives[deployment.accelerator_id] = deployment
- budgets = [self._budget(item) for item in representatives.values()]
- self._commit()
+ budgets = [
+ self._budget(item, update_pressure=update_pressure)
+ for item in representatives.values()
+ ]
+ if commit:
+ self._commit()
return budgets
def co_residency_matrix(self) -> list[CoResidencyEvidenceResponse]:
@@ -2100,6 +2324,16 @@ class ServingService:
raise ServingError(503, "NO_ELIGIBLE_NODE", "capability node is not online")
return deployment, route_key
+ def _store_invocation_payload(self, payload: dict[str, Any]) -> str:
+ try:
+ return cast(
+ str, self.payload_store.put(payload, self.settings.serving_payload_ttl_seconds)
+ )
+ except RedisError:
+ raise ServingError(
+ 503, "PAYLOAD_STORE_UNAVAILABLE", "transient payload store is unavailable"
+ ) from None
+
def _validate_inputs(self, request: EmbeddingInvokeRequest) -> list[str]:
inputs = request.inputs()
if len(inputs) > self.settings.gateway_max_batch_size:
@@ -2170,8 +2404,12 @@ class ServingService:
self,
deployment: CapabilityDeployment,
gateway_request: GatewayRequest,
- client: ServiceClient,
+ context: DispatchContext,
) -> ServingGpuLease:
+ if (gateway_request.service_client_id != context.service_client_id
+ or gateway_request.priority != context.workload_priority
+ or gateway_request.capability_deployment_id != deployment.id):
+ raise ServingError(409, "DISPATCH_CONTEXT_MISMATCH", "request does not match trusted dispatch context")
deadline = time.monotonic() + self.settings.gateway_queue_timeout_seconds
while time.monotonic() < deadline:
self.session.expire_all()
@@ -2182,6 +2420,10 @@ class ServingService:
envelope = self.repo.envelope(deployment.id)
if not current or not residency or not envelope:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "scheduler state is incomplete")
+ if (current.service_client_id != context.service_client_id
+ or current.priority != context.workload_priority
+ or current.capability_deployment_id != deployment.id):
+ raise ServingError(409, "DISPATCH_CONTEXT_MISMATCH", "queued dispatch context changed")
if envelope.stale:
raise ServingError(
503,
@@ -2203,7 +2445,7 @@ class ServingService:
resident = residency.measured_resident_vram_bytes if residency.state != "cold" else 0
plan = self._placement_plan(
deployment,
- priority=client.workload_priority,
+ priority=context.workload_priority,
deadline_remaining_ms=float(self.settings.gateway_request_timeout_seconds * 1000),
request_id=current.request_id,
dry_run=False,
@@ -2235,7 +2477,7 @@ class ServingService:
key_suffix=(
f"scheduler-{plan.decision_fingerprint[:16]}-{candidate.load_count}"
),
- priority=client.workload_priority,
+ priority=context.workload_priority,
)
self._wait_job(unload.id, deadline)
continue
@@ -2268,9 +2510,9 @@ class ServingService:
capability_deployment_id=deployment.id,
request_id=current.request_id,
reserved_vram_bytes=required,
- priority=client.workload_priority,
+ priority=context.workload_priority,
state="active",
- owner=client.name,
+ owner=context.lease_owner,
created_at=now,
acquired_at=now,
heartbeat_at=now,
@@ -2284,6 +2526,7 @@ class ServingService:
current.status = "running"
current.decision_evidence = {
**current.decision_evidence,
+ "dispatch_context": context.provenance(),
**self._placement_evidence(deployment),
"placement_plan_id": str(plan.id),
"decision": plan.verdict,
@@ -2291,6 +2534,12 @@ class ServingService:
"decision_fingerprint": plan.decision_fingerprint,
"required_vram_bytes": required,
}
+ lab_run_evidence = current.decision_evidence.get("lab_run_evidence")
+ if context.actor_kind == "operator" and isinstance(lab_run_evidence, dict):
+ current.decision_evidence = {
+ **current.decision_evidence,
+ "lab_run_evidence": {**lab_run_evidence, "status": "running"},
+ }
residency.active_requests += 1
if residency.state in {"warm", "idle"}:
residency.state = "busy"
@@ -2469,6 +2718,105 @@ class ServingService:
deployment: CapabilityDeployment | None = None,
experiment_route: str | None = None,
) -> EmbeddingInvokeResponse:
+ context = DispatchContext.for_service_client(client.id, client.name, client.workload_priority)
+ return self._invoke_embedding(request, context,
+ deployment=deployment,
+ experiment_route=experiment_route)
+
+ def _operator_lab_target_snapshot(
+ self, deployment: CapabilityDeployment, experiment_route_id: uuid.UUID
+ ) -> dict[str, Any]:
+ candidate = self.session.get(DeploymentCandidate, deployment.deployment_candidate_id)
+ profile = self.repo.profile(deployment.runtime_profile_id)
+ accelerator = self.repo.accelerator(deployment.accelerator_id)
+ contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
+ if (
+ candidate is None
+ or profile is None
+ or accelerator is None
+ or contract is None
+ or deployment.execution_approval_id is None
+ or deployment.embedding_space_id is None
+ or profile.artifact_set_id != deployment.artifact_set_id
+ ):
+ raise ServingError(
+ 409, "LAB_TARGET_EVIDENCE_MISSING", "selected embedding target evidence is incomplete"
+ )
+ return {
+ "deployment_id": str(deployment.id),
+ "candidate_id": str(candidate.id),
+ "experiment_route_id": str(experiment_route_id),
+ "capability_contract_id": str(contract.id),
+ "contract_version": contract.version,
+ "artifact_set_id": str(deployment.artifact_set_id),
+ "artifact_sha256": profile.artifact_sha256,
+ "runtime_profile_id": str(profile.id),
+ "runtime_profile_fingerprint": profile.fingerprint,
+ "compatibility_assessment_id": str(candidate.compatibility_assessment_id),
+ "runtime_probe_id": str(candidate.runtime_probe_id),
+ "compute_node_id": str(deployment.compute_node_id),
+ "accelerator_id": str(deployment.accelerator_id),
+ "accelerator_uuid": accelerator.device_uuid,
+ "execution_approval_id": str(deployment.execution_approval_id),
+ "embedding_space_id": str(deployment.embedding_space_id),
+ "deployment_fingerprint": deployment.config_fingerprint,
+ }
+
+ def invoke_operator_lab(
+ self, input_value: LabInput, context: DispatchContext, *,
+ deployment_id: uuid.UUID, experiment_route_id: uuid.UUID,
+ ) -> EmbeddingInvokeResponse:
+ """Internal synchronous mechanism only; no operator HTTP route or durable run is created."""
+ if context.actor_kind != "operator":
+ raise ServingError(403, "OPERATOR_CONTEXT_REQUIRED", "operator dispatch context is required")
+ if not isinstance(input_value, LabEmbeddingInput):
+ raise ServingError(422, "LAB_INPUT_NOT_SUPPORTED", "operator execution currently supports text embeddings only")
+ deployment, route_key = self._admit_operator_embedding(input_value, deployment_id, experiment_route_id)
+ return self._invoke_embedding(input_value.payload, context, deployment=deployment,
+ experiment_route=route_key, operator_route_id=experiment_route_id)
+
+ def _admit_operator_embedding(
+ self, input_value: LabEmbeddingInput, deployment_id: uuid.UUID, route_id: uuid.UUID,
+ ) -> tuple[CapabilityDeployment, str]:
+ self.session.expire_all()
+ deployment = self.repo.deployment(deployment_id)
+ route = self.repo.experiment_route_by_id(route_id)
+ if (not deployment or not route or route.status != "active"
+ or route.capability_deployment_id != deployment.id
+ or route.capability_contract_id != deployment.capability_contract_id
+ or deployment.production or deployment.channel != "experiment"
+ or deployment.status != "candidate"):
+ raise ServingError(409, "LAB_TARGET_NOT_ROUTABLE", "exact active nonproduction experiment is required")
+ contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
+ capability = self.session.get(Capability, contract.capability_id) if contract else None
+ if not contract or contract.version != 1 or not capability or capability.key != "rag.embedding":
+ raise ServingError(409, "LAB_CONTRACT_MISMATCH", "selected deployment must implement rag.embedding@1")
+ if deployment.execution_approval_id is None:
+ raise ServingError(409, "LAB_APPROVAL_STALE", "exact LAB approval is required")
+ preflight = LabPreflightService(self.session, self.settings).preflight(LabPreflightRequest(
+ target=LabTarget(deployment_candidate_id=deployment.deployment_candidate_id,
+ artifact_set_id=deployment.artifact_set_id, runtime_profile_id=deployment.runtime_profile_id,
+ compute_node_id=deployment.compute_node_id, execution_approval_id=deployment.execution_approval_id),
+ input=input_value))
+ if not preflight.checks_passed:
+ raise ServingError(409, "LAB_ADMISSION_BLOCKED", "selected LAB evidence or input is no longer admissible",
+ {"blockers": preflight.blockers})
+ if preflight.binding is None or preflight.binding.accelerator_id != deployment.accelerator_id:
+ raise ServingError(409, "LAB_TARGET_MISMATCH", "deployment accelerator differs from the proven target")
+ return deployment, route.route_key
+
+ def _invoke_embedding(
+ self, request: EmbeddingInvokeRequest, context: DispatchContext, *,
+ deployment: CapabilityDeployment | None, experiment_route: str | None = None,
+ operator_route_id: uuid.UUID | None = None,
+ ) -> EmbeddingInvokeResponse:
+ if context.actor_kind == "operator" and (operator_route_id is None or deployment is None):
+ raise ServingError(403, "LAB_TARGET_REQUIRED", "operator execution requires an explicitly admitted LAB route")
+ if context.actor_kind == "operator" and operator_route_id is not None and deployment is not None:
+ deployment, selected_route = self._admit_operator_embedding(
+ LabEmbeddingInput(capability="rag.embedding", payload=request), deployment.id, operator_route_id)
+ if experiment_route != selected_route:
+ raise ServingError(409, "LAB_TARGET_MISMATCH", "operator route does not match selected deployment")
started = time.perf_counter()
validation_started = time.perf_counter()
inputs = self._validate_inputs(request)
@@ -2488,20 +2836,40 @@ class ServingService:
raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
request_id = uuid.uuid4()
input_hash = _canonical_hash(inputs)
+ lab_run_evidence = None
+ if context.actor_kind == "operator":
+ if operator_route_id is None or context.operation_id is None:
+ raise ServingError(403, "LAB_TARGET_REQUIRED", "operator execution requires a persisted run identity")
+ lab_run_evidence = {
+ "schema_version": 1,
+ "status": "queued",
+ "capability": "rag.embedding@1",
+ "operation_id": str(context.operation_id),
+ "target": self._operator_lab_target_snapshot(deployment, operator_route_id),
+ "input": {
+ "sha256": input_hash,
+ "item_count": len(inputs),
+ "content_logged": False,
+ },
+ }
record = self.repo.add(
GatewayRequest(
request_id=request_id,
- service_client_id=client.id,
+ service_client_id=context.service_client_id,
capability_key="rag.embedding",
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
- priority=client.workload_priority,
+ priority=context.workload_priority,
input_sha256=input_hash,
input_count=len(inputs),
decision_evidence={
+ "dispatch_context": context.provenance(),
+ "operator_route_id": str(operator_route_id) if operator_route_id else None,
"input_persisted": False,
+ "service_credential_id": str(self.authenticated_credential_id)
+ if context.actor_kind == "service_client" and self.authenticated_credential_id else None,
"input_type": request.input_type,
"experiment_route": experiment_route,
"shape": {
@@ -2509,6 +2877,7 @@ class ServingService:
"item_count": len(inputs),
"maximum_characters": max(map(len, inputs), default=0),
},
+ **({"lab_run_evidence": lab_run_evidence} if lab_run_evidence else {}),
},
)
)
@@ -2516,26 +2885,34 @@ class ServingService:
deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
lease: ServingGpuLease | None = None
result_reference: str | None = None
+ payload_reference: str | None = None
+ job_enqueued = False
try:
scheduling_started = time.perf_counter()
- lease = self._acquire_request_lease(deployment, record, client)
+ lease = self._acquire_request_lease(deployment, record, context)
scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
+ if operator_route_id is not None:
+ self._admit_operator_embedding(LabEmbeddingInput(capability="rag.embedding", payload=request),
+ deployment.id, operator_route_id)
cold, load_ms = self._ensure_loaded(
- deployment, deadline, priority=client.workload_priority
+ deployment, deadline, priority=context.workload_priority
)
payload_started = time.perf_counter()
- payload_reference = self.payload_store.put(
- {"input": inputs, "input_type": request.input_type},
- self.settings.serving_payload_ttl_seconds,
+ payload_reference = self._store_invocation_payload(
+ {"input": inputs, "input_type": request.input_type}
)
+ if operator_route_id is not None:
+ self._admit_operator_embedding(LabEmbeddingInput(capability="rag.embedding", payload=request),
+ deployment.id, operator_route_id)
job = self._enqueue_job(
deployment,
"invoke",
request=record,
payload_reference=payload_reference,
key_suffix=f"request-{request_id}",
- priority=client.workload_priority,
+ priority=context.workload_priority,
)
+ job_enqueued = True
payload_ms = (time.perf_counter() - payload_started) * 1000
wait_started = time.perf_counter()
completed, result = self._wait_job(job.id, deadline)
@@ -2622,6 +2999,40 @@ class ServingService:
"latency_clock": "process-local-monotonic; control-plane timestamps for dispatch",
"latency_ms": latency_ms,
}
+ lab_run_evidence = current.decision_evidence.get("lab_run_evidence")
+ if context.actor_kind == "operator" and isinstance(lab_run_evidence, dict):
+ target = lab_run_evidence.get("target", {})
+ expected_gpu_uuid = (
+ target.get("accelerator_uuid") if isinstance(target, dict) else None
+ )
+ resource_observation = completed.result_summary.get("resource_observation", {})
+ raw_worker_metrics = (
+ resource_observation.get("metrics", {})
+ if isinstance(resource_observation, dict)
+ else {}
+ )
+ current.decision_evidence = {
+ **current.decision_evidence,
+ "lab_run_evidence": {
+ **lab_run_evidence,
+ "status": "completed",
+ "output": {
+ "type": "float_vector",
+ "dimension": space.dimension,
+ "item_count": len(vectors),
+ "embedding_space_id": str(space.id),
+ "sha256": _canonical_hash(vectors),
+ "content_logged": False,
+ },
+ "worker_observations": _worker_resource_observation(
+ raw_worker_metrics,
+ expected_gpu_uuid=(
+ expected_gpu_uuid if isinstance(expected_gpu_uuid, str) else None
+ ),
+ ),
+ "timing_ms": latency_ms,
+ },
+ }
self._commit()
selected_node = self.repo.node(deployment.compute_node_id)
if not selected_node:
@@ -2670,6 +3081,17 @@ class ServingService:
current.failure_code = exc.code
current.total_latency_ms = (time.perf_counter() - started) * 1000
current.finished_at = _now()
+ lab_run_evidence = current.decision_evidence.get("lab_run_evidence")
+ if context.actor_kind == "operator" and isinstance(lab_run_evidence, dict):
+ current.decision_evidence = {
+ **current.decision_evidence,
+ "lab_run_evidence": {
+ **lab_run_evidence,
+ "status": "failed",
+ "failure_code": exc.code,
+ "timing_ms": {"total": current.total_latency_ms},
+ },
+ }
if residency and residency.active_requests:
residency.active_requests -= 1
if residency.state == "busy":
@@ -2681,12 +3103,15 @@ class ServingService:
self._commit()
raise
finally:
+ if payload_reference and not job_enqueued:
+ self.payload_store.delete(payload_reference)
if result_reference:
self.payload_store.delete(result_reference)
def invoke_reranking(
self, request: RerankingInvokeRequest, client: ServiceClient
) -> RerankingInvokeResponse:
+ context = DispatchContext.for_service_client(client.id, client.name, client.workload_priority)
started = time.perf_counter()
validation_started = time.perf_counter()
documents = self._validate_reranking_input(request)
@@ -2713,19 +3138,20 @@ class ServingService:
record = self.repo.add(
GatewayRequest(
request_id=request_id,
- service_client_id=client.id,
+ service_client_id=context.service_client_id,
capability_key="rag.reranking",
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
- priority=client.workload_priority,
+ priority=context.workload_priority,
input_sha256=_canonical_hash(content_facts),
input_count=len(documents),
decision_evidence={
"input_persisted": False,
"content_logged": False,
"document_count": len(documents),
+ "service_credential_id": str(self.authenticated_credential_id) if self.authenticated_credential_id else None,
"top_n": request.top_n,
"experiment_route": experiment_route,
"shape": {
@@ -2746,15 +3172,14 @@ class ServingService:
result_reference: str | None = None
try:
scheduling_started = time.perf_counter()
- lease = self._acquire_request_lease(deployment, record, client)
+ lease = self._acquire_request_lease(deployment, record, context)
scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
cold, load_ms = self._ensure_loaded(
- deployment, deadline, priority=client.workload_priority
+ deployment, deadline, priority=context.workload_priority
)
payload_started = time.perf_counter()
- input_reference = self.payload_store.put(
- {"query": request.query, "documents": documents, "top_n": request.top_n},
- self.settings.serving_payload_ttl_seconds,
+ input_reference = self._store_invocation_payload(
+ {"query": request.query, "documents": documents, "top_n": request.top_n}
)
job = self._enqueue_job(
deployment,
@@ -2762,7 +3187,7 @@ class ServingService:
request=record,
payload_reference=input_reference,
key_suffix=f"request-{request_id}",
- priority=client.workload_priority,
+ priority=context.workload_priority,
)
payload_ms = (time.perf_counter() - payload_started) * 1000
wait_started = time.perf_counter()
@@ -2967,6 +3392,7 @@ class ServingService:
payload: dict[str, Any],
client: ServiceClient,
) -> tuple[dict[str, Any], uuid.UUID, GatewayExecution]:
+ context = DispatchContext.for_service_client(client.id, client.name, client.workload_priority)
started = time.perf_counter()
resolution_started = time.perf_counter()
deployment, experiment_route = self._experimental_deployment(capability_key)
@@ -2993,13 +3419,13 @@ class ServingService:
record = self.repo.add(
GatewayRequest(
request_id=request_id,
- service_client_id=client.id,
+ service_client_id=context.service_client_id,
capability_key=capability_key,
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
- priority=client.workload_priority,
+ priority=context.workload_priority,
input_sha256=_canonical_hash(payload),
input_count=input_count,
decision_evidence={
@@ -3007,6 +3433,7 @@ class ServingService:
"content_logged": False,
"experiment_route": experiment_route,
"modality": capability_key.split(".", 1)[0],
+ "service_credential_id": str(self.authenticated_credential_id) if self.authenticated_credential_id else None,
"shape": {
"bucket": _shape_bucket(approximate_bytes),
"encoded_payload_bytes_approximate": approximate_bytes,
@@ -3022,22 +3449,20 @@ class ServingService:
result_reference: str | None = None
try:
scheduling_started = time.perf_counter()
- lease = self._acquire_request_lease(deployment, record, client)
+ lease = self._acquire_request_lease(deployment, record, context)
scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
cold, load_ms = self._ensure_loaded(
- deployment, deadline, priority=client.workload_priority
+ deployment, deadline, priority=context.workload_priority
)
payload_started = time.perf_counter()
- input_reference = self.payload_store.put(
- payload, self.settings.serving_payload_ttl_seconds
- )
+ input_reference = self._store_invocation_payload(payload)
job = self._enqueue_job(
deployment,
"invoke",
request=record,
payload_reference=input_reference,
key_suffix=f"request-{request_id}",
- priority=client.workload_priority,
+ priority=context.workload_priority,
)
payload_ms = (time.perf_counter() - payload_started) * 1000
wait_started = time.perf_counter()
@@ -3234,7 +3659,7 @@ class ServingService:
rerank_documents: list[dict[str, str]] | None = None
top_n: int | None = None
modality_payload: dict[str, Any] | None = None
- if job.operation == "invoke":
+ if job.operation == "invoke" and job.lab_execution_run_id is None:
transient = (
self.payload_store.get(job.payload_reference) if job.payload_reference else None
)
@@ -3279,6 +3704,80 @@ class ServingService:
top_n = int(transient["top_n"])
else:
modality_payload = transient
+ lab_run: LabExecutionRun | None = None
+ if job.lab_execution_run_id is not None:
+ lab_run = self.session.scalar(
+ select(LabExecutionRun)
+ .where(LabExecutionRun.id == job.lab_execution_run_id)
+ .with_for_update()
+ )
+ expected_status = "queued" if job.operation == "invoke" else "preparing"
+ if lab_run is None or lab_run.status != expected_status:
+ job.status = (
+ "cancelled"
+ if lab_run and lab_run.status in {"cancel_requested", "cancelled"}
+ else "failed"
+ )
+ job.failure_code = (
+ "CANCELLED" if job.status == "cancelled" else "LAB_RUN_STATE_INVALID"
+ )
+ job.finished_at = _now()
+ if lab_run and lab_run.status == "cancel_requested":
+ from modelforge_api.services.lab_runs import LabRunService
+
+ run_service = LabRunService(self)
+ terminal = "failed" if lab_run.failure_code else "cancelled"
+ run_service._finish(lab_run, terminal, failure_code=lab_run.failure_code)
+ run_service._release_reservation(lab_run)
+ self._commit()
+ return None
+ from modelforge_api.services.lab_runs import LabRunService
+
+ run_service = LabRunService(self)
+ selected_input = run_service._input(lab_run)
+ if selected_input is None:
+ job.status = "failed"
+ job.failure_code = "REQUEST_PAYLOAD_EXPIRED"
+ job.finished_at = _now()
+ run_service._finish(lab_run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
+ run_service._release_reservation(lab_run, failed=True)
+ self._commit()
+ return None
+ if job.operation == "invoke":
+ if job.payload_reference != lab_run.payload_reference:
+ job.status = "failed"
+ job.failure_code = "LAB_PAYLOAD_REFERENCE_MISMATCH"
+ job.finished_at = _now()
+ run_service._finish(
+ lab_run, "failed", failure_code="LAB_PAYLOAD_REFERENCE_MISMATCH"
+ )
+ run_service._release_reservation(lab_run, failed=True)
+ self._commit()
+ return None
+ input_value = selected_input.payload.inputs()
+ input_type = selected_input.payload.input_type
+ try:
+ admitted, _ = self._admit_operator_embedding(
+ selected_input,
+ deployment.id,
+ uuid.UUID(str(lab_run.target_snapshot["experiment_route_id"])),
+ )
+ selected_snapshot = self._operator_lab_target_snapshot(
+ admitted,
+ uuid.UUID(str(lab_run.target_snapshot["experiment_route_id"])),
+ )
+ if selected_snapshot != lab_run.target_snapshot:
+ raise ServingError(
+ 409, "LAB_TARGET_CHANGED", "selected LAB target is no longer admissible"
+ )
+ except (ServingError, KeyError, ValueError):
+ job.status = "failed"
+ job.failure_code = "LAB_TARGET_CHANGED"
+ job.finished_at = _now()
+ run_service._finish(lab_run, "failed", failure_code="LAB_TARGET_CHANGED")
+ run_service._release_reservation(lab_run, failed=True)
+ self._commit()
+ return None
token = secrets.token_urlsafe(32)
now = _now()
job.status = "running"
@@ -3286,6 +3785,18 @@ class ServingService:
job.lease_token_hash = hashlib.sha256(token.encode()).hexdigest()
job.lease_expires_at = now + timedelta(seconds=self.settings.serving_job_lease_seconds)
job.started_at = job.started_at or now
+ if lab_run is not None and job.operation == "invoke":
+ lab_run.status = "running"
+ lab_run.started_at = lab_run.started_at or now
+ lab_run.heartbeat_at = now
+ gateway_request = (
+ self.session.get(GatewayRequest, job.gateway_request_id)
+ if job.gateway_request_id
+ else None
+ )
+ if gateway_request:
+ gateway_request.status = "running"
+ run_service._set_gateway_state(lab_run, "running")
self._commit()
runtime = RuntimeService(self.session, self.settings)
gateway_request = (
@@ -3329,6 +3840,7 @@ class ServingService:
top_n=top_n,
modality_payload=modality_payload,
request_id=gateway_request.request_id if gateway_request else None,
+ lab_run_id=lab_run.id if lab_run else None,
)
@staticmethod
@@ -3341,16 +3853,136 @@ class ServingService:
if not job.lease_expires_at or _aware(job.lease_expires_at) <= _now():
raise ServingError(409, "LEASE_TIMEOUT", "serving job lease expired")
+ def check_lab_cancellation(
+ self,
+ job_id: uuid.UUID,
+ node: ComputeNode,
+ request: AgentServingJobCancellationCheck,
+ ) -> AgentServingJobCancellationStatus:
+ job = self.session.scalar(
+ select(ServingJob).where(ServingJob.id == job_id).with_for_update()
+ )
+ if not job or job.lab_execution_run_id is None or job.status != "running":
+ raise ServingError(404, "SERVING_JOB_NOT_FOUND", "LAB serving job was not found")
+ self._assert_job_lease(job, node, request.lease_token)
+ run = self.session.scalar(
+ select(LabExecutionRun)
+ .where(LabExecutionRun.id == job.lab_execution_run_id)
+ .with_for_update()
+ )
+ if run is None:
+ raise ServingError(404, "LAB_RUN_NOT_FOUND", "LAB run was not found")
+ now = _now()
+ run.heartbeat_at = now
+ deadline = _aware(run.created_at) + timedelta(seconds=run.timeout_seconds)
+ job.lease_expires_at = min(
+ deadline,
+ now + timedelta(seconds=self.settings.serving_job_lease_seconds),
+ )
+ gateway = self.session.get(GatewayRequest, run.gateway_request_id)
+ if gateway:
+ lease = self.session.scalar(
+ select(ServingGpuLease)
+ .where(ServingGpuLease.request_id == gateway.request_id)
+ .order_by(ServingGpuLease.created_at.desc())
+ .limit(1)
+ )
+ if lease and lease.state in {"pending", "granted", "active", "releasing"}:
+ lease.heartbeat_at = now
+ lease.expires_at = job.lease_expires_at
+ self._commit()
+ return AgentServingJobCancellationStatus(cancel_requested=run.status == "cancel_requested")
+
+ def acknowledge_lab_cancellation(
+ self,
+ job_id: uuid.UUID,
+ node: ComputeNode,
+ request: AgentServingJobCancellationAck,
+ ) -> ServingJob:
+ job = self.session.scalar(
+ select(ServingJob).where(ServingJob.id == job_id).with_for_update()
+ )
+ if not job or job.lab_execution_run_id is None:
+ raise ServingError(404, "SERVING_JOB_NOT_FOUND", "LAB serving job was not found")
+ self._assert_job_lease(job, node, request.lease_token)
+ run = self.session.scalar(
+ select(LabExecutionRun)
+ .where(LabExecutionRun.id == job.lab_execution_run_id)
+ .with_for_update()
+ )
+ if run is None or run.status != "cancel_requested" or job.operation != "invoke":
+ raise ServingError(
+ 409, "LAB_CANCEL_NOT_REQUESTED", "worker cancellation was not requested"
+ )
+ now = _now()
+ run.cancellation_acknowledged_at = now
+ run.worker_observations = {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": "cancelled",
+ "safe_boundary_reached": True,
+ "worker_instance_id": request.worker_instance_id,
+ "worker_state": request.state,
+ }
+ job.status = "cancelled"
+ job.failure_code = "CANCELLED"
+ job.finished_at = now
+ job.lease_token_hash = None
+ job.lease_expires_at = None
+ job.result_reference = None
+ job.result_summary = {"cancelled": True, "safe_boundary_reached": True}
+ if job.payload_reference:
+ self.payload_store.delete(job.payload_reference)
+ job.payload_reference = None
+ from modelforge_api.services.lab_runs import LabRunService
+
+ run_service = LabRunService(self)
+ terminal = "failed" if run.failure_code else "cancelled"
+ run_service._finish(run, terminal, failure_code=run.failure_code)
+ run_service._release_reservation(run, failed=terminal == "failed")
+ deployment = self.repo.deployment(job.capability_deployment_id)
+ residency = self.repo.residency(job.capability_deployment_id, lock=True)
+ if residency:
+ if request.state in {"warm", "cold"}:
+ residency.state = request.state
+ elif request.state == "failed":
+ residency.state = "failed"
+ if deployment:
+ deployment.health_status = "unavailable"
+ residency.failure_code = (
+ None if residency.state in {"warm", "cold"} else residency.failure_code
+ )
+ self._commit()
+ return job
+
def complete_job(
self,
job_id: uuid.UUID,
node: ComputeNode,
request: AgentServingJobComplete,
) -> ServingJob:
- job = self.repo.job(job_id)
+ job = self.session.scalar(
+ select(ServingJob).where(ServingJob.id == job_id).with_for_update()
+ )
if not job:
raise ServingError(404, "SERVING_JOB_NOT_FOUND", "serving job was not found")
self._assert_job_lease(job, node, request.lease_token)
+ lab_run = (
+ self.session.scalar(
+ select(LabExecutionRun)
+ .where(LabExecutionRun.id == job.lab_execution_run_id)
+ .with_for_update()
+ )
+ if job.lab_execution_run_id
+ else None
+ )
+ if job.lab_execution_run_id and (
+ lab_run is None or lab_run.status not in {"running", "cancel_requested", "preparing"}
+ ):
+ raise ServingError(
+ 409, "LAB_RUN_STATE_INVALID", "LAB run is no longer awaiting this job"
+ )
+ lab_cancel_requested = bool(lab_run and lab_run.status == "cancel_requested")
deployment = self.repo.deployment(job.capability_deployment_id)
residency = self.repo.residency(job.capability_deployment_id, lock=True)
if not deployment or not residency:
@@ -3366,7 +3998,13 @@ class ServingService:
summary: dict[str, Any]
if capability.key == "rag.embedding":
vectors = result.get("vectors")
- expected_count = int(result.get("count", 1))
+ expected_count = lab_run.input_count if lab_run else int(result.get("count", 1))
+ if lab_run and (
+ result.get("count") != expected_count or result.get("normalized") is not True
+ ):
+ raise ServingError(
+ 502, "INVALID_OUTPUT", "runtime returned an invalid LAB embedding result"
+ )
space = self.repo.embedding_space(deployment.embedding_space_id)
if not space:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "embedding space is missing")
@@ -3442,10 +4080,15 @@ class ServingService:
}
else:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "unsupported capability")
- result_reference = self.payload_store.put(
- transient_result, self.settings.serving_payload_ttl_seconds
- )
- job.result_reference = result_reference
+ if lab_run is None:
+ result_reference = self.payload_store.put(
+ transient_result, self.settings.serving_payload_ttl_seconds
+ )
+ job.result_reference = result_reference
+ else:
+ # LAB run results are reduced to a digest and bounded metadata; vectors are
+ # never written to Redis or PostgreSQL for this operator path.
+ job.result_reference = None
job.result_summary = {
**summary,
"inference_time_ms": float(metrics.get("inference_time_ms", 0.0)),
@@ -3454,6 +4097,34 @@ class ServingService:
"worker_total_time_ms": float(metrics.get("worker_total_time_ms", 0.0)),
"prompt_tokens": int(result.get("prompt_tokens", 0)),
}
+ gateway_request = (
+ self.session.get(GatewayRequest, job.gateway_request_id)
+ if job.gateway_request_id
+ else None
+ )
+ dispatch_context = (
+ gateway_request.decision_evidence.get("dispatch_context", {})
+ if gateway_request
+ else {}
+ )
+ if (
+ job.operation == "invoke"
+ and capability.key == "rag.embedding"
+ and gateway_request is not None
+ and gateway_request.service_client_id is None
+ and gateway_request.priority == "lab"
+ and isinstance(dispatch_context, dict)
+ and dispatch_context.get("actor_kind") == "operator"
+ and dispatch_context.get("workload_priority") == "lab"
+ ):
+ accelerator = self.repo.accelerator(deployment.accelerator_id)
+ job.result_summary = {
+ **job.result_summary,
+ "resource_observation": _worker_resource_observation(
+ metrics,
+ expected_gpu_uuid=accelerator.device_uuid if accelerator else None,
+ ),
+ }
residency.state = "warm"
residency.last_used_at = _now()
residency.health = request.health
@@ -3540,6 +4211,72 @@ class ServingService:
if job.payload_reference:
self.payload_store.delete(job.payload_reference)
job.payload_reference = None
+ if lab_run is not None:
+ from modelforge_api.services.lab_runs import LabRunService
+
+ run_service = LabRunService(self)
+ now = _now()
+ if job.operation == "invoke":
+ accelerator = self.repo.accelerator(deployment.accelerator_id)
+ safe_observation = _worker_resource_observation(
+ metrics,
+ expected_gpu_uuid=accelerator.device_uuid if accelerator else None,
+ )
+ if lab_cancel_requested:
+ lab_run.cancellation_acknowledged_at = now
+ lab_run.worker_observations = {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": "cancelled",
+ "safe_boundary_reached": True,
+ "worker_instance_id": request.worker_instance_id,
+ "worker_state": request.state,
+ }
+ job.status = "cancelled"
+ job.failure_code = "CANCELLED"
+ job.result_reference = None
+ job.result_summary = {"cancelled": True, "safe_boundary_reached": True}
+ terminal = "failed" if lab_run.failure_code else "cancelled"
+ run_service._finish(lab_run, terminal, failure_code=lab_run.failure_code)
+ run_service._release_reservation(lab_run, failed=terminal == "failed")
+ else:
+ space = self.repo.embedding_space(deployment.embedding_space_id)
+ if not space:
+ raise ServingError(
+ 409, "CAPABILITY_UNAVAILABLE", "embedding space is missing"
+ )
+ lab_run.status = "completed"
+ lab_run.output_metadata = {
+ "type": "float_vector",
+ "dimension": space.dimension,
+ "item_count": lab_run.input_count,
+ "embedding_space_id": str(space.id),
+ "sha256": _canonical_hash(validated),
+ "content_logged": False,
+ }
+ lab_run.worker_observations = safe_observation
+ lab_run.failure_code = None
+ lab_run.finished_at = now
+ run_service._set_gateway_state(lab_run, "completed")
+ run_service._delete_payload(lab_run)
+ run_service._release_reservation(lab_run)
+ elif job.operation == "load":
+ if lab_cancel_requested:
+ lab_run.cancellation_acknowledged_at = now
+ lab_run.worker_observations = {
+ "source": "runtime-worker",
+ "phase": "after_load",
+ "availability": "cancelled",
+ "safe_boundary_reached": True,
+ "worker_instance_id": request.worker_instance_id,
+ "worker_state": request.state,
+ }
+ terminal = "failed" if lab_run.failure_code else "cancelled"
+ run_service._finish(lab_run, terminal, failure_code=lab_run.failure_code)
+ run_service._release_reservation(lab_run, failed=terminal == "failed")
+ else:
+ lab_run.heartbeat_at = now
+ run_service._set_gateway_state(lab_run, "preparing")
self._commit()
return job
@@ -3549,15 +4286,35 @@ class ServingService:
node: ComputeNode,
request: AgentServingJobFailure,
) -> ServingJob:
- job = self.repo.job(job_id)
+ job = self.session.scalar(
+ select(ServingJob).where(ServingJob.id == job_id).with_for_update()
+ )
if not job:
raise ServingError(404, "SERVING_JOB_NOT_FOUND", "serving job was not found")
self._assert_job_lease(job, node, request.lease_token)
+ lab_run = (
+ self.session.scalar(
+ select(LabExecutionRun)
+ .where(LabExecutionRun.id == job.lab_execution_run_id)
+ .with_for_update()
+ )
+ if job.lab_execution_run_id
+ else None
+ )
residency = self.repo.residency(job.capability_deployment_id, lock=True)
job.status = "failed"
- job.failure_code = request.failure_code
- job.failure_message = request.failure_message
- job.result_summary = {"details": request.details}
+ safe_failure_code = (
+ request.failure_code
+ if re.fullmatch(r"[A-Z0-9_]{1,64}", request.failure_code)
+ else "WORKER_FAILURE"
+ )
+ job.failure_code = safe_failure_code
+ job.failure_message = None if lab_run else request.failure_message
+ job.result_summary = (
+ {"operation": job.operation, "failure_code": safe_failure_code}
+ if lab_run
+ else {"details": request.details}
+ )
job.finished_at = _now()
job.lease_token_hash = None
job.lease_expires_at = None
@@ -3566,12 +4323,30 @@ class ServingService:
job.payload_reference = None
if residency:
residency.state = "cold" if job.operation in {"load", "invoke"} else "failed"
- residency.failure_code = request.failure_code
- residency.failure_message = request.failure_message
+ residency.failure_code = safe_failure_code
+ residency.failure_message = None if lab_run else request.failure_message
residency.measured_resident_vram_bytes = (
0 if residency.state == "cold" else residency.measured_resident_vram_bytes
)
residency.active_requests = max(0, residency.active_requests - 1)
+ if lab_run:
+ from modelforge_api.services.lab_runs import LabRunService
+
+ deployment = self.repo.deployment(job.capability_deployment_id)
+ if deployment:
+ deployment.health_status = "unavailable"
+ lab_run.worker_observations = {
+ "source": "runtime-worker",
+ "phase": "after_inference" if job.operation == "invoke" else "after_load",
+ "availability": "failed",
+ "safe_boundary_reached": True,
+ "worker_instance_id": request.worker_instance_id,
+ "worker_state": request.state,
+ }
+ lab_run.finished_at = _now()
+ run_service = LabRunService(self)
+ run_service._finish(lab_run, "failed", failure_code=safe_failure_code)
+ run_service._release_reservation(lab_run, failed=True)
self._commit()
return job
@@ -3685,16 +4460,39 @@ class ServingService:
self._commit()
return AgentServingStateAck()
+ def _advance_serving_generation(self, contract: CapabilityContract) -> None:
+ """Fence every stable route mutation, including the legacy first-promotion path."""
+ expected = contract.serving_generation
+ result = self.session.execute(
+ update(CapabilityContract).where(
+ CapabilityContract.id == contract.id,
+ CapabilityContract.serving_generation == expected,
+ ).values(serving_generation=expected + 1)
+ )
+ if rows_affected(result) != 1:
+ self.session.rollback()
+ raise ServingError(409, "SERVING_ROUTE_CHANGED", "stable route changed concurrently")
+
def request_unload(
self, deployment_id: uuid.UUID, *, drain: bool = False
) -> CapabilityDeploymentResponse:
deployment = self.repo.deployment(deployment_id)
+ route_contract = None
+ if deployment is not None and drain:
+ route_contract = self.repo.lock_contract_route(deployment.capability_contract_id)
+ if route_contract is None:
+ raise ServingError(409, "CAPABILITY_NOT_FOUND", "capability contract disappeared")
+ # Read status after the route lock: another switch may have drained this row.
+ self.session.refresh(deployment)
residency = self.repo.residency(deployment_id, lock=True) if deployment else None
if not deployment or not residency:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
if residency.active_requests:
raise ServingError(409, "DEPLOYMENT_BUSY", "active requests must finish before unload")
if drain:
+ if (route_contract is not None and deployment.production
+ and deployment.channel == "stable" and deployment.status == "stable"):
+ self._advance_serving_generation(route_contract)
deployment.status = "draining"
deployment.draining_at = _now()
deployment.health_status = "degraded"
@@ -3774,5 +4572,15 @@ class ServingService:
for lease in self.repo.expired_leases(now):
if lease.state == "expired":
expired += 1
+ from modelforge_api.services.benchmark_runs import BenchmarkRunService
+ from modelforge_api.services.lab_runs import LabRunService
+
+ lab_runs = LabRunService(self).reconcile()
+ benchmark_runs = BenchmarkRunService(self).reconcile()
self._commit()
- return {"expired_leases": expired, "unloads_queued": unloads}
+ return {
+ "expired_leases": expired,
+ "unloads_queued": unloads,
+ **lab_runs,
+ **benchmark_runs,
+ }
diff --git a/backend/src/modelforge_api/services/serving_reconciliation.py b/backend/src/modelforge_api/services/serving_reconciliation.py
index 2f6db83..3916331 100644
--- a/backend/src/modelforge_api/services/serving_reconciliation.py
+++ b/backend/src/modelforge_api/services/serving_reconciliation.py
@@ -1,9 +1,12 @@
from __future__ import annotations
import asyncio
+import uuid
import redis
+import structlog
from sqlalchemy.engine import Engine
+from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from modelforge_api.services.manifest_registry import ManifestRegistry
@@ -11,6 +14,8 @@ from modelforge_api.services.serving import ServingError, ServingService
from modelforge_api.services.transient_payloads import RedisPayloadStore
from modelforge_api.settings import Settings
+logger = structlog.get_logger()
+
class ServingReconciliationService:
def __init__(self, settings: Settings, engine: Engine) -> None:
@@ -30,9 +35,15 @@ class ServingReconciliationService:
actor_type="system",
actor_id="serving-reconciler",
).reconcile()
- except (OSError, RuntimeError, ValueError, ServingError, redis.RedisError):
- # Readiness and structured state expose persistent failures; the loop stays bounded.
- pass
+ except (OSError, RuntimeError, ValueError, ServingError, redis.RedisError,
+ SQLAlchemyError) as error:
+ # Database/Redis failures must not silently terminate reconciliation. Exception
+ # messages can contain SQL parameters or credentials, so retain only their type.
+ logger.error(
+ "serving_reconciliation_failed",
+ error_type=type(error).__name__,
+ correlation_id=str(uuid.uuid4()),
+ )
try:
await asyncio.wait_for(
self._stop.wait(), self.settings.serving_reconciliation_interval_seconds
diff --git a/backend/src/modelforge_api/services/startup_validation.py b/backend/src/modelforge_api/services/startup_validation.py
index 78d365f..7a63381 100644
--- a/backend/src/modelforge_api/services/startup_validation.py
+++ b/backend/src/modelforge_api/services/startup_validation.py
@@ -14,6 +14,7 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
+from datetime import UTC, datetime
from enum import StrEnum
from pathlib import Path
from typing import Any
@@ -21,6 +22,7 @@ from urllib.parse import unquote, urlparse
from sqlalchemy import Engine, text
+from modelforge_api.domain.operator_identity import parse_operator_identities
from modelforge_api.domain.release import (
MINIMUM_POSTGRES_MAJOR,
Compatibility,
@@ -323,15 +325,45 @@ def validate_settings(settings: Settings) -> ValidationReport:
_validate_url(report, "MODELFORGE_DATABASE_URL", settings.database_url, {"postgresql"})
_validate_url(report, "MODELFORGE_REDIS_URL", settings.redis_url, {"redis", "rediss"})
- if settings.operator_api_key is None:
- if production:
+ if settings.operator_password_hash is not None:
+ if (settings.operator_api_key is None
+ or len(settings.operator_api_key.get_secret_value()) < MINIMUM_OPERATOR_KEY_LENGTH):
report.fail(
StartupFailureCode.MISSING_REQUIRED_SETTING,
"MODELFORGE_OPERATOR_API_KEY",
- "production requires an operator API key; every admin route depends on it",
+ "password login requires a separate bootstrap key of at least 32 characters for session signing",
+ )
+ from modelforge_api.services.operator_password import parse_password_hash
+
+ try:
+ parse_password_hash(settings.operator_password_hash.get_secret_value())
+ except ValueError:
+ report.fail(
+ StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+ "MODELFORGE_OPERATOR_PASSWORD_HASH",
+ "must be a canonical salted scrypt password verifier from operator_password.py",
+ )
+
+ if settings.operator_api_key is None or not settings.operator_api_key.get_secret_value():
+ document = settings.operator_identities
+ named_admin = document is not None and bool(document.get_secret_value()) and any(
+ identity.role == "admin" and identity.expires_at > datetime.now(UTC)
+ for identity in parse_operator_identities(document.get_secret_value())
+ )
+ if production and not named_admin:
+ report.fail(
+ StartupFailureCode.MISSING_REQUIRED_SETTING,
+ "MODELFORGE_OPERATOR_API_KEY",
+ "production requires an operator API key or an unexpired named administrator",
)
else:
secret = settings.operator_api_key.get_secret_value()
+ if secret.startswith(("mfop_", "mfopm_", "mfsvc_", "mfnode_", "mfenroll_")):
+ report.fail(
+ StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+ "MODELFORGE_OPERATOR_API_KEY",
+ "reserved personal/machine key namespaces cannot be used as bootstrap credentials",
+ )
if production and len(secret) < MINIMUM_OPERATOR_KEY_LENGTH:
report.fail(
StartupFailureCode.INSECURE_PRODUCTION_SETTING,
diff --git a/backend/src/modelforge_api/settings.py b/backend/src/modelforge_api/settings.py
index 8119457..d707e1c 100644
--- a/backend/src/modelforge_api/settings.py
+++ b/backend/src/modelforge_api/settings.py
@@ -1,9 +1,55 @@
+from __future__ import annotations
+
+import ipaddress
+import re
+from collections.abc import Mapping
+from contextvars import ContextVar
from functools import lru_cache
from pathlib import Path
-from typing import Literal
+from typing import Any, Literal
+from urllib.parse import urlsplit
-from pydantic import Field, SecretStr
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import Field, PrivateAttr, SecretStr, field_validator, model_validator
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
+
+from modelforge_api.domain.operator_identity import parse_operator_identities
+
+_active_settings_source_capture: ContextVar[dict[str, str] | None] = ContextVar(
+ "modelforge_settings_source_capture", default=None
+)
+
+
+class _CapturedSettingsSource(PydanticBaseSettingsSource):
+ """Delegate a Pydantic source unchanged while capturing only its field names."""
+
+ def __init__(self, source: PydanticBaseSettingsSource, origin: str) -> None:
+ super().__init__(source.settings_cls, _init_state=source._init_state)
+ self._source = source
+ self._origin = origin
+
+ @property
+ def __name__(self) -> str:
+ return type(self._source).__name__
+
+ def get_field_value(
+ self, field: FieldInfo, field_name: str
+ ) -> tuple[Any, str, bool]:
+ raise NotImplementedError("captured sources delegate their complete call")
+
+ def __call__(self) -> dict[str, Any]:
+ self._source._set_current_state(self.current_state)
+ self._source._set_settings_sources_data(self.settings_sources_data)
+ values = self._source()
+
+ # Capture names and source labels only. Never retain source values, which may include
+ # SecretStr credentials or URLs containing userinfo.
+ captured = _active_settings_source_capture.get()
+ if captured is not None:
+ for field_name in values:
+ if field_name in self.settings_cls.model_fields:
+ captured.setdefault(field_name, self._origin)
+ return values
def default_config_root() -> Path:
@@ -13,12 +59,75 @@ def default_config_root() -> Path:
class Settings(BaseSettings):
+ _setting_sources: dict[str, str] = PrivateAttr(default_factory=dict)
+
model_config = SettingsConfigDict(
env_prefix="MODELFORGE_",
env_file=".env",
extra="ignore",
+ hide_input_in_errors=True,
)
+ def __init__(self, *args: Any, **values: Any) -> None:
+ captured: dict[str, str] = {}
+ token = _active_settings_source_capture.set(captured)
+ try:
+ super().__init__(*args, **values)
+ object.__setattr__(self, "_setting_sources", captured)
+ finally:
+ _active_settings_source_capture.reset(token)
+
+ @classmethod
+ def settings_customise_sources(
+ cls,
+ settings_cls: type[BaseSettings],
+ init_settings: PydanticBaseSettingsSource,
+ env_settings: PydanticBaseSettingsSource,
+ dotenv_settings: PydanticBaseSettingsSource,
+ file_secret_settings: PydanticBaseSettingsSource,
+ ) -> tuple[PydanticBaseSettingsSource, ...]:
+ """Track source provenance without changing Pydantic's built-in source priority."""
+
+ return (
+ _CapturedSettingsSource(init_settings, "initialization"),
+ _CapturedSettingsSource(env_settings, "process_environment"),
+ _CapturedSettingsSource(dotenv_settings, "dotenv"),
+ _CapturedSettingsSource(file_secret_settings, "secrets_file"),
+ )
+
+ def setting_source(self, field_name: str) -> str:
+ """Return the source that supplied a resolved field, or its Pydantic default."""
+
+ return self._setting_sources.get(field_name, "pydantic_default")
+
+ def _inherit_setting_sources(self, source_settings: Settings) -> None:
+ """Retain resolved origins after validated reconstruction from this settings object."""
+
+ object.__setattr__(self, "_setting_sources", dict(source_settings._setting_sources))
+
+ def __setattr__(self, name: str, value: Any) -> None:
+ super().__setattr__(name, value)
+ if (
+ name in type(self).model_fields
+ and _active_settings_source_capture.get() is None
+ and getattr(self, "__pydantic_private__", None) is not None
+ ):
+ sources = dict(self._setting_sources)
+ sources[name] = "runtime_update"
+ object.__setattr__(self, "_setting_sources", sources)
+
+ def model_copy(
+ self, *, update: Mapping[str, Any] | None = None, deep: bool = False
+ ) -> Settings:
+ copied = super().model_copy(update=update, deep=deep)
+ if update:
+ sources = dict(self._setting_sources)
+ for field_name in update:
+ if field_name in type(self).model_fields:
+ sources[field_name] = "runtime_update"
+ object.__setattr__(copied, "_setting_sources", sources)
+ return copied
+
env: Literal["development", "test", "production"] = "development"
api_host: str = "0.0.0.0"
api_port: int = 8000
@@ -29,6 +138,8 @@ class Settings(BaseSettings):
log_level: str = "INFO"
hf_home: str = "/data/hf-cache"
hf_token: SecretStr | None = None
+ secret_slot_root_key: SecretStr | None = None
+ secret_slot_root_key_id: str = Field(default="hf-control-plane-1", pattern=r"^[a-zA-Z0-9_-]{1,64}$")
hf_timeout_seconds: float = Field(default=30, gt=0, le=300)
hf_snapshot_ttl_seconds: int = Field(default=3600, ge=60, le=86400)
artifact_root: str = "/data/artifacts"
@@ -42,6 +153,25 @@ class Settings(BaseSettings):
node_identity_mode: Literal["auto", "persisted"] = "auto"
node_identity_file: Path = Path("/data/state/node-id")
operator_api_key: SecretStr | None = None
+ operator_identities: SecretStr | None = None
+
+ @field_validator("operator_identities")
+ @classmethod
+ def validate_operator_identities(cls, value: SecretStr | None) -> SecretStr | None:
+ if value is not None and value.get_secret_value():
+ parse_operator_identities(value.get_secret_value())
+ return value
+
+ @field_validator("alert_webhook_url", mode="before")
+ @classmethod
+ def unset_empty_alert_webhook_url(cls, value: object) -> object:
+ if value == "" or (
+ isinstance(value, SecretStr) and value.get_secret_value() == ""
+ ):
+ return None
+ return value
+
+ operator_password_hash: SecretStr | None = None
node_stale_after_seconds: int = Field(default=30, ge=10)
node_offline_after_seconds: int = Field(default=90, ge=20)
liveness_poll_interval_seconds: int = Field(default=5, ge=1)
@@ -84,6 +214,17 @@ class Settings(BaseSettings):
migration_reconciliation_enabled: bool = False
observability_monitor_enabled: bool = False
observability_poll_interval_seconds: int = Field(default=60, ge=30, le=3600)
+ alert_notifications_enabled: bool = False
+ alert_webhook_url: SecretStr | None = None
+ alert_webhook_allowed_hosts: str = ""
+ alert_webhook_min_severity: Literal["INFO", "WARNING", "CRITICAL"] = "WARNING"
+ alert_notification_timeout_seconds: float = Field(default=5.0, gt=0, le=30)
+ alert_notification_max_attempts: int = Field(default=8, ge=1, le=20)
+ alert_notification_batch_size: int = Field(default=20, ge=1, le=100)
+ alert_notification_lease_seconds: int = Field(default=60, ge=15, le=600)
+ alert_notification_retry_base_seconds: int = Field(default=15, ge=1, le=3600)
+ model_update_monitor_enabled: bool = True
+ model_update_check_interval_seconds: int = Field(default=86400, ge=3600, le=604800)
backup_root: Path = Path("/data/backups")
backup_restore_root: Path = Path("/data/restore")
backup_encryption_key: SecretStr | None = None
@@ -99,6 +240,51 @@ class Settings(BaseSettings):
recovery_reconciliation_enabled: bool = False
restore_allow_production_target: bool = False
+ @field_validator("alert_webhook_allowed_hosts")
+ @classmethod
+ def normalize_alert_webhook_allowed_hosts(cls, value: str) -> str:
+ hosts = [item.strip().casefold().rstrip(".") for item in value.split(",") if item.strip()]
+ if len(hosts) > 20 or len(hosts) != len(set(hosts)):
+ raise ValueError("alert webhook allowlist must contain up to 20 unique exact hosts")
+ for host in hosts:
+ try:
+ ipaddress.ip_address(host)
+ continue
+ except ValueError:
+ pass
+ if len(host) > 253 or not re.fullmatch(
+ r"(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?",
+ host,
+ ):
+ raise ValueError("alert webhook allowlist requires exact DNS names or IP addresses")
+ return ",".join(hosts)
+
+ @model_validator(mode="after")
+ def validate_alert_webhook_settings(self) -> Settings:
+ if not self.alert_notifications_enabled:
+ return self
+ if self.alert_webhook_url is None or not self.alert_webhook_allowed_hosts:
+ raise ValueError("enabled alert notifications require a webhook URL and exact host allowlist")
+ raw = self.alert_webhook_url.get_secret_value()
+ try:
+ parsed = urlsplit(raw)
+ host = (parsed.hostname or "").casefold().rstrip(".")
+ port = parsed.port
+ except ValueError:
+ raise ValueError("alert webhook URL must be a valid HTTPS URL on an allowed host") from None
+ allowed = set(self.alert_webhook_allowed_hosts.split(","))
+ if (
+ parsed.scheme.casefold() != "https"
+ or not host
+ or host not in allowed
+ or parsed.username is not None
+ or parsed.password is not None
+ or parsed.fragment
+ or port not in (None, 443)
+ ):
+ raise ValueError("alert webhook URL must be HTTPS on an exact allowlisted host")
+ return self
+
# Build identity. These are stamped into the image at build time and are never inferred at
# runtime: a build identity a running process can talk itself into is worth nothing during an
# incident. Absent values are reported as null rather than guessed.
@@ -106,6 +292,13 @@ class Settings(BaseSettings):
build_timestamp: str | None = None
build_image_digest: str | None = None
+ @field_validator("operator_password_hash", mode="before")
+ @classmethod
+ def unset_empty_password_hash(cls, value: object) -> object:
+ if value == "" or (isinstance(value, SecretStr) and value.get_secret_value() == ""):
+ return None
+ return value
+
@property
def cors_origin_list(self) -> list[str]:
return [item.strip() for item in self.cors_origins.split(",") if item.strip()]
diff --git a/backend/tests/http_contract_fakes.py b/backend/tests/http_contract_fakes.py
new file mode 100644
index 0000000..a74177b
--- /dev/null
+++ b/backend/tests/http_contract_fakes.py
@@ -0,0 +1,32 @@
+"""Opt-in HTTP fixture: real middleware/routes, no ambient startup or database."""
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager
+
+import pytest
+from fastapi import FastAPI
+
+from modelforge_api.db import engine as ambient_engine
+from modelforge_api.main import app
+
+
+@pytest.fixture
+def isolated_http_contract(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+ # These exercise the real HTTP stack with fake collectors, not application startup.
+ # Do not run reconciliation, schema validation or background polling against .env.
+ @asynccontextmanager
+ async def contract_lifespan(_app: FastAPI) -> AsyncIterator[None]:
+ yield
+
+ ambient_connections: list[str] = []
+
+ def reject_ambient_connection(*args: object, **kwargs: object) -> None:
+ ambient_connections.append("connection attempted")
+ raise AssertionError("HTTP contract tests must not use the ambient database")
+
+ monkeypatch.setattr(ambient_engine, "connect", reject_ambient_connection)
+ monkeypatch.setattr(ambient_engine, "raw_connection", reject_ambient_connection)
+ monkeypatch.setattr(app.router, "lifespan_context", contract_lifespan)
+ # Restore the original mapping (including other tests' overrides) on teardown.
+ monkeypatch.setattr(app, "dependency_overrides", {})
+ yield
+ assert not ambient_connections
diff --git a/backend/tests/recovery_fixtures.py b/backend/tests/recovery_fixtures.py
new file mode 100644
index 0000000..d73d60b
--- /dev/null
+++ b/backend/tests/recovery_fixtures.py
@@ -0,0 +1,35 @@
+"""Physical legacy schema helpers for recovery fingerprint tests."""
+
+from sqlalchemy.engine import Connection
+
+
+def recreate_0038_benchmark_runs(connection: Connection) -> None:
+ """Replace metadata's current table with the physical pre-0039 benchmark shape."""
+ connection.exec_driver_sql("DROP TABLE benchmark_runs")
+ connection.exec_driver_sql(
+ """
+ CREATE TABLE benchmark_runs (
+ id CHAR(32) NOT NULL,
+ deployment_id CHAR(32) NOT NULL,
+ suite_id CHAR(32) NOT NULL,
+ model_revision_sha VARCHAR(64) NOT NULL,
+ artifact_sha256 VARCHAR(64) NOT NULL,
+ runtime_type VARCHAR(32) NOT NULL,
+ runtime_version VARCHAR(128) NOT NULL,
+ runtime_image_digest VARCHAR(71),
+ launch_arguments JSON NOT NULL,
+ suite_revision VARCHAR(128) NOT NULL,
+ dataset_revision VARCHAR(255) NOT NULL,
+ environment_fingerprint JSON NOT NULL,
+ environment_digest VARCHAR(64) NOT NULL,
+ seed INTEGER,
+ results JSON NOT NULL,
+ status VARCHAR(32) NOT NULL,
+ comparable BOOLEAN NOT NULL,
+ stale_reason TEXT,
+ started_at DATETIME NOT NULL,
+ completed_at DATETIME,
+ PRIMARY KEY (id)
+ )
+ """
+ )
diff --git a/backend/tests/test_alert_notifications.py b/backend/tests/test_alert_notifications.py
new file mode 100644
index 0000000..390524d
--- /dev/null
+++ b/backend/tests/test_alert_notifications.py
@@ -0,0 +1,764 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+import os
+import threading
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import SimpleNamespace
+
+import httpx
+import pytest
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, func, inspect, select, text
+from sqlalchemy import event as sqlalchemy_event
+from sqlalchemy.engine import make_url
+from sqlalchemy.exc import IntegrityError, SQLAlchemyError
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.domain.alert_notifications import NotificationStatus
+from modelforge_api.domain.observability import AlertState
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ AlertHistoryEvent,
+ AlertNotificationDelivery,
+ AlertRuleRevision,
+ Base,
+ OperationalAlert,
+)
+from modelforge_api.services.alert_notifications import (
+ _claim,
+ dispatch_pending,
+ enqueue_alert_transition,
+)
+from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.services.recovery_fingerprint import (
+ ALERT_NOTIFICATION_SCHEMA,
+ PORTFOLIO_SCHEMA,
+ FingerprintCompatibility,
+ _without_alert_notifications,
+ _without_lab_execution,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ fingerprint_session,
+)
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN_TOKEN = "alert-notification-test-admin" # noqa: S105 - synthetic test credential
+HOOK_URL = "https://hooks.example.test/secret-route-value"
+
+
+@pytest.fixture
+def session() -> Session:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+def settings(**overrides: object) -> Settings:
+ values: dict[str, object] = {
+ "_env_file": None,
+ "env": "test",
+ "alert_notifications_enabled": True,
+ "alert_webhook_url": SecretStr(HOOK_URL),
+ "alert_webhook_allowed_hosts": "hooks.example.test",
+ "alert_notification_retry_base_seconds": 1,
+ "operator_api_key": SecretStr(ADMIN_TOKEN),
+ }
+ values.update(overrides)
+ return Settings(**values) # type: ignore[arg-type]
+
+
+def make_alert_event(
+ session: Session, *, severity: str = "CRITICAL", state: str = "FIRING"
+) -> tuple[OperationalAlert, AlertHistoryEvent]:
+ rule = AlertRuleRevision(
+ key=f"notification-test-{uuid.uuid4()}",
+ revision=1,
+ alert_type="BACKUP_STALE",
+ signal="backup_stale",
+ slo_policy_id=None,
+ condition={},
+ pending_seconds=0,
+ severity=severity,
+ labels={},
+ cooldown_seconds=0,
+ recovery_condition={},
+ active=True,
+ fingerprint=uuid.uuid4().hex,
+ created_by="notification-test",
+ )
+ session.add(rule)
+ session.flush()
+ alert = OperationalAlert(
+ rule_id=rule.id,
+ fingerprint=hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest(),
+ alert_type="BACKUP_STALE",
+ severity=severity,
+ state=state,
+ source="backup_sets",
+ subject_type="backup",
+ subject_ref="latest",
+ summary="Latest verified backup is stale",
+ details={"secret": "must-not-be-in-webhook"},
+ first_seen_at=datetime.now(UTC),
+ last_seen_at=datetime.now(UTC),
+ )
+ session.add(alert)
+ session.flush()
+ event = AlertHistoryEvent(
+ alert_id=alert.id,
+ from_state=None,
+ to_state=state,
+ actor="observability-engine",
+ reason="Signal condition detected",
+ evidence={"secret": "must-not-be-in-webhook"},
+ occurred_at=datetime.now(UTC),
+ )
+ session.add(event)
+ session.flush()
+ return alert, event
+
+
+def test_settings_require_https_exact_host_allowlist_and_standard_port() -> None:
+ with pytest.raises(ValidationError):
+ Settings(
+ _env_file=None,
+ alert_notifications_enabled=True,
+ alert_webhook_url=SecretStr("http://hooks.example.test/private"),
+ alert_webhook_allowed_hosts="hooks.example.test",
+ )
+ with pytest.raises(ValidationError):
+ Settings(
+ _env_file=None,
+ alert_notifications_enabled=True,
+ alert_webhook_url=SecretStr("https://evil.example.test/private"),
+ alert_webhook_allowed_hosts="hooks.example.test",
+ )
+ with pytest.raises(ValidationError):
+ Settings(
+ _env_file=None,
+ alert_notifications_enabled=True,
+ alert_webhook_url=SecretStr("https://hooks.example.test:8443/private"),
+ alert_webhook_allowed_hosts="hooks.example.test",
+ )
+ with pytest.raises(ValidationError):
+ Settings(
+ _env_file=None,
+ alert_notifications_enabled=True,
+ alert_webhook_url=SecretStr("https://hooks.example.test/private"),
+ alert_webhook_allowed_hosts="*.example.test",
+ )
+
+
+def test_enqueue_is_transactional_filtered_and_unique_per_history_event(session: Session) -> None:
+ config = settings(alert_webhook_min_severity="WARNING")
+ alert, event = make_alert_event(session)
+ enqueue_alert_transition(session, config, alert, event)
+ session.flush()
+ row = session.scalar(select(AlertNotificationDelivery))
+ assert row is not None
+ assert row.idempotency_key == f"alert:{event.id}:default"
+ assert row.event_kind == "FIRING"
+ assert row.payload["summary"] == alert.summary
+ assert "details" not in row.payload
+ assert "must-not-be-in-webhook" not in json.dumps(row.payload)
+
+ enqueue_alert_transition(session, config, alert, event)
+ with pytest.raises(IntegrityError):
+ session.flush()
+ session.rollback()
+ assert session.scalar(select(func.count()).select_from(AlertNotificationDelivery)) == 0
+
+
+def test_enqueue_only_notifies_firing_or_resolved_at_configured_severity(session: Session) -> None:
+ config = settings(alert_webhook_min_severity="WARNING")
+ alert, event = make_alert_event(session, severity="INFO", state="PENDING")
+ enqueue_alert_transition(session, config, alert, event)
+ assert session.scalar(select(func.count()).select_from(AlertNotificationDelivery)) == 0
+
+ event.to_state = "RESOLVED"
+ alert.severity = "CRITICAL"
+ enqueue_alert_transition(session, config, alert, event)
+ assert session.scalar(select(AlertNotificationDelivery)).event_kind == "RESOLVED" # type: ignore[union-attr]
+
+
+def test_firing_transition_and_outbox_commit_in_one_transaction(session: Session) -> None:
+ config = settings()
+ alert, _initial_event = make_alert_event(session, state="PENDING")
+ now = datetime.now(UTC)
+ ObservabilityService(session, config)._transition(
+ alert, AlertState.FIRING, "observability-engine", "pending duration satisfied", now, {}
+ )
+ session.commit()
+ delivery = session.scalar(select(AlertNotificationDelivery))
+ assert delivery is not None
+ assert delivery.status == NotificationStatus.PENDING.value
+ assert delivery.event_kind == "FIRING"
+ assert delivery.alert_history_event_id is not None
+
+ session.delete(delivery)
+ session.commit()
+ second, _event = make_alert_event(session, state="PENDING")
+ ObservabilityService(session, config)._transition(
+ second, AlertState.FIRING, "observability-engine", "pending duration satisfied", now, {}
+ )
+ session.flush()
+ session.rollback()
+ assert session.scalar(select(func.count()).select_from(AlertNotificationDelivery)) == 0
+
+
+def test_dispatch_retries_then_delivers_without_following_redirects_or_exposing_body(
+ session: Session,
+) -> None:
+ config = settings()
+ alert, event = make_alert_event(session)
+ enqueue_alert_transition(session, config, alert, event)
+ session.commit()
+ requests: list[httpx.Request] = []
+
+ def retry(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return httpx.Response(503, text="remote response contains sensitive diagnostic")
+
+ assert dispatch_pending(session, config, transport=httpx.MockTransport(retry)) == 1
+ row = session.scalar(select(AlertNotificationDelivery))
+ assert row is not None
+ assert row.status == NotificationStatus.RETRY_WAIT.value
+ assert row.attempt_count == 1
+ assert row.last_error_code == "REMOTE_RETRYABLE"
+ assert row.last_http_status == 503
+ assert HOOK_URL not in repr(row)
+ assert row.last_error_code not in "remote response contains sensitive diagnostic"
+ assert requests[0].url == httpx.URL(HOOK_URL)
+ assert requests[0].headers["idempotency-key"] == row.idempotency_key
+ assert requests[0].extensions.get("timeout") is not None
+
+ row.next_attempt_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ assert dispatch_pending(
+ session,
+ config,
+ transport=httpx.MockTransport(lambda request: httpx.Response(200, text="ignored")),
+ ) == 1
+ session.refresh(row)
+ assert row.status == NotificationStatus.DELIVERED.value
+ assert row.delivered_at is not None
+ assert row.last_error_code is None
+
+
+def test_dispatch_rejects_redirect_and_permanent_client_errors(session: Session) -> None:
+ config = settings()
+ alert, event = make_alert_event(session)
+ enqueue_alert_transition(session, config, alert, event)
+ session.commit()
+ requests: list[httpx.Request] = []
+
+ def redirect(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return httpx.Response(302, headers={"Location": "https://evil.example.test/"})
+
+ assert dispatch_pending(session, config, transport=httpx.MockTransport(redirect)) == 1
+ row = session.scalar(select(AlertNotificationDelivery))
+ assert row is not None
+ assert row.status == NotificationStatus.FAILED.value
+ assert row.last_error_code == "REDIRECT_REJECTED"
+ assert len(requests) == 1
+
+
+def test_dispatch_permanent_failure_after_attempt_limit(session: Session) -> None:
+ config = settings(alert_notification_max_attempts=1)
+ alert, event = make_alert_event(session)
+ enqueue_alert_transition(session, config, alert, event)
+ session.commit()
+ assert dispatch_pending(
+ session,
+ config,
+ transport=httpx.MockTransport(lambda request: httpx.Response(400, text="private body")),
+ ) == 1
+ row = session.scalar(select(AlertNotificationDelivery))
+ assert row is not None
+ assert row.status == NotificationStatus.FAILED.value
+ assert row.last_error_code == "REMOTE_REJECTED"
+ assert row.last_http_status == 400
+
+
+def test_schema_0037_recovery_holds_pending_and_rejects_unrelated_delivery_changes(
+ session: Session,
+) -> None:
+ config = settings()
+ alert, event = make_alert_event(session)
+ enqueue_alert_transition(session, config, alert, event)
+ pending = session.scalar(select(AlertNotificationDelivery))
+ assert pending is not None
+ pending.next_attempt_at = datetime.now(UTC) + timedelta(minutes=1)
+ pending.lease_expires_at = datetime.now(UTC) + timedelta(minutes=1)
+ pending.claim_token_hash = "a" * 64
+ delivered = AlertNotificationDelivery(
+ alert_history_event_id=None,
+ route_key="default",
+ idempotency_key="test:already-delivered:default",
+ event_kind="TEST",
+ severity=None,
+ payload={"event": "TEST"},
+ status="DELIVERED",
+ attempt_count=1,
+ last_attempt_at=datetime.now(UTC),
+ delivered_at=datetime.now(UTC),
+ )
+ session.add(delivered)
+ session.commit()
+ source = _without_lab_execution(fingerprint_session(session))
+
+ pending.status = NotificationStatus.HELD_FOR_REVIEW.value
+ pending.next_attempt_at = None
+ pending.lease_expires_at = None
+ pending.claim_token_hash = None
+ session.commit()
+ restored = fingerprint_session(session)
+ contract = fingerprint_compatibility(
+ source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert contract.compatible
+ assert contract.mode.startswith(
+ "BENCHMARK_RUNS_UPGRADE|20260923_0037|LAB_EXECUTION_UPGRADE|20260923_0037|"
+ "ALERT_NOTIFICATION_STRICT|"
+ )
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+ delivered.status = NotificationStatus.HELD_FOR_REVIEW.value
+ session.commit()
+ corrupted = fingerprint_session(session)
+ rejected = fingerprint_compatibility(
+ source, corrupted, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert not rejected.compatible
+
+
+def test_restore_reconciliation_holds_pending_deliveries_and_requires_review(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from modelforge_api.services.recovery import RecoveryService
+
+ statements: list[str] = []
+
+ class RecoveryEngine:
+ def scalar(self, _target: object, sql: str) -> str:
+ statements.append(sql)
+ if "from alert_notification_deliveries" in sql and "select count(*)" in sql:
+ return "2"
+ if "from audit_events where action" in sql:
+ return "0"
+ if "string_agg" in sql:
+ return ""
+ return "0"
+
+ service = RecoveryService(session, Settings(_env_file=None, env="test"))
+ service.engine = RecoveryEngine() # type: ignore[assignment]
+ monkeypatch.setattr(service, "_verify_restored_audit_chain", lambda _target: {})
+ monkeypatch.setattr(service, "_append_restored_audit_marker", lambda *_args, **_kwargs: "a" * 64)
+ monkeypatch.setattr(service, "_reconcile_selected_comparisons", lambda _target: {"interrupted": 0, "claims_cleared": 0})
+ monkeypatch.setattr(service, "_reconcile_creative_allocations", lambda _target: 0)
+ monkeypatch.setattr(service.audit, "write", lambda *_args, **_kwargs: None)
+ record = SimpleNamespace(id=uuid.uuid4())
+ plan = SimpleNamespace(
+ database_destination="postgresql+psycopg://modelforge@restore.example.test/restored",
+ backup_set=SimpleNamespace(backup_id="notification-recovery"),
+ )
+
+ result = service._reconcile_restored(record, plan) # type: ignore[arg-type]
+
+ assert any(
+ "update alert_notification_deliveries set status='HELD_FOR_REVIEW'" in sql
+ and "next_attempt_at=null" in sql
+ and "lease_expires_at=null" in sql
+ and "claim_token_hash=null" in sql
+ and "where status in ('PENDING','DISPATCHING','RETRY_WAIT')" in sql
+ for sql in statements
+ )
+ assert result["notification_deliveries_held_for_review"] == 2
+ assert result["outcome"] == "RECOVERY_RECONCILIATION_REQUIRED"
+
+
+def test_schema_0037_upgrade_allows_only_empty_outbox_and_exact_marker() -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ restored = fingerprint_session(session)
+ source = _without_alert_notifications(_without_lab_execution(restored))
+ contract = fingerprint_compatibility(
+ source, restored, source_schema_revision=PORTFOLIO_SCHEMA
+ )
+ assert contract.compatible
+ assert contract.mode.startswith(
+ "BENCHMARK_RUNS_UPGRADE|20260923_0036|LAB_EXECUTION_UPGRADE|20260923_0036|"
+ )
+ assert "ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_STRICT|" in contract.mode
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+ populated = json.loads(json.dumps(restored))
+ populated["tables"]["alert_notification_deliveries"]["row_count"] = 1
+ populated["tables"]["alert_notification_deliveries"]["digest"] = "f" * 64
+ rejected = fingerprint_compatibility(
+ source, populated, source_schema_revision=PORTFOLIO_SCHEMA
+ )
+ assert not rejected.compatible
+
+ unrelated = FingerprintCompatibility(
+ True,
+ "UNRELATED|" + contract.mode,
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ )
+ assert not audit_fingerprint_differences_are_compatible(source, restored, unrelated)
+ engine.dispose()
+
+
+def test_pre_0037_fingerprint_rejects_a_future_outbox_physical_table() -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ current = fingerprint_session(session)
+ source = _without_lab_execution(_without_alert_notifications(current))
+ source["physical_tables"] = sorted(
+ [*source["physical_tables"], "alert_notification_deliveries"]
+ )
+ rejected = fingerprint_compatibility(
+ source, source, source_schema_revision=PORTFOLIO_SCHEMA
+ )
+ assert not rejected.compatible
+ assert "table contract" in rejected.reason
+ engine.dispose()
+
+
+def test_schema_0037_migration_creates_indexes_and_refuses_history_downgrade() -> None:
+ migration_path = (
+ Path(__file__).resolve().parents[1]
+ / "alembic/versions/20260923_0037_alert_notification_outbox.py"
+ )
+ spec = importlib.util.spec_from_file_location("alert_notification_migration", migration_path)
+ assert spec is not None and spec.loader is not None
+ migration = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(migration)
+
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
+ migration.upgrade()
+ assert migration.down_revision == "20260923_0036"
+ assert "ix_alert_notification_dispatch_due" in {
+ item["name"] for item in inspect(connection).get_indexes("alert_notification_deliveries")
+ }
+ migration.downgrade()
+ migration.upgrade()
+ connection.execute(
+ text(
+ "insert into alert_notification_deliveries "
+ "(id, route_key, idempotency_key, event_kind, payload, status, attempt_count, created_at) "
+ "values (:id, 'default', 'test:migration:default', 'TEST', '{}', 'PENDING', 0, :created_at)"
+ ),
+ {"id": str(uuid.uuid4()), "created_at": datetime.now(UTC)},
+ )
+ with pytest.raises(RuntimeError, match="history exists"):
+ migration.downgrade()
+ engine.dispose()
+
+
+def test_admin_test_action_queues_safe_delivery_and_viewer_is_denied(session: Session) -> None:
+ config = settings()
+ app.dependency_overrides[get_session] = lambda: session
+ app.dependency_overrides[get_settings] = lambda: config
+ client = TestClient(app)
+ try:
+ assert client.post("/api/v1/admin/operations/notifications/test").status_code == 401
+ response = client.post(
+ "/api/v1/admin/operations/notifications/test",
+ headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN},
+ )
+ assert response.status_code == 202
+ assert HOOK_URL not in response.text
+ assert "secret-route-value" not in response.text
+ delivery_id = response.json()["delivery"]["id"]
+ listed = client.get(
+ "/api/v1/admin/operations/notification-deliveries",
+ headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN},
+ )
+ assert listed.status_code == 200
+ assert listed.json()[0]["id"] == delivery_id
+ assert "payload" not in listed.json()[0]
+
+ app.dependency_overrides[get_settings] = lambda: settings(
+ alert_notifications_enabled=False
+ )
+ unavailable = client.post(
+ "/api/v1/admin/operations/notifications/test",
+ headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN},
+ )
+ assert unavailable.status_code == 503
+ assert unavailable.json()["error"]["code"] == "NOTIFICATION_ROUTE_UNAVAILABLE"
+ assert unavailable.json()["error"]["message"] == (
+ "The deployment-managed alert notification route is unavailable"
+ )
+ assert HOOK_URL not in unavailable.text
+
+ viewer_token = "mfop_" + "v" * 43
+ viewer_settings = settings(
+ operator_api_key=None,
+ operator_identities=SecretStr(
+ json.dumps(
+ [
+ {
+ "credential_id": "viewer-test",
+ "subject": "alert-viewer",
+ "role": "viewer",
+ "token_sha256": hashlib.sha256(viewer_token.encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+ }
+ ]
+ )
+ ),
+ )
+ app.dependency_overrides[get_settings] = lambda: viewer_settings
+ denied = client.post(
+ "/api/v1/admin/operations/notifications/test",
+ headers={"X-ModelForge-Admin-Token": viewer_token},
+ )
+ assert denied.status_code == 403
+ finally:
+ app.dependency_overrides.clear()
+ client.close()
+
+
+def test_postgres_concurrent_claim_skips_the_row_held_by_another_worker() -> None:
+ admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+ if not admin_url:
+ pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ if not owner_url or not runtime_url:
+ pytest.fail("managed notification claim test requires owner and runtime URLs")
+ if admin_url.startswith("postgresql://"):
+ admin_url = admin_url.replace("postgresql://", "postgresql+psycopg://", 1)
+
+ database = f"test_alert_claim_{uuid.uuid4().hex}"
+ admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+ with admin_engine.connect() as connection:
+ connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+
+ owner_database_url = make_url(owner_url).set(database=database)
+ runtime_engine = create_engine(make_url(runtime_url).set(database=database), pool_pre_ping=True)
+ migration_config = Config()
+ migration_config.set_main_option(
+ "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+ )
+ migration_config.set_main_option(
+ "sqlalchemy.url",
+ owner_database_url.render_as_string(hide_password=False).replace("%", "%%"),
+ )
+ first_selected = threading.Event()
+ release_first = threading.Event()
+ results: dict[str, int] = {}
+ failures: list[SQLAlchemyError | TimeoutError] = []
+
+ @sqlalchemy_event.listens_for(runtime_engine, "after_cursor_execute")
+ def pause_first_claim(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ normalized = statement.lower()
+ if (
+ threading.current_thread().name == "notification-claim-a"
+ and "from alert_notification_deliveries" in normalized
+ and "for update" in normalized
+ ):
+ first_selected.set()
+ if not release_first.wait(timeout=10):
+ raise TimeoutError("first notification claim did not receive release signal")
+
+ try:
+ import importlib
+
+ importlib.import_module("alembic.command").upgrade(migration_config, "head")
+ with Session(runtime_engine) as session:
+ session.add(
+ AlertNotificationDelivery(
+ idempotency_key=f"test:{uuid.uuid4()}:default",
+ route_key="default",
+ event_kind="TEST",
+ payload={"schema_version": 1, "event": "TEST"},
+ status="PENDING",
+ attempt_count=0,
+ )
+ )
+ session.commit()
+
+ config = settings(alert_notification_batch_size=1)
+ now = datetime.now(UTC)
+
+ def claim(name: str) -> None:
+ try:
+ with Session(runtime_engine) as session:
+ results[name] = len(_claim(session, config, now))
+ except (SQLAlchemyError, TimeoutError) as error: # retained until both workers unblock
+ failures.append(error)
+
+ first = threading.Thread(target=claim, args=("first",), name="notification-claim-a")
+ second = threading.Thread(target=claim, args=("second",), name="notification-claim-b")
+ first.start()
+ assert first_selected.wait(timeout=10), "first worker did not lock the due row"
+ second.start()
+ second.join(timeout=10)
+ release_first.set()
+ first.join(timeout=10)
+ assert not first.is_alive() and not second.is_alive()
+ assert failures == []
+ assert results == {"first": 1, "second": 0}
+ finally:
+ release_first.set()
+ runtime_engine.dispose()
+ with admin_engine.connect() as connection:
+ connection.execute(text(f'DROP DATABASE IF EXISTS "{database}" WITH (FORCE)'))
+ admin_engine.dispose()
+
+
+def test_postgres_expired_lease_fences_a_stale_dispatch_result() -> None:
+ admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+ if not admin_url:
+ pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ if not owner_url or not runtime_url:
+ pytest.fail("managed notification lease test requires owner and runtime URLs")
+ if admin_url.startswith("postgresql://"):
+ admin_url = admin_url.replace("postgresql://", "postgresql+psycopg://", 1)
+
+ database = f"test_alert_lease_{uuid.uuid4().hex}"
+ admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+ with admin_engine.connect() as connection:
+ connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+
+ owner_database_url = make_url(owner_url).set(database=database)
+ runtime_engine = create_engine(make_url(runtime_url).set(database=database), pool_pre_ping=True)
+ migration_config = Config()
+ migration_config.set_main_option(
+ "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+ )
+ migration_config.set_main_option(
+ "sqlalchemy.url",
+ owner_database_url.render_as_string(hide_password=False).replace("%", "%%"),
+ )
+ first_delivery_started = threading.Event()
+ release_first_delivery = threading.Event()
+ results: dict[str, int] = {}
+ failures: list[Exception] = []
+ workers: list[threading.Thread] = []
+ delivery_id = uuid.uuid4()
+
+ class DelayedFailureTransport(httpx.BaseTransport):
+ def handle_request(self, request: httpx.Request) -> httpx.Response:
+ first_delivery_started.set()
+ if not release_first_delivery.wait(timeout=10):
+ raise AssertionError("first delivery did not receive release signal")
+ return httpx.Response(503, request=request)
+
+ class SuccessfulTransport(httpx.BaseTransport):
+ def handle_request(self, request: httpx.Request) -> httpx.Response:
+ return httpx.Response(200, request=request)
+
+ def dispatch(
+ name: str,
+ claim_time: datetime,
+ transport: httpx.BaseTransport,
+ ) -> None:
+ try:
+ with Session(runtime_engine) as session:
+ results[name] = dispatch_pending(
+ session, config, now=claim_time, transport=transport
+ )
+ except (AssertionError, RuntimeError, SQLAlchemyError, TimeoutError, ValueError) as error:
+ failures.append(error)
+
+ try:
+ import importlib
+
+ importlib.import_module("alembic.command").upgrade(migration_config, "head")
+ with Session(runtime_engine) as session:
+ session.add(
+ AlertNotificationDelivery(
+ id=delivery_id,
+ idempotency_key=f"test:{uuid.uuid4()}:default",
+ route_key="default",
+ event_kind="TEST",
+ payload={"schema_version": 1, "event": "TEST"},
+ status="PENDING",
+ attempt_count=0,
+ )
+ )
+ session.commit()
+
+ config = settings(alert_notification_batch_size=1, alert_notification_lease_seconds=15)
+ claim_time = datetime.now(UTC)
+ first = threading.Thread(
+ target=dispatch,
+ args=("first", claim_time, DelayedFailureTransport()),
+ name="notification-dispatch-stale",
+ )
+ second = threading.Thread(
+ target=dispatch,
+ args=(
+ "second",
+ claim_time + timedelta(seconds=config.alert_notification_lease_seconds + 1),
+ SuccessfulTransport(),
+ ),
+ name="notification-dispatch-reclaimed",
+ )
+ first.start()
+ workers.append(first)
+ assert first_delivery_started.wait(timeout=10), "first worker did not begin delivery"
+ second.start()
+ workers.append(second)
+ second.join(timeout=10)
+ assert not second.is_alive(), "replacement worker did not finish"
+ assert results.get("second") == 1
+ release_first_delivery.set()
+ first.join(timeout=10)
+ assert not first.is_alive(), "stale worker did not finish"
+ assert failures == []
+ assert results == {"second": 1, "first": 1}
+
+ with Session(runtime_engine) as session:
+ delivery = session.get(AlertNotificationDelivery, delivery_id)
+ assert delivery is not None
+ assert delivery.status == "DELIVERED"
+ assert delivery.attempt_count == 2
+ assert delivery.last_http_status == 200
+ assert delivery.last_error_code is None
+ assert delivery.delivered_at is not None
+ assert delivery.lease_expires_at is None
+ assert delivery.claim_token_hash is None
+ finally:
+ release_first_delivery.set()
+ for worker in workers:
+ worker.join(timeout=10)
+ runtime_engine.dispose()
+ with admin_engine.connect() as connection:
+ connection.execute(text(f'DROP DATABASE IF EXISTS "{database}" WITH (FORCE)'))
+ admin_engine.dispose()
diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py
index 4ebffcc..31c435d 100644
--- a/backend/tests/test_api.py
+++ b/backend/tests/test_api.py
@@ -1,15 +1,19 @@
+from typing import Any
+
from fastapi.testclient import TestClient
from pydantic import SecretStr
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session
from sqlalchemy.pool import StaticPool
+from modelforge_api.api.routes.catalog import list_capabilities, list_projects
from modelforge_api.api.routes.registry import get_registry_service
from modelforge_api.db import get_session
-from modelforge_api.domain.release import PRODUCT_VERSION
+from modelforge_api.domain.release import PRODUCT_VERSION, RELEASE_CHANNEL
from modelforge_api.main import app
-from modelforge_api.persistence.models import Base
+from modelforge_api.persistence.models import Base, Capability, CapabilityContract
from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.project_registry import sync_project_registry
from modelforge_api.services.registry import RegistryService, seed_candidate_registry
from modelforge_api.settings import Settings, get_settings
@@ -44,6 +48,32 @@ def test_readiness_validates_all_manifests() -> None:
assert response.json()["checks"] == {"manifests": "healthy"}
+def test_public_version_contract_supplies_console_release_identity_fields() -> None:
+ commit = "e472d444dd007f22db9f2eac0b97e4e97d0a2993"
+ built_at = "2026-09-24T12:00:00Z"
+ image_digest = f"sha256:{'f' * 64}"
+ app.dependency_overrides[get_settings] = lambda: Settings(
+ _env_file=None,
+ build_commit=commit,
+ build_timestamp=built_at,
+ build_image_digest=image_digest,
+ )
+ try:
+ response = client.get("/api/v1/version")
+ assert response.status_code == 200
+ payload = response.json()
+
+ assert payload["version"] == "1.3.0-rc.1"
+ assert payload["release_channel"] == "rc"
+ assert payload["source_commit"] == commit
+ assert payload["built_at"] == built_at
+ assert payload["image_digest"] == image_digest
+ assert payload["compatibility"]["schema_revision"] == "20260924_0039"
+ assert payload["compatibility"]["agent_protocol_version"] == 1
+ finally:
+ app.dependency_overrides.clear()
+
+
def test_system_metadata_reports_the_release_without_unpromoted_inference() -> None:
"""A released product reports its version and channel, not the milestone that built it."""
@@ -62,7 +92,7 @@ def test_system_metadata_reports_the_release_without_unpromoted_inference() -> N
headers=CONTROL_PLANE_HEADERS,
).json()
assert payload["version"] == PRODUCT_VERSION
- assert payload["release_channel"] == "stable"
+ assert payload["release_channel"] == RELEASE_CHANNEL
assert "milestone" not in payload
assert payload["production_inference_available"] is False
finally:
@@ -124,3 +154,88 @@ def test_capability_and_project_contract_endpoints() -> None:
assert {item["key"] for item in capabilities} >= {"rag.embedding", "vision.embedding"}
assert {item["id"] for item in projects} == {"examplerag", "examplevision", "example-ops"}
assert all("model_id" not in binding for project in projects for binding in project["bindings"])
+
+
+def test_capability_catalog_select_count_does_not_grow_with_contracts() -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ manifests = ManifestRegistry()
+ expected = manifests.capabilities()
+ assert len(expected) > 1
+ selects = 0
+
+ def count_selects(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ nonlocal selects
+ if statement.lstrip().upper().startswith("SELECT"):
+ selects += 1
+
+ event.listen(engine, "before_cursor_execute", count_selects)
+ try:
+ with Session(engine) as session:
+ result = list_capabilities(registry=manifests, session=session)
+ finally:
+ event.remove(engine, "before_cursor_execute", count_selects)
+ engine.dispose()
+
+ assert [(item.key, item.version) for item in result] == [
+ (item.capability, item.version) for item in expected
+ ]
+ assert selects == 1
+
+
+def test_project_catalog_select_count_does_not_grow_with_bindings() -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ manifests = ManifestRegistry()
+ with Session(engine) as session:
+ for manifest in manifests.capabilities():
+ if manifest.capability not in {"rag.embedding", "rag.reranking", "vision.embedding"}:
+ continue
+ capability = Capability(key=manifest.capability, description=manifest.description)
+ session.add(capability)
+ session.flush()
+ session.add(
+ CapabilityContract(
+ capability_id=capability.id,
+ version=manifest.version,
+ input_schema=manifest.input_schema,
+ output_schema=manifest.output_schema,
+ contract=manifest.model_dump(mode="json"),
+ upgrade_class=manifest.upgrade_class.value,
+ )
+ )
+ session.commit()
+ assert sync_project_registry(session, manifests).bindings_created == 3
+
+ selects = 0
+
+ def count_selects(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ nonlocal selects
+ if statement.lstrip().upper().startswith("SELECT"):
+ selects += 1
+
+ event.listen(engine, "before_cursor_execute", count_selects)
+ try:
+ with Session(engine) as session:
+ result = list_projects(registry=manifests, session=session)
+ finally:
+ event.remove(engine, "before_cursor_execute", count_selects)
+ engine.dispose()
+
+ assert {item.id for item in result} == {"examplerag", "examplevision", "example-ops"}
+ assert sum(len(item.bindings) for item in result) == 3
+ assert selects == 4
diff --git a/backend/tests/test_audit_integrity_rc.py b/backend/tests/test_audit_integrity_rc.py
index 225a91a..532e6ba 100644
--- a/backend/tests/test_audit_integrity_rc.py
+++ b/backend/tests/test_audit_integrity_rc.py
@@ -593,13 +593,17 @@ def test_exact_aliased_delete_and_head_rewrite_poc_is_refused(
"statement",
[
"/* leading /* nested */ comment */ DELETE\nFROM [main].[audit_events]",
- "-- leading decoy\nUPDATE `main`.`audit_chain_heads` "
- "SET event_count=0, last_sequence=0, last_event_hash=NULL",
+ (
+ "-- leading decoy\nUPDATE `main`.`audit_chain_heads` "
+ "SET event_count=0, last_sequence=0, last_event_hash=NULL"
+ ),
'INSERT /* gap */ INTO "public"."audit_events" (id) VALUES (NULL)',
"WITH doomed AS (SELECT 1) DELETE FROM audit_events",
"TRUNCATE TABLE harmless_table, audit_events",
- "MERGE INTO public.audit_chain_heads AS head USING incoming ON false "
- "WHEN MATCHED THEN DELETE",
+ (
+ "MERGE INTO public.audit_chain_heads AS head USING incoming ON false "
+ "WHEN MATCHED THEN DELETE"
+ ),
"COPY audit_events FROM STDIN",
"DROP TABLE harmless_table, audit_chain_heads",
"DO $$ BEGIN DELETE FROM audit_events; END $$",
@@ -736,8 +740,10 @@ def test_late_statement_rewrite_is_rolled_back_before_commit(tmp_path: Any) -> N
[
r'ALTER TABLE IF EXISTS U&"audit\005fevents" DISABLE TRIGGER ALL',
r'''ALTER TABLE IF EXISTS U&"audit!005fchain!005fheads" UESCAPE '!' DISABLE TRIGGER ALL''',
- "CREATE OR REPLACE FUNCTION reset_chain() RETURNS void LANGUAGE SQL AS $$ "
- "DELETE FROM audit_events $$",
+ (
+ "CREATE OR REPLACE FUNCTION reset_chain() RETURNS void LANGUAGE SQL AS $$ "
+ "DELETE FROM audit_events $$"
+ ),
"SELECT public.reset_chain()",
"CALL reset_chain()",
"DO $$ BEGIN EXECUTE 'DELETE FROM audit_events'; END $$",
@@ -756,8 +762,10 @@ def test_runtime_sql_classifier_defaults_procedural_and_obfuscated_mutation_to_d
[
"SELECT count(*) FROM audit_events",
"SELECT * FROM audit_chain_heads",
- "SELECT * FROM modelforge_audit.append_event_v2(NULL, NULL, NULL, NULL, NULL, "
- "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)",
+ (
+ "SELECT * FROM modelforge_audit.append_event_v2(NULL, NULL, NULL, NULL, NULL, "
+ "NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
+ ),
],
)
def test_runtime_sql_classifier_keeps_read_and_canonical_append_controls(
diff --git a/backend/tests/test_audit_migration_rc.py b/backend/tests/test_audit_migration_rc.py
index ccd2108..36fd221 100644
--- a/backend/tests/test_audit_migration_rc.py
+++ b/backend/tests/test_audit_migration_rc.py
@@ -12,6 +12,8 @@ from types import ModuleType
from unittest.mock import Mock
import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
from sqlalchemy import (
JSON,
BigInteger,
@@ -28,8 +30,6 @@ from sqlalchemy import (
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session
-from alembic.migration import MigrationContext
-from alembic.operations import Operations
from modelforge_api.domain.audit import AUDIT_EMPTY_LEGACY_PREFIX_SEAL
from modelforge_api.persistence.models import (
AuditChainHead,
diff --git a/backend/tests/test_audit_postgres_boundary_rc.py b/backend/tests/test_audit_postgres_boundary_rc.py
index c3a30c7..54aeb0e 100644
--- a/backend/tests/test_audit_postgres_boundary_rc.py
+++ b/backend/tests/test_audit_postgres_boundary_rc.py
@@ -9,21 +9,29 @@ from __future__ import annotations
import hashlib
import importlib.util
+import os
import re
from pathlib import Path
from types import ModuleType, SimpleNamespace
-from typing import Any, cast
-from unittest.mock import Mock
+from typing import Any, Self, cast
+import psycopg
import pytest
from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
from modelforge_api.persistence.audit_postgres import (
AUDIT_APPEND_BODY_SHA256,
AUDIT_GUARD_BODY_SHA256,
)
-from modelforge_api.persistence.models import _textual_audit_dml_targets
+from modelforge_api.persistence.models import (
+ AuditChainHead,
+ AuditEvent,
+ _textual_audit_dml_targets,
+)
+from modelforge_api.services.audit import AuditWriter
from modelforge_api.services.startup_validation import (
AUDIT_RUNTIME_BOUNDARY_SQL,
StartupFailureCode,
@@ -150,22 +158,20 @@ def test_generated_sql_is_executed_as_driver_safe_complete_statements() -> None:
assert append_function.rstrip().endswith("$append$")
assert all(statement.strip() and not statement.rstrip().endswith(";") for statement in statements)
+ class DriverConnection:
+ dialect = SimpleNamespace(name="postgresql", paramstyle="pyformat")
-def test_postgres_percent_syntax_is_escaped_only_at_the_dbapi_boundary() -> None:
- module = _migration()
- connection = SimpleNamespace(
- dialect=SimpleNamespace(name="postgresql", paramstyle="pyformat"),
- exec_driver_sql=Mock(),
- )
+ def __init__(self) -> None:
+ self.statements: list[str] = []
- module._exec_postgres_sql(
- connection,
- "declare value public.audit_events%rowtype; select format('%I', 'value')",
- )
+ def exec_driver_sql(self, statement: str) -> None:
+ self.statements.append(statement)
- connection.exec_driver_sql.assert_called_once_with(
- "declare value public.audit_events%%rowtype; select format('%%I', 'value')"
- )
+ driver = DriverConnection()
+ module._install_postgres_audit_boundary(driver)
+ assert driver.statements
+ assert any("%%rowtype" in statement.lower() for statement in driver.statements)
+ assert any("%%I" in statement for statement in driver.statements)
class _Rows:
@@ -324,10 +330,10 @@ class _StartupConnection:
def __init__(self, facts: dict[str, Any]) -> None:
self.facts = facts
- def __enter__(self) -> _StartupConnection:
+ def __enter__(self) -> Self:
return self
- def __exit__(self, *_args: Any) -> None:
+ def __exit__(self, *_args: object) -> None:
return None
def exec_driver_sql(self, _sql: str) -> _ScalarResult:
@@ -335,7 +341,7 @@ class _StartupConnection:
def execute(self, statement: Any) -> _ScalarResult:
if "alembic_version" in str(statement):
- return _ScalarResult("20260830_0024")
+ return _ScalarResult("20260909_0030")
return _ScalarResult(self.facts)
@@ -363,6 +369,118 @@ def _production_settings(tmp_path: Path, **overrides: Any) -> Settings:
return Settings(**values)
+def _psycopg_url(sqlalchemy_url: str) -> str:
+ return sqlalchemy_url.replace("postgresql+psycopg://", "postgresql://", 1)
+
+
+def _assert_postgres_statement_is_denied(database_url: str, statement: str) -> None:
+ with psycopg.connect(_psycopg_url(database_url)) as connection:
+ # Keep the cursor scope nested so rollback happens before the connection context can
+ # commit; combining these contexts would make an unexpectedly allowed DDL statement real.
+ with connection.cursor() as cursor: # noqa: SIM117
+ with pytest.raises(psycopg.errors.InsufficientPrivilege):
+ cursor.execute(statement)
+ connection.rollback()
+
+
+def test_managed_postgresql_executes_the_runtime_privilege_boundary(
+ tmp_path: Path,
+) -> None:
+ """Prove the generated grant contract against PostgreSQL, not merely against SQL text."""
+
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ if not owner_url or not runtime_url:
+ pytest.skip("managed PostgreSQL owner/runtime URLs are not configured")
+
+ with psycopg.connect(_psycopg_url(owner_url)) as owner_connection:
+ owner_facts = owner_connection.execute(
+ """
+ select current_user, session_user, rolsuper, rolcreaterole, rolcreatedb,
+ rolreplication, rolbypassrls
+ from pg_catalog.pg_roles
+ where rolname = current_user
+ """
+ ).fetchone()
+ assert owner_facts == ("modelforge", "modelforge", False, False, False, False, False)
+
+ with psycopg.connect(_psycopg_url(runtime_url)) as runtime_connection:
+ runtime_facts = runtime_connection.execute(
+ """
+ select current_user, session_user, rolsuper, rolcreaterole, rolcreatedb,
+ rolreplication, rolbypassrls, rolinherit
+ from pg_catalog.pg_roles
+ where rolname = current_user
+ """
+ ).fetchone()
+ memberships = runtime_connection.execute(
+ """
+ select count(*)
+ from pg_catalog.pg_auth_members
+ where member = (select oid from pg_catalog.pg_roles where rolname = current_user)
+ """
+ ).fetchone()
+ visible_events = runtime_connection.execute(
+ "select count(*) from public.audit_events"
+ ).fetchone()
+ assert runtime_facts == (
+ "modelforge_runtime",
+ "modelforge_runtime",
+ False,
+ False,
+ False,
+ False,
+ False,
+ False,
+ )
+ assert memberships == (0,)
+ assert visible_events == (0,)
+
+ _assert_postgres_statement_is_denied(runtime_url, "set role modelforge")
+ _assert_postgres_statement_is_denied(runtime_url, "truncate public.audit_events")
+ _assert_postgres_statement_is_denied(
+ runtime_url,
+ "insert into public.audit_events default values",
+ )
+ _assert_postgres_statement_is_denied(
+ runtime_url,
+ "create table public.modelforge_forbidden_runtime_ddl (id integer)",
+ )
+
+ engine = create_engine(runtime_url)
+ try:
+ with Session(engine) as session:
+ AuditWriter(session, "system", "managed-postgres-boundary").write(
+ "MANAGED_POSTGRES_BOUNDARY_VERIFIED",
+ "release_validation",
+ None,
+ {"credential": "runtime", "direct_audit_dml": "denied"},
+ )
+ session.commit()
+
+ with Session(engine) as session:
+ checkpoint = session.get(AuditChainHead, 1)
+ event_count = session.scalar(select(func.count()).select_from(AuditEvent))
+ assert checkpoint is not None
+ assert event_count == 1
+ assert checkpoint.event_count == 1
+ assert checkpoint.last_sequence == 1
+ assert checkpoint.last_event_hash is not None
+
+ report = validate_startup(
+ _production_settings(tmp_path, database_url=runtime_url),
+ engine,
+ )
+ incompatible = [
+ problem
+ for problem in report.problems
+ if problem.code is StartupFailureCode.INCOMPATIBLE_DATABASE
+ ]
+ assert incompatible == []
+ finally:
+ engine.dispose()
+
+
def test_production_startup_uses_catalog_policy_and_refuses_a_broken_trigger(
tmp_path: Path,
) -> None:
@@ -400,9 +518,12 @@ def test_production_configuration_rejects_owner_role_and_owner_secret_presence(
def test_compose_and_image_keep_admin_owner_credentials_out_of_the_api_process() -> None:
compose = (ROOT / "docker-compose.yml").read_text("utf-8")
dockerfile = (ROOT / "backend" / "Dockerfile").read_text("utf-8")
+ postgres_section = compose.split("\n postgres:", 1)[1].split("\n redis:", 1)[0]
api_section = compose.split("\n api:", 1)[1].split("\n migrate:", 1)[0]
migrate_section = compose.split("\n migrate:", 1)[1].split("\n web:", 1)[0]
+ assert "MODELFORGE_POSTGRES_ADMIN_USER:" in postgres_section
+ assert "MODELFORGE_POSTGRES_ADMIN_PASSWORD:" in postgres_section
assert "MODELFORGE_RUNTIME_DATABASE_URL" in api_section
assert "MODELFORGE_MIGRATION_DATABASE_URL" not in api_section
assert "MODELFORGE_POSTGRES_ADMIN_PASSWORD" not in api_section
@@ -413,21 +534,32 @@ def test_compose_and_image_keep_admin_owner_credentials_out_of_the_api_process()
def test_role_provisioning_contains_no_password_literal_and_demotes_both_app_roles() -> None:
- for relative_path in (
- "deploy/postgres/init/001-modelforge-roles.sql",
- "deploy/postgres/provision-existing-1.2.1.sql",
- ):
- provisioning = (ROOT / relative_path).read_text("utf-8")
+ provisioning = (
+ ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql"
+ ).read_text("utf-8")
- assert "\\getenv owner_password" in provisioning
- assert "\\getenv runtime_password" in provisioning
- assert "nosuperuser nocreatedb nocreaterole" in provisioning.lower()
- assert "modelforge_runtime" in provisioning
- assert "noinherit" in provisioning.lower()
- assert "pg_auth_members" in provisioning
- assert "revoke create, temporary on database %I from public" in provisioning
- assert "password 'modelforge'" not in provisioning.lower()
- assert "password 'postgres'" not in provisioning.lower()
+ assert "\\getenv owner_password" in provisioning
+ assert "\\getenv runtime_password" in provisioning
+ assert "nosuperuser nocreatedb nocreaterole" in provisioning.lower()
+ assert "modelforge_runtime" in provisioning
+ assert "noinherit" in provisioning.lower()
+ assert "pg_auth_members" in provisioning
+ assert "password 'modelforge'" not in provisioning.lower()
+ assert "password 'postgres'" not in provisioning.lower()
+
+
+def test_existing_cluster_provisioning_refuses_bootstrap_owner_before_mutation() -> None:
+ provisioning = (
+ ROOT / "deploy" / "postgres" / "provision-existing-1.2.1.sql"
+ ).read_text("utf-8")
+ lowered = provisioning.lower()
+
+ refusal = lowered.index("oid = 10")
+ first_role_mutation = lowered.index("create role")
+ assert refusal < first_role_mutation
+ assert "fresh cluster" in lowered
+ assert "--no-owner --no-privileges" in lowered
+ assert "pg_authid mutation" in lowered
@pytest.mark.parametrize(
diff --git a/backend/tests/test_audit_timeline_api_rc.py b/backend/tests/test_audit_timeline_api_rc.py
new file mode 100644
index 0000000..640d37e
--- /dev/null
+++ b/backend/tests/test_audit_timeline_api_rc.py
@@ -0,0 +1,252 @@
+"""Read-only, bounded audit timeline query contract regressions."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.main import app
+from modelforge_api.persistence.models import AuditEvent, Base
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.settings import Settings, get_settings
+
+TIMELINE_PATH = "/api/v1/admin/audit/events"
+ADMIN_CREDENTIAL = "audit-timeline-test-admin"
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+@pytest.fixture(autouse=True)
+def clean_overrides() -> Iterator[None]:
+ yield
+ app.dependency_overrides.clear()
+
+
+def _client(session: Session, role: str = "admin") -> tuple[TestClient, dict[str, str]]:
+ app.dependency_overrides[get_session] = lambda: session
+ if role == "admin":
+ settings = Settings(_env_file=None, operator_api_key=SecretStr(ADMIN_CREDENTIAL)) # type: ignore[call-arg]
+ token = ADMIN_CREDENTIAL
+ else:
+ token = "mfop_" + "A" * 43
+ identities = [
+ {
+ "credential_id": f"audit-{role}",
+ "subject": f"audit-{role}",
+ "role": role,
+ "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+ }
+ ]
+ settings = Settings(
+ _env_file=None,
+ operator_identities=SecretStr(json.dumps(identities)),
+ ) # type: ignore[call-arg]
+ app.dependency_overrides[get_settings] = lambda: settings
+ return TestClient(app), {"X-ModelForge-Admin-Token": token}
+
+
+def _append(
+ session: Session,
+ *,
+ principal: str = "alice",
+ action: str = "MODEL_UPDATED",
+ resource_type: str = "model",
+ resource_id: str = "model-1",
+ outcome: str = "success",
+ correlation_id: str = "correlation-1",
+ details: dict[str, object] | None = None,
+) -> AuditEvent:
+ event = AuditWriter(
+ session,
+ context=AuditContext("operator", principal, correlation_id),
+ ).write(
+ action,
+ resource_type,
+ resource_id,
+ details or {"changed_fields": ["display_name"]},
+ outcome=outcome,
+ )
+ session.commit()
+ return event
+
+
+@pytest.mark.parametrize("role", ["viewer", "operator"])
+def test_timeline_requires_admin_human_authentication(session: Session, role: str) -> None:
+ client, headers = _client(session, role)
+
+ assert client.get(TIMELINE_PATH).status_code == 401
+ assert client.get(TIMELINE_PATH, headers=headers).status_code == 403
+ assert (
+ client.get(
+ TIMELINE_PATH,
+ headers={"Authorization": "Bearer mfsvc_project-scoped"},
+ ).status_code
+ == 401
+ )
+ assert (
+ client.get(
+ TIMELINE_PATH,
+ headers={"Authorization": "Bearer mfnode_node-scoped"},
+ ).status_code
+ == 401
+ )
+
+
+def test_timeline_applies_typed_filters_and_omits_untrusted_details(session: Session) -> None:
+ _append(
+ session,
+ principal="alice-operator",
+ action="NODE_ENROLLMENT_COMPLETED",
+ resource_type="compute_node",
+ resource_id="gpu_node-node-42",
+ outcome="failure",
+ correlation_id="corr-audit-42",
+ details={
+ "access_token": "mfpw_NEVER_RETURN_THIS",
+ "nested": {"api-key": "NEVER_RETURN_THIS_EITHER", "changed": True},
+ "comment": "authorization: Bearer mfpw_NEVER_RETURN_THIS",
+ "operator_note": "opaque-credential-value-do-not-return-7f93",
+ },
+ )
+ _append(
+ session,
+ principal="bob-operator",
+ action="NODE_ENROLLMENT_COMPLETED",
+ resource_type="compute_node",
+ resource_id="gpu_node-node-43",
+ outcome="success",
+ correlation_id="corr-audit-43",
+ )
+ client, headers = _client(session)
+ moment = datetime.now(UTC)
+ response = client.get(
+ TIMELINE_PATH,
+ headers=headers,
+ params={
+ "principal": "alice",
+ "action": "enrollment",
+ "resource": "node-42",
+ "outcome": "failure",
+ "correlation_id": "corr-audit-42",
+ "occurred_after": (moment - timedelta(minutes=1)).isoformat(),
+ "occurred_before": (moment + timedelta(minutes=1)).isoformat(),
+ "limit": 10,
+ },
+ )
+
+ assert response.status_code == 200
+ payload = response.json()
+ assert len(payload["events"]) == 1
+ assert payload["events"][0]["actor_id"] == "alice-operator"
+ assert payload["events"][0]["action"] == "NODE_ENROLLMENT_COMPLETED"
+ assert "details" not in payload["events"][0]
+ for secret_like in (
+ "NEVER_RETURN_THIS",
+ "NEVER_RETURN_THIS_EITHER",
+ "opaque-credential-value-do-not-return-7f93",
+ ):
+ assert secret_like not in response.text
+
+
+def test_timeline_cursor_pages_stably_when_new_events_are_appended(session: Session) -> None:
+ seeded = [_append(session, principal=f"operator-{index}") for index in range(3)]
+ client, headers = _client(session)
+
+ first = client.get(TIMELINE_PATH, headers=headers, params={"limit": 2})
+ assert first.status_code == 200
+ first_payload = first.json()
+ assert [item["sequence"] for item in first_payload["events"]] == [
+ seeded[2].sequence,
+ seeded[1].sequence,
+ ]
+ cursor = first_payload["next_cursor"]
+ assert isinstance(cursor, str) and cursor
+
+ _append(session, principal="operator-new")
+ second = client.get(
+ TIMELINE_PATH,
+ headers=headers,
+ params={"limit": 2, "cursor": cursor},
+ )
+ assert second.status_code == 200
+ second_payload = second.json()
+ assert [item["sequence"] for item in second_payload["events"]] == [seeded[0].sequence]
+ assert second_payload["next_cursor"] is None
+ assert not (
+ {item["id"] for item in first_payload["events"]}
+ & {item["id"] for item in second_payload["events"]}
+ )
+
+ mismatched_filters = client.get(
+ TIMELINE_PATH,
+ headers=headers,
+ params={"limit": 2, "cursor": cursor, "principal": "another-principal"},
+ )
+ assert mismatched_filters.status_code == 422
+
+
+@pytest.mark.parametrize(
+ "params",
+ [
+ {"limit": 101},
+ {"cursor": "not-a-valid-cursor"},
+ {"principal": "p" * 256},
+ {"action": "a" * 129},
+ {"resource": "r" * 129},
+ {"outcome": "o" * 33},
+ {"correlation_id": "c" * 65},
+ {"occurred_after": "2026-01-01T00:00:00Z"},
+ {"occurred_after": "not-a-time", "occurred_before": "2026-01-02T00:00:00Z"},
+ {
+ "occurred_after": "2024-01-01T00:00:00Z",
+ "occurred_before": "2026-01-01T00:00:00Z",
+ },
+ {
+ "occurred_after": "2026-02-01T00:00:00Z",
+ "occurred_before": "2026-01-01T00:00:00Z",
+ },
+ {"unexpected_filter": "not-accepted"},
+ ],
+)
+def test_timeline_rejects_malformed_or_unbounded_query_input(
+ session: Session, params: dict[str, str | int]
+) -> None:
+ client, headers = _client(session)
+
+ response = client.get(TIMELINE_PATH, headers=headers, params=params)
+
+ assert response.status_code == 422
+
+
+def test_timeline_returns_a_typed_empty_page_and_does_not_append_audit_events(
+ session: Session,
+) -> None:
+ client, headers = _client(session)
+
+ response = client.get(TIMELINE_PATH, headers=headers)
+
+ assert response.status_code == 200
+ assert response.json() == {"events": [], "next_cursor": None, "limit": 50}
+ assert session.scalar(select(func.count()).select_from(AuditEvent)) == 0
diff --git a/backend/tests/test_benchmark_run_recovery_compatibility.py b/backend/tests/test_benchmark_run_recovery_compatibility.py
new file mode 100644
index 0000000..91afa80
--- /dev/null
+++ b/backend/tests/test_benchmark_run_recovery_compatibility.py
@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+from pathlib import Path
+from uuid import uuid4
+
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect, text
+
+from alembic import command
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+ BENCHMARK_RUN_SCHEMA,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+_PRIOR = "20260924_0038"
+
+
+def _migration_config(database_url: str) -> Config:
+ config = Config(str(Path(__file__).parents[1] / "alembic.ini"))
+ config.set_main_option("sqlalchemy.url", database_url)
+ return config
+
+
+def _create_0038_benchmark_table(engine) -> None:
+ with engine.begin() as connection:
+ recreate_0038_benchmark_runs(connection)
+ connection.exec_driver_sql(
+ "CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"
+ )
+ connection.exec_driver_sql(
+ "INSERT INTO alembic_version (version_num) VALUES (?)", (_PRIOR,)
+ )
+
+
+def _insert_legacy_benchmark_run(engine) -> str:
+ run_id = uuid4().hex
+ with engine.begin() as connection:
+ connection.execute(
+ text(
+ """
+ INSERT INTO benchmark_runs (
+ id, deployment_id, suite_id, model_revision_sha, artifact_sha256,
+ runtime_type, runtime_version, runtime_image_digest, launch_arguments,
+ suite_revision, dataset_revision, environment_fingerprint,
+ environment_digest, seed, results, status, comparable, stale_reason,
+ started_at, completed_at
+ ) VALUES (
+ :id, :deployment_id, :suite_id, :model_revision_sha, :artifact_sha256,
+ 'native', '1', NULL, '{}', 'suite-r1', 'dataset-r1', '{}',
+ :environment_digest, 7, '{"cases":[]}', 'completed', 1, NULL,
+ '2026-09-24T00:00:00', '2026-09-24T00:00:01'
+ )
+ """
+ ),
+ {
+ "id": run_id,
+ "deployment_id": uuid4().hex,
+ "suite_id": uuid4().hex,
+ "model_revision_sha": "a" * 64,
+ "artifact_sha256": "b" * 64,
+ "environment_digest": "c" * 64,
+ },
+ )
+ return run_id
+
+
+def _fingerprint(engine):
+ with engine.connect() as connection:
+ return semantic_fingerprint(connection)
+
+
+def _migrated_database(tmp_path):
+ database_path = tmp_path / "benchmark-recovery.db"
+ database_url = f"sqlite:///{database_path.as_posix()}"
+ engine = create_engine(database_url)
+ Base.metadata.create_all(engine)
+ _create_0038_benchmark_table(engine)
+ run_id = _insert_legacy_benchmark_run(engine)
+
+ prior_columns = {column["name"] for column in inspect(engine).get_columns("benchmark_runs")}
+ assert "actor_subject" not in prior_columns
+ source = _fingerprint(engine)
+ assert set(source["tables"]["benchmark_runs"]) == {
+ "row_count",
+ "digest",
+ "status",
+ "redacted_columns",
+ }
+
+ command.upgrade(_migration_config(database_url), "head")
+ assert _fingerprint(engine)["tables"]["benchmark_runs"]["row_count"] == 1
+ return engine, source, run_id
+
+
+def test_0038_benchmark_history_fingerprints_and_restores_through_0039(tmp_path) -> None:
+ engine, source, _run_id = _migrated_database(tmp_path)
+ try:
+ restored = _fingerprint(engine)
+ contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+ assert contract.compatible
+ assert contract.mode.startswith("BENCHMARK_RUNS_UPGRADE|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+ finally:
+ engine.dispose()
+
+
+def test_0039_upgrade_rejects_changed_legacy_benchmark_history(tmp_path) -> None:
+ engine, source, run_id = _migrated_database(tmp_path)
+ try:
+ with engine.begin() as connection:
+ connection.execute(
+ text("UPDATE benchmark_runs SET model_revision_sha=:sha WHERE id=:id"),
+ {"sha": "d" * 64, "id": run_id},
+ )
+ restored = _fingerprint(engine)
+
+ contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+ assert not contract.compatible
+ finally:
+ engine.dispose()
+
+
+def test_0039_upgrade_rejects_operator_state_absent_from_0038_backup(tmp_path) -> None:
+ engine, source, run_id = _migrated_database(tmp_path)
+ try:
+ with engine.begin() as connection:
+ connection.execute(
+ text(
+ "UPDATE benchmark_runs SET actor_subject='operator-a', "
+ "idempotency_key='run-1', request_fingerprint=:fingerprint, "
+ "capability_deployment_id=:deployment WHERE id=:id"
+ ),
+ {
+ "fingerprint": "e" * 64,
+ "deployment": uuid4().hex,
+ "id": run_id,
+ },
+ )
+ restored = _fingerprint(engine)
+
+ contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+ assert not contract.compatible
+ finally:
+ engine.dispose()
+
+
+def test_unversioned_current_database_keeps_operator_state_in_fingerprint(tmp_path) -> None:
+ engine, source, run_id = _migrated_database(tmp_path)
+ try:
+ with engine.begin() as connection:
+ connection.execute(
+ text(
+ "UPDATE benchmark_runs SET actor_subject='operator-a', "
+ "idempotency_key='run-1', request_fingerprint=:fingerprint, "
+ "capability_deployment_id=:deployment WHERE id=:id"
+ ),
+ {
+ "fingerprint": "e" * 64,
+ "deployment": uuid4().hex,
+ "id": run_id,
+ },
+ )
+ connection.execute(text("DROP TABLE alembic_version"))
+ restored = _fingerprint(engine)
+ projection = restored["tables"]["benchmark_runs"][
+ "benchmark_run_schema_projection"
+ ]
+
+ assert projection["non_legacy_row_count"] == 1
+ assert not fingerprint_compatibility(
+ source, restored, source_schema_revision=_PRIOR
+ ).compatible
+ finally:
+ engine.dispose()
+
+
+def test_unversioned_partial_benchmark_migration_fails_closed(tmp_path) -> None:
+ engine, source, _run_id = _migrated_database(tmp_path)
+ try:
+ with engine.begin() as connection:
+ connection.execute(text("DROP TABLE alembic_version"))
+ connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
+ restored = _fingerprint(engine)
+
+ assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+ assert not fingerprint_compatibility(
+ source, restored, source_schema_revision=_PRIOR
+ ).compatible
+ finally:
+ engine.dispose()
+
+
+def test_0038_revision_with_0039_columns_fails_closed(tmp_path) -> None:
+ engine, source, _run_id = _migrated_database(tmp_path)
+ try:
+ with engine.begin() as connection:
+ connection.execute(
+ text("UPDATE alembic_version SET version_num=:revision"),
+ {"revision": _PRIOR},
+ )
+ restored = _fingerprint(engine)
+
+ assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+ assert not fingerprint_compatibility(
+ source, restored, source_schema_revision=_PRIOR
+ ).compatible
+ finally:
+ engine.dispose()
+
+
+def test_0039_revision_with_incomplete_benchmark_columns_fails_closed(tmp_path) -> None:
+ engine, _source, _run_id = _migrated_database(tmp_path)
+ try:
+ with engine.begin() as connection:
+ connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
+ current = _fingerprint(engine)
+
+ assert current["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+ assert not fingerprint_compatibility(
+ current, current, source_schema_revision=BENCHMARK_RUN_SCHEMA
+ ).compatible
+ finally:
+ engine.dispose()
+
+
+def test_0039_benchmark_history_is_exact_when_both_sides_are_current(tmp_path) -> None:
+ engine, _legacy_source, run_id = _migrated_database(tmp_path)
+ try:
+ source = _fingerprint(engine)
+ contract = fingerprint_compatibility(
+ source, source, source_schema_revision=BENCHMARK_RUN_SCHEMA
+ )
+ assert contract.compatible
+ assert contract.mode.startswith("BENCHMARK_RUNS_STRICT|")
+ assert audit_fingerprint_differences_are_compatible(source, source, contract)
+
+ with engine.begin() as connection:
+ connection.execute(
+ text("UPDATE benchmark_runs SET generation=2 WHERE id=:id"), {"id": run_id}
+ )
+ changed = _fingerprint(engine)
+ assert not fingerprint_compatibility(
+ source, changed, source_schema_revision=BENCHMARK_RUN_SCHEMA
+ ).compatible
+ finally:
+ engine.dispose()
diff --git a/backend/tests/test_benchmark_runs.py b/backend/tests/test_benchmark_runs.py
new file mode 100644
index 0000000..a08722a
--- /dev/null
+++ b/backend/tests/test_benchmark_runs.py
@@ -0,0 +1,394 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import create_engine, event, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.benchmarking import BenchmarkRunStartRequest
+from modelforge_api.persistence.models import (
+ Base,
+ BenchmarkRun,
+ BenchmarkSuite,
+ GatewayRequest,
+ LabExecutionRun,
+ RuntimeProfile,
+)
+from modelforge_api.services.benchmark_runs import BenchmarkRunService
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingError, ServingService
+from tests.test_dispatch_context import operator_fixture
+
+
+@pytest.fixture
+def session():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+def _request(serving, target_request, selected) -> BenchmarkRunStartRequest:
+ suite = BenchmarkRunService(serving).suites()[0]
+ return BenchmarkRunStartRequest(
+ capability="rag.embedding@1",
+ suite_id=suite.id,
+ target=target_request.target,
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+
+
+def _run(serving, target_request, selected, *, actor="operator-one", key="benchmark-once"):
+ request = _request(serving, target_request, selected)
+ return BenchmarkRunService(serving).start(request, actor_subject=actor, idempotency_key=key)
+
+
+def test_suite_listing_is_read_only_and_advertises_declared_contract(session: Session) -> None:
+ serving, _target, _selected = operator_fixture(session)
+ writes: list[str] = []
+
+ def capture(_connection, _cursor, statement, _parameters, _context, _executemany):
+ if statement.lstrip().split(maxsplit=1)[0].upper() in {"INSERT", "UPDATE", "DELETE"}:
+ writes.append(statement)
+
+ event.listen(session.get_bind(), "before_cursor_execute", capture)
+ suites = BenchmarkRunService(serving).suites()
+
+ assert len(suites) == 1
+ assert suites[0].capability == "rag.embedding@1"
+ assert suites[0].immutable_at.year == 2026
+ assert not writes
+ assert session.scalar(select(func.count()).select_from(BenchmarkSuite)) == 0
+
+
+def test_operator_run_is_idempotent_actor_scoped_and_links_real_lab_gateway(
+ session: Session,
+) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ request = _request(serving, target_request, selected)
+ service = BenchmarkRunService(serving)
+
+ first = service.start(request, actor_subject="operator-one", idempotency_key="same-run")
+ replay = service.start(request, actor_subject="operator-one", idempotency_key="same-run")
+
+ assert first.id == replay.id
+ assert first.status == "running"
+ assert first.progress.active_case == "request-1"
+ assert first.contract.deployment_id == str(selected.capability_deployment_id)
+ assert first.contract.model_revision_sha
+ assert first.contract.artifact_sha256
+ assert first.target_snapshot["deployment_id"] == str(selected.capability_deployment_id)
+ assert session.scalar(select(func.count()).select_from(BenchmarkRun)) == 1
+ child = session.scalar(
+ select(LabExecutionRun).where(LabExecutionRun.actor_subject == "operator-one")
+ )
+ assert child is not None
+ gateway = session.get(GatewayRequest, child.gateway_request_id)
+ assert gateway is not None
+ assert gateway.capability_deployment_id == selected.capability_deployment_id
+ assert gateway.status == "queued"
+ assert gateway.service_client_id is None
+ assert service.list(actor_subject="someone-else") == []
+ with pytest.raises(ServingError) as raised:
+ service.get(first.id, actor_subject="someone-else")
+ assert raised.value.status_code == 404
+
+ changed = request.model_copy(update={"experiment_route_id": uuid4()})
+ with pytest.raises(ServingError) as raised:
+ service.start(changed, actor_subject="operator-one", idempotency_key="same-run")
+ assert raised.value.code == "BENCHMARK_IDEMPOTENCY_CONFLICT"
+
+
+def test_history_keyset_pages_tied_timestamps_without_skipping_when_new_run_arrives(
+ session: Session,
+) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ service = BenchmarkRunService(serving)
+ original = [
+ _run(
+ serving,
+ target_request,
+ selected,
+ key=f"history-page-{index}",
+ )
+ for index in range(5)
+ ]
+ _run(serving, target_request, selected, actor="operator-two", key="other-actor-run")
+
+ tied_created_at = datetime(2026, 9, 24, 12, tzinfo=UTC)
+ session.execute(
+ update(BenchmarkRun)
+ .where(BenchmarkRun.actor_subject == "operator-one")
+ .values(created_at=tied_created_at)
+ )
+ session.commit()
+ expected = sorted((run.id for run in original), key=lambda run_id: run_id.int, reverse=True)
+
+ first_page = service.list_page(actor_subject="operator-one", limit=2)
+ assert [run.id for run in first_page.runs] == expected[:2]
+ assert first_page.next_cursor is not None
+
+ inserted = _run(
+ serving,
+ target_request,
+ selected,
+ key="history-page-arrived-after-first-page",
+ )
+ session.execute(
+ update(BenchmarkRun)
+ .where(BenchmarkRun.id == inserted.id)
+ .values(created_at=tied_created_at + timedelta(days=1))
+ )
+ session.commit()
+
+ second_page = service.list_page(
+ actor_subject="operator-one", limit=2, cursor=first_page.next_cursor
+ )
+ third_page = service.list_page(
+ actor_subject="operator-one", limit=2, cursor=second_page.next_cursor
+ )
+ assert [run.id for run in second_page.runs] == expected[2:4]
+ assert [run.id for run in third_page.runs] == expected[4:]
+ assert third_page.next_cursor is None
+ assert service.list_page(actor_subject="operator-two", limit=2).runs[0].id != expected[0]
+
+ with pytest.raises(ServingError) as raised:
+ service.list_page(
+ actor_subject="operator-two", limit=2, cursor=first_page.next_cursor
+ )
+ assert raised.value.code == "BENCHMARK_CURSOR_INVALID"
+ with pytest.raises(ServingError) as raised:
+ service.list_page(actor_subject="operator-one", limit=2, cursor="")
+ assert raised.value.code == "BENCHMARK_CURSOR_INVALID"
+
+
+def test_target_mismatch_fails_before_persisting_parent(session: Session) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ request = _request(serving, target_request, selected)
+ request = request.model_copy(
+ update={"target": request.target.model_copy(update={"compute_node_id": uuid4()})}
+ )
+
+ with pytest.raises(ServingError) as raised:
+ BenchmarkRunService(serving).start(
+ request, actor_subject="operator-one", idempotency_key="bad-target"
+ )
+
+ assert raised.value.code == "BENCHMARK_TARGET_MISMATCH"
+ assert session.scalar(select(func.count()).select_from(BenchmarkRun)) == 0
+ assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 0
+
+
+def test_secret_like_launch_arguments_are_not_persisted_in_provenance(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ request = _request(serving, target_request, selected)
+ service = BenchmarkRunService(serving)
+ service._suite(request.suite_id)
+ original_snapshot = serving._operator_lab_target_snapshot
+
+ def snapshot_with_secret(*args, **kwargs):
+ snapshot = original_snapshot(*args, **kwargs)
+ serving.session.execute(
+ update(RuntimeProfile)
+ .where(RuntimeProfile.id == target_request.target.runtime_profile_id)
+ .values(launch_arguments={"environment": ["HF_TOKEN=private-canary"]})
+ )
+ serving.session.commit()
+ return snapshot
+
+ monkeypatch.setattr(serving, "_operator_lab_target_snapshot", snapshot_with_secret)
+
+ with pytest.raises(ServingError) as raised:
+ service.start(request, actor_subject="operator-one", idempotency_key="secret-arguments")
+
+ assert raised.value.code == "BENCHMARK_RUNTIME_ARGUMENTS_UNSAFE"
+ assert "private-canary" not in str(raised.value)
+ assert session.scalar(select(func.count()).select_from(BenchmarkRun)) == 0
+
+
+def test_lost_parent_link_reuses_same_child_idempotency_key(session: Session) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ response = _run(serving, target_request, selected, key="lost-link")
+ parent = session.get(BenchmarkRun, response.id)
+ assert parent is not None
+ execution = {**parent.results["execution"], "active_lab_run_id": None, "active_case": None}
+ parent.results = {**parent.results, "execution": execution}
+ parent.status = "accepted"
+ session.commit()
+
+ # A process crash after child commit but before parent relink replays the deterministic
+ # actor-scoped LAB key. The child uniqueness constraint returns the same durable run.
+ resumed = BenchmarkRunService(serving)
+ assert resumed.reconcile_one(parent.id)
+ parent = session.get(BenchmarkRun, response.id)
+ assert parent is not None
+ assert UUID(parent.results["execution"]["active_lab_run_id"]) == UUID(
+ response.results["execution"]["active_lab_run_id"]
+ )
+ assert (
+ session.scalar(
+ select(func.count())
+ .select_from(LabExecutionRun)
+ .where(
+ LabExecutionRun.actor_subject == "operator-one",
+ LabExecutionRun.idempotency_key == f"benchmark-{response.id.hex}-00",
+ )
+ )
+ == 1
+ )
+
+
+def test_vector_throughput_requires_valid_count_and_timing_from_same_attempt() -> None:
+ results = BenchmarkRunService._initial_results(comparable=True)
+ results["execution"]["attempts"] = [
+ {"status": "completed", "gateway": {"input_count": 4, "inference_time_ms": None}},
+ {"status": "completed", "gateway": {"input_count": True, "inference_time_ms": 20.0}},
+ {"status": "completed", "gateway": {"input_count": -2, "inference_time_ms": 10.0}},
+ {"status": "completed", "gateway": {"input_count": 1, "inference_time_ms": 10.0}},
+ ]
+
+ refreshed = BenchmarkRunService._refresh_metrics(results, {})
+
+ throughput = refreshed["metrics"]["throughput"]
+ assert throughput["successful_vectors_per_inference_second"] == 100.0
+ assert throughput["inference_samples"] == 1
+ assert throughput["measured_vectors"] == 1
+ assert throughput["successful_vectors_coverage"] == pytest.approx(1 / 5)
+
+
+def test_unmeasured_or_mismatched_worker_resources_remain_unknown() -> None:
+ results = BenchmarkRunService._initial_results(comparable=True)
+ results["execution"]["attempts"] = [
+ {
+ "worker_observations": {
+ "device_matches_target": True,
+ "metrics": {"temperature_c": 61},
+ }
+ },
+ {
+ "worker_observations": {
+ "device_matches_target": False,
+ "metrics": {"used_vram_bytes": 40_000},
+ }
+ },
+ ]
+
+ metrics = BenchmarkRunService._refresh_metrics(results, {"accelerator_uuid": "GPU-0"})[
+ "metrics"
+ ]["gpu_vram"]
+
+ assert metrics["telemetry_status"] == "UNKNOWN"
+ assert metrics["vram_status"] == "UNKNOWN"
+ assert metrics["gpu_utilization_status"] == "UNKNOWN"
+ assert metrics["peak_used_vram_bytes"] is None
+ assert metrics["target_accelerator_uuid"] == "GPU-0"
+
+
+def test_cancellation_racing_lab_admission_preserves_terminal_parent_and_stops_child(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ request = _request(serving, target_request, selected)
+ original_start = LabRunService.start
+
+ def start_then_cancel(lab_service, lab_request, *, actor_subject, idempotency_key):
+ child_response = original_start(
+ lab_service,
+ lab_request,
+ actor_subject=actor_subject,
+ idempotency_key=idempotency_key,
+ )
+ parent = session.scalar(
+ select(BenchmarkRun).where(
+ BenchmarkRun.actor_subject == "operator-one",
+ BenchmarkRun.idempotency_key == "racing-cancel",
+ )
+ )
+ assert parent is not None
+ BenchmarkRunService(serving).cancel(parent.id, actor_subject="operator-one")
+ return child_response
+
+ monkeypatch.setattr(LabRunService, "start", start_then_cancel)
+
+ # Start commits the parent before admission; the injected cancellation lands after the
+ # child is durable but before the benchmark reconciler can link it.
+ response = BenchmarkRunService(serving).start(
+ request, actor_subject="operator-one", idempotency_key="racing-cancel"
+ )
+ parent = session.get(BenchmarkRun, response.id)
+ assert parent is not None and parent.status == "cancelled"
+ execution = parent.results["execution"]
+ child = session.get(LabExecutionRun, UUID(execution["active_lab_run_id"]))
+ assert child is not None and child.status == "cancelled"
+ assert response.status == "cancelled"
+ assert response.progress.active_case is None
+
+
+def test_normal_reconciliation_resumes_children_and_aggregates_actual_gateway_fields(
+ session: Session,
+) -> None:
+ serving, target_request, selected = operator_fixture(session)
+ response = _run(serving, target_request, selected, key="resume-run")
+ parent_id = response.id
+ baseline = datetime.now(UTC).replace(microsecond=0)
+
+ for index in range(4):
+ parent = session.get(BenchmarkRun, parent_id)
+ assert parent is not None
+ execution = parent.results["execution"]
+ child_id = UUID(execution["active_lab_run_id"])
+ child = session.get(LabExecutionRun, child_id)
+ assert child is not None
+ gateway = session.get(GatewayRequest, child.gateway_request_id)
+ assert gateway is not None
+ gateway.created_at = baseline + timedelta(seconds=index * 2)
+ gateway.finished_at = gateway.created_at + timedelta(milliseconds=250)
+ gateway.cold = index == 0
+ gateway.load_time_ms = 75.0 if index == 0 else None
+ gateway.inference_time_ms = 100.0 if index == 0 else (None if index == 1 else 200.0)
+ gateway.total_latency_ms = 125.0 + index * 10
+ gateway.input_count = 1 if index != 1 else 4
+ child.status = "completed"
+ child.failure_code = None
+ child.finished_at = gateway.finished_at
+ child.worker_observations = (
+ {
+ "device_matches_target": True,
+ "metrics": {
+ "used_vram_bytes": 3_000,
+ "process_used_vram_bytes": 2_500,
+ "free_vram_bytes": 9_000,
+ "gpu_utilization_percent": 30,
+ },
+ }
+ if index == 0
+ else {"device_matches_target": False, "metrics": {"used_vram_bytes": 99}}
+ )
+ session.commit()
+
+ # Recreate the service object to prove accepted progress is recoverable from rows,
+ # then use the same normal reconciliation entry point as the background tick.
+ resumed = ServingService(
+ session, serving._deployment_settings, serving.manifests, serving.payload_store
+ )
+ summary = resumed.reconcile()
+ assert "benchmark_runs_driven" in summary
+
+ parent = session.get(BenchmarkRun, parent_id)
+ assert parent is not None and parent.status == "completed"
+ metrics = parent.results["metrics"]
+ assert metrics["cold_load_latency_ms"] == {"p50": 75.0, "p95": 75.0, "samples": 1}
+ assert metrics["warm_latency_ms"]["samples"] == 3
+ assert metrics["failures"] == {"attempted": 4, "failed": 0, "rate": 0.0}
+ assert metrics["throughput"]["inference_samples"] == 3
+ assert metrics["throughput"]["measured_vectors"] == 3
+ assert metrics["throughput"]["successful_vectors_coverage"] == pytest.approx(3 / 7)
+ assert metrics["gpu_vram"]["telemetry_status"] == "MEASURED"
+ assert metrics["gpu_vram"]["peak_used_vram_bytes"] == 3_000
+ assert metrics["quality"]["status"] == "UNKNOWN"
diff --git a/backend/tests/test_benchmark_runs_api.py b/backend/tests/test_benchmark_runs_api.py
new file mode 100644
index 0000000..fc3ac6e
--- /dev/null
+++ b/backend/tests/test_benchmark_runs_api.py
@@ -0,0 +1,192 @@
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import Mock
+from uuid import uuid4
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from modelforge_api.api.authorization import (
+ Principal,
+ PrincipalRole,
+ authenticate_operator_principal,
+)
+from modelforge_api.api.routes.lab import get_benchmark_run_service, router
+from modelforge_api.domain.benchmarking import (
+ BenchmarkRunProgress,
+ BenchmarkRunResponse,
+ BenchmarkSuiteResponse,
+)
+from modelforge_api.domain.contracts import BenchmarkEnvironmentFingerprint, BenchmarkRunContract
+
+
+def _responses() -> tuple[BenchmarkSuiteResponse, BenchmarkRunResponse]:
+ now = datetime.now(UTC)
+ suite = BenchmarkSuiteResponse(
+ id=uuid4(),
+ key="modelforge.rag-embedding.operator-performance",
+ revision="1",
+ capability="rag.embedding@1",
+ dataset_revision="builtin:rag-embedding-performance-v1",
+ definition={"schema_version": 1},
+ immutable_at=now,
+ )
+ environment = BenchmarkEnvironmentFingerprint(
+ runtime_type="onnx",
+ runtime_version="1.0",
+ runtime_image_digest=None,
+ launch_arguments={},
+ cuda_version=None,
+ driver_version=None,
+ accelerator_name=None,
+ accelerator_uuid=None,
+ context_length=None,
+ concurrency=1,
+ seed=None,
+ operating_system=None,
+ python_version=None,
+ )
+ run = BenchmarkRunResponse(
+ id=uuid4(),
+ capability="rag.embedding@1",
+ suite_id=suite.id,
+ capability_deployment_id=uuid4(),
+ status="accepted",
+ contract=BenchmarkRunContract(
+ deployment_id=str(uuid4()),
+ model_revision_sha="a" * 40,
+ artifact_sha256="b" * 64,
+ suite_key=suite.key,
+ suite_revision=suite.revision,
+ dataset_revision=suite.dataset_revision,
+ environment=environment,
+ started_at=now,
+ ),
+ target_snapshot={},
+ progress=BenchmarkRunProgress(
+ completed_invocations=0, total_invocations=4, active_case=None
+ ),
+ results={"metrics": {"gpu_vram": {"telemetry_status": "UNKNOWN"}}},
+ created_at=now,
+ )
+ return suite, run
+
+
+def _harness(
+ role: PrincipalRole,
+) -> tuple[TestClient, Mock, BenchmarkSuiteResponse, BenchmarkRunResponse]:
+ app = FastAPI()
+ app.include_router(router)
+ service = Mock()
+ suite, run = _responses()
+ service.suites.return_value = [suite]
+ service.list.return_value = [run]
+ service.list_page.return_value = SimpleNamespace(runs=[run], next_cursor=None)
+ service.start.return_value = run
+ service.get.return_value = run
+ service.cancel.return_value = run
+ app.dependency_overrides[authenticate_operator_principal] = lambda: Principal(
+ "operator-one", role, "test"
+ )
+ app.dependency_overrides[get_benchmark_run_service] = lambda: service
+ return TestClient(app), service, suite, run
+
+
+def _request(suite: BenchmarkSuiteResponse) -> dict[str, object]:
+ return {
+ "capability": "rag.embedding@1",
+ "suite_id": str(suite.id),
+ "target": {
+ key: str(uuid4())
+ for key in (
+ "deployment_candidate_id",
+ "artifact_set_id",
+ "runtime_profile_id",
+ "compute_node_id",
+ "execution_approval_id",
+ )
+ },
+ "deployment_id": str(uuid4()),
+ "experiment_route_id": str(uuid4()),
+ }
+
+
+def test_benchmark_endpoints_forward_operator_identity_and_disable_caching() -> None:
+ client, service, suite, run = _harness(PrincipalRole.OPERATOR)
+
+ suites = client.get("/api/v1/lab/benchmark-suites")
+ history = client.get("/api/v1/lab/benchmark-runs")
+ started = client.post(
+ "/api/v1/lab/benchmark-runs",
+ headers={"Idempotency-Key": "operator-benchmark"},
+ json=_request(suite),
+ )
+ status = client.get(f"/api/v1/lab/benchmark-runs/{run.id}")
+ cancelled = client.post(f"/api/v1/lab/benchmark-runs/{run.id}/cancel")
+
+ assert suites.status_code == history.status_code == 200
+ assert started.status_code == 202
+ assert status.status_code == cancelled.status_code == 200
+ assert all(
+ response.headers["cache-control"] == "no-store"
+ for response in (suites, history, started, status, cancelled)
+ )
+ assert service.start.call_args.kwargs == {
+ "actor_subject": "operator-one",
+ "idempotency_key": "operator-benchmark",
+ }
+ assert service.list_page.call_args.kwargs == {
+ "actor_subject": "operator-one",
+ "limit": 50,
+ "cursor": None,
+ }
+ assert service.get.call_args.kwargs == {"actor_subject": "operator-one"}
+ assert service.cancel.call_args.kwargs == {"actor_subject": "operator-one"}
+
+
+def test_benchmark_history_cursor_preserves_the_existing_list_response_contract() -> None:
+ client, service, _suite, run = _harness(PrincipalRole.OPERATOR)
+ service.list_page.return_value = SimpleNamespace(runs=[run], next_cursor="opaque-next-page")
+
+ response = client.get(
+ "/api/v1/lab/benchmark-runs",
+ params={"limit": 1, "cursor": "opaque-current-page"},
+ )
+
+ assert response.status_code == 200
+ assert isinstance(response.json(), list)
+ assert [item["id"] for item in response.json()] == [str(run.id)]
+ assert response.headers["x-next-cursor"] == "opaque-next-page"
+ assert response.headers["access-control-expose-headers"] == "X-Next-Cursor"
+ assert service.list_page.call_args.kwargs == {
+ "actor_subject": "operator-one",
+ "limit": 1,
+ "cursor": "opaque-current-page",
+ }
+
+
+def test_benchmark_write_routes_require_operator_and_strict_declared_capability() -> None:
+ client, service, suite, _run = _harness(PrincipalRole.VIEWER)
+
+ denied = client.post(
+ "/api/v1/lab/benchmark-runs",
+ headers={"Idempotency-Key": "viewer-benchmark"},
+ json=_request(suite),
+ )
+ invalid = _request(suite)
+ invalid["capability"] = "rag.rerank@1"
+ rejected = _harness(PrincipalRole.OPERATOR)[0].post(
+ "/api/v1/lab/benchmark-runs",
+ headers={"Idempotency-Key": "bad-capability"},
+ json=invalid,
+ )
+ operator_client, operator_service, operator_suite, _operator_run = _harness(
+ PrincipalRole.OPERATOR
+ )
+ missing_key = operator_client.post("/api/v1/lab/benchmark-runs", json=_request(operator_suite))
+
+ assert denied.status_code == 403
+ assert rejected.status_code == 422
+ assert missing_key.status_code == 422
+ service.start.assert_not_called()
+ operator_service.start.assert_not_called()
diff --git a/backend/tests/test_benchmark_runs_migration.py b/backend/tests/test_benchmark_runs_migration.py
new file mode 100644
index 0000000..11c4090
--- /dev/null
+++ b/backend/tests/test_benchmark_runs_migration.py
@@ -0,0 +1,89 @@
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect
+
+from alembic import command
+from modelforge_api.persistence.models import Base
+
+
+def test_operator_benchmark_migration_upgrades_and_downgrades_sqlite() -> None:
+ with TemporaryDirectory() as temp:
+ database_path = Path(temp) / "benchmark-migration.db"
+ url = f"sqlite:///{database_path.as_posix()}"
+ engine = create_engine(url)
+ Base.metadata.create_all(engine)
+ with engine.begin() as connection:
+ connection.exec_driver_sql("DROP TABLE benchmark_runs")
+ connection.exec_driver_sql(
+ """
+ CREATE TABLE benchmark_runs (
+ id CHAR(32) NOT NULL,
+ deployment_id CHAR(32) NOT NULL,
+ suite_id CHAR(32) NOT NULL,
+ model_revision_sha VARCHAR(64) NOT NULL,
+ artifact_sha256 VARCHAR(64) NOT NULL,
+ runtime_type VARCHAR(32) NOT NULL,
+ runtime_version VARCHAR(128) NOT NULL,
+ runtime_image_digest VARCHAR(71),
+ launch_arguments JSON NOT NULL,
+ suite_revision VARCHAR(128) NOT NULL,
+ dataset_revision VARCHAR(255) NOT NULL,
+ environment_fingerprint JSON NOT NULL,
+ environment_digest VARCHAR(64) NOT NULL,
+ seed INTEGER,
+ results JSON NOT NULL,
+ status VARCHAR(32) NOT NULL,
+ comparable BOOLEAN NOT NULL,
+ stale_reason TEXT,
+ started_at DATETIME NOT NULL,
+ completed_at DATETIME,
+ PRIMARY KEY (id),
+ FOREIGN KEY(deployment_id) REFERENCES deployments (id) ON DELETE RESTRICT,
+ FOREIGN KEY(suite_id) REFERENCES benchmark_suites (id) ON DELETE RESTRICT
+ )
+ """
+ )
+ connection.exec_driver_sql(
+ "CREATE INDEX ix_benchmark_runs_deployment_id ON benchmark_runs (deployment_id)"
+ )
+ connection.exec_driver_sql(
+ "CREATE INDEX ix_benchmark_runs_suite_id ON benchmark_runs (suite_id)"
+ )
+ connection.exec_driver_sql(
+ "CREATE INDEX ix_benchmark_runs_artifact_sha256 ON benchmark_runs (artifact_sha256)"
+ )
+ connection.exec_driver_sql(
+ "CREATE INDEX ix_benchmark_runs_environment_digest ON benchmark_runs (environment_digest)"
+ )
+ connection.exec_driver_sql(
+ "CREATE INDEX ix_benchmark_runs_deployment_completed "
+ "ON benchmark_runs (deployment_id, completed_at)"
+ )
+
+ config = Config(str(Path(__file__).parents[1] / "alembic.ini"))
+ config.set_main_option("sqlalchemy.url", url)
+ command.stamp(config, "20260924_0038")
+ command.upgrade(config, "head")
+
+ upgraded = inspect(engine)
+ columns = {column["name"]: column for column in upgraded.get_columns("benchmark_runs")}
+ assert columns["deployment_id"]["nullable"] is True
+ assert "capability_deployment_id" in columns
+ indexes = {index["name"] for index in upgraded.get_indexes("benchmark_runs")}
+ assert "ix_benchmark_runs_actor_subject" in indexes
+ assert "ix_benchmark_runs_capability_deployment_completed" in indexes
+
+ command.downgrade(config, "20260924_0038")
+ downgraded = {
+ column["name"]: column for column in inspect(engine).get_columns("benchmark_runs")
+ }
+ assert downgraded["deployment_id"]["nullable"] is False
+ assert "capability_deployment_id" not in downgraded
+
+ command.upgrade(config, "head")
+ assert "capability_deployment_id" in {
+ column["name"] for column in inspect(engine).get_columns("benchmark_runs")
+ }
+ engine.dispose()
diff --git a/backend/tests/test_benchmark_runs_postgres.py b/backend/tests/test_benchmark_runs_postgres.py
new file mode 100644
index 0000000..cece23f
--- /dev/null
+++ b/backend/tests/test_benchmark_runs_postgres.py
@@ -0,0 +1,446 @@
+"""Managed PostgreSQL qualification for benchmark-run persistence and restart behavior.
+
+This test is enabled only by the disposable PostgreSQL boundary in managed-validation.yml.
+It creates one uniquely named database on that boundary, then drops only that database if it
+still exists and is still owned by the expected validation role.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+import sqlalchemy as sa
+from alembic.config import Config
+from sqlalchemy import create_engine, func, inspect, select, text
+from sqlalchemy.engine import URL, make_url
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from alembic import command
+from modelforge_api.domain.benchmarking import BenchmarkRunStartRequest
+from modelforge_api.persistence.models import (
+ ArtifactSet,
+ ArtifactSetMember,
+ BenchmarkRun,
+ Deployment,
+ GatewayRequest,
+ LabExecutionRun,
+)
+from modelforge_api.services.benchmark_runs import BenchmarkRunService
+from modelforge_api.services.serving import ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_dispatch_context import operator_fixture
+
+_PRIOR = "20260924_0038"
+_CURRENT = "20260924_0039"
+_LIFECYCLE_OPT_IN = "managed-validation-v1"
+
+
+def _managed_test_urls() -> tuple[URL, URL, URL]:
+ """Fail closed unless the dedicated disposable PostgreSQL boundary opted in."""
+ if os.getenv("MODELFORGE_TEST_POSTGRES_DATABASE_LIFECYCLE") is None:
+ pytest.skip("requires the managed-validation disposable PostgreSQL lifecycle opt-in")
+ if os.getenv("MODELFORGE_TEST_POSTGRES_DATABASE_LIFECYCLE") != _LIFECYCLE_OPT_IN:
+ pytest.fail("managed PostgreSQL database lifecycle opt-in has an unexpected value")
+
+ raw_urls = (
+ os.getenv("MODELFORGE_TEST_POSTGRES_URL"),
+ os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL"),
+ os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL"),
+ )
+ if not all(raw_urls):
+ pytest.fail("managed PostgreSQL lifecycle opt-in requires all three test role URLs")
+
+ urls = tuple(make_url(value) for value in raw_urls if value is not None)
+ expected_users = ("postgres", "modelforge", "modelforge_runtime")
+ expected_passwords = (
+ os.getenv("MODELFORGE_POSTGRES_ADMIN_PASSWORD"),
+ os.getenv("MODELFORGE_MIGRATION_DB_PASSWORD"),
+ os.getenv("MODELFORGE_RUNTIME_DB_PASSWORD"),
+ )
+ labels = ("admin", "migration owner", "runtime")
+ if not all(expected_passwords):
+ pytest.fail("managed PostgreSQL role passwords are unavailable in this test process")
+
+ for url, username, password, label in zip(
+ urls, expected_users, expected_passwords, labels, strict=True
+ ):
+ if (
+ url.drivername != "postgresql+psycopg"
+ or url.host != "postgres"
+ or url.port != 5432
+ or url.database != "modelforge"
+ or url.username != username
+ or url.query
+ ):
+ pytest.fail(
+ f"managed PostgreSQL {label} URL does not match the isolated CI test boundary"
+ )
+ if url.password != password:
+ pytest.fail(
+ f"managed PostgreSQL {label} URL does not match its dedicated test role password"
+ )
+ return urls # type: ignore[return-value]
+
+
+def _current_revision(engine) -> str:
+ with engine.connect() as connection:
+ return connection.scalar(text("SELECT version_num FROM alembic_version"))
+
+
+def _alembic_config(database_url: URL) -> Config:
+ config = Config()
+ config.set_main_option("script_location", str(Path(__file__).resolve().parents[1] / "alembic"))
+ rendered_url = database_url.render_as_string(hide_password=False).replace("%", "%%")
+ config.set_main_option("sqlalchemy.url", rendered_url)
+ return config
+
+
+def _insert_legacy_run(
+ connection: sa.Connection,
+ *,
+ run_id: uuid.UUID,
+ deployment_id: uuid.UUID,
+ suite_id: uuid.UUID,
+) -> None:
+ """Seed the exact pre-0039 row shape to prove the additive migration preserves history."""
+ legacy_runs = sa.table(
+ "benchmark_runs",
+ sa.column("id", sa.Uuid()),
+ sa.column("deployment_id", sa.Uuid()),
+ sa.column("suite_id", sa.Uuid()),
+ sa.column("model_revision_sha", sa.String(64)),
+ sa.column("artifact_sha256", sa.String(64)),
+ sa.column("runtime_type", sa.String(32)),
+ sa.column("runtime_version", sa.String(128)),
+ sa.column("runtime_image_digest", sa.String(71)),
+ sa.column("launch_arguments", sa.JSON()),
+ sa.column("suite_revision", sa.String(128)),
+ sa.column("dataset_revision", sa.String(255)),
+ sa.column("environment_fingerprint", sa.JSON()),
+ sa.column("environment_digest", sa.String(64)),
+ sa.column("seed", sa.Integer()),
+ sa.column("results", sa.JSON()),
+ sa.column("status", sa.String(32)),
+ sa.column("comparable", sa.Boolean()),
+ sa.column("stale_reason", sa.Text()),
+ sa.column("started_at", sa.DateTime(timezone=True)),
+ sa.column("completed_at", sa.DateTime(timezone=True)),
+ )
+ now = datetime.now(UTC)
+ connection.execute(
+ legacy_runs.insert().values(
+ id=run_id,
+ deployment_id=deployment_id,
+ suite_id=suite_id,
+ model_revision_sha="a" * 40,
+ artifact_sha256="b" * 64,
+ runtime_type="onnx",
+ runtime_version="1.0",
+ runtime_image_digest=None,
+ launch_arguments={},
+ suite_revision="1",
+ dataset_revision="builtin:legacy-benchmark-fixture-v1",
+ environment_fingerprint={},
+ environment_digest="c" * 64,
+ seed=None,
+ results={"legacy_fixture": True},
+ status="completed",
+ comparable=True,
+ stale_reason=None,
+ started_at=now,
+ completed_at=now,
+ )
+ )
+
+
+def test_managed_postgresql_benchmark_migration_race_and_restart(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Exercise migration 0039 and concurrent/restarted starts against real PostgreSQL."""
+ admin_url, owner_base_url, runtime_base_url = _managed_test_urls()
+ database_name = f"test_benchmark_run_{uuid.uuid4().hex}"
+ admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT", pool_pre_ping=True)
+ created_database = False
+ owner_engine = None
+ runtime_engine = None
+
+ try:
+ with admin_engine.connect() as connection:
+ already_exists = connection.scalar(
+ text("SELECT 1 FROM pg_database WHERE datname = :database_name"),
+ {"database_name": database_name},
+ )
+ assert already_exists is None
+ connection.execute(text(f'CREATE DATABASE "{database_name}" OWNER modelforge'))
+ created_database = True
+
+ owner_url = owner_base_url.set(database=database_name)
+ runtime_url = runtime_base_url.set(database=database_name)
+ owner_engine = create_engine(owner_url, pool_pre_ping=True)
+ runtime_engine = create_engine(
+ runtime_url,
+ pool_pre_ping=True,
+ connect_args={
+ "connect_timeout": 15,
+ "options": "-c statement_timeout=30000 -c lock_timeout=15000",
+ },
+ )
+ migration = _alembic_config(owner_url)
+ command.upgrade(migration, _PRIOR)
+
+ legacy_run_id = uuid.uuid4()
+ with Session(owner_engine) as setup_session:
+ serving, target_request, selected = operator_fixture(setup_session)
+ benchmark_service = BenchmarkRunService(serving)
+ advertised_suite = benchmark_service.suites()[0]
+ suite = benchmark_service._suite(advertised_suite.id)
+ request = BenchmarkRunStartRequest(
+ capability="rag.embedding@1",
+ suite_id=suite.id,
+ target=target_request.target,
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+ artifact_set = setup_session.get(ArtifactSet, selected.deployment.artifact_set_id)
+ artifact_id = setup_session.scalar(
+ select(ArtifactSetMember.artifact_id)
+ .where(ArtifactSetMember.artifact_set_id == selected.deployment.artifact_set_id)
+ .limit(1)
+ )
+ assert artifact_set is not None and artifact_id is not None
+ legacy_deployment = Deployment(
+ name=f"benchmark-migration-fixture-{uuid.uuid4().hex}",
+ revision_id=artifact_set.revision_id,
+ artifact_id=artifact_id,
+ derived_artifact_id=None,
+ runtime_profile_id=selected.deployment.runtime_profile_id,
+ compute_node_id=selected.deployment.compute_node_id,
+ channel="candidate",
+ status="draft",
+ capability_contract_ids=[],
+ )
+ setup_session.add(legacy_deployment)
+ setup_session.flush()
+ legacy_deployment_id = legacy_deployment.id
+ setup_session.commit()
+ deployment_settings = serving._deployment_settings
+ manifests = serving.manifests
+ with owner_engine.begin() as connection:
+ _insert_legacy_run(
+ connection,
+ run_id=legacy_run_id,
+ deployment_id=legacy_deployment_id,
+ suite_id=suite.id,
+ )
+
+ command.upgrade(migration, "head")
+ assert _current_revision(owner_engine) == _CURRENT
+
+ column_metadata = {
+ column["name"]: column for column in inspect(owner_engine).get_columns("benchmark_runs")
+ }
+ upgraded_columns = set(column_metadata)
+ assert column_metadata["deployment_id"]["nullable"] is True
+ assert {
+ "capability_deployment_id",
+ "target_snapshot",
+ "actor_subject",
+ "idempotency_key",
+ "request_fingerprint",
+ "failure_code",
+ "cancellation_requested_at",
+ "created_at",
+ "generation",
+ } <= upgraded_columns
+ constraints = inspect(owner_engine)
+ assert "ck_benchmark_runs_operator_identity" in {
+ constraint["name"] for constraint in constraints.get_check_constraints("benchmark_runs")
+ }
+ assert "uq_benchmark_runs_actor_idempotency" in {
+ constraint["name"]
+ for constraint in constraints.get_unique_constraints("benchmark_runs")
+ }
+ capability_fk = next(
+ foreign_key
+ for foreign_key in constraints.get_foreign_keys("benchmark_runs")
+ if foreign_key["name"] == "fk_benchmark_runs_capability_deployment"
+ )
+ assert capability_fk["referred_table"] == "capability_deployments"
+ assert capability_fk["options"]["ondelete"] == "RESTRICT"
+
+ with Session(owner_engine) as session:
+ historical = session.get(BenchmarkRun, legacy_run_id)
+ assert historical is not None
+ assert historical.deployment_id == legacy_deployment_id
+ assert historical.actor_subject is None
+ assert historical.capability_deployment_id is None
+ assert historical.created_at is not None
+ assert historical.generation == 1
+
+ with owner_engine.connect() as connection:
+ historical_values = dict(
+ connection.execute(
+ select(BenchmarkRun.__table__).where(BenchmarkRun.id == legacy_run_id)
+ )
+ .mappings()
+ .one()
+ )
+ historical_values["id"] = uuid.uuid4()
+ historical_values["actor_subject"] = "partial-identity-must-fail"
+ with pytest.raises(IntegrityError), owner_engine.begin() as connection:
+ connection.execute(BenchmarkRun.__table__.insert().values(historical_values))
+
+ with runtime_engine.connect() as connection:
+ assert connection.scalar(text("SELECT version_num FROM alembic_version")) == _CURRENT
+ for privilege in ("SELECT", "INSERT", "UPDATE"):
+ assert connection.scalar(
+ text("SELECT has_table_privilege(current_user, 'benchmark_runs', :privilege)"),
+ {"privilege": privilege},
+ )
+ for privilege in ("DELETE", "TRUNCATE"):
+ assert not connection.scalar(
+ text("SELECT has_table_privilege(current_user, 'benchmark_runs', :privilege)"),
+ {"privilege": privilege},
+ )
+
+ original_provenance = BenchmarkRunService._provenance
+ provenance_barrier = threading.Barrier(2)
+
+ def synchronized_provenance(service, benchmark_request):
+ provenance = original_provenance(service, benchmark_request)
+ provenance_barrier.wait(timeout=30)
+ return provenance
+
+ monkeypatch.setattr(BenchmarkRunService, "_provenance", synchronized_provenance)
+
+ def start_once() -> uuid.UUID:
+ with Session(runtime_engine) as session:
+ serving = ServingService(
+ session,
+ deployment_settings,
+ manifests,
+ MemoryPayloadStore(),
+ )
+ return (
+ BenchmarkRunService(serving)
+ .start(
+ request,
+ actor_subject="managed-postgres-operator",
+ idempotency_key="same-benchmark-start",
+ )
+ .id
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ run_ids = list(pool.map(lambda _index: start_once(), range(2)))
+ assert run_ids[0] == run_ids[1]
+
+ benchmark_run_id = run_ids[0]
+ with Session(runtime_engine) as session:
+ benchmark = session.get(BenchmarkRun, benchmark_run_id)
+ assert benchmark is not None
+ assert benchmark.actor_subject == "managed-postgres-operator"
+ assert benchmark.idempotency_key == "same-benchmark-start"
+ assert benchmark.capability_deployment_id == request.deployment_id
+ assert benchmark.target_snapshot["deployment_id"] == str(request.deployment_id)
+ assert (
+ session.scalar(
+ select(func.count())
+ .select_from(BenchmarkRun)
+ .where(
+ BenchmarkRun.actor_subject == "managed-postgres-operator",
+ BenchmarkRun.idempotency_key == "same-benchmark-start",
+ )
+ )
+ == 1
+ )
+ child_id = uuid.UUID(benchmark.results["execution"]["active_lab_run_id"])
+ child = session.get(LabExecutionRun, child_id)
+ assert child is not None
+ assert child.actor_subject == "managed-postgres-operator"
+ assert child.idempotency_key == f"benchmark-{benchmark_run_id.hex}-00"
+ assert session.get(GatewayRequest, child.gateway_request_id) is not None
+ assert (
+ session.scalar(
+ select(func.count())
+ .select_from(LabExecutionRun)
+ .where(
+ LabExecutionRun.actor_subject == "managed-postgres-operator",
+ LabExecutionRun.idempotency_key == f"benchmark-{benchmark_run_id.hex}-00",
+ )
+ )
+ == 1
+ )
+
+ # Recreate the engine and service after the race: the committed parent/child link must
+ # remain readable and replaying the same actor/key must not create another child.
+ runtime_engine.dispose()
+ runtime_engine = create_engine(
+ runtime_url,
+ pool_pre_ping=True,
+ connect_args={
+ "connect_timeout": 15,
+ "options": "-c statement_timeout=30000 -c lock_timeout=15000",
+ },
+ )
+ with Session(runtime_engine) as session:
+ serving = ServingService(
+ session,
+ deployment_settings,
+ manifests,
+ MemoryPayloadStore(),
+ )
+ service = BenchmarkRunService(serving)
+ recovered = service.get(benchmark_run_id, actor_subject="managed-postgres-operator")
+ replay = service.start(
+ request,
+ actor_subject="managed-postgres-operator",
+ idempotency_key="same-benchmark-start",
+ )
+ assert recovered.id == replay.id == benchmark_run_id
+ assert replay.contract.deployment_id == str(request.deployment_id)
+ assert len(service.list(actor_subject="managed-postgres-operator")) == 1
+ assert (
+ session.scalar(
+ select(func.count())
+ .select_from(LabExecutionRun)
+ .where(LabExecutionRun.actor_subject == "managed-postgres-operator")
+ )
+ == 1
+ )
+
+ # The downgrade must refuse to erase operator history and leave the migrated schema live.
+ with pytest.raises(RuntimeError, match="Operator benchmark history exists"):
+ command.downgrade(migration, _PRIOR)
+ assert _current_revision(owner_engine) == _CURRENT
+ with Session(owner_engine) as session:
+ assert session.get(BenchmarkRun, benchmark_run_id) is not None
+ assert session.get(BenchmarkRun, legacy_run_id) is not None
+ finally:
+ if runtime_engine is not None:
+ runtime_engine.dispose()
+ if owner_engine is not None:
+ owner_engine.dispose()
+ if created_database:
+ with admin_engine.connect() as connection:
+ database_owner = connection.scalar(
+ text(
+ "SELECT pg_get_userbyid(datdba) FROM pg_database "
+ "WHERE datname = :database_name"
+ ),
+ {"database_name": database_name},
+ )
+ if database_owner is not None:
+ if database_owner != "modelforge":
+ raise RuntimeError(
+ "refusing to remove the qualification database after its owner changed"
+ )
+ connection.execute(text(f'DROP DATABASE "{database_name}" WITH (FORCE)'))
+ admin_engine.dispose()
diff --git a/backend/tests/test_catalog_query_performance.py b/backend/tests/test_catalog_query_performance.py
new file mode 100644
index 0000000..16297a2
--- /dev/null
+++ b/backend/tests/test_catalog_query_performance.py
@@ -0,0 +1,523 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime
+from typing import Any
+from uuid import UUID, uuid4
+
+from sqlalchemy import create_engine, event, select
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.catalog import capability_estate, model_installation_rationale
+from modelforge_api.domain.schemas import (
+ CapabilityEstateResponse,
+ InstallationDependencyResponse,
+ ModelInstallationRationaleResponse,
+)
+from modelforge_api.persistence.models import (
+ ArtifactSet,
+ Base,
+ Capability,
+ CapabilityContract,
+ CapabilityDeployment,
+ CapabilityEvaluationRun,
+ CapabilityResourceEnvelope,
+ Model,
+ ModelRevision,
+ Project,
+ ProjectBinding,
+ ProjectFitEvidence,
+ ResidencyAllocation,
+ RuntimeProfile,
+ ServiceClient,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+
+
+def _catalog_session() -> tuple[Engine, Session, dict[str, UUID]]:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ session = Session(engine)
+ now = datetime(2026, 1, 1, tzinfo=UTC)
+ ids: dict[str, UUID] = {}
+
+ models: dict[str, Model] = {}
+ revisions: dict[str, ModelRevision] = {}
+ artifact_sets: dict[str, ArtifactSet] = {}
+ for key, display_name in (("model-z", "Zulu model"), ("model-a", "Alpha model"), ("model-b", "Beta model")):
+ model = Model(
+ key=key,
+ display_name=display_name,
+ upstream_provider="huggingface",
+ upstream_source=f"itworx/{key}",
+ )
+ session.add(model)
+ session.flush()
+ revision = ModelRevision(
+ model_id=model.id,
+ upstream_revision="main",
+ resolved_commit_sha=("a" if key == "model-a" else "b" if key == "model-b" else "c") * 40,
+ )
+ session.add(revision)
+ session.flush()
+ models[key] = model
+ revisions[key] = revision
+ ids[key] = model.id
+
+ if key != "model-z":
+ artifact_set = ArtifactSet(
+ revision_id=revision.id,
+ snapshot_id=uuid4(),
+ variant_key="full",
+ label=f"{display_name} full set",
+ selection_reason="installed local test fixture",
+ selected_paths=["weights.safetensors"],
+ total_size_bytes=1000 if key == "model-a" else 2000,
+ file_count=1,
+ availability="local",
+ status="complete",
+ completeness="complete",
+ security_status="verified",
+ license_status="approved",
+ immutable_at=now,
+ )
+ session.add(artifact_set)
+ session.flush()
+ artifact_sets[key] = artifact_set
+
+ capability_manifests = {
+ item.capability: item for item in ManifestRegistry().capabilities()
+ }
+ contracts: dict[str, CapabilityContract] = {}
+ for capability_key in ("rag.embedding", "vision.embedding", "speech.transcription"):
+ manifest = capability_manifests[capability_key]
+ capability = Capability(key=capability_key, description=manifest.description)
+ session.add(capability)
+ session.flush()
+ contract = CapabilityContract(
+ capability_id=capability.id,
+ version=manifest.version,
+ input_schema=manifest.input_schema,
+ output_schema=manifest.output_schema,
+ contract=manifest.model_dump(mode="json"),
+ upgrade_class=manifest.upgrade_class.value,
+ )
+ session.add(contract)
+ session.flush()
+ contracts[capability_key] = contract
+ ids[f"contract:{capability_key}"] = contract.id
+
+ profiles: list[RuntimeProfile] = []
+ for profile_ix, key in enumerate(("model-a", "model-b")):
+ profile = RuntimeProfile(
+ name=f"profile-{key}",
+ runtime_type="vllm",
+ runtime_version="1.0",
+ artifact_sha256=("d" if profile_ix == 0 else "e") * 64,
+ fingerprint=("f" if profile_ix == 0 else "1") * 64,
+ )
+ session.add(profile)
+ session.flush()
+ profiles.append(profile)
+
+ deployments: dict[str, CapabilityDeployment] = {}
+ deployment_specs = (
+ ("alpha-rag-prod", "model-a", "rag.embedding", profiles[0], True, "stable", "ready", 1),
+ ("alpha-rag-lab", "model-a", "rag.embedding", profiles[0], False, "lab", "lab_ready", 2),
+ ("alpha-vision", "model-a", "vision.embedding", profiles[0], False, "lab", "lab_ready", 3),
+ ("beta-rag", "model-b", "rag.embedding", profiles[1], False, "lab", "lab_ready", 4),
+ ("beta-speech", "model-b", "speech.transcription", profiles[1], False, "lab", "lab_ready", 5),
+ )
+ for name, model_key, capability_key, profile, production, channel, status, seconds in deployment_specs:
+ deployment = CapabilityDeployment(
+ capability_contract_id=contracts[capability_key].id,
+ deployment_candidate_id=uuid4(),
+ artifact_set_id=artifact_sets[model_key].id,
+ runtime_profile_id=profile.id,
+ compute_node_id=uuid4(),
+ accelerator_id=uuid4(),
+ channel=channel,
+ status=status,
+ production=production,
+ health_status="ready_on_demand",
+ config_fingerprint=(f"{seconds:02x}" * 32),
+ provenance={"fixture": name},
+ created_at=now.replace(second=seconds),
+ )
+ session.add(deployment)
+ session.flush()
+ deployments[name] = deployment
+ ids[f"deployment:{name}"] = deployment.id
+ ids[f"node:{name}"] = deployment.compute_node_id
+
+ first_rag = deployments["alpha-rag-prod"]
+ vision = deployments["alpha-vision"]
+ speech = deployments["beta-speech"]
+ for deployment in deployments.values():
+ session.add(
+ CapabilityResourceEnvelope(
+ capability_deployment_id=deployment.id,
+ runtime_probe_id=uuid4(),
+ accelerator_kind="gpu",
+ accelerator_uuid=f"gpu-{deployment.id}",
+ environment_fingerprint="a" * 64,
+ concurrency=1,
+ batch_size=1,
+ max_sequence_length=512,
+ baseline_vram_bytes=100,
+ resident_vram_bytes=200,
+ peak_vram_bytes=300,
+ required_vram_bytes=400,
+ cold_load_time_ms=10.0,
+ inference_latency_ms=20.0,
+ evidence={"fixture": True},
+ immutable_at=now,
+ )
+ )
+ for ix, (deployment, status) in enumerate(
+ ((first_rag, "passed"), (first_rag, "failed"), (vision, "passed"), (speech, "passed"))
+ ):
+ evaluation = CapabilityEvaluationRun(
+ suite_id=uuid4(),
+ capability_deployment_id=deployment.id,
+ status=status,
+ metric_values={},
+ case_results=[],
+ resource_metrics={},
+ environment_fingerprint="b" * 64,
+ evidence_digest=f"{ix:064x}",
+ evidence={"fixture": True},
+ started_at=now.replace(second=ix),
+ completed_at=now.replace(second=ix),
+ # Equal times exercise the old ORDER BY created_at DESC LIMIT 1 behavior.
+ created_at=now if deployment is first_rag else now.replace(second=ix),
+ )
+ session.add(evaluation)
+ session.flush()
+ ids[f"evaluation:{ix}"] = evaluation.id
+
+ projects = [
+ Project(key="alpha-project", name="Alpha", description="active consumer"),
+ Project(key="beta-project", name="Beta", description="deprecated consumer"),
+ Project(key="gamma-project", name="Gamma", description="second active consumer"),
+ ]
+ session.add_all(projects)
+ session.flush()
+ project_by_key = {item.key: item for item in projects}
+ bindings = [
+ ProjectBinding(
+ project_id=project_by_key["alpha-project"].id,
+ capability_contract_id=contracts["rag.embedding"].id,
+ channel="stable",
+ priority="interactive",
+ ),
+ ProjectBinding(
+ project_id=project_by_key["beta-project"].id,
+ capability_contract_id=contracts["rag.embedding"].id,
+ channel="lab",
+ priority="background",
+ deprecated_at=now,
+ ),
+ ProjectBinding(
+ project_id=project_by_key["gamma-project"].id,
+ capability_contract_id=contracts["vision.embedding"].id,
+ channel="lab",
+ priority="interactive",
+ ),
+ ]
+ session.add_all(bindings)
+ session.flush()
+ session.add_all(
+ [
+ ServiceClient(
+ name="alpha-client",
+ status="active",
+ allowed_capabilities=["rag.embedding"],
+ project_binding_id=bindings[0].id,
+ ),
+ ServiceClient(
+ name="deprecated-client",
+ status="active",
+ allowed_capabilities=["rag.embedding"],
+ project_binding_id=bindings[1].id,
+ ),
+ ServiceClient(
+ name="inactive-client",
+ status="disabled",
+ allowed_capabilities=["vision.embedding"],
+ project_binding_id=bindings[2].id,
+ ),
+ ]
+ )
+ fit_evidence = [
+ ProjectFitEvidence(
+ project_binding_id=bindings[0].id,
+ capability_deployment_id=first_rag.id,
+ environment="shadow",
+ recommendation="keep_lab",
+ evidence_class="project_fit",
+ engineering_integration="partial",
+ production_validation="not_validated",
+ production_action="no_change",
+ case_count=1,
+ metric_values={},
+ critical_errors=0,
+ blockers=[],
+ evidence_digest="c" * 64,
+ evidence={"fixture": True},
+ ),
+ ProjectFitEvidence(
+ project_binding_id=bindings[2].id,
+ capability_deployment_id=vision.id,
+ environment="lab",
+ recommendation="keep_lab",
+ evidence_class="project_fit",
+ engineering_integration="partial",
+ production_validation="not_validated",
+ production_action="no_change",
+ case_count=1,
+ metric_values={},
+ critical_errors=0,
+ blockers=[],
+ evidence_digest="d" * 64,
+ evidence={"fixture": True},
+ ),
+ ]
+ session.add_all(fit_evidence)
+ session.flush()
+ ids["fit:rag"] = fit_evidence[0].id
+ ids["fit:vision"] = fit_evidence[1].id
+ session.add(
+ ResidencyAllocation(
+ capability_deployment_id=first_rag.id,
+ compute_node_id=first_rag.compute_node_id,
+ accelerator_id=first_rag.accelerator_id,
+ state="cold",
+ last_used_at=now,
+ )
+ )
+ session.commit()
+ session.close()
+ return engine, Session(engine), ids
+
+
+def _count_selects[T](engine: Engine, call: Callable[[], T]) -> tuple[T, int]:
+ statements: list[str] = []
+
+ def count(
+ _conn: Any,
+ _cursor: Any,
+ statement: str,
+ _params: Any,
+ _context: Any,
+ _many: bool,
+ ) -> None:
+ if statement.lstrip().upper().startswith("SELECT"):
+ statements.append(statement)
+
+ event.listen(engine, "before_cursor_execute", count)
+ try:
+ response = call()
+ finally:
+ event.remove(engine, "before_cursor_execute", count)
+ return response, len(statements)
+
+
+def test_capability_estate_uses_batched_reads_for_deployed_capabilities() -> None:
+ engine, session, ids = _catalog_session()
+ try:
+ latest_rag_evaluation = session.scalar(
+ select(CapabilityEvaluationRun)
+ .where(
+ CapabilityEvaluationRun.capability_deployment_id
+ == ids["deployment:alpha-rag-prod"]
+ )
+ .order_by(CapabilityEvaluationRun.created_at.desc())
+ .limit(1)
+ )
+ assert latest_rag_evaluation is not None
+ result, reads = _count_selects(
+ engine,
+ lambda: capability_estate(ManifestRegistry(), session),
+ )
+
+ assert reads == 8
+ deployment_facts = {
+ "rag.embedding": (
+ "alpha-rag-prod",
+ "Alpha model",
+ "a" * 40,
+ "ready",
+ latest_rag_evaluation.status,
+ ),
+ "vision.embedding": (
+ "alpha-vision",
+ "Alpha model",
+ "a" * 40,
+ "lab_ready",
+ "passed",
+ ),
+ "speech.transcription": (
+ "beta-speech",
+ "Beta model",
+ "b" * 40,
+ "lab_ready",
+ "passed",
+ ),
+ }
+ expected = []
+ for manifest in ManifestRegistry().capabilities():
+ facts = deployment_facts.get(manifest.capability)
+ deployment_name, model_name, revision, state, evaluation_state = facts or (
+ None,
+ None,
+ None,
+ "blocked" if manifest.estate.stability == "blocked" else "not_deployed",
+ "not_evaluated",
+ )
+ expected.append(
+ CapabilityEstateResponse(
+ capability=manifest.capability,
+ version=manifest.version,
+ category=manifest.estate.category,
+ purpose=manifest.estate.purpose,
+ declared_stability=manifest.estate.stability,
+ operational_state=state,
+ current_deployment_id=(
+ ids[f"deployment:{deployment_name}"] if deployment_name else None
+ ),
+ model=model_name,
+ revision=revision,
+ runtime="vllm" if deployment_name else None,
+ node=str(ids[f"node:{deployment_name}"]) if deployment_name else None,
+ resource_class=manifest.estate.resource_class,
+ measured_required_vram_bytes=400 if deployment_name else None,
+ consumers=manifest.estate.consumers,
+ privacy_class=manifest.privacy.classification,
+ evaluation_type=manifest.estate.evaluation_type,
+ evaluation_state=evaluation_state,
+ ).model_dump(mode="json")
+ )
+ assert [item.model_dump(mode="json") for item in result] == expected
+ finally:
+ session.close()
+ engine.dispose()
+
+
+def test_installation_rationale_uses_batched_reads_across_models_and_dependencies() -> None:
+ engine, session, ids = _catalog_session()
+ try:
+ result, reads = _count_selects(engine, lambda: model_installation_rationale(session))
+
+ assert reads == 9
+ expected_alpha_dependencies = [
+ InstallationDependencyResponse(
+ capability="rag.embedding",
+ version=1,
+ deployment_id=ids["deployment:alpha-rag-prod"],
+ channel="stable",
+ production=True,
+ project_consumers=["alpha-project"],
+ active_project_consumers=["alpha-project", "beta-project"],
+ project_fit_evidence_ids=[ids["fit:rag"]],
+ evaluation_run_ids=[ids["evaluation:0"], ids["evaluation:1"]],
+ last_used_at=datetime.fromisoformat("2026-01-01T00:00:00"),
+ ),
+ InstallationDependencyResponse(
+ capability="rag.embedding",
+ version=1,
+ deployment_id=ids["deployment:alpha-rag-lab"],
+ channel="lab",
+ production=False,
+ project_consumers=["alpha-project"],
+ active_project_consumers=["alpha-project", "beta-project"],
+ project_fit_evidence_ids=[ids["fit:rag"]],
+ evaluation_run_ids=[],
+ last_used_at=None,
+ ),
+ InstallationDependencyResponse(
+ capability="vision.embedding",
+ version=1,
+ deployment_id=ids["deployment:alpha-vision"],
+ channel="lab",
+ production=False,
+ project_consumers=["gamma-project"],
+ active_project_consumers=[],
+ project_fit_evidence_ids=[ids["fit:vision"]],
+ evaluation_run_ids=[ids["evaluation:2"]],
+ last_used_at=None,
+ ),
+ ]
+ expected_beta_dependencies = [
+ InstallationDependencyResponse(
+ capability="rag.embedding",
+ version=1,
+ deployment_id=ids["deployment:beta-rag"],
+ channel="lab",
+ production=False,
+ project_consumers=["alpha-project"],
+ active_project_consumers=["alpha-project", "beta-project"],
+ project_fit_evidence_ids=[ids["fit:rag"]],
+ evaluation_run_ids=[],
+ last_used_at=None,
+ ),
+ InstallationDependencyResponse(
+ capability="speech.transcription",
+ version=1,
+ deployment_id=ids["deployment:beta-speech"],
+ channel="lab",
+ production=False,
+ project_consumers=[],
+ active_project_consumers=[],
+ project_fit_evidence_ids=[],
+ evaluation_run_ids=[ids["evaluation:3"]],
+ last_used_at=None,
+ ),
+ ]
+ expected = [
+ ModelInstallationRationaleResponse(
+ model_id=ids["model-a"],
+ display_name="Alpha model",
+ upstream_source="itworx/model-a",
+ installed=True,
+ installed_bytes=1000,
+ dependencies=expected_alpha_dependencies,
+ can_delete=False,
+ deletion_blockers=[
+ "capability_deployment_dependency",
+ "production_dependency",
+ ],
+ ),
+ ModelInstallationRationaleResponse(
+ model_id=ids["model-b"],
+ display_name="Beta model",
+ upstream_source="itworx/model-b",
+ installed=True,
+ installed_bytes=2000,
+ dependencies=expected_beta_dependencies,
+ can_delete=False,
+ deletion_blockers=["capability_deployment_dependency"],
+ ),
+ ModelInstallationRationaleResponse(
+ model_id=ids["model-z"],
+ display_name="Zulu model",
+ upstream_source="itworx/model-z",
+ installed=False,
+ installed_bytes=0,
+ dependencies=[],
+ can_delete=False,
+ deletion_blockers=[],
+ ),
+ ]
+ assert [item.model_dump(mode="json") for item in result] == [
+ item.model_dump(mode="json") for item in expected
+ ]
+ finally:
+ session.close()
+ engine.dispose()
diff --git a/backend/tests/test_concurrency_m16.py b/backend/tests/test_concurrency_m16.py
index 019d50e..666810d 100644
--- a/backend/tests/test_concurrency_m16.py
+++ b/backend/tests/test_concurrency_m16.py
@@ -12,6 +12,7 @@ the live chaos evidence; what these tests pin down is that the *code* claims a s
from __future__ import annotations
+import importlib
import os
import threading
import uuid
@@ -22,9 +23,10 @@ from pathlib import Path
from typing import Any
import pytest
+from alembic.config import Config
from pydantic import SecretStr
from sqlalchemy import create_engine, event, func, select, text
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import Engine, make_url
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.orm import Session
@@ -151,7 +153,9 @@ def test_a_single_use_enrolment_token_survives_a_concurrent_storm(
with Session(engine) as session:
assert session.scalar(select(func.count()).select_from(ComputeNode)) == 1
active = session.scalar(
- select(func.count()).select_from(NodeCredential).where(NodeCredential.revoked_at.is_(None))
+ select(func.count())
+ .select_from(NodeCredential)
+ .where(NodeCredential.revoked_at.is_(None))
)
assert active == 1
enrollment = session.scalar(select(NodeEnrollment))
@@ -168,7 +172,7 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
"""Managed gate for the PostgreSQL row-lock semantics SQLite cannot reproduce.
Set MODELFORGE_TEST_POSTGRES_URL to an isolated, disposable PostgreSQL database. The test
- creates and removes one UUID-named schema, pauses enrolment after its conditional claim has
+ creates and removes one UUID-named database, pauses enrolment after its conditional claim has
acquired the row lock, then proves a concurrent revocation loses after the claim commits.
"""
@@ -180,15 +184,17 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
if database_url.startswith("postgresql://"):
database_url = database_url.replace("postgresql://", "postgresql+psycopg://", 1)
- schema = f"test_enrollment_linearization_{uuid.uuid4().hex}"
- admin_engine = create_engine(database_url, pool_pre_ping=True)
- with admin_engine.begin() as connection:
- connection.execute(text(f'CREATE SCHEMA "{schema}"'))
- engine = create_engine(
- database_url,
- pool_pre_ping=True,
- connect_args={"options": f"-csearch_path={schema}"},
- )
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ if not owner_url or not runtime_url:
+ pytest.fail("managed enrollment test requires owner and runtime URLs")
+ database = f"test_enrollment_linearization_{uuid.uuid4().hex}"
+ admin_engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
+ with admin_engine.connect() as connection:
+ connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+ # The canonical audit function deliberately targets public tables. A search_path-only
+ # fixture would read a private checkpoint but append to another schema's audit chain.
+ engine = create_engine(make_url(runtime_url).set(database=database), pool_pre_ping=True)
claim_written = threading.Event()
allow_claim_commit = threading.Event()
revoke_update_started = threading.Event()
@@ -207,7 +213,18 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
revoke_update_started.set()
try:
- Base.metadata.create_all(engine)
+ migration = Config()
+ migration.set_main_option(
+ "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+ )
+ migration.set_main_option(
+ "sqlalchemy.url",
+ make_url(owner_url)
+ .set(database=database)
+ .render_as_string(hide_password=False)
+ .replace("%", "%%"),
+ )
+ importlib.import_module("alembic.command").upgrade(migration, "head")
with Session(engine) as session:
created = NodeAgentService(session, settings()).create_enrollment(
EnrollmentTokenCreate(display_name="PostgreSQL row-lock target")
@@ -293,8 +310,8 @@ def test_managed_postgresql_claim_and_revocation_linearize_on_the_row_lock() ->
finally:
allow_claim_commit.set()
engine.dispose()
- with admin_engine.begin() as connection:
- connection.execute(text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE'))
+ with admin_engine.connect() as connection:
+ connection.execute(text(f'DROP DATABASE IF EXISTS "{database}"'))
admin_engine.dispose()
@@ -360,9 +377,7 @@ def test_re_enrolment_of_one_identity_under_load_keeps_one_active_credential(
with Session(engine) as session:
service = NodeAgentService(session, settings())
tokens = [
- service.create_enrollment(
- EnrollmentTokenCreate(display_name="GPU Node")
- ).enrollment_token
+ service.create_enrollment(EnrollmentTokenCreate(display_name="GPU Node")).enrollment_token
for _ in range(rounds)
]
@@ -421,9 +436,7 @@ def test_the_same_idempotency_key_in_parallel_produces_one_logical_operation(
def attempt(_index: int) -> str:
with Session(engine) as session:
- existing = session.scalar(
- select(ServingJob).where(ServingJob.idempotency_key == key)
- )
+ existing = session.scalar(select(ServingJob).where(ServingJob.idempotency_key == key))
if existing is not None:
return "REUSED"
session.add(
@@ -478,9 +491,7 @@ def test_concurrent_audit_appends_form_one_strict_chain(engine: Engine) -> None:
assert len(events) == attempts
assert audit_chain_violations(events, session.get(AuditChainHead, 1)) == []
result = next(
- item
- for item in check_invariants(session).results
- if item.key == "audit_chain_intact"
+ item for item in check_invariants(session).results if item.key == "audit_chain_intact"
)
assert result.status is InvariantStatus.HOLDS
diff --git a/backend/tests/test_configuration_catalog.py b/backend/tests/test_configuration_catalog.py
new file mode 100644
index 0000000..f82ee62
--- /dev/null
+++ b/backend/tests/test_configuration_catalog.py
@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from pytest import MonkeyPatch
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.domain.configuration_policy import SERVING_OVERRIDE_KEYS
+from modelforge_api.domain.configuration_reference import SETTING_DOCS, Sensitivity
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.configuration import build_configuration_catalog
+from modelforge_api.settings import Settings, get_settings
+
+OPERATOR_TOKEN = "configuration-test-operator-token" # noqa: S105 - synthetic test credential
+DATABASE_SECRET = (
+ "postgresql+psycopg://runtime:do-not-leak@db:5432/modelforge" # noqa: S105
+)
+HF_SECRET = "hf_do-not-leak" # noqa: S105 - synthetic test credential
+REDIS_PASSWORD = "redis-do-not-leak" # noqa: S105 - synthetic test credential
+REDIS_URL = f"redis://cache-user:{REDIS_PASSWORD}@redis.internal:6379/0"
+ALERT_WEBHOOK_SECRET = "https://hooks.example.test/token-that-must-not-leak" # noqa: S105
+
+
+def _settings() -> Settings:
+ return Settings(
+ _env_file=None,
+ env="test",
+ operator_api_key=SecretStr(OPERATOR_TOKEN),
+ database_url=DATABASE_SECRET,
+ hf_token=SecretStr(HF_SECRET),
+ redis_url=REDIS_URL,
+ alert_webhook_url=SecretStr(ALERT_WEBHOOK_SECRET),
+ log_level="WARNING",
+ )
+
+
+def test_catalog_covers_every_runtime_setting_and_never_serializes_secrets() -> None:
+ catalog = build_configuration_catalog(_settings())
+
+ by_key = {entry.key: entry for entry in catalog.settings}
+ assert set(by_key) == set(SETTING_DOCS)
+ assert catalog.read_only is False
+ assert {entry.key for entry in catalog.settings if entry.editable} == SERVING_OVERRIDE_KEYS
+
+ secret_keys = {
+ key
+ for key, documentation in SETTING_DOCS.items()
+ if documentation.sensitivity is Sensitivity.SECRET
+ }
+ assert secret_keys
+ assert all(by_key[key].value is None for key in secret_keys)
+ assert by_key["database_url"].configured is True
+ assert by_key["hf_token"].configured is True
+ assert by_key["operator_api_key"].configured is True
+ assert by_key["alert_webhook_url"].configured is True
+
+ payload = catalog.model_dump_json()
+ assert DATABASE_SECRET not in payload
+ assert HF_SECRET not in payload
+ assert OPERATOR_TOKEN not in payload
+ assert REDIS_PASSWORD not in payload
+ assert ALERT_WEBHOOK_SECRET not in payload
+
+
+def test_catalog_reports_the_actual_pydantic_source_at_each_priority(
+ tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ dotenv = tmp_path / "model-forge.env"
+ dotenv.write_text(
+ "MODELFORGE_LOG_LEVEL=INFO\n"
+ "MODELFORGE_GATEWAY_MAX_BATCH_SIZE=12\n"
+ "MODELFORGE_GATEWAY_REQUEST_TIMEOUT_SECONDS=100\n"
+ "MODELFORGE_HF_TOKEN=hf_dotenv_secret\n"
+ "MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS=\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("MODELFORGE_LOG_LEVEL", "WARNING")
+ monkeypatch.setenv("MODELFORGE_GATEWAY_REQUEST_TIMEOUT_SECONDS", "111")
+ monkeypatch.setenv("MODELFORGE_HF_TOKEN", "hf_process_secret")
+
+ settings = Settings(
+ _env_file=dotenv,
+ env="test",
+ gateway_request_timeout_seconds=123,
+ )
+ catalog = build_configuration_catalog(settings)
+ by_key = {entry.key: entry for entry in catalog.settings}
+
+ assert settings.log_level == "WARNING"
+ assert by_key["log_level"].source == "process_environment"
+ assert by_key["log_level"].value == "WARNING"
+ assert settings.gateway_max_batch_size == 12
+ assert by_key["gateway_max_batch_size"].source == "dotenv"
+ assert by_key["gateway_max_batch_size"].value == 12
+ assert settings.gateway_request_timeout_seconds == 123
+ assert by_key["gateway_request_timeout_seconds"].source == "initialization"
+ assert by_key["cors_origins"].source == "pydantic_default"
+ assert by_key["hf_token"].source == "process_environment"
+ assert by_key["hf_token"].value is None
+ assert by_key["alert_webhook_allowed_hosts"].source == "dotenv"
+ assert by_key["alert_webhook_allowed_hosts"].value == ""
+ assert settings.setting_source("hf_token") == "process_environment"
+ assert "hf_process_secret" not in repr(settings._setting_sources)
+ assert "hf_dotenv_secret" not in repr(settings._setting_sources)
+ assert "hf_process_secret" not in catalog.model_dump_json()
+ assert "hf_dotenv_secret" not in catalog.model_dump_json()
+
+ effective = settings.model_copy(update={"gateway_max_batch_size": 24})
+ overridden = build_configuration_catalog(effective, overrides={"gateway_max_batch_size": 24})
+ overridden_setting = next(item for item in overridden.settings if item.key == "gateway_max_batch_size")
+ assert overridden_setting.source == "operator_override"
+ assert overridden_setting.value == 24
+
+ monkeypatch.setenv("MODELFORGE_LOG_LEVEL", "")
+ empty_environment_value = Settings(_env_file=dotenv, env="test")
+ empty_catalog = build_configuration_catalog(empty_environment_value)
+ empty_log_level = next(item for item in empty_catalog.settings if item.key == "log_level")
+ assert empty_environment_value.log_level == ""
+ assert empty_log_level.source == "process_environment"
+ assert empty_log_level.value == ""
+
+
+def test_catalog_reports_a_secrets_file_source_without_returning_its_value(
+ tmp_path: Path, monkeypatch: MonkeyPatch
+) -> None:
+ secret_dir = tmp_path / "settings-secrets"
+ secret_dir.mkdir()
+ (secret_dir / "MODELFORGE_LOG_LEVEL").write_text("ERROR", encoding="utf-8")
+ monkeypatch.delenv("MODELFORGE_LOG_LEVEL", raising=False)
+
+ settings = Settings(_env_file=None, _secrets_dir=secret_dir, env="test")
+ catalog = build_configuration_catalog(settings)
+ log_level = next(item for item in catalog.settings if item.key == "log_level")
+
+ assert settings.log_level == "ERROR"
+ assert log_level.source == "secrets_file"
+ assert log_level.value == "ERROR"
+
+
+def test_settings_copy_and_mutation_never_keep_a_stale_field_source(
+ monkeypatch: MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("MODELFORGE_LOG_LEVEL", "WARNING")
+ settings = Settings(_env_file=None, env="test")
+
+ unchanged_copy = settings.model_copy()
+ assert unchanged_copy.setting_source("log_level") == "process_environment"
+
+ copied_update = settings.model_copy(update={"log_level": "DEBUG"})
+ assert copied_update.setting_source("log_level") == "runtime_update"
+ assert settings.setting_source("log_level") == "process_environment"
+
+ settings.log_level = "ERROR"
+ assert settings.setting_source("log_level") == "runtime_update"
+
+ reconstructed = Settings.model_validate(settings.model_dump())
+ assert reconstructed.setting_source("log_level") == "initialization"
+
+
+def test_catalog_exposes_the_queue_timeout_cross_setting_validation() -> None:
+ catalog = build_configuration_catalog(_settings())
+ by_key = {entry.key: entry for entry in catalog.settings}
+
+ assert by_key["gateway_queue_timeout_seconds"].validation_rules["less_than_setting"] == (
+ "gateway_request_timeout_seconds"
+ )
+
+
+def test_configuration_catalog_endpoint_requires_control_plane_auth_and_stays_secret_safe() -> None:
+ settings = _settings()
+ engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+ Base.metadata.create_all(engine)
+ session = Session(engine)
+ app.dependency_overrides[get_session] = lambda: session
+ app.dependency_overrides[get_settings] = lambda: settings
+ client = TestClient(app)
+ try:
+ anonymous = client.get("/api/v1/configuration/settings")
+ assert anonymous.status_code == 401
+
+ response = client.get(
+ "/api/v1/configuration/settings",
+ headers={"X-ModelForge-Admin-Token": OPERATOR_TOKEN},
+ )
+ assert response.status_code == 200
+ body = response.text
+ assert DATABASE_SECRET not in body
+ assert HF_SECRET not in body
+ assert OPERATOR_TOKEN not in body
+ assert REDIS_PASSWORD not in body
+ payload = response.json()
+ assert payload["environment"] == "test"
+ assert payload["read_only"] is False
+ assert {item["key"] for item in payload["settings"]} == set(SETTING_DOCS)
+ finally:
+ app.dependency_overrides.clear()
+ session.close()
+ engine.dispose()
diff --git a/backend/tests/test_configuration_changes.py b/backend/tests/test_configuration_changes.py
new file mode 100644
index 0000000..5eae1e1
--- /dev/null
+++ b/backend/tests/test_configuration_changes.py
@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import (
+ Principal,
+ PrincipalRole,
+ authenticate_operator_principal,
+)
+from modelforge_api.db import get_session
+from modelforge_api.domain.configuration import (
+ ConfigurationApply,
+ ConfigurationChange,
+ ConfigurationImport,
+)
+from modelforge_api.domain.serving import EmbeddingInvokeRequest
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ Base,
+ ConfigurationRevision,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.configuration_changes import (
+ ConfigurationError,
+ ConfigurationService,
+ configuration_state,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from modelforge_api.settings import Settings, get_settings
+
+TOKEN = "rc01-synthetic-administrator" # noqa: S105
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+ engine = create_engine(
+ "sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as database:
+ yield database
+ engine.dispose()
+
+
+def settings() -> Settings:
+ return Settings(
+ _env_file=None,
+ env="test",
+ operator_api_key=SecretStr(TOKEN),
+ config_root=Path(__file__).resolve().parents[2] / "config",
+ )
+
+
+def service(session: Session) -> ConfigurationService:
+ return ConfigurationService(
+ session, settings(), AuditContext("operator", "reviewer", "config-test")
+ )
+
+
+def reviewed_apply(
+ subject: ConfigurationService, change: ConfigurationChange
+) -> ConfigurationApply:
+ preview = subject.preview(change)
+ return ConfigurationApply(
+ **change.model_dump(), preview_digest=preview.preview_digest, confirm=True
+ )
+
+
+def test_preview_apply_rollback_and_real_serving_consumer(session: Session) -> None:
+ subject = service(session)
+ old_request = ServingService(
+ session, settings(), ManifestRegistry(settings().config_root), MemoryPayloadStore()
+ )
+ assert old_request._validate_inputs(EmbeddingInvokeRequest(input=["one", "two"])) == ["one", "two"]
+ change = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 1})
+ reviewed = reviewed_apply(subject, change)
+ assert configuration_state(session) == (0, {})
+ assert session.scalar(select(func.count()).select_from(AuditEvent)) == 0
+ result = subject.apply(reviewed)
+ assert result.status == "accepted_for_new_requests"
+ session.expire_all()
+ assert configuration_state(session) == (1, {"gateway_max_batch_size": 1})
+ request = EmbeddingInvokeRequest(input=["one", "two"])
+ assert old_request._validate_inputs(request) == ["one", "two"]
+ new_request = ServingService(
+ session, settings(), ManifestRegistry(settings().config_root), MemoryPayloadStore()
+ )
+ with pytest.raises(ServingError, match="configured maximum"):
+ new_request._validate_inputs(request)
+ event = session.scalar(select(AuditEvent).where(AuditEvent.action == "configuration.applied"))
+ assert (
+ event is not None and event.actor_id == "reviewer" and event.correlation_id == "config-test"
+ )
+ assert TOKEN not in str(event.details)
+ rollback = subject.rollback_change(0, 1)
+ subject.apply(reviewed_apply(subject, rollback))
+ assert configuration_state(session) == (2, {})
+ assert [row.revision for row in subject.history()] == [2, 1]
+
+
+def test_preview_reports_the_exact_resolved_source_across_override_changes(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setenv("MODELFORGE_GATEWAY_MAX_BATCH_SIZE", "16")
+ base_settings = Settings(_env_file=None, env="test")
+ subject = ConfigurationService(
+ session, base_settings, AuditContext("operator", "reviewer", "config-source-test")
+ )
+
+ add_override = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 12})
+ add_preview = subject.preview(add_override)
+ assert [
+ (item.source_before, item.source_after) for item in add_preview.differences
+ ] == [("process_environment", "operator_override")]
+ subject.apply(reviewed_apply(subject, add_override))
+
+ remove_override = ConfigurationChange(expected_revision=1, changes={"gateway_max_batch_size": None})
+ remove_preview = subject.preview(remove_override)
+ assert [
+ (item.source_before, item.source_after) for item in remove_preview.differences
+ ] == [("operator_override", "process_environment")]
+
+
+def test_preview_cannot_be_reused_or_changed_and_baseline_drift_conflicts(session: Session) -> None:
+ subject = service(session)
+ reviewed = reviewed_apply(
+ subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+ )
+ tampered = reviewed.model_copy(update={"changes": {"gateway_max_batch_size": 3}})
+ with pytest.raises(ConfigurationError, match="proposal changed"):
+ subject.apply(tampered)
+ changed_baseline = ConfigurationService(
+ session,
+ settings().model_copy(update={"gateway_request_timeout_seconds": 90}),
+ subject.context,
+ )
+ with pytest.raises(ConfigurationError, match="proposal changed"):
+ changed_baseline.apply(reviewed)
+ subject.apply(reviewed)
+ with pytest.raises(ConfigurationError, match="Configuration changed"):
+ subject.apply(reviewed)
+ assert len(subject.history()) == 1
+
+
+def test_audit_failure_rolls_back_pointer_and_history(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ subject = service(session)
+ reviewed = reviewed_apply(
+ subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+ )
+
+ def fail(*args: object, **kwargs: object) -> None:
+ raise RuntimeError("audit unavailable")
+
+ monkeypatch.setattr(AuditWriter, "write", fail)
+ with pytest.raises(RuntimeError, match="audit unavailable"):
+ subject.apply(reviewed)
+ assert configuration_state(session) == (0, {})
+ assert session.scalar(select(func.count()).select_from(ConfigurationRevision)) == 0
+
+
+@pytest.mark.parametrize(
+ "changes",
+ [
+ {"operator_api_key": 10},
+ {"allow_remote_code": 1},
+ {"scheduler_safety_reserve_bytes": 1},
+ {"gateway_max_batch_size": 0},
+ {"gateway_max_batch_size": 2.5},
+ {"gateway_queue_timeout_seconds": 50},
+ {"gateway_max_input_characters": 10**310},
+ {"gateway_max_batch_size": float("inf")},
+ ],
+)
+def test_unsafe_and_invalid_changes_fail_before_persistence(
+ session: Session, changes: dict[str, int | float]
+) -> None:
+ with pytest.raises(ConfigurationError):
+ service(session).preview(ConfigurationChange(expected_revision=0, changes=changes))
+ assert configuration_state(session) == (0, {})
+
+
+def test_export_import_compatibility_and_replacement(session: Session) -> None:
+ subject = service(session)
+ subject.apply(
+ reviewed_apply(
+ subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+ )
+ )
+ exported = subject.export()
+ assert TOKEN not in exported.model_dump_json()
+ assert "database_url" in exported.secret_slots
+ assert "postgresql" not in exported.model_dump_json()
+ exported.overrides = {"gateway_max_input_characters": 300}
+ change = subject.import_change(ConfigurationImport(expected_revision=1, document=exported))
+ assert change.changes == {"gateway_max_batch_size": None, "gateway_max_input_characters": 300}
+ subject.apply(reviewed_apply(subject, change))
+ assert configuration_state(session)[1] == exported.overrides
+ for replacement in (
+ {"format": "unknown"},
+ {"source_version": "2.0.0"},
+ {"secret_slots": []},
+ {"overrides": {"database_url": 1}},
+ ):
+ incompatible = exported.model_copy(update=replacement)
+ with pytest.raises(ConfigurationError):
+ subject.import_change(ConfigurationImport(expected_revision=2, document=incompatible))
+
+
+def test_api_authorization_validation_redaction_and_audited_apply(session: Session) -> None:
+ app.dependency_overrides[get_settings] = settings
+ app.dependency_overrides[get_session] = lambda: session
+ client = TestClient(app)
+ headers = {"X-ModelForge-Admin-Token": TOKEN}
+ change = {"expected_revision": 0, "changes": {"gateway_max_batch_size": 2}}
+ try:
+ assert client.post("/api/v1/configuration/preview", json=change).status_code == 401
+ for role in (PrincipalRole.VIEWER, PrincipalRole.OPERATOR):
+ app.dependency_overrides[authenticate_operator_principal] = lambda role=role: Principal(
+ "limited", role, "test"
+ )
+ assert (
+ client.post("/api/v1/configuration/apply", headers=headers, json={}).status_code
+ == 403
+ )
+ app.dependency_overrides.pop(authenticate_operator_principal)
+ preview = client.post("/api/v1/configuration/preview", headers=headers, json=change)
+ assert preview.status_code == 200
+ body = {**change, "preview_digest": preview.json()["preview_digest"], "confirm": True}
+ assert (
+ client.post("/api/v1/configuration/apply", headers=headers, json=body).status_code
+ == 200
+ )
+ catalog = client.get("/api/v1/configuration/settings", headers=headers).json()
+ entry = next(row for row in catalog["settings"] if row["key"] == "gateway_max_batch_size")
+ assert (catalog["revision"], entry["value"], entry["source"]) == (1, 2, "operator_override")
+ log_level = next(row for row in catalog["settings"] if row["key"] == "log_level")
+ assert log_level["source"] == "pydantic_default"
+ for malicious in (
+ {"operator_api_key": "secret-must-not-leak"},
+ {"secret-must-not-leak": "secret-must-not-leak"},
+ ):
+ rejected = client.post(
+ "/api/v1/configuration/preview",
+ headers=headers,
+ json={"expected_revision": 1, "changes": malicious},
+ )
+ assert rejected.status_code == 422
+ assert "secret-must-not-leak" not in rejected.text
+ finally:
+ app.dependency_overrides.clear()
+
+
+def test_configuration_state_survives_a_new_connection(tmp_path: Path) -> None:
+ database_url = f"sqlite:///{tmp_path / 'config.db'}"
+ first_engine = create_engine(database_url)
+ Base.metadata.create_all(first_engine)
+ with Session(first_engine) as first:
+ subject = service(first)
+ subject.apply(
+ reviewed_apply(
+ subject,
+ ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2}),
+ )
+ )
+ first_engine.dispose()
+ second_engine = create_engine(database_url)
+ with Session(second_engine) as second:
+ assert configuration_state(second) == (1, {"gateway_max_batch_size": 2})
+ second_engine.dispose()
diff --git a/backend/tests/test_configuration_postgres.py b/backend/tests/test_configuration_postgres.py
new file mode 100644
index 0000000..cd86dab
--- /dev/null
+++ b/backend/tests/test_configuration_postgres.py
@@ -0,0 +1,254 @@
+"""Real PostgreSQL migration, backup recovery, role and concurrent-write qualification."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import re
+import subprocess
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.configuration import ConfigurationApply, ConfigurationChange
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import AuditEvent, ConfigurationRevision, Project
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.configuration_changes import (
+ ConfigurationError,
+ ConfigurationService,
+ configuration_state,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+
+
+def _database_tool(tool: str, arguments: list[str], password: str | None, archive: Path) -> None:
+ container = os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+ environment = dict(os.environ)
+ if password:
+ environment["PGPASSWORD"] = password
+ command = [tool, *arguments]
+ if container:
+ assert re.fullmatch(r"[0-9a-f]{12,64}", container)
+ command = ["docker", "exec", "-i", "-e", "PGPASSWORD", container, *command]
+ with archive.open("wb" if tool == "pg_dump" else "rb") as handle:
+ result = subprocess.run( # noqa: S603 - bounded test tool and UUID database arguments
+ command,
+ env=environment,
+ timeout=120,
+ check=False,
+ stdin=handle if tool == "pg_restore" else subprocess.DEVNULL,
+ stdout=handle if tool == "pg_dump" else subprocess.DEVNULL,
+ stderr=subprocess.PIPE,
+ )
+ assert result.returncode == 0, f"{tool} failed during isolated configuration qualification"
+
+
+def test_postgres_configuration_upgrade_backup_recovery_and_concurrency(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+ if not admin_url:
+ pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ assert owner_url and runtime_url, (
+ "managed qualification requires separate owner and runtime roles"
+ )
+ database = f"test_config_{uuid.uuid4().hex}"
+ restored_database = f"test_config_restore_{uuid.uuid4().hex}"
+ admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+ owner = make_url(owner_url).set(database=database)
+ runtime = create_engine(make_url(runtime_url).set(database=database))
+ restored_runtime = create_engine(make_url(runtime_url).set(database=restored_database))
+ migration = Config()
+ migration.set_main_option(
+ "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+ )
+ migration.set_main_option(
+ "sqlalchemy.url", owner.render_as_string(hide_password=False).replace("%", "%%")
+ )
+ alembic = importlib.import_module("alembic.command")
+ with admin.connect() as connection:
+ connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
+ connection.execute(text(f'CREATE DATABASE "{restored_database}" OWNER modelforge'))
+ try:
+ alembic.upgrade(migration, "20260830_0024")
+ with Session(runtime) as session:
+ session.add(
+ Project(
+ key="existing-consumer",
+ name="Existing consumer",
+ description="Prior-release fixture",
+ )
+ )
+ AuditWriter(session, "operator", "prior-release").write(
+ "qualification.baseline", "project", None, {"source_schema": "20260830_0024"}
+ )
+ session.commit()
+ baseline = session.scalar(select(AuditEvent.event_hash))
+ archive = tmp_path / "pre-configuration.dump"
+ host = (
+ "127.0.0.1"
+ if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+ else (owner.host or "localhost")
+ )
+ args = [
+ "--host",
+ host,
+ "--port",
+ str(owner.port or 5432),
+ "--username",
+ owner.username or "modelforge",
+ "--dbname",
+ database,
+ ]
+ _database_tool("pg_dump", [*args, "--format=custom"], owner.password, archive)
+ assert archive.stat().st_size > 0
+ alembic.upgrade(migration, "head")
+ with Session(runtime) as session:
+ assert session.scalar(select(Project.key)) == "existing-consumer"
+ assert session.scalar(select(AuditEvent.event_hash)) == baseline
+ assert configuration_state(session) == (0, {})
+ # Mechanical round-trip is qualified before operator changes. Once changes exist,
+ # rollback uses the retained backup rather than destroying revision history.
+ alembic.downgrade(migration, "20260830_0024")
+ alembic.upgrade(migration, "head")
+ apply_preview_barrier = threading.Barrier(2)
+ synchronize_apply_preview = threading.local()
+ original_preview = ConfigurationService.preview
+
+ def preview_at_cas(
+ subject: ConfigurationService, change: ConfigurationChange
+ ):
+ preview = original_preview(subject, change)
+ if getattr(synchronize_apply_preview, "enabled", False):
+ synchronize_apply_preview.enabled = False
+ # Both real PostgreSQL transactions have validated the same revision and digest;
+ # release them together immediately before their conditional UPDATE.
+ apply_preview_barrier.wait(timeout=30)
+ return preview
+
+ monkeypatch.setattr(ConfigurationService, "preview", preview_at_cas)
+
+ def contender(expected_revision: int, value: int) -> str:
+ with Session(runtime) as session:
+ subject = ConfigurationService(
+ session,
+ Settings(_env_file=None, env="test"),
+ AuditContext("operator", f"contender-{value}", "configuration-race"),
+ )
+ change = ConfigurationChange(
+ expected_revision=expected_revision,
+ changes={"gateway_max_batch_size": value},
+ )
+ preview = subject.preview(change)
+ session.rollback() # Start both writes from independent fresh transactions.
+ synchronize_apply_preview.enabled = True
+ try:
+ subject.apply(
+ ConfigurationApply(
+ **change.model_dump(),
+ preview_digest=preview.preview_digest,
+ confirm=True,
+ )
+ )
+ return "applied"
+ except ConfigurationError as error:
+ assert error.code == "CONFIGURATION_CONFLICT"
+ return "conflict"
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert sorted(pool.map(lambda value: contender(0, value), [2, 3])) == [
+ "applied",
+ "conflict",
+ ]
+ with Session(runtime) as session:
+ assert configuration_state(session)[0] == 1
+ assert len(list(session.scalars(select(ConfigurationRevision)))) == 1
+ assert (
+ len(
+ list(
+ session.scalars(
+ select(AuditEvent).where(AuditEvent.action == "configuration.applied")
+ )
+ )
+ )
+ == 1
+ )
+ # Exercise the existing-row UPDATE path as well as the empty-singleton insert race.
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert sorted(pool.map(lambda value: contender(1, value), [4, 5])) == [
+ "applied",
+ "conflict",
+ ]
+ with Session(runtime) as session:
+ assert configuration_state(session)[0] == 2
+ assert [row.revision for row in session.scalars(
+ select(ConfigurationRevision).order_by(ConfigurationRevision.revision)
+ )] == [1, 2]
+ assert len(list(session.scalars(
+ select(AuditEvent).where(AuditEvent.action == "configuration.applied")
+ ))) == 2
+ with pytest.raises(RuntimeError, match="Configuration history exists"):
+ alembic.downgrade(migration, "20260830_0024")
+ with Session(runtime) as session:
+ assert configuration_state(session)[0] == 2
+ assert (
+ session.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
+ == TARGET_SCHEMA_REVISION
+ )
+ with runtime.connect() as connection:
+ for statement in (
+ "UPDATE configuration_revisions SET actor = 'tampered'",
+ "DELETE FROM configuration_revisions",
+ "TRUNCATE configuration_revisions",
+ ):
+ with pytest.raises(DBAPIError):
+ connection.execute(text(statement))
+ connection.rollback()
+ args[-1] = restored_database
+ _database_tool("pg_restore", [*args, "--exit-on-error"], owner.password, archive)
+ with Session(restored_runtime) as session:
+ assert session.scalar(select(Project.key)) == "existing-consumer"
+ assert session.scalar(select(AuditEvent.event_hash)) == baseline
+ assert (
+ session.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
+ == "20260830_0024"
+ )
+ migration.set_main_option(
+ "sqlalchemy.url",
+ owner.set(database=restored_database)
+ .render_as_string(hide_password=False)
+ .replace("%", "%%"),
+ )
+ alembic.upgrade(migration, "head")
+ with Session(restored_runtime) as session:
+ assert configuration_state(session) == (0, {})
+ with runtime.connect() as connection:
+ current = semantic_fingerprint(connection)
+ contract = fingerprint_compatibility(
+ current, current, source_schema_revision=TARGET_SCHEMA_REVISION
+ )
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(current, current, contract)
+ finally:
+ runtime.dispose()
+ restored_runtime.dispose()
+ with admin.connect() as connection:
+ for target in (database, restored_database):
+ connection.execute(text(f'DROP DATABASE IF EXISTS "{target}" WITH (FORCE)'))
+ admin.dispose()
diff --git a/backend/tests/test_configuration_recovery.py b/backend/tests/test_configuration_recovery.py
new file mode 100644
index 0000000..dd480a5
--- /dev/null
+++ b/backend/tests/test_configuration_recovery.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import copy
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.configuration import ConfigurationApply, ConfigurationChange
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.configuration_changes import ConfigurationService
+from modelforge_api.services.recovery_fingerprint import (
+ CONFIGURATION_TABLES,
+ SELECTED_COMPARISON_SCHEMA,
+ _without_creative,
+ _without_lab_execution,
+ _without_managed_identities,
+ _without_model_changes,
+ _without_model_updates,
+ _without_projects,
+ _without_secrets,
+ _without_selected_comparisons,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+
+
+def test_upgrade_introduces_only_empty_configuration_and_current_restore_preserves_every_revision() -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ AuditWriter(session, "operator", "baseline").write("test.baseline", "configuration", None, {})
+ session.commit()
+ fresh = semantic_fingerprint(session.connection())
+ previous = _without_lab_execution(_without_projects(
+ _without_managed_identities(
+ _without_secrets(
+ _without_model_updates(
+ _without_selected_comparisons(
+ _without_creative(_without_model_changes(copy.deepcopy(fresh)))
+ )
+ )
+ )
+ ),
+ legacy=True,
+ ))
+ previous["tables"] = {key: value for key, value in previous["tables"].items() if key not in CONFIGURATION_TABLES}
+ previous["physical_tables"] = [name for name in previous["physical_tables"] if name not in CONFIGURATION_TABLES]
+ upgrade = fingerprint_compatibility(previous, fresh, source_schema_revision="20260830_0024")
+ assert upgrade.compatible
+ assert audit_fingerprint_differences_are_compatible(previous, fresh, upgrade)
+ subject = ConfigurationService(session, Settings(_env_file=None), AuditContext("operator", "config", "restore-test"))
+ change = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
+ preview = subject.preview(change)
+ subject.apply(ConfigurationApply(**change.model_dump(), preview_digest=preview.preview_digest, confirm=True))
+ configured = semantic_fingerprint(session.connection())
+ configured_0033 = _without_lab_execution(_without_model_changes(configured))
+ assert not fingerprint_compatibility(previous, configured, source_schema_revision="20260830_0024").compatible
+ current = fingerprint_compatibility(
+ configured_0033,
+ configured_0033,
+ source_schema_revision=SELECTED_COMPARISON_SCHEMA,
+ )
+ assert current.compatible
+ assert audit_fingerprint_differences_are_compatible(configured_0033, configured_0033, current)
+ for table in CONFIGURATION_TABLES:
+ altered = copy.deepcopy(configured_0033)
+ altered["tables"][table]["digest"] = "0" * 64
+ assert not audit_fingerprint_differences_are_compatible(configured_0033, altered, current)
+ missing = copy.deepcopy(configured_0033)
+ missing["tables"].pop(table)
+ assert not fingerprint_compatibility(
+ configured_0033,
+ missing,
+ source_schema_revision=SELECTED_COMPARISON_SCHEMA,
+ ).compatible
+ engine.dispose()
diff --git a/backend/tests/test_deployed_recovery_convergence.py b/backend/tests/test_deployed_recovery_convergence.py
new file mode 100644
index 0000000..43cd63b
--- /dev/null
+++ b/backend/tests/test_deployed_recovery_convergence.py
@@ -0,0 +1,192 @@
+"""Fresh deployed-source proofs retain history across the explicit 0033 convergence."""
+from __future__ import annotations
+
+import copy
+import importlib
+import os
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, ModelRevision, ProjectBinding
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+ CREATIVE_RESTORE_ERROR,
+ CREATIVE_TABLES,
+ _actual_table,
+ _without_model_changes,
+ _without_portfolio,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ semantic_fingerprint as _current_semantic_fingerprint,
+)
+from modelforge_api.services.recovery_postgres import PostgresTarget
+from modelforge_api.settings import Settings
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+from tests.test_deployed_schema_convergence import _migration
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_selected_comparison_recovery import SQLiteRecoveryTarget
+from tests.test_serving_generation_recovery import (
+ _remove_lab_serving_job_link,
+ legacy_generation_fixture,
+)
+
+__all__ = ["identity_database"]
+TARGET = PostgresTarget("localhost", 5432, "fixture", "fixture", None)
+
+
+def semantic_fingerprint(connection):
+ return _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
+
+
+def _populate(session: Session, state: str = "granted"):
+ request, client_id = _valid_selection(session)
+ connection = session.connection()
+ project_id = session.scalar(select(ProjectBinding.project_id).where(ProjectBinding.id == request.project_binding_id))
+ tables = {name: _actual_table(connection, name) for name in (*CREATIVE_TABLES, "model_update_watches")}
+ revision = session.execute(select(ModelRevision.id, ModelRevision.model_id, ModelRevision.resolved_commit_sha).order_by(ModelRevision.id).limit(1)).one()
+ from uuid import UUID
+ project_id = UUID(str(project_id))
+ revision_id, model_id = UUID(str(revision.id)), UUID(str(revision.model_id))
+ allocation_id, watch_id = uuid4(), uuid4()
+ connection.execute(tables["external_creative_allocations"].insert().values(id=allocation_id,
+ service_client_id=client_id, project_id=project_id, request_id="preserve", idempotency_key="preserve",
+ request_sha256="a" * 64, workload="audio", resource_ref="audio.fixture", provider_resource_id="fixture",
+ required_vram_bytes=1024, priority="background", ttl_seconds=600, state=state,
+ decision_code="PRIOR_DECISION", decision_evidence={"preserve": "exact"}, required_actions=[],
+ acquired_at=datetime.now(UTC)))
+ connection.execute(tables["external_creative_resources"].insert().values(id=uuid4(), project_id=project_id,
+ resource_ref="audio.fixture", workload="audio", provider_resource_id="fixture", approved=False,
+ configured_vram_bytes=1024))
+ connection.execute(tables["external_creative_action_attestations"].insert().values(id=uuid4(), allocation_id=allocation_id,
+ action="release_idle_comfy", before_vram_bytes=1024, after_vram_bytes=0,
+ observed_at=datetime.now(UTC), validated=True))
+ connection.execute(tables["external_creative_profile_observations"].insert().values(id=uuid4(), allocation_id=allocation_id,
+ resource_ref="audio.fixture", provider_resource_id="fixture", workload="audio", observed_peak_vram_bytes=512,
+ reserved_vram_bytes=1024, duration_ms=1, outcome="succeeded"))
+ connection.execute(tables["model_update_watches"].insert().values(id=watch_id, model_id=model_id,
+ baseline_revision_id=revision_id, baseline_commit_sha=revision.resolved_commit_sha, upstream_ref="main",
+ check_token=uuid4(), check_until=datetime.now(UTC)))
+ session.commit()
+ return allocation_id, watch_id
+
+
+@pytest.fixture
+def legacy():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with engine.begin() as connection:
+ # `create_all` uses today's metadata; this fixture represents schema 0032, before the
+ # notification outbox and LAB evidence existed, so remove those future physical artifacts.
+ connection.execute(text('DROP TABLE "alert_notification_deliveries"'))
+ connection.execute(text("DROP TABLE selected_model_comparisons"))
+ recreate_0038_benchmark_runs(connection)
+ _remove_lab_serving_job_link(connection)
+ connection.execute(text("ALTER TABLE model_update_watches DROP COLUMN enabled"))
+ connection.execute(text("ALTER TABLE model_update_watches DROP COLUMN version"))
+ connection.execute(text("CREATE TABLE alembic_version(version_num VARCHAR(32))"))
+ connection.execute(text("INSERT INTO alembic_version VALUES ('20260909_0032')"))
+ with Session(engine) as session:
+ _populate(session)
+ source = semantic_fingerprint(session.connection())
+ module = _migration()
+ module.op = Operations(MigrationContext.configure(session.connection()))
+ module.upgrade()
+ session.execute(text("UPDATE alembic_version SET version_num='20260909_0033'"))
+ service = RecoveryService(session, Settings(_env_file=None))
+ service.engine = SQLiteRecoveryTarget(session) # type: ignore[assignment]
+ yield session, source, service
+ engine.dispose()
+
+
+def test_populated_source_projection_and_restore_preserve_all_other_identity(legacy):
+ session, source, service = legacy
+ assert "hub_controls_projection" in source["tables"]["model_update_watches"]
+ assert service._creative_allocation_residuals(TARGET) == 1
+ assert service._reconcile_creative_allocations(TARGET) == 1
+ session.execute(text("UPDATE model_update_watches SET check_token=NULL,check_until=NULL"))
+ restored = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+ row = session.execute(text("SELECT state,decision_code,uncertain_at,decision_evidence FROM external_creative_allocations")).one()
+ assert row.state == "uncertain" and row.decision_code == CREATIVE_RESTORE_ERROR
+ assert row.uncertain_at is None
+ assert service._creative_allocation_residuals(TARGET) == 0
+ assert service._reconcile_creative_allocations(TARGET) == 0
+ for name in CREATIVE_TABLES - {"external_creative_allocations"}:
+ assert source["tables"][name] == restored["tables"][name]
+
+
+def test_legacy_opaque_populated_watch_digest_is_not_projectable(legacy):
+ session, source, _ = legacy
+ opaque = copy.deepcopy(source)
+ del opaque["tables"]["model_update_watches"]["hub_controls_projection"]
+ contract = fingerprint_compatibility(opaque, semantic_fingerprint(session.connection()), source_schema_revision="20260909_0032")
+ assert not contract.compatible and "opaque" in contract.reason
+
+
+@pytest.mark.parametrize("table", sorted(CREATIVE_TABLES | {"model_update_watches"}))
+def test_populated_state_tampering_and_missing_tables_fail_closed(legacy, table):
+ session, source, service = legacy
+ service._reconcile_creative_allocations(TARGET)
+ session.execute(text("UPDATE model_update_watches SET check_token=NULL,check_until=NULL"))
+ restored = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+ changed = copy.deepcopy(restored)
+ changed["tables"][table]["digest"] = "0" * 64
+ assert not audit_fingerprint_differences_are_compatible(source, changed, contract)
+ del changed["tables"][table]
+ assert not fingerprint_compatibility(source, changed, source_schema_revision="20260909_0032").compatible
+
+
+def test_unreconciled_grant_and_unknown_physical_table_block_proof(legacy):
+ session, source, _ = legacy
+ restored = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+ assert not audit_fingerprint_differences_are_compatible(source, restored, contract)
+ restored["physical_tables"] = sorted([*restored["physical_tables"], "unknown_private_state"])
+ assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032").compatible
+
+
+def test_unknown_revision_and_forged_current_watch_projection_fail_closed(legacy):
+ session, source, _ = legacy
+ restored = semantic_fingerprint(session.connection())
+ assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0031x").compatible
+ restored["tables"]["model_update_watches"]["hub_controls_projection"] = source["tables"]["model_update_watches"]["hub_controls_projection"]
+ assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032").compatible
+
+
+def test_actual_postgres_populated_source_dump_upgrade_restore_projection(identity_database: IdentityDatabase, tmp_path):
+ database = identity_database
+ alembic = importlib.import_module("alembic.command")
+ alembic.upgrade(database.migration(), "20260909_0032")
+ with legacy_generation_fixture(database.owner), Session(database.runtime) as session:
+ _populate(session, "queued")
+ with Session(database.runtime) as session:
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "populated-deployed0032.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ alembic.upgrade(database.migration(restored=True), "20260910_0034")
+ with Session(database.restored_runtime) as session:
+ from tests.test_managed_identities_postgres import ManagedPostgresTools
+ recovery = RecoveryService(session, Settings(_env_file=None))
+ recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+ target = PostgresTarget(database.restored_owner.host or "localhost", database.restored_owner.port or 5432,
+ database.restored_owner.database or "", database.restored_owner.username or "", database.restored_owner.password)
+ assert recovery._reconcile_creative_allocations(target) == 1
+ recovery.engine.scalar(target, "UPDATE model_update_watches SET check_token=NULL,check_until=NULL")
+ restored = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+ assert recovery._creative_allocation_residuals(target) == 0
diff --git a/backend/tests/test_deployed_schema_convergence.py b/backend/tests/test_deployed_schema_convergence.py
new file mode 100644
index 0000000..5265342
--- /dev/null
+++ b/backend/tests/test_deployed_schema_convergence.py
@@ -0,0 +1,255 @@
+"""Preserve deployed migration meaning and prove additive convergence on isolated databases."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import importlib.util
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from alembic.script import ScriptDirectory
+from sqlalchemy import Column, Integer, MetaData, Table, create_engine, inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import AuditEvent, ModelRevision, ProjectBinding
+from modelforge_api.services.recovery_fingerprint import _actual_table
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_serving_generation_recovery import legacy_generation_fixture
+
+__all__ = ["identity_database"]
+ROOT = Path(__file__).resolve().parents[1]
+VERSIONS = ROOT / "alembic" / "versions"
+FROZEN = {
+ "20260909_0029_external_creative_coordination.py": "ac41923b2bd7a8729abeaae5c872afa3186d4b188fe002015d77c9bc23b9c5db",
+ "20260909_0030_model_update_inbox.py": "265379bd5e1c2cee78a9c4a4e6004ec802a6dbbc57e5f30c422e8eb91fea99cf",
+ "20260909_0031_external_creative_governance.py": "397d4b288bb7564eb50da5950e1478db237f2a91d801de3d3786112b638f2c55",
+ "20260909_0032_audio_creative_workloads.py": "ede826fa5ada798e5730ef99b0a8380d1831d3ed11a1baccda9ed5bc0e60d362",
+}
+HISTORY_TABLES = (
+ "model_update_watches",
+ "model_update_notices",
+ "external_creative_allocations",
+ "external_creative_resources",
+ "external_creative_action_attestations",
+ "external_creative_profile_observations",
+)
+
+
+def _migration():
+ specification = importlib.util.spec_from_file_location(
+ "convergence", VERSIONS / "20260909_0033_deployed_hub_convergence.py"
+ )
+ assert specification is not None and specification.loader is not None
+ module = importlib.util.module_from_spec(specification)
+ specification.loader.exec_module(module)
+ return module
+
+
+def test_deployed_migration_content_remains_frozen_and_graph_is_linear() -> None:
+ for name, digest in FROZEN.items():
+ # Normalize checkout line endings only; the deployed repository blobs are LF.
+ assert (
+ hashlib.sha256((VERSIONS / name).read_bytes().replace(b"\r\n", b"\n")).hexdigest()
+ == digest
+ )
+ config = Config()
+ config.set_main_option("script_location", str(ROOT / "alembic"))
+ graph = ScriptDirectory.from_config(config)
+ assert graph.get_heads() == ["20260924_0039"]
+ assert Path(graph.get_revision("20260909_0029").path).name in FROZEN
+ assert Path(graph.get_revision("20260909_0030").path).name in FROZEN
+ assert graph.get_revision("20260909_0033").down_revision == "20260909_0032"
+ assert graph.get_revision("20260923_0036").down_revision == "20260911_0035"
+ assert graph.get_revision("20260923_0037").down_revision == "20260923_0036"
+ assert graph.get_revision("20260924_0038").down_revision == "20260923_0037"
+ assert graph.get_revision("20260924_0039").down_revision == "20260924_0038"
+ assert not (VERSIONS / "20260909_0029_model_update_inbox.py").exists()
+ assert not (VERSIONS / "20260909_0030_selected_model_comparisons.py").exists()
+
+
+@pytest.mark.parametrize("populated", HISTORY_TABLES)
+def test_new_downgrade_refuses_populated_deployed_history(populated: str) -> None:
+ engine = create_engine("sqlite://")
+ with engine.begin() as connection:
+ for name in HISTORY_TABLES:
+ Table(name, MetaData(), Column("id", Integer)).create(connection)
+ connection.execute(text("ALTER TABLE model_update_watches ADD COLUMN upstream_ref TEXT"))
+ module = _migration()
+ module.op = Operations(MigrationContext.configure(connection))
+ module.upgrade()
+ table = Table(populated, MetaData(), autoload_with=connection)
+ connection.execute(table.insert().values(id=1))
+ with pytest.raises(RuntimeError, match="history exists"):
+ module.downgrade()
+ assert inspect(connection).has_table("selected_model_comparisons")
+ assert {
+ column["name"] for column in inspect(connection).get_columns("model_update_watches")
+ } >= {"enabled", "version"}
+ engine.dispose()
+
+
+def test_populated_0032_upgrade_preserves_creative_watch_audit_history_and_restricts_runtime(
+ identity_database: IdentityDatabase,
+) -> None:
+ database = identity_database
+ alembic = importlib.import_module("alembic.command")
+ alembic.upgrade(database.migration(), "20260909_0032")
+ with legacy_generation_fixture(database.owner), Session(database.runtime) as session:
+ request, client_id = _valid_selection(session)
+ connection = session.connection()
+ tables = {name: _actual_table(connection, name) for name in HISTORY_TABLES}
+ project_id = connection.scalar(
+ select(ProjectBinding.project_id).where(ProjectBinding.id == request.project_binding_id),
+ )
+ revision = connection.execute(
+ select(ModelRevision.id, ModelRevision.model_id, ModelRevision.resolved_commit_sha).order_by(ModelRevision.id).limit(1)
+ ).one()
+ watch_id, allocation_id, resource_id = uuid4(), uuid4(), uuid4()
+ connection.execute(
+ tables["model_update_watches"]
+ .insert()
+ .values(
+ id=watch_id,
+ model_id=revision.model_id,
+ baseline_revision_id=revision.id,
+ baseline_commit_sha=revision.resolved_commit_sha,
+ upstream_ref="main",
+ )
+ )
+ connection.execute(
+ tables["model_update_notices"]
+ .insert()
+ .values(
+ id=uuid4(),
+ watch_id=watch_id,
+ model_id=revision.model_id,
+ baseline_commit_sha=revision.resolved_commit_sha,
+ candidate_revision_id=revision.id,
+ candidate_commit_sha=revision.resolved_commit_sha,
+ status="dismissed",
+ )
+ )
+ connection.execute(
+ tables["external_creative_allocations"]
+ .insert()
+ .values(
+ id=allocation_id,
+ service_client_id=client_id,
+ project_id=project_id,
+ request_id="preserved",
+ idempotency_key="preserved",
+ request_sha256="a" * 64,
+ workload="audio",
+ resource_ref="fixture.audio@1",
+ provider_resource_id="fixture",
+ required_vram_bytes=1024,
+ priority="background",
+ ttl_seconds=600,
+ state="released",
+ decision_code="OFFLINE_FIXTURE",
+ decision_evidence={"preserve": True},
+ required_actions=[],
+ )
+ )
+ connection.execute(
+ tables["external_creative_resources"]
+ .insert()
+ .values(
+ id=resource_id,
+ project_id=project_id,
+ resource_ref="fixture.audio@1",
+ workload="audio",
+ provider_resource_id="fixture",
+ approved=False,
+ configured_vram_bytes=1024,
+ )
+ )
+ connection.execute(
+ tables["external_creative_action_attestations"]
+ .insert()
+ .values(
+ id=uuid4(),
+ allocation_id=allocation_id,
+ action="release_idle_comfy",
+ before_vram_bytes=1024,
+ after_vram_bytes=0,
+ observed_at=datetime.now(UTC),
+ validated=True,
+ )
+ )
+ connection.execute(
+ tables["external_creative_profile_observations"]
+ .insert()
+ .values(
+ id=uuid4(),
+ allocation_id=allocation_id,
+ resource_ref="fixture.audio@1",
+ provider_resource_id="fixture",
+ workload="audio",
+ observed_peak_vram_bytes=512,
+ reserved_vram_bytes=1024,
+ duration_ms=1,
+ outcome="succeeded",
+ )
+ )
+ session.commit()
+ before = {
+ name: session.execute(select(table).order_by(table.c.id)).mappings().all()
+ for name, table in tables.items()
+ }
+ audit_before = session.execute(
+ select(AuditEvent.id, AuditEvent.canonical_payload, AuditEvent.event_hash).order_by(AuditEvent.sequence)
+ ).all()
+ assert "enabled" not in {
+ column["name"]
+ for column in inspect(session.connection()).get_columns("model_update_watches")
+ }
+ alembic.upgrade(database.migration(), "20260909_0033")
+ with database.runtime.connect() as connection:
+ for name, table in tables.items():
+ assert (
+ connection.execute(select(table).order_by(table.c.id)).mappings().all()
+ == before[name]
+ )
+ assert (
+ connection.execute(
+ select(AuditEvent.id, AuditEvent.canonical_payload, AuditEvent.event_hash).order_by(AuditEvent.sequence)
+ ).all()
+ == audit_before
+ )
+ assert connection.execute(
+ text("SELECT enabled, version FROM model_update_watches WHERE id=:id"), {"id": watch_id}
+ ).one() == (True, 1)
+ assert connection.scalar(text("SELECT count(*) FROM selected_model_comparisons")) == 0
+ for table_name in (
+ "model_update_watches",
+ "model_update_notices",
+ "selected_model_comparisons",
+ ):
+ for privilege in ("SELECT", "INSERT", "UPDATE"):
+ assert connection.scalar(
+ text("SELECT has_table_privilege(current_user, :table, :privilege)"),
+ {"table": table_name, "privilege": privilege},
+ )
+ for privilege in ("DELETE", "TRUNCATE"):
+ assert not connection.scalar(
+ text("SELECT has_table_privilege(current_user, :table, :privilege)"),
+ {"table": table_name, "privilege": privilege},
+ )
+ with pytest.raises(DBAPIError):
+ connection.execute(
+ text("DELETE FROM model_update_watches WHERE id=:id"), {"id": watch_id}
+ )
+ connection.rollback()
+ with pytest.raises(RuntimeError, match="history exists"):
+ alembic.downgrade(database.migration(), "20260909_0032")
+ with database.runtime.connect() as connection:
+ assert connection.scalar(text("SELECT version_num FROM alembic_version")) == "20260909_0033"
diff --git a/backend/tests/test_dispatch_context.py b/backend/tests/test_dispatch_context.py
new file mode 100644
index 0000000..e3fc6e5
--- /dev/null
+++ b/backend/tests/test_dispatch_context.py
@@ -0,0 +1,472 @@
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.lab import LabRerankingInput
+from modelforge_api.domain.serving import (
+ CapabilityExperimentCreate,
+ EmbeddingInvokeRequest,
+ RerankingInvokeRequest,
+ ServiceClientCreate,
+)
+from modelforge_api.persistence.models import (
+ Base,
+ DeploymentCandidate,
+ GatewayRequest,
+ SchedulerPolicyRevision,
+ ServiceClient,
+ ServingJob,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_lab_preflight import setup_lab
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+@pytest.mark.parametrize("priority", ["production", "interactive", "background", "benchmark"])
+def test_persisted_service_priority_and_lease_owner_are_preserved(priority: str) -> None:
+ client_id = uuid4()
+ context = DispatchContext.for_service_client(client_id, "project-client", priority)
+ assert context.service_client_id == client_id
+ assert context.workload_priority == priority
+ assert context.lease_owner == "project-client"
+ assert context.provenance()["actor_id"] == str(client_id)
+ assert context.operation_id is None
+ with pytest.raises(ValidationError, match="frozen"):
+ context.workload_priority = "lab"
+
+
+@pytest.mark.parametrize("priority", ["lab", "maintenance", "admin", "", "PRODUCTION"])
+def test_service_dispatch_rejects_unconfigured_priority_domain(priority: str) -> None:
+ with pytest.raises(ValidationError):
+ DispatchContext.for_service_client(uuid4(), "project-client", priority)
+
+
+def test_operator_identity_is_separate_and_cannot_elevate_priority() -> None:
+ context = DispatchContext.for_operator("jens", uuid4())
+ assert context.service_client_id is None
+ assert context.workload_priority == "lab"
+ assert context.lease_owner == "operator:jens"
+ for changes in (
+ {"workload_priority": "production"}, {"service_client_id": uuid4()},
+ {"operation_id": None}, {"actor_kind": "service_client"},
+ {"lease_owner": "a-different-owner"}, {"token": "not-accepted"},
+ ):
+ with pytest.raises(ValidationError):
+ DispatchContext.model_validate({**context.model_dump(), **changes})
+ assert set(context.provenance()) == {
+ "actor_kind", "actor_id", "service_client_id", "operation_id", "workload_priority",
+ }
+
+
+def test_operator_scheduler_admission_requires_no_fabricated_service_client(session: Session) -> None:
+ service, candidate, _ = setup_m5(session)
+ _, response = approve_and_promote(service, candidate.id)
+ deployment = service.repo.deployment(response.id)
+ assert deployment is not None
+ context = DispatchContext.for_operator("operator-test", uuid4())
+ request = GatewayRequest(request_id=uuid4(), service_client_id=None,
+ capability_key="rag.embedding", capability_version=1,
+ capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
+ status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
+ session.add(request)
+ session.commit()
+ lease = service._acquire_request_lease(deployment, request, context)
+ assert lease.priority == "lab"
+ assert lease.owner == "operator:operator-test"
+ assert lease.request_id == request.request_id
+ assert request.decision_evidence["dispatch_context"] == context.provenance()
+ assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
+
+
+def test_operator_context_cannot_bypass_paused_lab_policy(session: Session) -> None:
+ service, candidate, _ = setup_m5(session)
+ _, response = approve_and_promote(service, candidate.id)
+ deployment = service.repo.deployment(response.id)
+ assert deployment is not None
+ session.add(SchedulerPolicyRevision(revision="lab-paused-test", active=True,
+ configuration={"lab_paused": True}))
+ context = DispatchContext.for_operator("operator-test", uuid4())
+ request = GatewayRequest(request_id=uuid4(), service_client_id=None,
+ capability_key="rag.embedding", capability_version=1,
+ capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
+ status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
+ session.add(request)
+ session.commit()
+ with pytest.raises(ServingError) as raised:
+ service._acquire_request_lease(deployment, request, context)
+ assert raised.value.code == "POLICY_BLOCKED"
+ assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
+
+
+@pytest.mark.parametrize("mismatch", ["client", "priority", "deployment"])
+def test_context_mismatch_fails_before_scheduler_mutation(session: Session, mismatch: str) -> None:
+ service, candidate, _ = setup_m5(session)
+ _, response = approve_and_promote(service, candidate.id)
+ deployment = service.repo.deployment(response.id)
+ assert deployment is not None
+ context = DispatchContext.for_operator("operator-test", uuid4())
+ request = GatewayRequest(request_id=uuid4(), service_client_id=uuid4() if mismatch == "client" else None,
+ capability_key="rag.embedding", capability_version=1,
+ capability_deployment_id=uuid4() if mismatch == "deployment" else deployment.id,
+ compute_node_id=deployment.compute_node_id, status="queued",
+ priority="production" if mismatch == "priority" else "lab", input_sha256="a" * 64, input_count=1)
+ with pytest.raises(ServingError) as raised:
+ service._acquire_request_lease(deployment, request, context)
+ assert raised.value.code == "DISPATCH_CONTEXT_MISMATCH"
+ assert request.status == "queued"
+ assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
+
+
+def operator_fixture(session: Session):
+ preflight, request = setup_lab(session)
+ service = InlineWorkerServingService(session, preflight.settings,
+ ManifestRegistry(), MemoryPayloadStore())
+ experiment = service.create_experiment(request.target.deployment_candidate_id,
+ CapabilityExperimentCreate(route_key="operator-selected", purpose="Bounded exact-target unit test",
+ execution_approval_id=request.target.execution_approval_id))
+ return service, request, experiment
+
+
+def test_operator_embedding_executes_exact_selected_experiment_without_client(session: Session) -> None:
+ service, request, selected = operator_fixture(session)
+ service.synthetic_worker_metrics = {
+ "gpu_uuid": service.repo.accelerator(selected.deployment.accelerator_id).device_uuid,
+ "used_vram_bytes": 4_096,
+ "process_used_vram_bytes": 2_048,
+ "free_vram_bytes": 8_192,
+ "gpu_utilization_percent": 32,
+ "temperature_c": 63,
+ "power_draw_w": 101.25,
+ "private_metric_canary": "must-not-be-persisted",
+ }
+ other = service.create_experiment(request.target.deployment_candidate_id,
+ CapabilityExperimentCreate(route_key="operator-other", purpose="Unselected comparison fixture",
+ execution_approval_id=request.target.execution_approval_id))
+ service.authenticated_credential_id = uuid4() # stale service-instance state must not cross domains
+ operation_id = uuid4()
+ context = DispatchContext.for_operator("operator-test", operation_id)
+ result = service.invoke_operator_lab(request.input, context,
+ deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+ record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+ assert record is not None
+ assert record.capability_deployment_id == selected.capability_deployment_id != other.capability_deployment_id
+ assert record.service_client_id is None and record.priority == "lab"
+ assert record.decision_evidence["service_credential_id"] is None
+ assert record.decision_evidence["dispatch_context"] == context.provenance()
+ assert record.decision_evidence["operator_route_id"] == str(selected.id)
+ assert result.embedding_space_id == selected.deployment.embedding_space.id
+ evidence = record.decision_evidence["lab_run_evidence"]
+ candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
+ profile = service.repo.profile(request.target.runtime_profile_id)
+ deployment = service.repo.deployment(selected.capability_deployment_id)
+ assert candidate is not None and profile is not None and deployment is not None
+ accelerator = service.repo.accelerator(deployment.accelerator_id)
+ assert accelerator is not None
+ assert evidence["schema_version"] == 1
+ assert evidence["status"] == "completed"
+ assert evidence["capability"] == "rag.embedding@1"
+ assert evidence["operation_id"] == str(operation_id)
+ assert evidence["input"] == {
+ "sha256": record.input_sha256,
+ "item_count": 1,
+ "content_logged": False,
+ }
+ assert evidence["target"] == {
+ "deployment_id": str(selected.capability_deployment_id),
+ "candidate_id": str(candidate.id),
+ "experiment_route_id": str(selected.id),
+ "capability_contract_id": str(deployment.capability_contract_id),
+ "contract_version": 1,
+ "artifact_set_id": str(deployment.artifact_set_id),
+ "artifact_sha256": profile.artifact_sha256,
+ "runtime_profile_id": str(profile.id),
+ "runtime_profile_fingerprint": profile.fingerprint,
+ "compatibility_assessment_id": str(candidate.compatibility_assessment_id),
+ "runtime_probe_id": str(candidate.runtime_probe_id),
+ "compute_node_id": str(deployment.compute_node_id),
+ "accelerator_id": str(deployment.accelerator_id),
+ "accelerator_uuid": accelerator.device_uuid,
+ "execution_approval_id": str(request.target.execution_approval_id),
+ "embedding_space_id": str(deployment.embedding_space_id),
+ "deployment_fingerprint": deployment.config_fingerprint,
+ }
+ expected_output_hash = hashlib.sha256(
+ json.dumps(result.data, sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+ assert evidence["output"] == {
+ "type": "float_vector",
+ "dimension": result.dimension,
+ "item_count": len(result.data),
+ "embedding_space_id": str(result.embedding_space_id),
+ "sha256": expected_output_hash,
+ "content_logged": False,
+ }
+ serving_job = session.scalar(
+ select(ServingJob).where(
+ ServingJob.gateway_request_id == record.id,
+ ServingJob.operation == "invoke",
+ )
+ )
+ expected_observations = {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": "complete",
+ "device_matches_target": True,
+ "metrics": {
+ "gpu_uuid": accelerator.device_uuid,
+ "used_vram_bytes": 4_096,
+ "process_used_vram_bytes": 2_048,
+ "free_vram_bytes": 8_192,
+ "gpu_utilization_percent": 32,
+ "temperature_c": 63,
+ "power_draw_w": 101.25,
+ },
+ }
+ assert evidence["worker_observations"] == expected_observations
+ assert serving_job is not None
+ assert serving_job.result_summary["resource_observation"] == expected_observations
+ assert "vectors" not in evidence["output"] and "data" not in evidence["output"]
+ assert "private input content" not in json.dumps(evidence)
+ assert "must-not-be-persisted" not in json.dumps(evidence)
+ assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
+
+
+def test_operator_lab_without_worker_resource_metrics_does_not_invent_gpu_evidence(
+ session: Session,
+) -> None:
+ service, request, selected = operator_fixture(session)
+ result = service.invoke_operator_lab(
+ request.input,
+ DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+ record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+ assert record is not None
+ observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
+ assert observations == {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": "unavailable",
+ "device_matches_target": None,
+ "metrics": {},
+ }
+
+
+def test_operator_lab_rejects_unbounded_worker_resource_metrics(session: Session) -> None:
+ service, request, selected = operator_fixture(session)
+ deployment = service.repo.deployment(selected.capability_deployment_id)
+ assert deployment is not None
+ accelerator = service.repo.accelerator(deployment.accelerator_id)
+ assert accelerator is not None
+ service.synthetic_worker_metrics = {
+ "gpu_uuid": accelerator.device_uuid,
+ "used_vram_bytes": True,
+ "process_used_vram_bytes": 2**100,
+ "free_vram_bytes": 8_192,
+ "gpu_utilization_percent": 101,
+ "temperature_c": 10**1000,
+ "power_draw_w": 10**1000,
+ "private_metric_canary": "must-not-be-persisted",
+ }
+ result = service.invoke_operator_lab(
+ request.input,
+ DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+ record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+ assert record is not None
+ observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
+ assert observations == {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": "partial",
+ "device_matches_target": True,
+ "metrics": {"gpu_uuid": accelerator.device_uuid, "free_vram_bytes": 8_192},
+ }
+
+
+def test_operator_lab_scheduler_failure_is_durable_and_redacted(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ service, request, selected = operator_fixture(session)
+
+ def reject_admission(*_args, **_kwargs):
+ raise ServingError(429, "LAB_QUEUE_REJECTED", "private scheduler detail")
+
+ monkeypatch.setattr(service, "_acquire_request_lease", reject_admission)
+ with pytest.raises(ServingError, match="private scheduler detail"):
+ service.invoke_operator_lab(
+ request.input,
+ DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+
+ record = session.scalar(select(GatewayRequest).where(GatewayRequest.priority == "lab"))
+ assert record is not None and record.status == "failed"
+ assert record.failure_code == "LAB_QUEUE_REJECTED"
+ evidence = record.decision_evidence["lab_run_evidence"]
+ assert evidence["status"] == "failed"
+ assert evidence["failure_code"] == "LAB_QUEUE_REJECTED"
+ assert "output" not in evidence and "worker_observations" not in evidence
+ assert "private input content" not in json.dumps(evidence)
+ assert "private scheduler detail" not in json.dumps(evidence)
+
+
+@pytest.mark.parametrize("bad_target", ["missing_deployment", "missing_route", "inactive_route", "stable"])
+def test_operator_embedding_rejects_non_exact_or_non_lab_target(session: Session, bad_target: str) -> None:
+ service, request, selected = operator_fixture(session)
+ deployment_id, route_id = selected.capability_deployment_id, selected.id
+ if bad_target == "missing_deployment":
+ deployment_id = uuid4()
+ elif bad_target == "missing_route":
+ route_id = uuid4()
+ elif bad_target == "inactive_route":
+ service.deactivate_experiment(route_id)
+ else:
+ _, stable = approve_and_promote(service, request.target.deployment_candidate_id)
+ deployment_id = stable.id
+ with pytest.raises(ServingError) as raised:
+ service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=deployment_id, experiment_route_id=route_id)
+ assert raised.value.code == "LAB_TARGET_NOT_ROUTABLE"
+ assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+def test_operator_entry_rejects_service_context_and_unimplemented_family(session: Session) -> None:
+ service, request, selected = operator_fixture(session)
+ with pytest.raises(ServingError) as denied:
+ service.invoke_operator_lab(request.input, DispatchContext.for_service_client(uuid4(), "test-client", "production"),
+ deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+ assert denied.value.code == "OPERATOR_CONTEXT_REQUIRED"
+ reranking = LabRerankingInput(capability="rag.reranking", payload=RerankingInvokeRequest(
+ query="test", documents=[{"id": "a", "text": "test"}], top_n=1))
+ with pytest.raises(ServingError) as unsupported:
+ service.invoke_operator_lab(reranking, DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+ assert unsupported.value.code == "LAB_INPUT_NOT_SUPPORTED"
+ assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+@pytest.mark.parametrize("invalid", ["approval_expired", "wrong_contract"])
+def test_operator_entry_rechecks_exact_approval_and_contract(session: Session, invalid: str) -> None:
+ service, request, selected = operator_fixture(session)
+ if invalid == "approval_expired":
+ approval = service.repo.execution_approval(request.target.execution_approval_id)
+ assert approval is not None
+ approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ else:
+ route = service.repo.experiment_route_by_id(selected.id)
+ assert route is not None
+ route.capability_contract_id = service.ensure_contract("rag.reranking", 1).id
+ session.commit()
+ with pytest.raises(ServingError) as denied:
+ service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+ assert denied.value.code == ("LAB_ADMISSION_BLOCKED" if invalid == "approval_expired" else "LAB_TARGET_NOT_ROUTABLE")
+ assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+def test_route_deactivation_after_load_prevents_operator_inference(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+ service, request, selected = operator_fixture(session)
+ original = service._ensure_loaded
+ original_put = service.payload_store.put
+ input_references = []
+ def track_put(payload, ttl_seconds):
+ reference = original_put(payload, ttl_seconds)
+ if "input" in payload:
+ input_references.append(reference)
+ return reference
+ monkeypatch.setattr(service.payload_store, "put", track_put)
+ def load_then_deactivate(deployment, deadline, *, priority):
+ loaded = original(deployment, deadline, priority=priority)
+ route = service.repo.experiment_route_by_id(selected.id)
+ assert route is not None
+ route.status = "inactive"
+ session.commit()
+ return loaded
+ monkeypatch.setattr(service, "_ensure_loaded", load_then_deactivate)
+ with pytest.raises(ServingError) as denied:
+ service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+ assert denied.value.code == "LAB_TARGET_NOT_ROUTABLE"
+ assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
+ record = session.scalar(select(GatewayRequest))
+ assert record is not None and record.status == "failed"
+ assert len(input_references) == 1
+ assert service.payload_store.get(input_references[0]) is None
+
+
+def test_gateway_times_validation_once_before_stable_resolution(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+ service, request, _ = operator_fixture(session)
+ approve_and_promote(service, request.target.deployment_candidate_id)
+ created = service.create_client(ServiceClientCreate(name="timing-client", allowed_capabilities=["rag.embedding@1"]))
+ client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+ events = []
+ clock = [1000.0]
+ original_validate = service._validate_inputs
+ original_resolve = service._stable_embedding_deployment
+ def validate(value):
+ events.append("validate")
+ clock[0] += 0.125
+ return original_validate(value)
+ def resolve():
+ events.append("resolve")
+ clock[0] += 0.375
+ return original_resolve()
+ monkeypatch.setattr(service, "_validate_inputs", validate)
+ monkeypatch.setattr(service, "_stable_embedding_deployment", resolve)
+ monkeypatch.setattr("modelforge_api.services.serving.time.perf_counter", lambda: clock[0])
+ response = service.invoke(EmbeddingInvokeRequest(input="timed request"), client)
+ assert events == ["validate", "resolve"]
+ record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == response.request_id))
+ assert record is not None
+ latency = record.decision_evidence["latency_ms"]
+ assert latency["validation"] == 125.0
+ assert latency["resolution"] == 375.0
+ assert latency["total"] >= 500.0
+
+
+def test_failed_enqueue_deletes_input_without_a_job_owner(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+ service, request, selected = operator_fixture(session)
+ original_enqueue = service._enqueue_job
+ original_put = service.payload_store.put
+ input_references = []
+ def track_put(payload, ttl_seconds):
+ reference = original_put(payload, ttl_seconds)
+ if "input" in payload:
+ input_references.append(reference)
+ return reference
+ def enqueue(deployment, operation, **kwargs):
+ if operation == "invoke":
+ raise ServingError(409, "ENQUEUE_REJECTED", "isolated enqueue failure")
+ return original_enqueue(deployment, operation, **kwargs)
+ monkeypatch.setattr(service.payload_store, "put", track_put)
+ monkeypatch.setattr(service, "_enqueue_job", enqueue)
+ with pytest.raises(ServingError, match="isolated enqueue failure"):
+ service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+ deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+ assert len(input_references) == 1
+ assert service.payload_store.get(input_references[0]) is None
+ assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
diff --git a/backend/tests/test_evaluation_project_boundary.py b/backend/tests/test_evaluation_project_boundary.py
new file mode 100644
index 0000000..288a151
--- /dev/null
+++ b/backend/tests/test_evaluation_project_boundary.py
@@ -0,0 +1,120 @@
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import func, select, update
+
+from modelforge_api.domain.evaluation import (
+ EvaluationCaseResultCreate,
+ EvaluationComparisonCreate,
+ EvaluationRunComplete,
+ EvaluationRunCreate,
+)
+from modelforge_api.persistence.models import (
+ EvaluationCase,
+ EvaluationCaseResult,
+ EvaluationComparison,
+ EvaluationRun,
+ Project,
+)
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from tests.test_evaluation import _run, _session, _suite
+
+
+def test_run_creation_rejects_another_projects_suite_without_writing_run() -> None:
+ with _session() as session:
+ owner = Project(key="owner", name="Owner", description="test")
+ other = Project(key="other", name="Other", description="test")
+ session.add_all([owner, other])
+ session.commit()
+ service = EvaluationService(session)
+ suite = _suite(service, owner.id, uuid4())
+ request = EvaluationRunCreate(project_id=other.id, suite_revision_id=suite.latest_revision_id,
+ target_kind="current", target_index_ref="current", embedding_space_ref="space",
+ corpus_revision="corpus", retrieval_config_digest="a" * 64,
+ environment_fingerprint={"engine": "test"})
+ with pytest.raises(EvaluationError) as rejected:
+ service.create_run(request)
+ assert rejected.value.code == "suite_project_mismatch"
+ assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_legacy_wrong_project_run_cannot_complete_or_become_eligible() -> None:
+ with _session() as session:
+ owner = Project(key="owner", name="Owner", description="test")
+ other = Project(key="other", name="Other", description="test")
+ session.add_all([owner, other])
+ session.commit()
+ service = EvaluationService(session)
+ relevant = uuid4()
+ suite = _suite(service, owner.id, relevant)
+ case = session.scalar(select(EvaluationCase))
+ assert case is not None
+ baseline = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
+ target="current", rank=1)
+ candidate = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
+ target="shadow", rank=1)
+ # Model historical rows admitted before the ownership check, without changing their suite.
+ session.execute(update(EvaluationRun).values(project_id=other.id))
+ session.commit()
+ comparison = service.compare(EvaluationComparisonCreate(
+ baseline_run_id=baseline.id, candidate_run_id=candidate.id))
+ assert comparison.comparability == "not_comparable"
+ assert comparison.promotion_eligibility == "requires_review"
+ assert comparison.comparability_evidence["baseline_suite_project"] is False
+ session.execute(update(EvaluationRun).where(EvaluationRun.id == candidate.id)
+ .values(status="running"))
+ session.commit()
+ with pytest.raises(EvaluationError) as rejected:
+ service.complete_run(candidate.id, EvaluationRunComplete(results=[
+ EvaluationCaseResultCreate(case_id=case.id, ranked_results=[], latency_ms=1)]))
+ assert rejected.value.code == "suite_project_mismatch"
+
+
+def test_failed_baseline_remains_reportable_but_never_eligible_even_with_error_tolerance() -> None:
+ with _session() as session:
+ project = Project(key="owner", name="Owner", description="test")
+ session.add(project)
+ session.commit()
+ service = EvaluationService(session)
+ relevant = uuid4()
+ suite = _suite(service, project.id, relevant)
+ case = session.scalar(select(EvaluationCase))
+ assert case is not None
+ baseline = service.create_run(EvaluationRunCreate(project_id=project.id,
+ suite_revision_id=suite.latest_revision_id, target_kind="current", target_index_ref="current",
+ embedding_space_ref="space", corpus_revision="corpus-v1", retrieval_config_digest="a" * 64,
+ environment_fingerprint={"engine": "test-v1"}))
+ service.complete_run(baseline.id, EvaluationRunComplete(results=[EvaluationCaseResultCreate(
+ case_id=case.id, ranked_results=[], latency_ms=1, error_code="upstream_failed")]))
+ candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+ target="shadow", rank=1)
+ comparison = service.compare(EvaluationComparisonCreate(baseline_run_id=baseline.id,
+ candidate_run_id=candidate.id, maximum_error_rate=1))
+ assert comparison.comparability == "comparable"
+ assert comparison.metric_deltas["recall_at_10"] == 1
+ assert comparison.promotion_eligibility == "not_eligible"
+ assert comparison.eligibility_evidence["baseline_error_count"] == 1
+ assert comparison.eligibility_evidence["baseline_error_free"] is False
+
+
+def test_caller_owned_transaction_rolls_back_runs_case_results_and_comparison() -> None:
+ with _session() as session:
+ project = Project(key="owner", name="Owner", description="test")
+ session.add(project)
+ session.commit()
+ relevant = uuid4()
+ suite = _suite(EvaluationService(session), project.id, relevant)
+ case = session.scalar(select(EvaluationCase))
+ assert case is not None
+ service = EvaluationService(session, commit=False)
+ baseline = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+ target="current", rank=2)
+ candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+ target="shadow", rank=1)
+ comparison = service.compare(EvaluationComparisonCreate(
+ baseline_run_id=baseline.id, candidate_run_id=candidate.id))
+ assert comparison.promotion_eligibility == "eligible"
+ assert session.scalar(select(func.count()).select_from(EvaluationCaseResult)) == 2
+ session.rollback()
+ for model in (EvaluationRun, EvaluationCaseResult, EvaluationComparison):
+ assert session.scalar(select(func.count()).select_from(model)) == 0
diff --git a/backend/tests/test_external_creative_admission.py b/backend/tests/test_external_creative_admission.py
new file mode 100644
index 0000000..3978632
--- /dev/null
+++ b/backend/tests/test_external_creative_admission.py
@@ -0,0 +1,456 @@
+"""Defensive allocation authority and reservation regressions; no GPU invocation."""
+
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import create_engine, event, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.routes.external_creative import operator_allocations
+from modelforge_api.domain.external_creative import (
+ ExternalActionAttestation,
+ ExternalAllocationCreate,
+ ExternalProfileObservation,
+ ExternalResourceApproval,
+)
+from modelforge_api.persistence.models import (
+ Accelerator,
+ Base,
+ ExternalCreativeAllocation,
+ Project,
+ ProjectBinding,
+ ServiceClient,
+)
+from modelforge_api.services import external_creative as external_creative_module
+from modelforge_api.services.external_creative import (
+ ExternalCreativeService,
+ attestations,
+ observations,
+ resources,
+)
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import setup_m5
+
+
+@pytest.fixture
+def allocation_setup():
+ engine = create_engine("sqlite://")
+
+ @event.listens_for(engine, "connect")
+ def foreign_keys(connection, _record):
+ connection.execute("PRAGMA foreign_keys=ON")
+
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ serving, _candidate, node = setup_m5(session)
+ contract = serving.ensure_contract("rag.embedding", 1)
+ project = Project(key="creative-fixture", name="Creative", description="Offline fixture")
+ session.add(project)
+ session.flush()
+ binding = ProjectBinding(
+ project_id=project.id,
+ capability_contract_id=contract.id,
+ channel="stable",
+ priority="interactive",
+ )
+ session.add(binding)
+ session.flush()
+ client = ServiceClient(
+ name="Creative client",
+ project_binding_id=binding.id,
+ allowed_capabilities=["creative.resource@1"],
+ workload_priority="interactive",
+ )
+ session.add(client)
+ session.commit()
+ accelerator = session.scalar(
+ select(Accelerator).where(Accelerator.compute_node_id == node.id)
+ )
+ budget = SimpleNamespace(
+ accelerator_id=accelerator.id,
+ attribution_confidence="HIGH",
+ pressure_state="LOW",
+ schedulable_free_vram_bytes=100,
+ policy_revision="fixture",
+ observed_at=None,
+ )
+
+ # Contract matches root's shared accounting: subtract retained reservations once.
+ def overview(*, commit):
+ assert commit is False
+ retained = sum(
+ row.required_vram_bytes
+ for row in session.scalars(
+ select(ExternalCreativeAllocation).where(
+ ExternalCreativeAllocation.state.in_(["granted", "uncertain"])
+ )
+ )
+ )
+ return [
+ SimpleNamespace(
+ **{
+ **vars(budget),
+ "schedulable_free_vram_bytes": budget.schedulable_free_vram_bytes
+ - retained,
+ }
+ )
+ ]
+
+ serving.scheduler_overview = Mock(side_effect=overview)
+ service = ExternalCreativeService(session, serving)
+ service.upsert_resource(
+ ExternalResourceApproval(
+ project_id=project.id,
+ resource_ref="chosen-model",
+ workload="image",
+ provider_resource_id="private-provider",
+ configured_vram_bytes=60,
+ ),
+ "fixture",
+ )
+ request = ExternalAllocationCreate(
+ request_id="one",
+ idempotency_key="one",
+ installation_id="fixture",
+ workload="image",
+ resource_ref="chosen-model",
+ provider_resource_id="private-provider",
+ required_vram_bytes=60,
+ )
+ yield session, service, request, client, budget, binding, project
+ engine.dispose()
+
+
+def test_attestation_requires_evidence_and_records_authorized_action(allocation_setup):
+ session, service, request, client, _, _, _ = allocation_setup
+ allocation = service.acquire(request, client)
+ assert allocation.state == "granted"
+ assert allocation.required_actions == []
+ # The normal free-capacity grant needs no eviction. Exercise the retained
+ # action attestation contract with an explicitly action-bearing fixture.
+ row = session.get(ExternalCreativeAllocation, allocation.allocation_id)
+ row.required_actions = ["release_idle_comfy"]
+ session.commit()
+ allocation = service.acquire(request, client)
+
+ with pytest.raises(ServingError, match="attestation is required"):
+ service.action(allocation.allocation_id, request.request_id, "attest", client)
+ assert session.execute(select(attestations)).all() == []
+
+ service.action(
+ allocation.allocation_id,
+ request.request_id,
+ "attest",
+ client,
+ attestation=ExternalActionAttestation(
+ action=allocation.required_actions[0],
+ observed_before=100,
+ observed_after=40,
+ completed_at=datetime.now(UTC),
+ ),
+ )
+ recorded = session.execute(select(attestations)).one()
+ assert recorded.action == allocation.required_actions[0]
+ assert recorded.before_vram_bytes == 100
+ assert recorded.after_vram_bytes == 40
+
+
+def test_priority_cannot_exceed_either_client_or_binding(allocation_setup):
+ session, service, request, client, _, binding, _ = allocation_setup
+ with pytest.raises(ServingError, match="priority"):
+ service.acquire(request.model_copy(update={"priority": "production"}), client)
+ client.workload_priority = "production"
+ session.commit()
+ with pytest.raises(ServingError, match="priority"):
+ service.acquire(request.model_copy(update={"priority": "production"}), client)
+ assert list(session.scalars(select(ExternalCreativeAllocation))) == []
+ binding.priority = "production"
+ session.commit()
+ assert (
+ service.acquire(request.model_copy(update={"priority": "production"}), client).priority
+ == "production"
+ )
+
+
+@pytest.mark.parametrize(
+ "revocation", ["client", "scope", "binding", "project", "resource", "priority"]
+)
+def test_queue_rechecks_current_authority(allocation_setup, revocation):
+ session, service, request, client, budget, binding, project = allocation_setup
+ budget.schedulable_free_vram_bytes = 0
+ created = service.acquire(request, client)
+ assert created.state == "queued"
+ if revocation == "client":
+ client.status = "disabled"
+ elif revocation == "scope":
+ client.allowed_capabilities = []
+ elif revocation == "binding":
+ binding.deprecated_at = datetime.now(UTC)
+ elif revocation == "project":
+ project.active = False
+ elif revocation == "priority":
+ client.workload_priority = "background"
+ else:
+ session.execute(resources.update().values(approved=False))
+ session.commit()
+ budget.schedulable_free_vram_bytes = 100
+ service._promote_queue()
+ session.commit()
+ row = session.get(ExternalCreativeAllocation, created.allocation_id)
+ assert row.state == "blocked" and row.acquired_at is None and row.accelerator_id is None
+
+
+def test_queue_recalculates_resource_floor_and_accounts_new_grants_once(allocation_setup):
+ session, service, request, client, budget, _, _ = allocation_setup
+ budget.schedulable_free_vram_bytes = 0
+ first = service.acquire(request, client)
+ second = service.acquire(
+ request.model_copy(update={"idempotency_key": "two", "request_id": "two"}), client
+ )
+ session.execute(resources.update().values(configured_vram_bytes=80))
+ session.commit()
+ budget.schedulable_free_vram_bytes = 150
+ service._promote_queue()
+ session.commit()
+ pair = [session.get(ExternalCreativeAllocation, item.allocation_id) for item in (first, second)]
+ assert sorted(row.state for row in pair) == ["granted", "queued"]
+ assert all(row.required_vram_bytes == 80 for row in pair)
+ assert all(row.required_actions == [] for row in pair)
+
+
+def test_permanently_oversized_queue_head_does_not_starve_fitting_request(allocation_setup):
+ _, service, request, client, _, _, _ = allocation_setup
+ oversized = service.acquire(
+ request.model_copy(
+ update={
+ "idempotency_key": "oversized",
+ "request_id": "oversized",
+ "required_vram_bytes": 120,
+ }
+ ),
+ client,
+ )
+ fitting = service.acquire(
+ request.model_copy(update={"idempotency_key": "fitting", "request_id": "fitting"}),
+ client,
+ )
+
+ assert oversized.state == "queued"
+ assert fitting.state == "granted"
+ assert fitting.required_actions == []
+ assert fitting.decision_evidence["headroom_before_bytes"] == 100
+
+
+def test_new_request_promotes_fitting_queue_head_before_competing_for_capacity(allocation_setup):
+ session, service, request, client, budget, _, _ = allocation_setup
+ budget.schedulable_free_vram_bytes = 0
+ waiting = service.acquire(request, client)
+ budget.schedulable_free_vram_bytes = 100
+
+ newcomer = service.acquire(
+ request.model_copy(update={"idempotency_key": "new", "request_id": "new"}), client
+ )
+
+ session.refresh(waiting_row := session.get(ExternalCreativeAllocation, waiting.allocation_id))
+ assert waiting_row.state == "granted"
+ assert newcomer.state == "queued"
+
+
+def test_queued_decision_reports_observed_capacity_instead_of_zero_sentinel(allocation_setup):
+ _, service, request, client, budget, _, _ = allocation_setup
+ observed_at = datetime(2026, 9, 22, 8, 30, tzinfo=UTC)
+ budget.schedulable_free_vram_bytes = 50
+ budget.observed_at = observed_at
+
+ queued = service.acquire(request, client)
+
+ assert queued.state == "queued"
+ assert queued.decision_evidence["headroom_before_bytes"] == 50
+ assert queued.decision_evidence["observed_at"] == observed_at.isoformat()
+
+
+def test_expired_heartbeat_cannot_resurrect_or_release_reservation(allocation_setup):
+ session, service, request, client, _, _, _ = allocation_setup
+ first = service.acquire(request, client)
+ row = session.get(ExternalCreativeAllocation, first.allocation_id)
+ row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ with pytest.raises(ServingError, match="expired"):
+ service.action(row.id, request.request_id, "heartbeat", client)
+ session.expire_all()
+ assert row.state == "uncertain" and row.required_vram_bytes == 60 and row.accelerator_id
+ second = service.acquire(
+ request.model_copy(update={"idempotency_key": "two", "request_id": "two"}), client
+ )
+ assert second.state == "queued"
+ service.action(row.id, request.request_id, "release", client)
+ assert session.get(ExternalCreativeAllocation, second.allocation_id).state == "granted"
+
+
+def test_confirming_expired_grant_released_promotes_waiting_allocation(allocation_setup):
+ session, service, request, client, _, _, _ = allocation_setup
+ granted = service.acquire(request, client)
+ expired = session.get(ExternalCreativeAllocation, granted.allocation_id)
+ expired.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+
+ waiting = service.acquire(
+ request.model_copy(update={"idempotency_key": "waiting", "request_id": "waiting"}),
+ client,
+ )
+ session.refresh(expired)
+ assert expired.state == "uncertain"
+ assert expired.released_at is None
+ assert waiting.state == "queued"
+
+ service.reconcile(expired.id, "confirm_released", "provider verified idle", "operator")
+
+ waiting_row = session.get(ExternalCreativeAllocation, waiting.allocation_id)
+ session.refresh(waiting_row)
+ assert waiting_row.state == "granted"
+
+
+@pytest.mark.parametrize("mutation", ["queued", "reserved", "expired"])
+def test_profile_requires_actual_grant_and_authoritative_reservation(allocation_setup, mutation):
+ session, service, request, client, budget, _, _ = allocation_setup
+ if mutation == "queued":
+ budget.schedulable_free_vram_bytes = 0
+ first = service.acquire(request, client)
+ row = session.get(ExternalCreativeAllocation, first.allocation_id)
+ if mutation == "expired":
+ row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ body = ExternalProfileObservation(
+ allocation_id=row.id,
+ resource_ref=request.resource_ref,
+ provider_resource_id=request.provider_resource_id,
+ workload="image",
+ observed_peak_vram_bytes=50,
+ reserved_vram_bytes=61 if mutation == "reserved" else 60,
+ duration_ms=1000,
+ outcome="succeeded",
+ )
+ with pytest.raises(ServingError, match="actual grant"):
+ service.observe(body, client)
+ assert list(session.execute(select(observations))) == []
+
+
+def test_released_grant_can_report_exact_profile_without_reopening_capacity(allocation_setup):
+ session, service, request, client, _, _, _ = allocation_setup
+ first = service.acquire(request, client)
+ service.action(first.allocation_id, request.request_id, "release", client)
+ body = ExternalProfileObservation(
+ allocation_id=first.allocation_id,
+ resource_ref=request.resource_ref,
+ provider_resource_id=request.provider_resource_id,
+ workload="image",
+ observed_peak_vram_bytes=50,
+ reserved_vram_bytes=60,
+ duration_ms=1000,
+ outcome="succeeded",
+ )
+ assert service.observe(body, client).measurement_count == 1
+ assert service.observe(body, client).measurement_count == 1
+ assert session.get(ExternalCreativeAllocation, first.allocation_id).state == "released"
+
+
+def test_history_read_projects_expiry_without_inventing_release(allocation_setup):
+ session, service, request, client, _, _, _ = allocation_setup
+ first = service.acquire(request, client)
+ row = session.get(ExternalCreativeAllocation, first.allocation_id)
+ row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ page = service.list_allocations("uncertain", 20, 0)
+ assert page.total == 1 and page.items[0].allocation_id == row.id
+ assert page.items[0].state == "uncertain"
+ assert page.items[0].decision_code == "ALLOCATION_EXPIRED_UNCERTAIN"
+ assert page.items[0].expires_at is None
+ assert page.items[0].uncertain_at is None
+ session.expire_all()
+ stored = session.get(ExternalCreativeAllocation, first.allocation_id)
+ assert stored.state == "granted" and stored.released_at is None
+ assert stored.expires_at.replace(tzinfo=UTC) < datetime.now(UTC)
+ assert stored.uncertain_at is None
+ assert stored.required_vram_bytes == 60 and stored.accelerator_id is not None
+
+
+def test_operator_allocation_get_performs_no_dml_or_commit(allocation_setup):
+ session, service, request, client, _, _, _ = allocation_setup
+ first = service.acquire(request, client)
+ row = session.get(ExternalCreativeAllocation, first.allocation_id)
+ row.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ client.name = "pending read-only probe"
+
+ statements = []
+ commits = []
+
+ def record_statement(_connection, _cursor, statement, _parameters, _context, _many):
+ operation = statement.lstrip().split(None, 1)[0].upper()
+ if operation in {"INSERT", "UPDATE", "DELETE", "REPLACE"}:
+ statements.append(operation)
+
+ def record_commit(_session):
+ commits.append(True)
+
+ bind = session.get_bind()
+ event.listen(bind, "before_cursor_execute", record_statement)
+ event.listen(session, "before_commit", record_commit)
+ try:
+ page = operator_allocations(service, object(), "uncertain", 20, 0)
+ finally:
+ event.remove(bind, "before_cursor_execute", record_statement)
+ event.remove(session, "before_commit", record_commit)
+
+ assert page.total == 1 and page.items[0].state == "uncertain"
+ assert statements == []
+ assert commits == []
+ session.rollback()
+ session.refresh(client)
+ assert client.name == "Creative client"
+
+
+def test_history_read_expiry_boundary_is_inclusive(allocation_setup, monkeypatch):
+ session, service, request, client, _, _, _ = allocation_setup
+ first = service.acquire(request, client)
+ row = session.get(ExternalCreativeAllocation, first.allocation_id)
+ boundary = datetime(2030, 1, 1, tzinfo=UTC)
+ row.expires_at = boundary
+ session.commit()
+ monkeypatch.setattr(external_creative_module, "_now", lambda: boundary)
+
+ expired = service.list_allocations("uncertain", 20, 0)
+ assert expired.total == 1
+ assert expired.items[0].state == "uncertain"
+ assert expired.items[0].decision_code == "ALLOCATION_EXPIRED_UNCERTAIN"
+ assert expired.items[0].uncertain_at is None
+
+ row.expires_at = boundary + timedelta(microseconds=1)
+ session.commit()
+ before_expiry = service.list_allocations("uncertain", 20, 0)
+ active = service.list_allocations("granted", 20, 0)
+
+ assert before_expiry.total == 0
+ assert active.total == 1
+ assert active.items[0].state == "granted"
+ assert active.items[0].expires_at.replace(tzinfo=UTC) == boundary + timedelta(microseconds=1)
+
+
+def test_restored_queued_uncertainty_requires_explicit_reconciliation(allocation_setup):
+ session, service, request, client, budget, _, _ = allocation_setup
+ budget.schedulable_free_vram_bytes = 0
+ first = service.acquire(request, client)
+ row = session.get(ExternalCreativeAllocation, first.allocation_id)
+ row.state, row.decision_code = "uncertain", "CREATIVE_RESTORE_INTERRUPTED"
+ session.commit()
+ budget.schedulable_free_vram_bytes = 100
+ assert service.get(row.id, client).state == "uncertain"
+ assert row.accelerator_id is None and row.acquired_at is None
+ assert (
+ service.reconcile(
+ row.id, "confirm_released", "External owner verified idle", "operator"
+ ).state
+ == "released"
+ )
diff --git a/backend/tests/test_external_creative_governance.py b/backend/tests/test_external_creative_governance.py
new file mode 100644
index 0000000..8a11900
--- /dev/null
+++ b/backend/tests/test_external_creative_governance.py
@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from modelforge_api.domain.external_creative import (
+ ExternalActionAttestation,
+ ExternalAllocationCreate,
+ ExternalAllocationReconcile,
+ ExternalProfileObservation,
+ ExternalResourceApproval,
+)
+from modelforge_api.services.external_creative import PRIORITY, _digest
+
+AUDIO_WORKLOADS = (
+ "speech_generation",
+ "voice_clone",
+ "music_generation",
+ "sound_generation",
+ "foley_generation",
+ "transcription",
+ "audio_enhancement",
+ "lip_sync",
+)
+
+
+def test_priority_order_keeps_production_before_interactive_and_background() -> None:
+ assert PRIORITY["production"] < PRIORITY["interactive"] < PRIORITY["background"]
+ assert PRIORITY["background"] < PRIORITY["benchmark"]
+
+
+def test_resource_binding_requires_bounded_positive_profile() -> None:
+ binding = ExternalResourceApproval(
+ project_id=uuid.uuid4(),
+ resource_ref="wan.video@2.2",
+ workload="video",
+ provider_resource_id="configured-comfyui",
+ configured_vram_bytes=12 * 1024**3,
+ )
+ assert binding.approved is True
+ with pytest.raises(ValidationError):
+ binding.model_copy(update={"configured_vram_bytes": 0}).model_validate(
+ binding.model_copy(update={"configured_vram_bytes": 0}).model_dump()
+ )
+
+
+@pytest.mark.parametrize("workload", AUDIO_WORKLOADS)
+def test_audio_workloads_are_explicitly_admitted_by_the_contract(workload: str) -> None:
+ binding = ExternalResourceApproval(
+ project_id=uuid.uuid4(),
+ resource_ref="candidate.audio@exact-revision",
+ workload=workload,
+ provider_resource_id="configured-audio-provider",
+ approved=False,
+ configured_vram_bytes=4 * 1024**3,
+ )
+ assert binding.workload == workload
+ assert binding.approved is False
+
+
+def test_unknown_audio_workload_is_rejected() -> None:
+ with pytest.raises(ValidationError):
+ ExternalResourceApproval(
+ project_id=uuid.uuid4(),
+ resource_ref="candidate.audio@exact-revision",
+ workload="generic_audio",
+ provider_resource_id="configured-audio-provider",
+ approved=False,
+ configured_vram_bytes=4 * 1024**3,
+ )
+
+
+def test_workload_is_part_of_the_idempotent_allocation_identity() -> None:
+ common = {
+ "request_id": "director-shot-42",
+ "idempotency_key": "director-shot-42-sound",
+ "installation_id": "private-muse",
+ "resource_ref": "candidate.audio@exact-revision",
+ "provider_resource_id": "configured-audio-provider",
+ "required_vram_bytes": 4 * 1024**3,
+ }
+ speech = ExternalAllocationCreate(**common, workload="speech_generation")
+ clone = ExternalAllocationCreate(**common, workload="voice_clone")
+ assert _digest(speech) != _digest(clone)
+
+
+def test_attestation_and_reconciliation_are_closed_enums() -> None:
+ attestation = ExternalActionAttestation(
+ action="release_idle_comfy",
+ observed_before=10,
+ observed_after=4,
+ completed_at="2026-09-09T12:00:00Z",
+ )
+ assert attestation.observed_before - attestation.observed_after == 6
+ assert ExternalAllocationReconcile(action="confirm_released", reason="provider verified idle")
+ with pytest.raises(ValidationError):
+ ExternalAllocationReconcile(action="force", reason="provider verified idle")
+
+
+def test_profile_observation_is_bounded_and_allocation_bound() -> None:
+ observation = ExternalProfileObservation(
+ allocation_id=uuid.uuid4(),
+ resource_ref="flux.image@9b",
+ provider_resource_id="configured-comfyui",
+ workload="image",
+ observed_peak_vram_bytes=14 * 1024**3,
+ reserved_vram_bytes=16 * 1024**3,
+ duration_ms=120_000,
+ outcome="succeeded",
+ )
+ assert observation.observed_peak_vram_bytes <= observation.reserved_vram_bytes
+
+
+def test_migration_retains_each_profile_observation() -> None:
+ repository_root = Path(__file__).resolve().parents[2]
+ migration = (
+ repository_root / "backend/alembic/versions/20260909_0031_external_creative_governance.py"
+ ).read_text()
+ assert '"external_creative_profile_observations"' in migration
+ assert '"uq_external_creative_profile_observation_allocation"' in migration
+ assert 'down_revision = "20260909_0030"' in migration
+
+
+def test_audio_workload_migration_widens_every_persisted_route_component() -> None:
+ repository_root = Path(__file__).resolve().parents[2]
+ migration = (
+ repository_root / "backend/alembic/versions/20260909_0032_audio_creative_workloads.py"
+ ).read_text()
+ assert 'down_revision = "20260909_0031"' in migration
+ assert '"external_creative_allocations"' in migration
+ assert '"external_creative_resources"' in migration
+ assert '"external_creative_profile_observations"' in migration
+ assert "type_=sa.String(32)" in migration
diff --git a/backend/tests/test_external_creative_postgres.py b/backend/tests/test_external_creative_postgres.py
new file mode 100644
index 0000000..5573655
--- /dev/null
+++ b/backend/tests/test_external_creative_postgres.py
@@ -0,0 +1,302 @@
+"""Real PostgreSQL admission serialization with deterministic offline capacity."""
+
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import Mock
+from uuid import uuid4
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.external_creative import (
+ ExternalAllocationCreate,
+ ExternalResourceApproval,
+)
+from modelforge_api.persistence.models import (
+ Accelerator,
+ ExternalCreativeAllocation,
+ GatewayRequest,
+ ServiceClient,
+ ServingGpuLease,
+)
+from modelforge_api.services import external_creative as external_creative_module
+from modelforge_api.services.external_creative import ExternalCreativeService
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_serving import MemoryPayloadStore
+
+__all__ = ["identity_database"]
+
+
+def test_competing_external_admissions_retain_one_capacity_reservation(
+ identity_database: IdentityDatabase,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ with Session(database.runtime) as session:
+ _request, client_id = _valid_selection(session)
+ client = session.get(ServiceClient, client_id)
+ client.allowed_capabilities = ["creative.resource@1"]
+ session.commit()
+ accelerator_id = session.scalar(select(Accelerator.id))
+ service = ExternalCreativeService(session, Mock())
+ project_id = service._project(client)
+ service.upsert_resource(
+ ExternalResourceApproval(
+ project_id=project_id,
+ resource_ref="race-fixture",
+ workload="image",
+ provider_resource_id="offline-provider",
+ configured_vram_bytes=60,
+ ),
+ "qualification",
+ )
+
+ barrier = threading.Barrier(2)
+
+ def contender(index: int) -> str:
+ with Session(database.runtime) as session:
+
+ def overview(*, commit):
+ assert commit is False
+ retained = sum(
+ session.scalars(
+ select(ExternalCreativeAllocation.required_vram_bytes).where(
+ ExternalCreativeAllocation.state.in_(["granted", "uncertain"])
+ )
+ )
+ )
+ return [
+ SimpleNamespace(
+ accelerator_id=accelerator_id,
+ attribution_confidence="HIGH",
+ pressure_state="LOW",
+ schedulable_free_vram_bytes=100 - retained,
+ policy_revision="offline-fixture",
+ observed_at=None,
+ )
+ ]
+
+ serving = Mock()
+ serving.scheduler_overview.side_effect = overview
+ service = ExternalCreativeService(session, serving)
+ client = session.get(ServiceClient, client_id)
+ session.commit()
+ barrier.wait(timeout=30)
+ return service.acquire(
+ ExternalAllocationCreate(
+ request_id=f"race-{index}",
+ idempotency_key=f"race-{index}",
+ installation_id="fixture",
+ workload="image",
+ resource_ref="race-fixture",
+ provider_resource_id="offline-provider",
+ required_vram_bytes=60,
+ ),
+ client,
+ ).state
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert sorted(pool.map(contender, [1, 2])) == ["granted", "queued"]
+ with Session(database.runtime) as session:
+ rows = list(session.scalars(select(ExternalCreativeAllocation)))
+ assert len(rows) == 2
+ assert sum(row.required_vram_bytes for row in rows if row.state == "granted") == 60
+
+
+def test_native_and_external_admission_share_actual_postgres_capacity(
+ identity_database: IdentityDatabase,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ settings = Settings(
+ _env_file=None,
+ env="test",
+ gateway_queue_timeout_seconds=1,
+ gateway_request_timeout_seconds=60,
+ )
+
+ def serving(session):
+ return ServingService(session, settings, ManifestRegistry(), MemoryPayloadStore())
+
+ with Session(database.runtime) as session:
+ selection, client_id = _valid_selection(session)
+ client = session.get(ServiceClient, client_id)
+ client.allowed_capabilities = ["rag.embedding@1", "creative.resource@1"]
+ session.commit()
+ service = serving(session)
+ deployment_id = selection.current_deployment_id
+ deployment = service.repo.deployment(deployment_id)
+ baseline = service._budget(deployment)
+ plan = service._placement_plan(
+ deployment, priority=client.workload_priority, dry_run=True, persist=False, commit=False
+ )
+ assert plan.verdict == "ADMIT"
+ available, native_required = baseline.schedulable_free_vram_bytes, plan.required_vram_bytes
+ assert 0 < native_required < available
+ external_required = available - native_required // 2
+ assert external_required <= available < external_required + native_required
+ external = ExternalCreativeService(session, service)
+ external.upsert_resource(
+ ExternalResourceApproval(
+ project_id=external._project(client),
+ resource_ref="mixed-race",
+ workload="image",
+ provider_resource_id="offline-provider",
+ configured_vram_bytes=external_required,
+ ),
+ "qualification",
+ )
+ gateway = GatewayRequest(
+ request_id=uuid4(),
+ service_client_id=client.id,
+ capability_key="rag.embedding",
+ capability_version=1,
+ capability_deployment_id=deployment.id,
+ compute_node_id=deployment.compute_node_id,
+ status="queued",
+ priority=client.workload_priority,
+ input_sha256="a" * 64,
+ input_count=1,
+ )
+ session.add(gateway)
+ session.commit()
+ gateway_id = gateway.id
+
+ barrier = threading.Barrier(2)
+
+ def contender(native):
+ with Session(database.runtime) as session:
+ service = serving(session)
+ client = session.get(ServiceClient, client_id)
+ deployment = service.repo.deployment(deployment_id)
+ gateway = session.get(GatewayRequest, gateway_id)
+ context = DispatchContext.for_service_client(
+ client.id, client.name, client.workload_priority
+ )
+ session.commit()
+ barrier.wait(timeout=30)
+ if native:
+ try:
+ service._acquire_request_lease(deployment, gateway, context)
+ return "native_granted"
+ except ServingError as error:
+ assert error.code in {
+ "REQUEST_TIMEOUT",
+ "INSUFFICIENT_SCHEDULABLE_VRAM",
+ "INSUFFICIENT_CAPACITY",
+ }, error.code
+ return "native_blocked"
+ result = ExternalCreativeService(session, service).acquire(
+ ExternalAllocationCreate(
+ request_id="mixed-race",
+ idempotency_key="mixed-race",
+ installation_id="fixture",
+ resource_ref="mixed-race",
+ workload="image",
+ provider_resource_id="offline-provider",
+ required_vram_bytes=external_required,
+ ttl_seconds=60,
+ ),
+ client,
+ )
+ return "external_" + result.state
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ outcomes = list(pool.map(contender, [True, False]))
+ assert outcomes in (
+ ["native_granted", "external_queued"],
+ ["native_blocked", "external_granted"],
+ )
+ with Session(database.runtime) as session:
+ native = list(
+ session.scalars(
+ select(ServingGpuLease).where(ServingGpuLease.state.in_(["active", "granted"]))
+ )
+ )
+ external = list(
+ session.scalars(
+ select(ExternalCreativeAllocation).where(
+ ExternalCreativeAllocation.state.in_(["granted", "uncertain"])
+ )
+ )
+ )
+ assert len(native) + len(external) == 1
+ assert (
+ sum(row.reserved_vram_bytes for row in native)
+ + sum(row.required_vram_bytes for row in external)
+ <= available
+ )
+
+
+def test_expired_external_listing_keeps_postgres_count_and_rows_in_sync(
+ identity_database: IdentityDatabase, monkeypatch
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ with Session(database.runtime) as session:
+ _request, client_id = _valid_selection(session)
+ client = session.get(ServiceClient, client_id)
+ client.allowed_capabilities = ["creative.resource@1"]
+ session.commit()
+
+ accelerator_id = session.scalar(select(Accelerator.id))
+ serving = Mock()
+ serving.scheduler_overview.return_value = [
+ SimpleNamespace(
+ accelerator_id=accelerator_id,
+ attribution_confidence="HIGH",
+ pressure_state="LOW",
+ schedulable_free_vram_bytes=100,
+ policy_revision="postgres-boundary-fixture",
+ observed_at=None,
+ )
+ ]
+ service = ExternalCreativeService(session, serving)
+ project_id = service._project(client)
+ service.upsert_resource(
+ ExternalResourceApproval(
+ project_id=project_id,
+ resource_ref="expiry-boundary-fixture",
+ workload="image",
+ provider_resource_id="offline-provider",
+ configured_vram_bytes=60,
+ ),
+ "qualification",
+ )
+ acquired = service.acquire(
+ ExternalAllocationCreate(
+ request_id="expiry-boundary",
+ idempotency_key="expiry-boundary",
+ installation_id="fixture",
+ workload="image",
+ resource_ref="expiry-boundary-fixture",
+ provider_resource_id="offline-provider",
+ required_vram_bytes=60,
+ ),
+ client,
+ )
+ row = session.get(ExternalCreativeAllocation, acquired.allocation_id)
+ boundary = datetime(2030, 1, 1, tzinfo=UTC)
+ row.expires_at = boundary
+ session.commit()
+ monkeypatch.setattr(external_creative_module, "_now", lambda: boundary)
+
+ uncertain = service.list_allocations("uncertain", 20, 0)
+ granted = service.list_allocations("granted", 20, 0)
+
+ assert uncertain.total == len(uncertain.items) == 1
+ assert uncertain.items[0].allocation_id == acquired.allocation_id
+ assert uncertain.items[0].state == "uncertain"
+ assert uncertain.items[0].decision_code == "ALLOCATION_EXPIRED_UNCERTAIN"
+ assert uncertain.items[0].uncertain_at is None
+ assert granted.total == len(granted.items) == 0
+ session.refresh(row)
+ assert row.state == "granted" and row.expires_at == boundary
diff --git a/backend/tests/test_hardware_api.py b/backend/tests/test_hardware_api.py
index ad1946b..9604366 100644
--- a/backend/tests/test_hardware_api.py
+++ b/backend/tests/test_hardware_api.py
@@ -1,5 +1,7 @@
+import pytest
from fastapi.testclient import TestClient
from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector, accelerator
+from http_contract_fakes import isolated_http_contract # noqa: F401 - opt-in pytest fixture
from pydantic import SecretStr
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
@@ -11,6 +13,8 @@ from modelforge_api.persistence.models import Base
from modelforge_api.services.hardware_inventory import HardwareInventoryService, HardwareRefreshBusy
from modelforge_api.settings import Settings, get_settings
+pytestmark = pytest.mark.usefixtures("isolated_http_contract")
+
def _test_operator_credential() -> str:
return "hardware-test-operator"
@@ -86,6 +90,7 @@ def test_hardware_refresh_and_resource_endpoints_serialize_real_and_unknown_valu
finally:
app.dependency_overrides.clear()
session.close()
+ engine.dispose()
def test_concurrent_refresh_returns_normalized_conflict() -> None:
@@ -119,3 +124,16 @@ def test_refresh_failure_returns_normalized_service_unavailable() -> None:
assert response.json()["error"]["message"] == "hardware inventory failed: RuntimeError"
finally:
app.dependency_overrides.clear()
+
+
+def test_hardware_contract_preserves_operator_authentication_and_correlation() -> None:
+ _configure_operator_auth()
+ with TestClient(app) as client:
+ response = client.post(
+ "/api/v1/hardware/refresh", headers={"X-Correlation-ID": "hardware-auth-test"}
+ )
+ assert response.status_code == 401
+ assert response.json()["error"]["code"] == "http_401"
+ assert response.json()["error"]["correlation_id"] == "hardware-auth-test"
+ assert response.headers["x-correlation-id"] == "hardware-auth-test"
+ assert response.headers["cache-control"] == "no-store"
diff --git a/backend/tests/test_huggingface_provider.py b/backend/tests/test_huggingface_provider.py
index daeae26..2d6a740 100644
--- a/backend/tests/test_huggingface_provider.py
+++ b/backend/tests/test_huggingface_provider.py
@@ -55,6 +55,19 @@ def test_search_uses_official_client_and_preserves_unknowns() -> None:
assert calls["search"] == "model" and calls["full"] is True
+def test_provider_uses_only_explicit_credential_and_official_origin(monkeypatch) -> None:
+ from huggingface_hub import constants
+ from huggingface_hub.utils import build_hf_headers
+
+ monkeypatch.setenv("HF_TOKEN", "hf_ambient_test_only")
+ monkeypatch.setattr(constants, "ENDPOINT", "https://untrusted.invalid")
+ anonymous = OfficialHuggingFaceProvider()
+ assert anonymous.api.endpoint == "https://huggingface.co"
+ assert "authorization" not in build_hf_headers(token=anonymous.token)
+ explicit = OfficialHuggingFaceProvider(token="hf_explicit_test_only") # noqa: S106
+ assert build_hf_headers(token=explicit.token)["authorization"] == "Bearer hf_explicit_test_only"
+
+
def test_snapshot_resolves_sha_inventory_card_and_scanner_evidence() -> None:
provider = OfficialHuggingFaceProvider()
provider.api.model_info = lambda *args, **kwargs: SimpleNamespace( # type: ignore[method-assign]
diff --git a/backend/tests/test_lab_execution_api.py b/backend/tests/test_lab_execution_api.py
new file mode 100644
index 0000000..cb9f38a
--- /dev/null
+++ b/backend/tests/test_lab_execution_api.py
@@ -0,0 +1,106 @@
+from datetime import UTC, datetime
+from unittest.mock import Mock
+from uuid import uuid4
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from modelforge_api.api.authorization import (
+ Principal,
+ PrincipalRole,
+ authenticate_operator_principal,
+)
+from modelforge_api.api.routes.lab import get_lab_run_service, router
+from modelforge_api.domain.lab import LabExecutionRunResponse, LabInputSummary
+
+
+def start_body() -> dict[str, object]:
+ return {
+ "target": {
+ name: str(uuid4())
+ for name in (
+ "deployment_candidate_id",
+ "artifact_set_id",
+ "runtime_profile_id",
+ "compute_node_id",
+ "execution_approval_id",
+ )
+ },
+ "input": {"capability": "rag.embedding", "payload": {"input": "private-lab-canary"}},
+ "deployment_id": str(uuid4()),
+ "experiment_route_id": str(uuid4()),
+ }
+
+
+def harness(role: PrincipalRole) -> tuple[TestClient, Mock]:
+ app = FastAPI()
+ app.include_router(router)
+ service = Mock()
+ service.start.return_value = LabExecutionRunResponse(
+ id=uuid4(),
+ status="accepted",
+ target_snapshot={"deployment_id": str(uuid4())},
+ input_summary=LabInputSummary(sha256="a" * 64, item_count=1, encoded_bytes=45),
+ input_payload_expires_at=datetime.now(UTC),
+ retention_seconds=300,
+ created_at=datetime.now(UTC),
+ )
+ app.dependency_overrides[authenticate_operator_principal] = lambda: Principal(
+ "operator-one", role, "test"
+ )
+ app.dependency_overrides[get_lab_run_service] = lambda: service
+ return TestClient(app), service
+
+
+def test_operator_lab_start_route_is_registered_and_returns_accepted_run() -> None:
+ client, service = harness(PrincipalRole.OPERATOR)
+
+ response = client.post(
+ "/api/v1/lab/runs",
+ headers={"Idempotency-Key": "start-once"},
+ json=start_body(),
+ )
+
+ assert response.status_code == 202
+ payload = response.json()
+ assert payload["status"] == "accepted"
+ assert payload["capability"] == "rag.embedding@1"
+ assert "private-lab-canary" not in response.text
+ assert response.headers["cache-control"] == "no-store"
+ service.start.assert_called_once()
+ assert service.start.call_args.kwargs["actor_subject"] == "operator-one"
+ assert service.start.call_args.kwargs["idempotency_key"] == "start-once"
+
+
+def test_lab_run_start_requires_operator_role() -> None:
+ client, service = harness(PrincipalRole.VIEWER)
+
+ response = client.post(
+ "/api/v1/lab/runs",
+ headers={"Idempotency-Key": "start-viewer"},
+ json=start_body(),
+ )
+
+ assert response.status_code == 403
+ service.start.assert_not_called()
+
+
+def test_lab_run_status_and_cancel_are_no_store_and_actor_scoped() -> None:
+ client, service = harness(PrincipalRole.OPERATOR)
+ run_id = service.start.return_value.id
+ service.get.return_value = service.start.return_value
+ service.cancel.return_value = service.start.return_value
+
+ status_response = client.get(f"/api/v1/lab/runs/{run_id}")
+ cancel_response = client.post(f"/api/v1/lab/runs/{run_id}/cancel")
+
+ assert status_response.status_code == 200
+ assert cancel_response.status_code == 200
+ assert status_response.headers["cache-control"] == "no-store"
+ assert cancel_response.headers["cache-control"] == "no-store"
+ assert "payload_reference" not in status_response.text
+ assert "private-lab-canary" not in status_response.text
+ assert service.get.call_args.args == (run_id,)
+ assert service.get.call_args.kwargs == {"actor_subject": "operator-one"}
+ assert service.cancel.call_args.args == (run_id,)
+ assert service.cancel.call_args.kwargs == {"actor_subject": "operator-one"}
diff --git a/backend/tests/test_lab_execution_postgres.py b/backend/tests/test_lab_execution_postgres.py
new file mode 100644
index 0000000..d4239fb
--- /dev/null
+++ b/backend/tests/test_lab_execution_postgres.py
@@ -0,0 +1,167 @@
+"""Managed PostgreSQL migration and actor-scoped LAB idempotency qualification."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, func, select, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabExecutionStartRequest
+from modelforge_api.persistence.models import GatewayRequest, LabExecutionRun
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingService
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_dispatch_context import operator_fixture
+
+_PRIOR = "20260923_0037"
+_CURRENT = "20260924_0039"
+
+
+def test_managed_postgresql_lab_migration_and_concurrent_idempotent_start(
+) -> None:
+ """The database constraint must linearize simultaneous starts for one actor/key."""
+ admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+ if not admin_url:
+ pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ assert owner_url and runtime_url, "managed qualification requires owner and runtime URLs"
+
+ if admin_url.startswith("postgresql://"):
+ admin_url = admin_url.replace("postgresql://", "postgresql+psycopg://", 1)
+ owner = make_url(owner_url)
+ runtime = make_url(runtime_url)
+ assert owner.get_backend_name() == runtime.get_backend_name() == "postgresql"
+ assert owner.username == "modelforge"
+ assert runtime.username == "modelforge_runtime"
+
+ database_name = f"test_lab_runs_{uuid.uuid4().hex}"
+ admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+ owner_engine = create_engine(owner.set(database=database_name))
+ runtime_engine = create_engine(
+ runtime.set(database=database_name),
+ connect_args={"connect_timeout": 15, "options": "-c statement_timeout=30000 -c lock_timeout=15000"},
+ )
+ migration = Config()
+ migration.set_main_option(
+ "script_location", str(Path(__file__).resolve().parents[1] / "alembic")
+ )
+ migration.set_main_option(
+ "sqlalchemy.url",
+ owner.set(database=database_name)
+ .render_as_string(hide_password=False)
+ .replace("%", "%%"),
+ )
+ alembic = importlib.import_module("alembic.command")
+ barrier = threading.Barrier(2)
+
+ with admin.connect() as connection:
+ connection.execute(text(f'CREATE DATABASE "{database_name}" OWNER modelforge'))
+ try:
+ alembic.upgrade(migration, _PRIOR)
+ with runtime_engine.connect() as connection:
+ assert connection.scalar(text("SELECT to_regclass('public.lab_execution_runs')")) is None
+
+ alembic.upgrade(migration, "head")
+ with runtime_engine.connect() as connection:
+ assert connection.scalar(text("SELECT version_num FROM alembic_version")) == _CURRENT
+ assert connection.scalar(
+ text("SELECT has_table_privilege(current_user, 'lab_execution_runs', 'SELECT')")
+ )
+ for privilege in ("INSERT", "UPDATE"):
+ assert connection.scalar(
+ text(
+ "SELECT has_table_privilege(current_user, 'lab_execution_runs', :privilege)"
+ ),
+ {"privilege": privilege},
+ )
+ for privilege in ("DELETE", "TRUNCATE"):
+ assert not connection.scalar(
+ text(
+ "SELECT has_table_privilege(current_user, 'lab_execution_runs', :privilege)"
+ ),
+ {"privilege": privilege},
+ )
+
+ with Session(owner_engine) as setup_session:
+ setup_serving, preflight, selected = operator_fixture(setup_session)
+ request = LabExecutionStartRequest(
+ target=preflight.target,
+ input=preflight.input,
+ limits=preflight.limits,
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+ deployment_settings = setup_serving._deployment_settings
+ manifests = setup_serving.manifests
+
+ def start_once() -> uuid.UUID:
+ with Session(runtime_engine) as session:
+ serving = ServingService(
+ session,
+ deployment_settings,
+ manifests,
+ MemoryPayloadStore(),
+ )
+ service = LabRunService(serving)
+ select_target = service._selected_target
+
+ def select_same_target(_request: LabExecutionStartRequest):
+ selection = select_target(_request)
+ barrier.wait(timeout=30)
+ return selection
+
+ service._selected_target = select_same_target # type: ignore[method-assign]
+ result = service.start(
+ request,
+ actor_subject="postgres-race-operator",
+ idempotency_key="same-selected-run",
+ )
+ return result.id
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ run_ids = list(pool.map(lambda _index: start_once(), range(2)))
+ assert run_ids[0] == run_ids[1]
+
+ cancel_barrier = threading.Barrier(2)
+
+ def cancel_once() -> str:
+ with Session(runtime_engine) as session:
+ serving = SimpleNamespace(session=session, payload_store=MemoryPayloadStore())
+ service = LabRunService(serving)
+ cancel_barrier.wait(timeout=30)
+ return service.cancel(
+ run_ids[0], actor_subject="postgres-race-operator"
+ ).status
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ cancel_statuses = list(pool.map(lambda _index: cancel_once(), range(2)))
+ assert cancel_statuses == ["cancelled", "cancelled"]
+
+ with Session(runtime_engine) as session:
+ assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 1
+ assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 1
+ run = session.scalar(select(LabExecutionRun))
+ assert run is not None
+ assert run.actor_subject == "postgres-race-operator"
+ assert run.idempotency_key == "same-selected-run"
+ assert run.status == "cancelled"
+ assert run.cancellation_requested_at is not None
+ assert run.cancellation_acknowledged_at is None
+ assert run.input_sha256 and "private postgres race input" not in str(run.target_snapshot)
+ finally:
+ runtime_engine.dispose()
+ owner_engine.dispose()
+ with admin.connect() as connection:
+ connection.execute(text(f'DROP DATABASE IF EXISTS "{database_name}" WITH (FORCE)'))
+ admin.dispose()
diff --git a/backend/tests/test_lab_execution_races_postgres.py b/backend/tests/test_lab_execution_races_postgres.py
new file mode 100644
index 0000000..56b6953
--- /dev/null
+++ b/backend/tests/test_lab_execution_races_postgres.py
@@ -0,0 +1,198 @@
+"""PostgreSQL qualification for durable LAB cancellation versus worker completion."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session
+
+pytest_plugins = ("tests.test_managed_identities_postgres",)
+
+from modelforge_api.domain.lab import LabExecutionStartRequest
+from modelforge_api.domain.serving import AgentServingJobComplete
+from modelforge_api.persistence.models import GatewayRequest, LabExecutionRun, ServingJob
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingService
+from tests.test_dispatch_context import operator_fixture
+from tests.test_managed_identities_postgres import IdentityDatabase
+
+
+@pytest.mark.parametrize("first_writer", ["cancel", "complete"])
+def test_lab_cancel_and_worker_completion_linearize_on_job_then_run_locks(
+ identity_database: IdentityDatabase, first_writer: str
+) -> None:
+ """Hold the winning transaction open and prove the other writer observes its result."""
+ importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+ engine = identity_database.runtime
+
+ with Session(engine) as session:
+ serving, preflight, selected = operator_fixture(session)
+ request = LabExecutionStartRequest(
+ target=preflight.target,
+ input=preflight.input,
+ limits=preflight.limits,
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+ payload_store = serving.payload_store
+ run = LabRunService(serving).start(
+ request,
+ actor_subject=f"postgres-lab-race-{first_writer}",
+ idempotency_key=f"worker-completion-{first_writer}",
+ )
+ persisted_run = session.get(LabExecutionRun, run.id)
+ assert persisted_run is not None
+ gateway = session.get(GatewayRequest, persisted_run.gateway_request_id)
+ assert gateway is not None
+ token = f"lab-race-token-{first_writer}-" + "t" * 32
+ now = datetime.now(UTC)
+ job = ServingJob(
+ capability_deployment_id=selected.capability_deployment_id,
+ compute_node_id=selected.deployment.compute_node_id,
+ gateway_request_id=gateway.id,
+ lab_execution_run_id=persisted_run.id,
+ operation="invoke",
+ status="running",
+ priority="lab",
+ idempotency_key=f"worker-job-{first_writer}",
+ payload_reference=persisted_run.payload_reference,
+ attempt_count=1,
+ lease_token_hash=hashlib.sha256(token.encode()).hexdigest(),
+ lease_expires_at=now + timedelta(minutes=2),
+ result_summary={},
+ started_at=now,
+ )
+ persisted_run.status = "running"
+ persisted_run.started_at = now
+ persisted_run.heartbeat_at = now
+ gateway.status = "running"
+ session.add(job)
+ session.commit()
+
+ run_id = run.id
+ job_id = job.id
+ deployment_id = selected.capability_deployment_id
+ node_id = selected.deployment.compute_node_id
+ embedding_space = selected.deployment.embedding_space
+ assert embedding_space is not None
+ dimension = embedding_space.dimension
+ deployment_settings = serving._deployment_settings
+ manifests = serving.manifests
+
+ actor_subject = f"postgres-lab-race-{first_writer}"
+ first_writer_has_locks = threading.Event()
+ allow_first_writer_to_commit = threading.Event()
+ second_writer_requested_job_lock = threading.Event()
+
+ def make_serving(session: Session) -> ServingService:
+ return ServingService(session, deployment_settings, manifests, payload_store)
+
+ def complete(service: ServingService) -> str:
+ deployment = service.repo.deployment(deployment_id)
+ node = service.repo.node(node_id)
+ assert deployment is not None and node is not None
+ vector = [1.0] + [0.0] * (dimension - 1)
+ service.complete_job(
+ job_id,
+ node,
+ AgentServingJobComplete(
+ lease_token=token,
+ worker_instance_id="postgres-race-worker",
+ state="warm",
+ result={"vectors": [vector], "count": 1, "normalized": True},
+ metrics={"inference_time_ms": 8.0},
+ health={},
+ runtime_facts={},
+ ),
+ )
+ return "completed"
+
+ def write(kind: str, *, hold_commit: bool) -> str:
+ with Session(engine) as session:
+ service = make_serving(session)
+ original_commit = session.commit
+
+ def commit_after_gate() -> None:
+ if hold_commit:
+ first_writer_has_locks.set()
+ if not allow_first_writer_to_commit.wait(timeout=20):
+ raise RuntimeError("timed out holding the first LAB writer locks")
+ original_commit()
+
+ session.commit = commit_after_gate # type: ignore[method-assign]
+ if kind == "cancel":
+ return LabRunService(service).cancel(
+ run_id, actor_subject=actor_subject
+ ).status
+ return complete(service)
+
+ @event.listens_for(engine, "before_cursor_execute")
+ def observe_second_job_lock(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ normalized = statement.lower()
+ if (
+ threading.current_thread().name.startswith("lab-race-second")
+ and "serving_jobs" in normalized
+ and "for update" in normalized
+ ):
+ second_writer_requested_job_lock.set()
+
+ second_writer = "complete" if first_writer == "cancel" else "cancel"
+ try:
+ with ThreadPoolExecutor(max_workers=1, thread_name_prefix="lab-race-first") as first_pool:
+ first_future = first_pool.submit(write, first_writer, hold_commit=True)
+ if not first_writer_has_locks.wait(timeout=10):
+ first_future.result(timeout=1)
+ pytest.fail("first writer never reached commit")
+ with ThreadPoolExecutor(
+ max_workers=1, thread_name_prefix="lab-race-second"
+ ) as second_pool:
+ second_future = second_pool.submit(write, second_writer, hold_commit=False)
+ if not second_writer_requested_job_lock.wait(timeout=10):
+ second_future.result(timeout=1)
+ pytest.fail("second writer never requested the serving-job lock")
+ with pytest.raises(FutureTimeoutError):
+ second_future.result(timeout=0.25)
+ allow_first_writer_to_commit.set()
+ first_future.result(timeout=30)
+ second_future.result(timeout=30)
+ finally:
+ allow_first_writer_to_commit.set()
+ event.remove(engine, "before_cursor_execute", observe_second_job_lock)
+
+ expected_status = "cancelled" if first_writer == "cancel" else "completed"
+ with Session(engine) as session:
+ persisted_run = session.get(LabExecutionRun, run_id)
+ persisted_job = session.get(ServingJob, job_id)
+ gateway = session.scalar(
+ select(GatewayRequest).where(GatewayRequest.id == persisted_run.gateway_request_id)
+ ) if persisted_run else None
+ assert persisted_run is not None and persisted_job is not None and gateway is not None
+ assert persisted_run.status == expected_status
+ assert persisted_job.status == expected_status
+ assert gateway.status == expected_status
+ assert persisted_job.lease_token_hash is None
+ if first_writer == "cancel":
+ assert persisted_run.cancellation_requested_at is not None
+ assert persisted_run.cancellation_acknowledged_at is not None
+ assert persisted_run.output_metadata is None
+ assert persisted_job.result_reference is None
+ assert persisted_job.result_summary["cancelled"] is True
+ else:
+ assert persisted_run.cancellation_requested_at is None
+ assert persisted_run.cancellation_acknowledged_at is None
+ assert persisted_run.output_metadata is not None
+ assert persisted_job.result_reference is None
diff --git a/backend/tests/test_lab_execution_recovery_compatibility.py b/backend/tests/test_lab_execution_recovery_compatibility.py
new file mode 100644
index 0000000..0ebbf48
--- /dev/null
+++ b/backend/tests/test_lab_execution_recovery_compatibility.py
@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import copy
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+ ALERT_NOTIFICATION_SCHEMA,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ fingerprint_session,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+LAB_EXECUTION_SCHEMA = "20260924_0038"
+LAB_EXECUTION_TABLE = "lab_execution_runs"
+
+
+def _as_schema_0037(fingerprint: dict[str, object]) -> dict[str, object]:
+ source = copy.deepcopy(fingerprint)
+ tables = source["tables"]
+ assert isinstance(tables, dict)
+ tables.pop(LAB_EXECUTION_TABLE, None)
+ serving_jobs = tables.get("serving_jobs")
+ if isinstance(serving_jobs, dict):
+ projection = serving_jobs.pop("lab_execution_projection", None)
+ if isinstance(projection, dict):
+ base = projection.get("base")
+ assert isinstance(base, dict)
+ tables["serving_jobs"] = base
+ physical_tables = source.get("physical_tables")
+ assert isinstance(physical_tables, list)
+ source["physical_tables"] = [
+ name for name in physical_tables if name != LAB_EXECUTION_TABLE
+ ]
+ return source
+
+
+def _database_fingerprint() -> dict[str, object]:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with engine.begin() as connection:
+ recreate_0038_benchmark_runs(connection)
+ connection.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32))"))
+ connection.execute(text("INSERT INTO alembic_version VALUES (:revision)"),
+ {"revision": LAB_EXECUTION_SCHEMA})
+ with Session(engine) as session:
+ result = fingerprint_session(session)
+ engine.dispose()
+ return result
+
+
+def _schema_0037_fingerprint_with_future_job_link() -> dict[str, object]:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with engine.begin() as connection:
+ recreate_0038_benchmark_runs(connection)
+ connection.execute(text("DROP TABLE lab_execution_runs"))
+ connection.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32))"))
+ connection.execute(text("INSERT INTO alembic_version VALUES ('20260923_0037')"))
+ with Session(engine) as session:
+ result = fingerprint_session(session)
+ engine.dispose()
+ return result
+
+
+def test_0037_backup_upgrades_only_with_empty_lab_history_and_unlinked_jobs() -> None:
+ restored = _database_fingerprint()
+ source = _as_schema_0037(restored)
+
+ contract = fingerprint_compatibility(
+ source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+
+ assert contract.compatible
+ assert contract.mode.startswith("LAB_EXECUTION_UPGRADE|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+ populated = copy.deepcopy(restored)
+ populated["tables"][LAB_EXECUTION_TABLE]["row_count"] = 1
+ populated["tables"][LAB_EXECUTION_TABLE]["digest"] = "f" * 64
+ rejected_history = fingerprint_compatibility(
+ source, populated, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert not rejected_history.compatible
+
+ linked = copy.deepcopy(restored)
+ linked["tables"]["serving_jobs"]["lab_execution_projection"]["run_link_count"] = 1
+ rejected_link = fingerprint_compatibility(
+ source, linked, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert not rejected_link.compatible
+
+ wrong_empty_link_digest = copy.deepcopy(restored)
+ wrong_empty_link_digest["tables"]["serving_jobs"]["lab_execution_projection"][
+ "run_link_digest"
+ ] = "f" * 64
+ rejected_digest = fingerprint_compatibility(
+ source, wrong_empty_link_digest, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert not rejected_digest.compatible
+
+
+def test_0037_upgrade_preserves_preexisting_jobs_with_null_lab_links() -> None:
+ restored = _database_fingerprint()
+ serving_jobs = restored["tables"]["serving_jobs"]
+ projection = serving_jobs["lab_execution_projection"]
+ serving_jobs["row_count"] = 1
+ serving_jobs["digest"] = "a" * 64
+ projection["base"]["row_count"] = 1
+ projection["base"]["digest"] = "b" * 64
+ source = _as_schema_0037(restored)
+
+ contract = fingerprint_compatibility(
+ source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+
+ assert contract.compatible
+ assert contract.mode.startswith("LAB_EXECUTION_UPGRADE|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+
+def test_pre_0038_source_with_future_lab_table_fails_closed() -> None:
+ fingerprint = _database_fingerprint()
+
+ contract = fingerprint_compatibility(
+ fingerprint, fingerprint, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+
+ assert not contract.compatible
+ assert "future LAB" in contract.reason
+
+
+def test_pre_0038_source_with_future_job_link_column_fails_closed() -> None:
+ source = _schema_0037_fingerprint_with_future_job_link()
+ restored = _database_fingerprint()
+
+ contract = fingerprint_compatibility(
+ source, restored, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+
+ assert source["tables"]["serving_jobs"]["lab_execution_schema_observed"] is True
+ assert not contract.compatible
+ assert "future LAB" in contract.reason
+
+
+def test_0038_strict_restore_preserves_lab_run_history_and_job_links() -> None:
+ source = _database_fingerprint()
+ restored = copy.deepcopy(source)
+
+ contract = fingerprint_compatibility(
+ source, restored, source_schema_revision=LAB_EXECUTION_SCHEMA
+ )
+
+ assert contract.compatible
+ assert contract.mode.startswith("LAB_EXECUTION_STRICT|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+
+ altered = copy.deepcopy(restored)
+ altered["tables"][LAB_EXECUTION_TABLE]["digest"] = "f" * 64
+ rejected = fingerprint_compatibility(
+ source, altered, source_schema_revision=LAB_EXECUTION_SCHEMA
+ )
+ assert not rejected.compatible
+
+
+def test_pre_0038_fingerprint_rejects_lab_history_or_job_links() -> None:
+ current = _database_fingerprint()
+ populated = copy.deepcopy(current)
+ populated["tables"][LAB_EXECUTION_TABLE]["row_count"] = 1
+ populated["tables"][LAB_EXECUTION_TABLE]["digest"] = "f" * 64
+
+ rejected_history = fingerprint_compatibility(
+ populated, populated, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert not rejected_history.compatible
+
+ linked = copy.deepcopy(current)
+ linked["tables"]["serving_jobs"]["lab_execution_projection"]["run_link_count"] = 1
+ rejected_link = fingerprint_compatibility(
+ linked, linked, source_schema_revision=ALERT_NOTIFICATION_SCHEMA
+ )
+ assert not rejected_link.compatible
+
+ malformed_projection = copy.deepcopy(current)
+ malformed_projection["tables"]["serving_jobs"]["lab_execution_projection"][
+ "base"
+ ] = {"row_count": 0, "digest": "bad"}
+ rejected_shape = fingerprint_compatibility(
+ malformed_projection,
+ malformed_projection,
+ source_schema_revision=LAB_EXECUTION_SCHEMA,
+ )
+ assert not rejected_shape.compatible
diff --git a/backend/tests/test_lab_execution_runs.py b/backend/tests/test_lab_execution_runs.py
new file mode 100644
index 0000000..9e6860c
--- /dev/null
+++ b/backend/tests/test_lab_execution_runs.py
@@ -0,0 +1,585 @@
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabExecutionStartRequest
+from modelforge_api.domain.serving import (
+ AgentServingJobCancellationAck,
+ AgentServingJobCancellationCheck,
+ AgentServingJobComplete,
+)
+from modelforge_api.persistence.models import (
+ Base,
+ GatewayRequest,
+ LabExecutionRun,
+ ServingJob,
+)
+from modelforge_api.services.lab_runs import LabRunService
+from modelforge_api.services.serving import ServingError
+from tests.test_dispatch_context import operator_fixture
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+def make_start_request(request, selected) -> LabExecutionStartRequest:
+ return LabExecutionStartRequest(
+ target=request.target,
+ input=request.input,
+ limits=request.limits,
+ deployment_id=selected.capability_deployment_id,
+ experiment_route_id=selected.id,
+ )
+
+
+def test_same_actor_key_and_fingerprint_returns_one_durable_run(session: Session) -> None:
+ serving, preflight, selected = operator_fixture(session)
+ service = LabRunService(serving)
+ request = make_start_request(preflight, selected)
+
+ first = service.start(request, actor_subject="operator-test", idempotency_key="run-001")
+ retry = service.start(request, actor_subject="operator-test", idempotency_key="run-001")
+
+ assert retry.id == first.id
+ assert retry.status == "accepted"
+ assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 1
+ assert "private input content" not in first.model_dump_json()
+ assert "payload_reference" not in first.model_dump()
+
+
+def test_reused_actor_key_with_different_input_is_a_conflict(session: Session) -> None:
+ serving, preflight, selected = operator_fixture(session)
+ service = LabRunService(serving)
+ request = make_start_request(preflight, selected)
+ changed = LabExecutionStartRequest.model_validate(
+ {
+ **request.model_dump(mode="json"),
+ "input": {"capability": "rag.embedding", "payload": {"input": "different"}},
+ }
+ )
+ service.start(request, actor_subject="operator-test", idempotency_key="run-002")
+
+ with pytest.raises(ServingError) as raised:
+ service.start(changed, actor_subject="operator-test", idempotency_key="run-002")
+
+ assert raised.value.status_code == 409
+ assert raised.value.code == "LAB_IDEMPOTENCY_CONFLICT"
+ assert session.scalar(select(func.count()).select_from(LabExecutionRun)) == 1
+
+
+def test_run_status_and_cancel_are_scoped_to_authenticated_actor(session: Session) -> None:
+ serving, preflight, selected = operator_fixture(session)
+ service = LabRunService(serving)
+ run = service.start(
+ make_start_request(preflight, selected),
+ actor_subject="operator-owner",
+ idempotency_key="run-003",
+ )
+
+ for action in (
+ lambda: service.get(run.id, actor_subject="operator-other"),
+ lambda: service.cancel(run.id, actor_subject="operator-other"),
+ ):
+ with pytest.raises(ServingError) as raised:
+ action()
+ assert raised.value.status_code == 404
+ assert raised.value.code == "LAB_RUN_NOT_FOUND"
+
+ assert service.get(run.id, actor_subject="operator-owner").id == run.id
+ cancelled = service.cancel(run.id, actor_subject="operator-owner")
+ assert cancelled.status == "cancelled"
+ assert service.cancel(run.id, actor_subject="operator-owner").status == "cancelled"
+
+
+def _claimed_run(session: Session, actor: str, key: str):
+ serving, preflight, selected = operator_fixture(session)
+ run_service = LabRunService(serving)
+ run_response = run_service.start(
+ make_start_request(preflight, selected), actor_subject=actor, idempotency_key=key
+ )
+ run = session.get(LabExecutionRun, run_response.id)
+ assert run is not None
+ gateway = session.get(GatewayRequest, run.gateway_request_id)
+ assert gateway is not None
+ token = "t" * 32
+ job = ServingJob(
+ capability_deployment_id=selected.capability_deployment_id,
+ compute_node_id=selected.deployment.compute_node_id,
+ gateway_request_id=gateway.id,
+ lab_execution_run_id=run.id,
+ operation="invoke",
+ status="running",
+ priority="lab",
+ idempotency_key=f"test-{key}",
+ payload_reference=run.payload_reference,
+ attempt_count=1,
+ lease_token_hash=hashlib.sha256(token.encode()).hexdigest(),
+ lease_expires_at=datetime.now(UTC) + timedelta(minutes=2),
+ result_summary={},
+ started_at=datetime.now(UTC),
+ )
+ session.add(job)
+ run.status = "running"
+ run.started_at = datetime.now(UTC)
+ run.heartbeat_at = datetime.now(UTC)
+ gateway.status = "running"
+ session.commit()
+ return serving, run_service, run, job, selected, token
+
+
+def _queued_run_with_job(session: Session, actor: str, key: str):
+ serving, preflight, selected = operator_fixture(session)
+ run_service = LabRunService(serving)
+ response = run_service.start(
+ make_start_request(preflight, selected), actor_subject=actor, idempotency_key=key
+ )
+ run = session.get(LabExecutionRun, response.id)
+ assert run is not None
+ gateway = session.get(GatewayRequest, run.gateway_request_id)
+ assert gateway is not None
+ job = ServingJob(
+ capability_deployment_id=selected.capability_deployment_id,
+ compute_node_id=selected.deployment.compute_node_id,
+ gateway_request_id=gateway.id,
+ lab_execution_run_id=run.id,
+ operation="invoke",
+ status="queued",
+ priority="lab",
+ idempotency_key=f"queued-{key}",
+ payload_reference=run.payload_reference,
+ attempt_count=0,
+ result_summary={},
+ )
+ run.status = "queued"
+ gateway.status = "queued"
+ session.add(job)
+ session.commit()
+ return serving, run_service, run, job, selected, preflight
+
+
+def test_cancel_racing_with_worker_completion_discards_vectors_and_acknowledges(
+ session: Session,
+) -> None:
+ serving, run_service, run, job, selected, token = _claimed_run(
+ session, "operator-race", "run-race"
+ )
+ requested = run_service.cancel(run.id, actor_subject="operator-race")
+ assert requested.status == "cancel_requested"
+
+ deployment = serving.repo.deployment(selected.capability_deployment_id)
+ assert deployment is not None
+ node = serving.repo.node(deployment.compute_node_id)
+ space = serving.repo.embedding_space(deployment.embedding_space_id)
+ assert node is not None and space is not None
+ vector = [1.0] + [0.0] * (space.dimension - 1)
+ serving.complete_job(
+ job.id,
+ node,
+ AgentServingJobComplete(
+ lease_token=token,
+ worker_instance_id="worker-cancel-race",
+ state="warm",
+ result={"vectors": [vector], "count": 1, "normalized": True},
+ metrics={"inference_time_ms": 12.0, "private_metric": "must-not-persist"},
+ health={},
+ runtime_facts={"private_runtime_fact": "must-not-persist"},
+ ),
+ )
+
+ session.refresh(run)
+ session.refresh(job)
+ assert run.status == "cancelled"
+ assert run.cancellation_acknowledged_at is not None
+ assert run.output_metadata is None
+ assert job.status == "cancelled"
+ assert job.result_reference is None
+ persisted = f"{run.worker_observations!r} {job.result_summary!r}"
+ assert "private_metric" not in persisted
+ assert "private_runtime_fact" not in persisted
+ assert "must-not-persist" not in persisted
+ assert serving.payload_store.get(run.payload_reference) is None
+
+
+def test_successful_lab_completion_persists_hash_and_allowlisted_metadata_only(
+ session: Session,
+) -> None:
+ serving, _run_service, run, job, selected, token = _claimed_run(
+ session, "operator-success", "run-success"
+ )
+ deployment = serving.repo.deployment(selected.capability_deployment_id)
+ assert deployment is not None
+ node = serving.repo.node(deployment.compute_node_id)
+ space = serving.repo.embedding_space(deployment.embedding_space_id)
+ assert node is not None and space is not None
+ vector = [1.0] + [0.0] * (space.dimension - 1)
+ serving.complete_job(
+ job.id,
+ node,
+ AgentServingJobComplete(
+ lease_token=token,
+ worker_instance_id="worker-success",
+ state="warm",
+ result={"vectors": [vector], "count": 1, "normalized": True},
+ metrics={
+ "inference_time_ms": 12.0,
+ "gpu_uuid": "GPU-test",
+ "private_metric": "must-not-persist",
+ },
+ health={},
+ runtime_facts={"private_runtime_fact": "must-not-persist"},
+ ),
+ )
+
+ session.refresh(run)
+ session.refresh(job)
+ assert run.status == "completed"
+ assert run.output_metadata is not None
+ assert run.output_metadata["dimension"] == space.dimension
+ assert run.output_metadata["item_count"] == 1
+ assert len(run.output_metadata["sha256"]) == 64
+ assert (
+ run.output_metadata["sha256"]
+ == hashlib.sha256(
+ json.dumps([vector], sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+ )
+ assert run.output_metadata["content_logged"] is False
+ assert run.worker_observations is not None
+ assert run.worker_observations["source"] == "runtime-worker"
+ assert "private_metric" not in str(run.worker_observations)
+ assert "private_runtime_fact" not in str(run.worker_observations)
+ assert job.status == "completed"
+ assert job.result_reference is None
+ assert "vectors" not in str(job.result_summary)
+ gateway = session.get(GatewayRequest, run.gateway_request_id)
+ assert gateway is not None
+ persisted_evidence = gateway.decision_evidence["lab_run_evidence"]
+ assert persisted_evidence["status"] == "completed"
+ assert persisted_evidence["output"] == run.output_metadata
+ assert "private_metric" not in str(gateway.decision_evidence)
+ assert "private_runtime_fact" not in str(gateway.decision_evidence)
+ assert "vectors" not in str(gateway.decision_evidence)
+ assert "private input content" not in str(gateway.decision_evidence)
+ assert serving.payload_store.get(run.payload_reference) is None
+
+
+def test_reconciler_dispatches_exact_cold_target_through_load_and_invoke_jobs(
+ session: Session,
+) -> None:
+ serving, preflight, selected = operator_fixture(session)
+ run_service = LabRunService(serving)
+ created = run_service.start(
+ make_start_request(preflight, selected),
+ actor_subject="operator-reconcile",
+ idempotency_key="run-reconcile",
+ )
+
+ serving.reconcile()
+ run = session.get(LabExecutionRun, created.id)
+ assert run is not None
+ assert run.status == "preparing"
+ load_job = session.scalar(
+ select(ServingJob).where(
+ ServingJob.lab_execution_run_id == run.id,
+ ServingJob.operation == "load",
+ )
+ )
+ assert load_job is not None and load_job.status == "queued"
+ node = serving.repo.node(selected.deployment.compute_node_id)
+ assert node is not None
+ load_lease = serving.claim_next(node)
+ assert load_lease is not None and load_lease.job_id == load_job.id
+ assert load_lease.lab_run_id == run.id
+ serving.complete_job(
+ load_job.id,
+ node,
+ AgentServingJobComplete(
+ lease_token=load_lease.lease_token,
+ worker_instance_id="synthetic-worker",
+ state="warm",
+ result={},
+ metrics={
+ "baseline_vram_bytes": 1_000,
+ "loaded_vram_bytes": 3_000,
+ },
+ health={"model": "healthy"},
+ runtime_facts={"offline_local_only": True},
+ ),
+ )
+
+ serving.reconcile()
+ session.refresh(run)
+ assert run.status == "queued"
+ invoke_job = session.scalar(
+ select(ServingJob).where(
+ ServingJob.lab_execution_run_id == run.id,
+ ServingJob.operation == "invoke",
+ )
+ )
+ assert invoke_job is not None and invoke_job.status == "queued"
+ invoke_lease = serving.claim_next(node)
+ assert invoke_lease is not None and invoke_lease.job_id == invoke_job.id
+ assert invoke_lease.deployment_id == selected.capability_deployment_id
+ assert invoke_lease.lab_run_id == run.id
+ assert invoke_lease.input == ["private input content"]
+ assert invoke_lease.expected_dimension is not None
+ deployment = serving.repo.deployment(selected.capability_deployment_id)
+ assert deployment is not None
+ accelerator = serving.repo.accelerator(deployment.accelerator_id)
+ assert accelerator is not None
+ vector = [1.0] + [0.0] * (invoke_lease.expected_dimension - 1)
+ serving.complete_job(
+ invoke_job.id,
+ node,
+ AgentServingJobComplete(
+ lease_token=invoke_lease.lease_token,
+ worker_instance_id="synthetic-worker",
+ state="warm",
+ result={"vectors": [vector], "count": 1, "normalized": True},
+ metrics={
+ "inference_time_ms": 12.0,
+ "gpu_uuid": accelerator.device_uuid,
+ "used_vram_bytes": 3_000,
+ "process_used_vram_bytes": 2_000,
+ "free_vram_bytes": 7_000,
+ "gpu_utilization_percent": 35,
+ "temperature_c": 60,
+ "power_draw_w": 90,
+ "private_metric": "must-not-persist",
+ },
+ health={"model": "healthy"},
+ runtime_facts={"private_runtime_fact": "must-not-persist"},
+ ),
+ )
+
+ session.refresh(run)
+ session.refresh(invoke_job)
+ assert run.status == "completed"
+ assert run.output_metadata is not None
+ assert run.output_metadata["item_count"] == 1
+ assert len(run.output_metadata["sha256"]) == 64
+ assert run.worker_observations is not None
+ assert run.worker_observations["device_matches_target"] is True
+ assert "private_metric" not in str(run.worker_observations)
+ assert "private_runtime_fact" not in str(run.worker_observations)
+ assert invoke_job.result_reference is None
+ assert "vectors" not in str(invoke_job.result_summary)
+ assert (
+ "private input content"
+ not in run_service.get(run.id, actor_subject="operator-reconcile").model_dump_json()
+ )
+
+
+def test_expired_claim_is_interrupted_and_never_reclaimed(session: Session) -> None:
+ serving, _run_service, run, job, selected, _token = _claimed_run(
+ session, "operator-stale", "run-stale"
+ )
+ job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+
+ serving.reconcile()
+
+ session.refresh(run)
+ session.refresh(job)
+ assert run.status == "interrupted"
+ assert run.failure_code == "LAB_WORKER_LEASE_STALE"
+ assert job.status == "failed"
+ assert job.failure_code == "LAB_WORKER_LEASE_STALE"
+ node = serving.repo.node(selected.deployment.compute_node_id)
+ assert node is not None
+ assert serving.claim_next(node) is None
+
+
+@pytest.mark.parametrize("expiry_mode", ["redis_key_missing", "durable_expiry_elapsed"])
+def test_expired_or_missing_redis_input_fails_closed_without_dispatch(
+ session: Session, expiry_mode: str
+) -> None:
+ serving, preflight, selected = operator_fixture(session)
+ run_service = LabRunService(serving)
+ created = run_service.start(
+ make_start_request(preflight, selected),
+ actor_subject=f"operator-expired-{expiry_mode}",
+ idempotency_key=f"expired-{expiry_mode}",
+ )
+ run = session.get(LabExecutionRun, created.id)
+ assert run is not None and run.payload_reference is not None
+ if expiry_mode == "redis_key_missing":
+ # Redis TTL expiry is represented by the payload store returning no key.
+ serving.payload_store.delete(run.payload_reference)
+ else:
+ run.input_payload_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+
+ serving.reconcile()
+
+ session.refresh(run)
+ gateway = session.get(GatewayRequest, run.gateway_request_id)
+ assert gateway is not None
+ assert run.status == "failed"
+ assert run.failure_code == "REQUEST_PAYLOAD_EXPIRED"
+ assert run.payload_reference is None
+ assert gateway.status == "failed"
+ assert session.scalar(
+ select(func.count()).select_from(ServingJob).where(
+ ServingJob.lab_execution_run_id == run.id
+ )
+ ) == 0
+
+
+def test_claim_with_missing_redis_key_fails_run_instead_of_leaving_it_queued(
+ session: Session,
+) -> None:
+ serving, _run_service, run, job, selected, _preflight = _queued_run_with_job(
+ session, "operator-claim-expired", "claim-expired"
+ )
+ assert run.payload_reference is not None
+ serving.payload_store.delete(run.payload_reference)
+ node = serving.repo.node(selected.deployment.compute_node_id)
+ assert node is not None
+
+ lease = serving.claim_next(node)
+
+ session.refresh(run)
+ session.refresh(job)
+ assert lease is None
+ assert run.status == "failed"
+ assert run.failure_code == "REQUEST_PAYLOAD_EXPIRED"
+ assert job.status == "failed"
+ assert job.failure_code == "REQUEST_PAYLOAD_EXPIRED"
+
+
+def test_lab_claim_rejects_another_runs_live_redis_payload(session: Session) -> None:
+ serving, run_service, run, job, selected, preflight = _queued_run_with_job(
+ session, "operator-payload-isolation", "payload-isolation"
+ )
+ changed_request = LabExecutionStartRequest.model_validate(
+ {
+ **make_start_request(preflight, selected).model_dump(mode="json"),
+ "input": {
+ "capability": "rag.embedding",
+ "payload": {"input": "another run's private input"},
+ },
+ }
+ )
+ other = run_service.start(
+ changed_request,
+ actor_subject="operator-payload-isolation",
+ idempotency_key="payload-isolation-other",
+ )
+ assert other.id != run.id
+ assert other.input_summary.sha256 != run.input_sha256
+ other_run_payload = session.get(LabExecutionRun, other.id)
+ assert other_run_payload is not None and other_run_payload.payload_reference is not None
+ job.payload_reference = other_run_payload.payload_reference
+ session.commit()
+ node = serving.repo.node(selected.deployment.compute_node_id)
+ assert node is not None
+
+ lease = serving.claim_next(node)
+
+ session.refresh(run)
+ session.refresh(job)
+ session.refresh(other_run_payload)
+ assert lease is None
+ assert run.status == "failed"
+ assert run.failure_code == "LAB_PAYLOAD_REFERENCE_MISMATCH"
+ assert job.failure_code == "LAB_PAYLOAD_REFERENCE_MISMATCH"
+ assert other_run_payload.status == "accepted"
+
+
+def test_worker_poll_renews_lease_and_ack_completes_cancel_transition(session: Session) -> None:
+ serving, run_service, run, job, selected, token = _claimed_run(
+ session, "operator-ack", "run-ack"
+ )
+ run.timeout_seconds = 300
+ job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
+ session.commit()
+ node = serving.repo.node(selected.deployment.compute_node_id)
+ assert node is not None
+ before = job.lease_expires_at
+ requested = run_service.cancel(run.id, actor_subject="operator-ack")
+ assert requested.status == "cancel_requested"
+
+ state = serving.check_lab_cancellation(
+ job.id, node, AgentServingJobCancellationCheck(lease_token=token)
+ )
+ assert state.cancel_requested is True
+ session.refresh(job)
+ assert job.lease_expires_at is not None and job.lease_expires_at >= before
+ serving.acknowledge_lab_cancellation(
+ job.id,
+ node,
+ AgentServingJobCancellationAck(
+ lease_token=token,
+ worker_instance_id="worker-ack",
+ state="warm",
+ ),
+ )
+
+ session.refresh(run)
+ session.refresh(job)
+ assert run.status == "cancelled"
+ assert run.cancellation_acknowledged_at is not None
+ assert run.worker_observations == {
+ "source": "runtime-worker",
+ "phase": "after_inference",
+ "availability": "cancelled",
+ "safe_boundary_reached": True,
+ "worker_instance_id": "worker-ack",
+ "worker_state": "warm",
+ }
+ assert job.status == "cancelled"
+ assert job.result_reference is None
+
+
+def test_queued_run_cancel_prevents_worker_claim_and_deletes_ttl_input(session: Session) -> None:
+ serving, preflight, selected = operator_fixture(session)
+ run_service = LabRunService(serving)
+ response = run_service.start(
+ make_start_request(preflight, selected),
+ actor_subject="operator-queued-cancel",
+ idempotency_key="queued-cancel",
+ )
+ run = session.get(LabExecutionRun, response.id)
+ assert run is not None
+ payload_reference = run.payload_reference
+ gateway = session.get(GatewayRequest, run.gateway_request_id)
+ assert gateway is not None
+ job = ServingJob(
+ capability_deployment_id=selected.capability_deployment_id,
+ compute_node_id=selected.deployment.compute_node_id,
+ gateway_request_id=gateway.id,
+ lab_execution_run_id=run.id,
+ operation="invoke",
+ status="queued",
+ priority="lab",
+ idempotency_key="queued-cancel-job",
+ payload_reference=run.payload_reference,
+ attempt_count=0,
+ result_summary={},
+ )
+ session.add(job)
+ run.status = "queued"
+ gateway.status = "queued"
+ session.commit()
+
+ cancelled = run_service.cancel(run.id, actor_subject="operator-queued-cancel")
+
+ session.refresh(job)
+ assert cancelled.status == "cancelled"
+ assert job.status == "cancelled"
+ assert job.lease_token_hash is None
+ assert "input_payload_expires_at" in cancelled.model_dump()
+ assert payload_reference is not None
+ assert serving.payload_store.get(payload_reference) is None
diff --git a/backend/tests/test_lab_preflight.py b/backend/tests/test_lab_preflight.py
new file mode 100644
index 0000000..5e29eaf
--- /dev/null
+++ b/backend/tests/test_lab_preflight.py
@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+import uuid
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabPreflightRequest
+from modelforge_api.persistence.models import Base, DeploymentCandidate, RuntimeProfile
+from modelforge_api.services.lab import FAMILY_MODALITIES, LabPreflightService
+from tests.test_serving import setup_m5
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with Session(engine) as db:
+ yield db
+ engine.dispose()
+
+
+INPUTS: dict[str, dict[str, Any]] = {
+ "rag.embedding": {"input": "private input content"},
+ "rag.reranking": {
+ "query": "private query",
+ "documents": [{"id": "a", "text": "private document"}],
+ "top_n": 1,
+ },
+ "document.ocr": {"content_base64": "cHJpdmF0ZQ==", "media_type": "image/png"},
+ "vision.embedding": {"items": [{"text": "private visual text"}]},
+ "speech.transcription": {"audio_base64": "cHJpdmF0ZQ=="},
+}
+
+
+def setup_lab(
+ db: Session, capability: str = "rag.embedding"
+) -> tuple[LabPreflightService, LabPreflightRequest]:
+ serving, candidate_response, _ = setup_m5(db)
+ svc = LabPreflightService(db, serving.settings)
+ candidate = db.get(DeploymentCandidate, candidate_response.id)
+ assert candidate is not None
+ profile = svc.runtime.repo.profile(candidate.runtime_profile_id)
+ probe = svc.runtime.repo.probe(candidate.runtime_probe_id)
+ assert profile is not None and probe is not None
+ # Synthetic completed family evidence, not a hardware qualification.
+ # Construct synthetic family fixture below ORM sealing; production immutability remains tested elsewhere.
+ db.execute(
+ update(RuntimeProfile)
+ .where(RuntimeProfile.id == profile.id)
+ .values(modality=FAMILY_MODALITIES[capability])
+ )
+ if capability == "rag.reranking":
+ probe.inference_result = {"output_type": "ranked_scores", "finite": True}
+ probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
+ elif capability in {"document.ocr", "speech.transcription"}:
+ probe.inference_result = {
+ "output_type": "ocr_text" if capability == "document.ocr" else "transcription",
+ "output_valid": True,
+ }
+ db.commit()
+ request = LabPreflightRequest.model_validate(
+ {
+ "target": {
+ "deployment_candidate_id": candidate.id,
+ "artifact_set_id": candidate.artifact_set_id,
+ "runtime_profile_id": candidate.runtime_profile_id,
+ "compute_node_id": candidate.compute_node_id,
+ "execution_approval_id": probe.execution_approval_id,
+ },
+ "input": {"capability": capability, "payload": INPUTS[capability]},
+ }
+ )
+ return svc, request
+
+
+def table_counts(db: Session) -> dict[str, int]:
+ return {
+ name: db.scalar(select(func.count()).select_from(table)) or 0
+ for name, table in Base.metadata.tables.items()
+ }
+
+
+@pytest.mark.parametrize("capability", INPUTS)
+def test_all_families_return_redacted_read_only_preflight(
+ session: Session, capability: str
+) -> None:
+ svc, request = setup_lab(session, capability)
+ before = table_counts(session)
+ response = svc.preflight(request)
+ assert response.checks_passed, response.blockers
+ assert response.binding is not None
+ assert response.binding.deployment_candidate_id == request.target.deployment_candidate_id
+ assert response.binding.capability == capability
+ assert response.permission_granted is False and response.execution_started is False
+ assert len(response.input_summary.sha256) == 64
+ assert response.input_summary.item_count == 1
+ assert response.input_summary.encoded_bytes > 0
+ encoded = response.model_dump_json()
+ assert "private" not in encoded and "cHJpdmF0ZQ==" not in encoded
+ assert "Reviewed exact" not in encoded
+ assert table_counts(session) == before
+ assert not session.new and not session.dirty and not session.deleted
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "deployment_candidate_id",
+ "artifact_set_id",
+ "runtime_profile_id",
+ "compute_node_id",
+ "execution_approval_id",
+ ],
+)
+def test_target_ids_never_select_another_entity(session: Session, field: str) -> None:
+ svc, request = setup_lab(session)
+ data = request.model_dump()
+ data["target"][field] = uuid.uuid4()
+ response = svc.preflight(LabPreflightRequest.model_validate(data))
+ assert not response.checks_passed and response.binding is None
+
+
+@pytest.mark.parametrize(
+ "change,code",
+ [
+ ("revoked", "LAB_APPROVAL_STALE"),
+ ("expired", "LAB_APPROVAL_STALE"),
+ ("approval_fingerprint", "LAB_APPROVAL_STALE"),
+ ("assessment", "ASSESSMENT_STALE_OR_BLOCKED"),
+ ("profile_fingerprint", "ASSESSMENT_STALE_OR_BLOCKED"),
+ ("production", "CANDIDATE_NOT_LAB_READY"),
+ ("channel", "CANDIDATE_NOT_LAB_READY"),
+ ("offline", "NODE_NOT_READY"),
+ ("lab_ineligible", "NODE_NOT_READY"),
+ ("assessment_blocker", "ASSESSMENT_STALE_OR_BLOCKED"),
+ ("assessment_status", "ASSESSMENT_STALE_OR_BLOCKED"),
+ ("node_protocol", "NODE_NOT_READY"),
+ ("modality", "CAPABILITY_PROFILE_MISMATCH"),
+ ("probe", "PROBE_NOT_COMPLETED"),
+ ("probe_identity", "EVIDENCE_IDENTITY_MISMATCH"),
+ ("invalid_output", "PROBE_OUTPUT_INVALID"),
+ ("wrong_dimension", "PROBE_OUTPUT_INVALID"),
+ ("gpu", "PROBED_ACCELERATOR_MISSING"),
+ ("remote_code", "RUNTIME_POLICY_UNSUPPORTED"),
+ ],
+)
+def test_stale_and_incoherent_evidence_blocks(session: Session, change: str, code: str) -> None:
+ svc, request = setup_lab(session)
+ candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
+ assert candidate is not None
+ repo = svc.runtime.repo
+ approval = repo.approval(request.target.execution_approval_id)
+ assessment = repo.assessment(candidate.compatibility_assessment_id)
+ profile = repo.profile(candidate.runtime_profile_id)
+ node = repo.node(candidate.compute_node_id)
+ probe = repo.probe(candidate.runtime_probe_id)
+ assert approval and assessment and profile and node and probe
+ if change == "revoked":
+ approval.revoked_at = datetime.now(UTC)
+ elif change == "expired":
+ approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ elif change == "approval_fingerprint":
+ approval.evidence_fingerprint = "0" * 64
+ elif change == "assessment":
+ assessment.stale = True
+ elif change == "profile_fingerprint":
+ session.execute(
+ update(RuntimeProfile)
+ .where(RuntimeProfile.id == profile.id)
+ .values(fingerprint="0" * 64)
+ )
+ elif change == "production":
+ candidate.production = True
+ elif change == "channel":
+ candidate.channel = "stable"
+ elif change == "offline":
+ node.liveness_state = "offline"
+ elif change == "lab_ineligible":
+ node.lab_eligible = False
+ elif change == "assessment_blocker":
+ assessment.blockers = ["private reason"]
+ elif change == "assessment_status":
+ assessment.status = "future_unknown"
+ elif change == "node_protocol":
+ node.agent_capabilities = []
+ elif change == "modality":
+ session.execute(
+ update(RuntimeProfile).where(RuntimeProfile.id == profile.id).values(modality="audio")
+ )
+ elif change == "probe":
+ probe.status = "running"
+ elif change == "probe_identity":
+ probe.execution_approval_id = uuid.uuid4()
+ elif change == "invalid_output":
+ probe.inference_result = {"dimension": True, "finite": True}
+ elif change == "wrong_dimension":
+ probe.inference_result = {"dimension": 768, "finite": True}
+ elif change == "gpu":
+ probe.runtime_facts = {"gpu_uuid": "missing"}
+ elif change == "remote_code":
+ session.execute(
+ update(RuntimeProfile)
+ .where(RuntimeProfile.id == profile.id)
+ .values(trust_remote_code=True)
+ )
+ session.commit()
+ response = svc.preflight(request)
+ assert not response.checks_passed and code in response.blockers
+ assert not response.execution_started and not response.permission_granted
+
+
+@pytest.mark.parametrize(
+ "capability,payload",
+ [
+ ("rag.embedding", {"input": []}),
+ ("rag.reranking", {"query": "x", "documents": [], "top_n": 1}),
+ ("document.ocr", {"content_base64": "!!!", "media_type": "image/png"}),
+ (
+ "vision.embedding",
+ {"items": [{"text": "x", "image_base64": "eA==", "media_type": "image/png"}]},
+ ),
+ ("speech.transcription", {"audio_base64": "!!!"}),
+ ],
+)
+def test_malformed_family_input_rejected(
+ session: Session, capability: str, payload: dict[str, Any]
+) -> None:
+ _, request = setup_lab(session)
+ data = request.model_dump()
+ data["input"] = {"capability": capability, "payload": payload}
+ with pytest.raises(ValidationError):
+ LabPreflightRequest.model_validate(data)
+
+
+@pytest.mark.parametrize(
+ "limits",
+ [
+ {"timeout_seconds": 301},
+ {"timeout_seconds": 4},
+ {"timeout_seconds": 5, "queue_timeout_seconds": 6},
+ {"queue_timeout_seconds": 0},
+ {"retention_seconds": 3601},
+ {"max_output_bytes": 1048577},
+ {"concurrency": 2},
+ {"timeout_seconds": True},
+ ],
+)
+def test_budget_limits_are_bounded(session: Session, limits: dict[str, Any]) -> None:
+ _, request = setup_lab(session)
+ data = request.model_dump()
+ data["limits"].update(limits)
+ with pytest.raises(ValidationError):
+ LabPreflightRequest.model_validate(data)
+
+
+def test_embedding_server_limits_and_payload_limit(session: Session) -> None:
+ svc, request = setup_lab(session)
+ data = request.model_dump()
+ data["input"]["payload"]["input"] = ["x" * 9000] * 9
+ response = svc.preflight(LabPreflightRequest.model_validate(data))
+ assert {"INPUT_LIMIT_EXCEEDED", "PAYLOAD_LIMIT_EXCEEDED"}.issubset(response.blockers)
+
+
+def test_preflight_rechecks_evidence_and_does_not_accept_reason(session: Session) -> None:
+ svc, request = setup_lab(session)
+ assert svc.preflight(request).checks_passed
+ approval = svc.runtime.repo.approval(request.target.execution_approval_id)
+ assert approval is not None
+ approval.revoked_at = datetime.now(UTC)
+ session.commit()
+ assert "LAB_APPROVAL_STALE" in svc.preflight(request).blockers
+ with pytest.raises(ValidationError):
+ LabPreflightRequest.model_validate({**request.model_dump(), "reason": "secret reason"})
+
+
+def test_preflight_uses_injected_manifest_root(session: Session, tmp_path: Path) -> None:
+ svc, request = setup_lab(session)
+ isolated_settings = svc.settings.model_copy(update={"config_root": tmp_path})
+ response = LabPreflightService(session, isolated_settings).preflight(request)
+ assert "CAPABILITY_CONTRACT_UNAVAILABLE" in response.blockers
+ assert not response.checks_passed
diff --git a/backend/tests/test_lab_preflight_api.py b/backend/tests/test_lab_preflight_api.py
new file mode 100644
index 0000000..4c4cb91
--- /dev/null
+++ b/backend/tests/test_lab_preflight_api.py
@@ -0,0 +1,80 @@
+from unittest.mock import Mock
+from uuid import uuid4
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.exceptions import RequestValidationError
+from fastapi.testclient import TestClient
+
+from modelforge_api.api.authorization import (
+ Principal,
+ PrincipalRole,
+ authenticate_operator_principal,
+)
+from modelforge_api.api.routes.lab import get_lab_service, router
+from modelforge_api.domain.lab import LabInputSummary, LabLimits, LabPreflightResponse
+from modelforge_api.main import validation_error
+
+
+def request_body() -> dict:
+ return {
+ "target": {name: str(uuid4()) for name in (
+ "deployment_candidate_id", "artifact_set_id", "runtime_profile_id",
+ "compute_node_id", "execution_approval_id",
+ )},
+ "input": {"capability": "rag.embedding", "payload": {"input": "private-input-canary"}},
+ }
+
+
+def harness(role: PrincipalRole | None) -> tuple[TestClient, Mock]:
+ app = FastAPI()
+ app.include_router(router)
+ app.add_exception_handler(RequestValidationError, validation_error) # type: ignore[arg-type]
+
+ def principal() -> Principal:
+ if role is None:
+ raise HTTPException(status_code=401, detail="Authentication required")
+ return Principal("test-operator", role, "test")
+
+ service = Mock()
+ service.preflight.return_value = LabPreflightResponse(
+ input_summary=LabInputSummary(sha256="a" * 64, item_count=1, encoded_bytes=40),
+ limits=LabLimits(), blockers=["CANDIDATE_NOT_FOUND"], checks_passed=False,
+ )
+ app.dependency_overrides[authenticate_operator_principal] = principal
+ app.dependency_overrides[get_lab_service] = lambda: service
+ return TestClient(app), service
+
+
+@pytest.mark.parametrize("role,status", [
+ (None, 401), (PrincipalRole.VIEWER, 403),
+ (PrincipalRole.OPERATOR, 200), (PrincipalRole.ADMIN, 200),
+])
+def test_preflight_requires_operator_without_execution(role: PrincipalRole | None, status: int) -> None:
+ client, service = harness(role)
+ response = client.post("/api/v1/lab/preflight", json=request_body())
+ assert response.status_code == status
+ assert service.preflight.call_count == int(status == 200)
+ if status == 200:
+ assert response.json()["execution_started"] is False
+ assert response.json()["permission_granted"] is False
+ assert "private-input-canary" not in response.text
+
+
+@pytest.mark.parametrize("location", ["discriminator", "payload", "extra"])
+def test_validation_never_reflects_payload_or_discriminator(location: str) -> None:
+ client, service = harness(PrincipalRole.OPERATOR)
+ body = request_body()
+ canary = "sensitive-lab-validation-canary"
+ if location == "discriminator":
+ body["input"]["capability"] = canary
+ elif location == "payload":
+ body["input"]["payload"]["input"] = {canary: canary}
+ else:
+ body[canary] = canary
+ response = client.post("/api/v1/lab/preflight", json=body)
+ assert response.status_code == 422
+ assert canary not in response.text
+ assert "private-input-canary" not in response.text
+ assert response.json()["error"]["code"] == "request_validation_failed"
+ service.preflight.assert_not_called()
diff --git a/backend/tests/test_lifecycle_m12.py b/backend/tests/test_lifecycle_m12.py
index 0ec80aa..1c1968f 100644
--- a/backend/tests/test_lifecycle_m12.py
+++ b/backend/tests/test_lifecycle_m12.py
@@ -2,13 +2,18 @@ from __future__ import annotations
import uuid
from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, delete, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
+from starlette.requests import Request
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.api.routes.lifecycle import get_lifecycle_service
+from modelforge_api.api.routes.lifecycle import observe_canary as observe_canary_route
from modelforge_api.domain.lifecycle_contracts import (
ApprovalDecision,
ApprovalPolicyCreate,
@@ -31,12 +36,14 @@ from modelforge_api.persistence.models import (
ArtifactLocationRemovalRecord,
ArtifactSet,
ArtifactSetMember,
+ AuditEvent,
Base,
CapabilityDeployment,
CapabilityResourceEnvelope,
ComputeNode,
DeploymentCandidate,
LifecycleApprovalRequest,
+ LifecycleCanaryRun,
LifecycleEvent,
LifecyclePolicyRevision,
LifecyclePromotionPlan,
@@ -48,6 +55,7 @@ from modelforge_api.persistence.models import (
RuntimeProfile,
StorageRoot,
)
+from modelforge_api.services import audit as audit_service
from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
from modelforge_api.settings import Settings, get_settings
from tests.test_runtime import setup_runtime
@@ -402,6 +410,128 @@ def test_canary_auto_abort_restores_snapshot(session: Session) -> None:
)
+@pytest.mark.parametrize("scope_change", [
+ "subject_environment", "subject_target", "approval_environment", "approval_subject",
+ "approval_target", "approval_ref", "approval_transition", "policy_scope", "plan_desired",
+])
+@pytest.mark.parametrize("legacy_ready", [False, True])
+def test_manual_canary_rejects_persisted_production_scope(
+ session: Session, scope_change: str, legacy_ready: bool,
+) -> None:
+ service, subject, plan = approved_lab_plan(session)
+ operation = service.execute_plan(
+ plan.id,
+ PlanExecutionCreate(
+ executor="executor-a", idempotency_key="scope-test",
+ expected_subject_version=subject.version,
+ ),
+ )
+ stored_plan = session.get(LifecyclePromotionPlan, plan.id)
+ assert stored_plan is not None
+ approval = session.get(LifecycleApprovalRequest, stored_plan.approval_request_id)
+ assert approval is not None
+ changes = {
+ "subject_environment": (LifecycleSubject, subject.id, {"environment": "PRODUCTION"}),
+ "subject_target": (LifecycleSubject, subject.id, {"target_type": "CAPABILITY"}),
+ "approval_environment": (
+ LifecycleApprovalRequest, approval.id, {"environment": "PRODUCTION"},
+ ),
+ "approval_subject": (LifecycleApprovalRequest, approval.id, {"subject_id": None}),
+ "approval_target": (
+ LifecycleApprovalRequest, approval.id, {"target_type": "CAPABILITY"},
+ ),
+ "approval_ref": (LifecycleApprovalRequest, approval.id, {"target_ref": "another"}),
+ "approval_transition": (
+ LifecycleApprovalRequest, approval.id, {"requested_transition": "STABLE"},
+ ),
+ "policy_scope": (
+ LifecyclePolicyRevision, approval.policy_revision_id, {"scope": "PRODUCTION"},
+ ),
+ "plan_desired": (LifecyclePromotionPlan, plan.id, {"desired_state": "STABLE"}),
+ }
+ model, identity, values = changes[scope_change]
+ session.execute(update(model).where(model.id == identity).values(**values))
+ if legacy_ready:
+ session.execute(update(LifecycleCanaryRun).where(
+ LifecycleCanaryRun.operation_id == operation.id,
+ ).values(status="READY_FOR_PROMOTION_REVIEW", request_count=2, result={}))
+ session.commit()
+ before = service.operation(operation.id).model_dump(mode="json")
+ before_events = list(session.scalars(select(LifecycleEvent.id)))
+ before_audit = list(session.scalars(select(AuditEvent.id)))
+ with pytest.raises(LifecycleError) as denied:
+ if legacy_ready:
+ service.commit_operation(operation.id, ApprovalDecision(
+ actor="executor-a", reason="Legacy ready observation is not authoritative",
+ ))
+ else:
+ service.observe_canary(operation.id, CanaryObservation(
+ request_count=2, error_count=0, latency_p95_ms=10,
+ capability_health="HEALTHY", scheduler_ready=True,
+ ))
+ assert denied.value.status_code == 409
+ assert denied.value.code == "MANUAL_CANARY_REHEARSAL_ONLY"
+ assert service.operation(operation.id).model_dump(mode="json") == before
+ assert list(session.scalars(select(LifecycleEvent.id))) == before_events
+ assert list(session.scalars(select(AuditEvent.id))) == before_audit
+
+
+@pytest.mark.parametrize("abort", [False, True])
+@pytest.mark.parametrize("audit_failure", [False, True])
+def test_manual_lab_canary_route_attribution_and_atomic_audit(
+ session: Session, monkeypatch: pytest.MonkeyPatch, abort: bool, audit_failure: bool,
+) -> None:
+ service, subject, plan = approved_lab_plan(session)
+ operation = service.execute_plan(plan.id, PlanExecutionCreate(
+ executor="executor-a", idempotency_key="attribution-test",
+ expected_subject_version=subject.version,
+ ))
+ before = service.operation(operation.id).model_dump(mode="json")
+ before_events = list(session.scalars(select(LifecycleEvent.id)))
+ http_request = Request({"type": "http"})
+ http_request.state.correlation_id = "manual-lab-request-42"
+ principal = Principal(subject="named-admin", role=PrincipalRole.ADMIN,
+ authentication_method="named_operator_key")
+
+ def invoke() -> object:
+ return observe_canary_route(operation.id, CanaryObservation(
+ request_count=2, error_count=1 if abort else 0, latency_p95_ms=10,
+ capability_health="HEALTHY", scheduler_ready=True,
+ ), service, principal, http_request)
+
+ if audit_failure:
+ def fail_append(*args: object, **kwargs: object) -> None:
+ raise RuntimeError("audit unavailable")
+ monkeypatch.setattr(audit_service, "_append_canonical_audit_event", fail_append)
+ with pytest.raises(RuntimeError, match="audit unavailable"):
+ invoke()
+ assert service.operation(operation.id).model_dump(mode="json") == before
+ assert list(session.scalars(select(LifecycleEvent.id))) == before_events
+ assert session.get(LifecycleSubject, subject.id).state == "CANARY"
+ else:
+ invoke()
+ audit = session.scalar(select(AuditEvent).where(
+ AuditEvent.action == "lifecycle.manual_lab_canary_observed",
+ ))
+ assert audit is not None
+ assert audit.actor_id == "named-admin"
+ assert audit.correlation_id == "manual-lab-request-42"
+ canary = session.scalar(select(LifecycleCanaryRun).where(
+ LifecycleCanaryRun.operation_id == operation.id,
+ ))
+ assert canary is not None
+ assert canary.result["provenance"] == {
+ "source": "MANUAL_LAB_REHEARSAL", "authoritative": False,
+ "actor": "named-admin", "correlation_id": "manual-lab-request-42",
+ }
+ event = session.scalar(select(LifecycleEvent).where(
+ LifecycleEvent.event_type == (
+ "CANARY_ABORTED" if abort else "CANARY_READY_FOR_PROMOTION_REVIEW"
+ ),
+ ))
+ assert event is not None and event.actor == "named-admin"
+
+
def test_restart_reconciliation_and_cas_are_conservative(session: Session) -> None:
service, subject_value, plan = approved_lab_plan(session, target="restart-lab")
with pytest.raises(LifecycleError, match="changed concurrently"):
@@ -806,6 +936,40 @@ def test_lifecycle_routes_deny_capability_credentials() -> None:
app.dependency_overrides.clear()
+def test_manual_canary_http_denial_is_typed_and_preserves_authentication() -> None:
+ contexts = []
+
+ def deny_observation(operation_id: object, observation: object, *, audit_context: object) -> None:
+ contexts.append(audit_context)
+ raise LifecycleError(409, "MANUAL_CANARY_REHEARSAL_ONLY", "LAB rehearsal required")
+
+ app.dependency_overrides[get_settings] = lambda: Settings(operator_api_key="operator-secret")
+ app.dependency_overrides[get_lifecycle_service] = lambda: SimpleNamespace(
+ observe_canary=deny_observation,
+ )
+ try:
+ client = TestClient(app)
+ url = f"/api/v1/admin/lifecycle/operations/{uuid.uuid4()}/canary"
+ body = {"request_count": 2, "error_count": 0, "latency_p95_ms": 10,
+ "capability_health": "HEALTHY", "scheduler_ready": True}
+ assert client.post(url, json=body).status_code == 401
+ assert client.post(url, json=body, headers={
+ "X-ModelForge-Admin-Token": "mfsvc_project-key",
+ }).status_code == 401
+ assert contexts == []
+ result = client.post(url, json=body, headers={
+ "X-ModelForge-Admin-Token": "operator-secret",
+ "X-Correlation-ID": "canary-denial-42",
+ })
+ assert result.status_code == 409
+ assert result.json()["error"]["code"] == "MANUAL_CANARY_REHEARSAL_ONLY"
+ assert result.json()["error"]["correlation_id"] == "canary-denial-42"
+ assert contexts[0].actor_id == "legacy-operator"
+ assert contexts[0].correlation_id == "canary-denial-42"
+ finally:
+ app.dependency_overrides.clear()
+
+
def test_blocked_approval_is_audited_without_payload_content(session: Session) -> None:
service = LifecycleService(session)
result = service.request_approval(
diff --git a/backend/tests/test_lifecycle_promotion_races_postgres.py b/backend/tests/test_lifecycle_promotion_races_postgres.py
new file mode 100644
index 0000000..38a79c0
--- /dev/null
+++ b/backend/tests/test_lifecycle_promotion_races_postgres.py
@@ -0,0 +1,131 @@
+"""Real PostgreSQL qualification for approval revocation versus serving promotion."""
+
+from __future__ import annotations
+
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FutureTimeoutError
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session
+
+pytest_plugins = ("tests.test_managed_identities_postgres",)
+
+from modelforge_api.domain.lifecycle_contracts import ApprovalDecision
+from modelforge_api.persistence.models import (
+ CapabilityContract,
+ LifecycleApprovalRequest,
+ LifecycleEvent,
+ LifecycleOperation,
+)
+from modelforge_api.persistence.serving_repository import ServingRepository
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from tests.test_lifecycle_serving import approved_plan, build_pair
+from tests.test_managed_identities_postgres import IdentityDatabase
+
+
+def test_revoked_approval_fences_inflight_serving_promotion(
+ identity_database: IdentityDatabase,
+) -> None:
+ """A commit waiting on the approval row must see the revocation, not switch the route."""
+ importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+ engine = identity_database.runtime
+ with Session(engine) as session:
+ pair = build_pair(session)
+ _, _, current, candidate, selection, _ = pair
+ _, plan, operation, _ = approved_plan(pair)
+ approval_id = plan.approval_request_id
+ operation_id = operation.id
+ contract_id = current.capability_contract_id
+ current_id, candidate_id = current.id, candidate.id
+ original_generation = selection.expected_route_generation
+ baseline_events = set(session.scalars(select(LifecycleEvent.id)))
+
+ revoker_holds_lock = threading.Event()
+ release_revoker = threading.Event()
+ promoter_requested_lock = threading.Event()
+
+ def revoke() -> None:
+ with Session(engine) as session:
+ original_commit = session.commit
+
+ def commit_after_gate() -> None:
+ revoker_holds_lock.set()
+ if not release_revoker.wait(timeout=20):
+ raise RuntimeError("timed out holding the approval revocation")
+ original_commit()
+
+ session.commit = commit_after_gate # type: ignore[method-assign]
+ LifecycleService(session).decide_approval(
+ approval_id,
+ ApprovalDecision(actor="qualification", reason="Revoke before serving promotion"),
+ "revoke",
+ )
+
+ def promote() -> str:
+ with Session(engine) as session:
+ try:
+ LifecycleService(session).commit_operation(
+ operation_id,
+ ApprovalDecision(actor="qualification", reason="Attempt reviewed route change"),
+ )
+ except LifecycleError as error:
+ session.rollback()
+ return error.code
+ return "COMMITTED"
+
+ @event.listens_for(engine, "before_cursor_execute")
+ def observe_approval_lock(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ if (
+ threading.current_thread().name.startswith("promotion-race-promoter")
+ and "lifecycle_approval_requests" in statement.lower()
+ and "for update" in statement.lower()
+ ):
+ promoter_requested_lock.set()
+
+ try:
+ with ThreadPoolExecutor(
+ max_workers=1, thread_name_prefix="promotion-race-revoker"
+ ) as first:
+ revoked = first.submit(revoke)
+ if not revoker_holds_lock.wait(timeout=10):
+ revoked.result(timeout=1)
+ pytest.fail("revoker did not reach the held commit")
+ with ThreadPoolExecutor(
+ max_workers=1, thread_name_prefix="promotion-race-promoter"
+ ) as second:
+ promoted = second.submit(promote)
+ if not promoter_requested_lock.wait(timeout=10):
+ promoted.result(timeout=1)
+ pytest.fail("promoter did not request the approval lock")
+ with pytest.raises(FutureTimeoutError):
+ promoted.result(timeout=0.25)
+ release_revoker.set()
+ revoked.result(timeout=30)
+ assert promoted.result(timeout=30) == "APPROVAL_STALE"
+ finally:
+ release_revoker.set()
+ event.remove(engine, "before_cursor_execute", observe_approval_lock)
+
+ with Session(engine) as session:
+ approval = session.get(LifecycleApprovalRequest, approval_id)
+ operation = session.get(LifecycleOperation, operation_id)
+ contract = session.get(CapabilityContract, contract_id)
+ assert approval is not None and approval.status == "REVOKED"
+ assert operation is not None and operation.stage == "VERIFYING"
+ assert contract is not None and contract.serving_generation == original_generation
+ route = ServingRepository(session).stable_deployment(contract_id)
+ assert route is not None and route.id == current_id and route.id != candidate_id
+ events = list(
+ session.scalars(select(LifecycleEvent).where(LifecycleEvent.id.not_in(baseline_events)))
+ )
+ assert [item.event_type for item in events] == ["APPROVAL_REVOKED"]
diff --git a/backend/tests/test_lifecycle_serving.py b/backend/tests/test_lifecycle_serving.py
new file mode 100644
index 0000000..0d075b2
--- /dev/null
+++ b/backend/tests/test_lifecycle_serving.py
@@ -0,0 +1,422 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+ EvaluationCaseCreate,
+ EvaluationRevisionCreate,
+ EvaluationSuiteCreate,
+)
+from modelforge_api.domain.lifecycle_contracts import (
+ ApprovalDecision,
+ ApprovalRequestCreate,
+ LifecycleEvidenceBundle,
+ LifecycleSubjectCreate,
+ PlanExecutionCreate,
+ PromotionPlanCreate,
+ ServingTransitionSelection,
+)
+from modelforge_api.domain.selected_model_comparison import ComparisonCreate
+from modelforge_api.domain.serving import EmbeddingInvokeRequest, ServiceClientCreate
+from modelforge_api.persistence.models import (
+ ArtifactSet,
+ Base,
+ CapabilityContract,
+ CapabilityDeployment,
+ CapabilityExperimentRoute,
+ CapabilityResourceEnvelope,
+ DeploymentCandidate,
+ EmbeddingSpace,
+ GatewayRequest,
+ LifecycleApprovalRequest,
+ LifecycleEvent,
+ LifecycleOperation,
+ LifecycleRetentionRecord,
+ LifecycleSubject,
+ Project,
+ ProjectBinding,
+ ProjectFitEvidence,
+ ResidencyAllocation,
+ RuntimeProfile,
+ SelectedModelComparison,
+ ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from modelforge_api.services.lifecycle_serving import LifecycleServing, digest
+from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
+from modelforge_api.services.serving import ServingError
+from tests.test_selected_model_comparison import _claimed
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+def _clone(model, row, **changes):
+ return model(**{column.name: getattr(row, column.name) for column in model.__table__.columns
+ if column.name not in {"id", "created_at", "updated_at"}} | changes)
+
+
+@pytest.fixture
+def serving_pair():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ yield build_pair(session)
+ engine.dispose()
+
+
+def build_pair(session):
+ serving, candidate_ref, _ = setup_m5(session)
+ production_approval, deployed = approve_and_promote(serving, candidate_ref.id)
+ current = session.get(CapabilityDeployment, deployed.id)
+ candidate = _clone(CapabilityDeployment, current, config_fingerprint="b" * 64,
+ channel="experiment", status="candidate", production=False, production_approval_id=None,
+ promoted_at=None)
+ session.add(candidate)
+ session.flush()
+ envelope = session.scalar(select(CapabilityResourceEnvelope).where(
+ CapabilityResourceEnvelope.capability_deployment_id == current.id))
+ next_envelope = _clone(CapabilityResourceEnvelope, envelope, capability_deployment_id=candidate.id)
+ session.add(next_envelope)
+ residency = session.scalar(select(ResidencyAllocation).where(
+ ResidencyAllocation.capability_deployment_id == current.id))
+ session.add(_clone(ResidencyAllocation, residency, capability_deployment_id=candidate.id))
+ project = Project(key="selected-owner", name="Owner", description="test")
+ session.add(project)
+ session.flush()
+ binding = ProjectBinding(project_id=project.id, capability_contract_id=current.capability_contract_id,
+ channel="stable", priority="production")
+ session.add(binding)
+ session.flush()
+ client = ServiceClient(name="selected-consumer", allowed_capabilities=["rag.embedding@1"],
+ project_binding_id=binding.id)
+ route = CapabilityExperimentRoute(route_key="selected-test", capability_contract_id=current.capability_contract_id,
+ capability_deployment_id=candidate.id, purpose="Bounded comparison", evidence={})
+ session.add_all((client, route))
+ session.commit()
+ from modelforge_api.services.evaluation import EvaluationService
+
+ relevant = uuid4()
+ suite = EvaluationService(session).create_suite(EvaluationSuiteCreate(project_id=project.id,
+ key="selected", name="Selected", description="One case", revision=EvaluationRevisionCreate(
+ revision="v1", dataset_revision="selected-v1", retrieval_settings={"top_k": 10}, cases=[
+ EvaluationCaseCreate(case_key="one", query="Exact unchanged query", relevant_chunk_ids=[relevant],
+ critical=True, review_status="reviewed", label_provenance={"source": "review", "corpus_revision": "a" * 64})])))
+ comparisons = SelectedModelComparisonService(session, AuditContext("operator", "test"))
+ request = ComparisonCreate(project_binding_id=binding.id, consumer_client_id=client.id,
+ suite_revision_id=suite.latest_revision_id, current_deployment_id=current.id,
+ candidate_deployment_id=candidate.id, candidate_route_id=route.id, corpus_sha256="a" * 64,
+ retrieval_config_sha256=digest({"top_k": 10}), current_index_ref="current-index",
+ candidate_index_ref="isolated-test-index", idempotency_key="selected-one")
+ claim, result, _ = _claimed((session, comparisons, request, client, relevant))
+ completed = comparisons.result(claim.comparison.id, result, client)
+ contract = session.get(CapabilityContract, current.capability_contract_id)
+ selection = ServingTransitionSelection(current_deployment_id=current.id, candidate_deployment_id=candidate.id,
+ production_approval_id=production_approval.id, expected_route_generation=contract.serving_generation,
+ comparison_ids={binding.id: completed.id})
+ artifact = session.get(ArtifactSet, candidate.artifact_set_id)
+ profile = session.get(RuntimeProfile, candidate.runtime_profile_id)
+ source = session.get(DeploymentCandidate, candidate.deployment_candidate_id)
+ evidence = LifecycleEvidenceBundle(serving_transition=selection,
+ artifact_set_id=artifact.id, model_revision_id=artifact.revision_id,
+ runtime_profile_id=profile.id, runtime_probe_id=source.runtime_probe_id,
+ runtime_image_digest=profile.runtime_image_digest, capability_deployment_id=candidate.id,
+ production_execution_approval_id=production_approval.id, resource_envelope_id=next_envelope.id,
+ embedding_space_id=candidate.embedding_space_id, rollback_target_ref=str(current.id),
+ evaluation_run_ids=[completed.baseline_run_id, completed.candidate_run_id],
+ integrity="VERIFIED", security="APPROVED", license="APPROVED", runtime="PROVEN",
+ evaluation="PROMOTION_ELIGIBLE", production_validation="SATISFIED",
+ scheduler_readiness="READY", evidence_revisions={"scheduler": "fixture"})
+ return session, serving, current, candidate, selection, evidence
+
+
+def approved_plan(pair):
+ session, _, current, candidate, _, evidence = pair
+ service = LifecycleService(session)
+ subject = service.create_subject(LifecycleSubjectCreate(target_type="CAPABILITY_DEPLOYMENT",
+ target_ref=str(candidate.id), environment="PRODUCTION", state="PROMOTION_ELIGIBLE"))
+ approval = service.request_approval(ApprovalRequestCreate(policy_key="capability-production",
+ subject_id=subject.id, target_type=subject.target_type, target_ref=subject.target_ref,
+ environment="PRODUCTION", requested_transition="STABLE", evidence=evidence,
+ requested_by="owner", reason="Review exact pair and all affected projects"))
+ assert approval.status == "PENDING", approval.blockers
+ service.decide_approval(approval.id, ApprovalDecision(actor="owner", reason="Evidence reviewed for every project"), "approve")
+ plan = service.create_plan(PromotionPlanCreate(approval_request_id=approval.id, subject_id=subject.id,
+ desired_state="STABLE", migration_class="transparent", candidate_deployment_id=candidate.id,
+ rollback_target_ref=str(current.id), created_by="owner"))
+ plan = service.approve_plan(plan.id, ApprovalDecision(actor="owner", reason="Exact immutable transition approved"))
+ operation = service.execute_plan(plan.id, PlanExecutionCreate(executor="owner", idempotency_key="execute-one",
+ expected_subject_version=subject.version))
+ assert operation.stage == "VERIFYING"
+ return service, plan, operation, subject
+
+
+def test_commit_and_rollback_switch_real_route_and_retain_exact_assets(serving_pair):
+ session, serving, current, candidate, selection, _ = serving_pair
+ before = (LifecycleServing.route_snapshot(current), LifecycleServing.route_snapshot(candidate))
+ service, plan, operation, subject = approved_plan(serving_pair)
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+ result = service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit permanent replacement"))
+ assert result.stage == "COMMITTED"
+ assert service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Idempotent committed result retry")).stage == "COMMITTED"
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
+ assert current.status == "draining" and candidate.production is True
+ assert session.get(LifecycleSubject, subject.id).state == "STABLE"
+ assert session.get(CapabilityContract, current.capability_contract_id).serving_generation == selection.expected_route_generation + 1
+ retained = list(session.scalars(select(LifecycleRetentionRecord)))
+ assert any(item.target_type == "ARTIFACT_SET" and item.target_ref == str(current.artifact_set_id) for item in retained)
+ assert plan.project_consumers == [session.get(ProjectBinding, next(iter(selection.comparison_ids))).project_id.__str__()]
+ rolled = service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Restore exact previous serving route"))
+ assert rolled.stage == "ROLLED_BACK"
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+ assert (LifecycleServing.route_snapshot(current), LifecycleServing.route_snapshot(candidate)) == before
+ assert session.get(CapabilityContract, current.capability_contract_id).serving_generation == selection.expected_route_generation + 2
+ assert service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Idempotent rollback repeat")).stage == "ROLLED_BACK"
+
+
+def test_commit_and_rollback_route_live_gateway_requests_to_exact_deployments(serving_pair):
+ session, serving, current, candidate, _, _ = serving_pair
+ gateway = InlineWorkerServingService(
+ session,
+ serving.settings,
+ serving.manifests,
+ serving.payload_store,
+ )
+ created = gateway.create_client(
+ ServiceClientCreate(
+ name="lifecycle-route-proof",
+ allowed_capabilities=["rag.embedding@1"],
+ )
+ )
+ service, _, operation, _ = approved_plan(serving_pair)
+
+ service.commit_operation(
+ operation.id,
+ ApprovalDecision(actor="owner", reason="Route proof commits the reviewed replacement"),
+ )
+ candidate_client = gateway.authenticate(
+ f"Bearer {created.credential}", "rag.embedding@1"
+ )
+ candidate_response = gateway.invoke(
+ EmbeddingInvokeRequest(input="request after replacement"), candidate_client
+ )
+ candidate_request = session.scalar(
+ select(GatewayRequest).where(GatewayRequest.request_id == candidate_response.request_id)
+ )
+ assert candidate_response.embedding_space_id == candidate.embedding_space_id
+ assert candidate_request is not None
+ assert candidate_request.capability_deployment_id == candidate.id
+
+ service.rollback_operation(
+ operation.id,
+ ApprovalDecision(actor="owner", reason="Route proof restores the exact prior deployment"),
+ )
+ current_client = gateway.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+ current_response = gateway.invoke(
+ EmbeddingInvokeRequest(input="request after rollback"), current_client
+ )
+ current_request = session.scalar(
+ select(GatewayRequest).where(GatewayRequest.request_id == current_response.request_id)
+ )
+ assert current_response.embedding_space_id == current.embedding_space_id
+ assert current_request is not None
+ assert current_request.capability_deployment_id == current.id
+
+
+@pytest.mark.parametrize("mutation", ["generation", "bindings", "revoked", "comparison", "envelope"])
+def test_changed_authority_or_route_blocks_commit(serving_pair, mutation):
+ session, serving, current, candidate, selection, _ = serving_pair
+ service, _, operation, _ = approved_plan(serving_pair)
+ if mutation == "generation":
+ session.get(CapabilityContract, current.capability_contract_id).serving_generation += 1
+ elif mutation == "bindings":
+ project = Project(key="second", name="Second", description="test")
+ session.add(project)
+ session.flush()
+ session.add(ProjectBinding(project_id=project.id, capability_contract_id=current.capability_contract_id,
+ channel="stable", priority="production"))
+ elif mutation == "revoked":
+ serving.repo.approval(selection.production_approval_id).revoked_at = datetime.now(UTC)
+ elif mutation == "comparison":
+ session.get(SelectedModelComparison, next(iter(selection.comparison_ids.values()))).status = "failed"
+ else:
+ session.scalar(select(CapabilityResourceEnvelope).where(
+ CapabilityResourceEnvelope.capability_deployment_id == candidate.id)).stale = True
+ session.commit()
+ with pytest.raises((LifecycleError, ServingError)):
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Must reject changed authority"))
+ session.rollback()
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+ assert session.get(LifecycleOperation, operation.id).stage == "VERIFYING"
+
+
+def test_equal_dimension_different_space_requires_consumer_cutover(serving_pair):
+ session, _, current, candidate, selection, _ = serving_pair
+ old_space = session.get(EmbeddingSpace, current.embedding_space_id)
+ space = _clone(EmbeddingSpace, old_space, identity_digest="d" * 64)
+ session.add(space)
+ session.flush()
+ candidate.embedding_space_id = space.id
+ session.commit()
+ with pytest.raises(LifecycleError, match="consumer reindex"):
+ LifecycleServing(session).prepare(selection)
+
+
+@pytest.mark.parametrize("mutation", ["generation", "binding", "config"])
+def test_rollback_preserves_later_route_or_configuration(serving_pair, mutation):
+ session, serving, current, candidate, _, _ = serving_pair
+ service, _, operation, _ = approved_plan(serving_pair)
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit serving replacement"))
+ if mutation == "generation":
+ # Even an A->B->A route cycle must fence the old operation.
+ session.get(CapabilityContract, current.capability_contract_id).serving_generation += 2
+ elif mutation == "binding":
+ session.scalar(select(ProjectBinding).where(ProjectBinding.capability_contract_id == current.capability_contract_id)).priority = "background"
+ else:
+ candidate.max_queue_depth += 1
+ session.commit()
+ with pytest.raises(LifecycleError, match="preserved|changed"):
+ service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Old operation cannot overwrite later state"))
+ session.rollback()
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
+ assert session.get(LifecycleOperation, operation.id).stage == "COMMITTED"
+
+
+def test_subject_conflict_rolls_back_both_route_and_generation(serving_pair):
+ session, serving, current, _, selection, _ = serving_pair
+ service, _, operation, subject = approved_plan(serving_pair)
+ session.execute(update(LifecycleSubject).where(LifecycleSubject.id == subject.id).values(version=100))
+ session.commit()
+ with pytest.raises(LifecycleError, match="subject changed"):
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Reject concurrent subject transition"))
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+ assert session.get(CapabilityContract, current.capability_contract_id).serving_generation == selection.expected_route_generation
+
+
+def test_revoked_lifecycle_approval_cannot_commit(serving_pair):
+ session, serving, current, _, _, _ = serving_pair
+ service, plan, operation, _ = approved_plan(serving_pair)
+ session.get(LifecycleApprovalRequest, plan.approval_request_id).status = "REVOKED"
+ session.commit()
+ with pytest.raises(LifecycleError, match="no longer current"):
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Reject revoked lifecycle approval"))
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+@pytest.mark.parametrize("mutation", ["revoked", "quarantined", "offline"])
+def test_rollback_cannot_reactivate_invalid_prior_production_authority(serving_pair, mutation):
+ session, serving, current, candidate, _, _ = serving_pair
+ service, _, operation, _ = approved_plan(serving_pair)
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit serving replacement"))
+ if mutation == "revoked":
+ serving.repo.approval(current.production_approval_id).revoked_at = datetime.now(UTC)
+ elif mutation == "quarantined":
+ serving.repo.set_artifacts(current.artifact_set_id)[0].quarantined = True
+ else:
+ serving.repo.node(current.compute_node_id).liveness_state = "offline"
+ session.commit()
+ with pytest.raises((LifecycleError, ServingError)):
+ service.rollback_operation(operation.id, ApprovalDecision(actor="owner", reason="Refuse a revoked rollback target"))
+ session.rollback()
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
+
+
+@pytest.mark.parametrize("validation", ["DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "NOT_REQUIRED"])
+def test_actual_serving_approval_requires_satisfied_validation(serving_pair, validation):
+ serving_pair[-1].production_validation = validation
+ with pytest.raises(LifecycleError, match="satisfied production validation"):
+ approved_plan(serving_pair)
+
+
+def test_deferred_historical_approval_cannot_commit_actual_route(serving_pair):
+ session, serving, current, _, _, _ = serving_pair
+ service, plan, operation, _ = approved_plan(serving_pair)
+ approval = session.get(LifecycleApprovalRequest, plan.approval_request_id)
+ # Simulate a previously accepted capability-only snapshot; its declared deferred
+ # validation must not gain operational authority under the new serving adapter.
+ approval.evidence_snapshot = {**approval.evidence_snapshot, "declared": {
+ **approval.evidence_snapshot["declared"], "production_validation": "DEFERRED_EXTERNAL_VALIDATION"}}
+ session.commit()
+ with pytest.raises(LifecycleError, match="no longer current"):
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Reject explicitly deferred validation"))
+ session.rollback()
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+def _fit(pair, *, deployment_id=None, **changes):
+ session, _, _, candidate, selection, _ = pair
+ values = {"project_binding_id": next(iter(selection.comparison_ids)),
+ "capability_deployment_id": deployment_id or candidate.id, "environment": "production",
+ "recommendation": "PROMOTION_ELIGIBLE", "evidence_class": "OWNER_PHOTO",
+ "engineering_integration": "PASS", "production_validation": "SATISFIED", "production_action": "NONE",
+ "case_count": 1, "metric_values": {}, "critical_errors": 0, "blockers": [], "evidence_digest": uuid4().hex * 2,
+ "evidence": {"source": "offline test fixture"}, "created_at": datetime.now(UTC)}
+ row = ProjectFitEvidence(**(values | changes))
+ session.add(row)
+ session.commit()
+ return row
+
+
+@pytest.mark.parametrize("changes", [
+ {"recommendation": "KEEP_LAB", "production_validation": "DEFERRED_EXTERNAL_VALIDATION"},
+ {"recommendation": "BLOCKED", "production_validation": "REQUIRED"},
+ {"recommendation": "REQUIRES_MORE_EVIDENCE"},
+ {"engineering_integration": "BLOCKED"},
+ {"blockers": ["EXTERNAL_VALIDATION_REQUIRED"]},
+ {"critical_errors": 1},
+])
+def test_later_negative_exact_project_fit_blocks_commit(serving_pair, changes):
+ session, serving, current, _, _, _ = serving_pair
+ service, _, operation, _ = approved_plan(serving_pair)
+ _fit(serving_pair, **changes)
+ with pytest.raises(LifecycleError, match="no longer current"):
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Respect later project-specific blocker"))
+ session.rollback()
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+def test_project_fit_is_exactly_scoped_and_bound_into_plan(serving_pair):
+ _, _, current, candidate, selection, _ = serving_pair
+ _fit(serving_pair, deployment_id=current.id, recommendation="BLOCKED")
+ fit = _fit(serving_pair)
+ service, plan, operation, _ = approved_plan(serving_pair)
+ contract = plan.affected_identities["serving_transition"]
+ assert contract["production_validation"] == "SATISFIED"
+ assert contract["comparisons"][str(next(iter(selection.comparison_ids)))]["project_fit"]["id"] == str(fit.id)
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Exact candidate project validation reviewed"))
+ assert candidate.status == "stable"
+
+
+def test_newer_positive_fit_still_requires_reviewed_plan_refresh(serving_pair):
+ session, serving, current, _, _, _ = serving_pair
+ _fit(serving_pair)
+ service, _, operation, _ = approved_plan(serving_pair)
+ _fit(serving_pair)
+ with pytest.raises(LifecycleError, match="no longer current"):
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="New evidence needs a new review"))
+ session.rollback()
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == current.id
+
+
+@pytest.mark.parametrize("read_before_revoke", [False, True])
+def test_committed_serving_approval_remains_readable_and_revocable(serving_pair, read_before_revoke):
+ session, serving, current, candidate, _, _ = serving_pair
+ service, plan, operation, _ = approved_plan(serving_pair)
+ service.commit_operation(operation.id, ApprovalDecision(actor="owner", reason="Explicit permanent replacement"))
+ if read_before_revoke:
+ historical = service.approval(plan.approval_request_id)
+ assert historical.status == "STALE"
+ assert "SERVING_ROUTE_CHANGED" in historical.blockers
+ revoked = service.decide_approval(plan.approval_request_id,
+ ApprovalDecision(actor="owner", reason="Withdraw obsolete replacement authority"), "revoke")
+ assert revoked.status == "REVOKED"
+ assert service.approval(plan.approval_request_id).status == "REVOKED"
+ events = list(session.scalars(select(LifecycleEvent).where(
+ LifecycleEvent.event_type == "APPROVAL_STALE")))
+ assert len(events) == 1
+ assert "SERVING_ROUTE_CHANGED" in str(events[0].details)
+ assert serving.repo.stable_deployment(current.capability_contract_id).id == candidate.id
diff --git a/backend/tests/test_lifecycle_serving_postgres.py b/backend/tests/test_lifecycle_serving_postgres.py
new file mode 100644
index 0000000..0cd4534
--- /dev/null
+++ b/backend/tests/test_lifecycle_serving_postgres.py
@@ -0,0 +1,113 @@
+"""Disposable PostgreSQL route CAS and runtime-role transaction qualification."""
+
+import importlib
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lifecycle_contracts import ApprovalDecision
+from modelforge_api.persistence.models import CapabilityContract, LifecycleOperation
+from modelforge_api.persistence.serving_repository import ServingRepository
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from tests.test_lifecycle_serving import approved_plan, build_pair
+from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
+
+__all__ = ["identity_database"]
+
+
+def test_two_committers_switch_once_and_exact_rollback_increments_generation(
+ identity_database: IdentityDatabase,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ with Session(database.runtime) as session:
+ pair = build_pair(session)
+ _, _, current, candidate, selection, _ = pair
+ _, _, operation, _ = approved_plan(pair)
+ operation_id, current_id, candidate_id = operation.id, current.id, candidate.id
+ contract_id, generation = (
+ current.capability_contract_id,
+ selection.expected_route_generation,
+ )
+
+ barrier = threading.Barrier(2)
+
+ def contender(_index: int) -> str:
+ with Session(database.runtime) as session:
+ barrier.wait(timeout=30)
+ try:
+ return (
+ LifecycleService(session)
+ .commit_operation(
+ operation_id,
+ ApprovalDecision(
+ actor="qualification", reason="Concurrent explicit commit qualification"
+ ),
+ )
+ .stage
+ )
+ except LifecycleError:
+ session.rollback()
+ return "CONFLICT"
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ # Same-operation retries return its persisted result, without a second switch.
+ assert list(pool.map(contender, [1, 2])) == ["COMMITTED", "COMMITTED"]
+ with Session(database.runtime) as session:
+ assert ServingRepository(session).stable_deployment(contract_id).id == candidate_id
+ assert session.get(CapabilityContract, contract_id).serving_generation == generation + 1
+ assert session.get(LifecycleOperation, operation_id).stage == "COMMITTED"
+ restored = LifecycleService(session).rollback_operation(
+ operation_id,
+ ApprovalDecision(actor="qualification", reason="Restore exact prior serving route"),
+ )
+ assert restored.stage == "ROLLED_BACK"
+ assert ServingRepository(session).stable_deployment(contract_id).id == current_id
+ assert session.get(CapabilityContract, contract_id).serving_generation == generation + 2
+ with pytest.raises(RuntimeError, match="Serving route history exists"):
+ importlib.import_module("alembic.command").downgrade(database.migration(), "20260909_0033")
+
+
+def test_current_route_generation_survives_postgres_dump_restore(
+ identity_database: IdentityDatabase, tmp_path
+):
+ from modelforge_api.services.recovery_fingerprint import (
+ BENCHMARK_RUN_SCHEMA,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+ )
+
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ with Session(database.runtime) as session:
+ pair = build_pair(session)
+ _, _, operation, _ = approved_plan(pair)
+ LifecycleService(session).commit_operation(
+ operation.id,
+ ApprovalDecision(
+ actor="qualification", reason="Qualify retained committed route after restore"
+ ),
+ )
+ contract_id = pair[2].capability_contract_id
+ generation = session.get(CapabilityContract, contract_id).serving_generation
+ assert generation > 0
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "committed-route-current.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ with Session(database.restored_runtime) as session:
+ restored = semantic_fingerprint(session.connection())
+ compatibility = fingerprint_compatibility(
+ source, restored, source_schema_revision=BENCHMARK_RUN_SCHEMA
+ )
+ assert compatibility.compatible
+ assert audit_fingerprint_differences_are_compatible(source, restored, compatibility)
+ assert session.get(CapabilityContract, contract_id).serving_generation == generation
+ session.get(CapabilityContract, contract_id).serving_generation = 0
+ session.flush()
+ changed = semantic_fingerprint(session.connection())
+ assert not audit_fingerprint_differences_are_compatible(source, changed, compatibility)
+ session.rollback()
diff --git a/backend/tests/test_managed_identities.py b/backend/tests/test_managed_identities.py
new file mode 100644
index 0000000..f365c91
--- /dev/null
+++ b/backend/tests/test_managed_identities.py
@@ -0,0 +1,510 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import importlib.util
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, delete, event, select, update
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import Principal, PrincipalRole, authenticate_operator_token
+from modelforge_api.db import get_session
+from modelforge_api.domain.managed_identity import (
+ ManagedIdentityChange,
+ ManagedIdentityCreate,
+ ManagedIdentityRevoke,
+ ManagedIdentityRotate,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ Base,
+ ManagedOperatorCredential,
+ ManagedOperatorCredentialActivation,
+ ManagedOperatorIdentity,
+ ManagedOperatorState,
+)
+from modelforge_api.services import audit as audit_module
+from modelforge_api.services.operator_identities import (
+ ManagedIdentityError,
+ ManagedIdentityService,
+ authenticate_managed_operator,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ CURRENT_TRUTH_TABLES,
+ MANAGED_ACTIVATION_TABLE,
+ MANAGED_IDENTITY_SCHEMA,
+ SECRET_SCHEMA,
+ _without_creative,
+ _without_lab_execution,
+ _without_managed_identities,
+ _without_model_changes,
+ _without_model_updates,
+ _without_secrets,
+ _without_selected_comparisons,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ fingerprint_session,
+)
+from modelforge_api.settings import Settings, get_settings
+
+ANCHOR = Principal("legacy-operator", PrincipalRole.ADMIN, "legacy_admin_token")
+REASON = "Reviewed operator identity lifecycle"
+
+
+@pytest.fixture
+def session():
+ engine = create_engine(
+ "sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ yield session
+ engine.dispose()
+
+
+def service(session: Session, principal: Principal = ANCHOR) -> ManagedIdentityService:
+ return ManagedIdentityService(session, principal, "managed-request-42")
+
+
+def create(session: Session, slug: str = "alice", role: str = "admin"):
+ return service(session).create(
+ ManagedIdentityCreate(
+ slug=slug,
+ role=role,
+ expires_in_days=30,
+ expected_revision=service(session).list().revision,
+ reason=REASON,
+ )
+ )
+
+
+def test_issue_metadata_auth_and_audit(session: Session) -> None:
+ issued = create(session)
+ assert issued.one_time_key.startswith("mfopm_") and len(issued.one_time_key) == 49
+ assert issued.revision == 1 and issued.identity.subject == "managed:alice"
+ principal = authenticate_managed_operator(session, issued.one_time_key)
+ session.commit()
+ assert principal.subject == "managed:alice" and principal.role == PrincipalRole.ADMIN
+ metadata = service(session).list().model_dump_json()
+ digest = hashlib.sha256(issued.one_time_key.encode()).hexdigest()
+ assert (
+ issued.one_time_key not in metadata
+ and digest not in metadata
+ and "token_sha256" not in metadata
+ )
+ assert service(session).list().identities[0].credentials[0].last_used_at is not None
+ audit = session.scalar(
+ select(AuditEvent).where(AuditEvent.action == "managed_identity.created")
+ )
+ assert audit is not None and audit.actor_id == "legacy-operator"
+ assert audit.correlation_id == "managed-request-42" and issued.one_time_key not in str(
+ audit.details
+ )
+
+
+def test_credential_metadata_preserves_utc_after_database_reload(session: Session) -> None:
+ create(session)
+ instant = datetime(2026, 9, 8, 10, 30, tzinfo=UTC)
+ session.execute(update(ManagedOperatorCredential).values(
+ revoked_at=instant, superseded_at=instant, last_used_at=instant,
+ ))
+ session.commit()
+ session.expire_all()
+ credential = service(session).list().identities[0].credentials[0]
+ assert credential.revoked_at == instant
+ assert credential.superseded_at == instant
+ assert credential.last_used_at == instant
+
+
+@pytest.mark.parametrize("schema", ["20260908_0026", MANAGED_IDENTITY_SCHEMA])
+def test_recovery_allows_one_marker_but_preserves_history(session: Session, schema: str) -> None:
+ create(session)
+ before = _without_lab_execution(_without_model_changes(
+ _without_secrets(
+ _without_model_updates(
+ _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+ )
+ )
+ ))
+ audit_module.AuditWriter(session, "operator", "recovery").write(
+ "RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {"restored": True},
+ )
+ session.commit()
+ after = _without_lab_execution(_without_model_changes(
+ _without_secrets(
+ _without_model_updates(
+ _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+ )
+ )
+ ))
+ if schema == "20260908_0026":
+ before, after = _without_managed_identities(before), _without_managed_identities(after)
+ contract = fingerprint_compatibility(before, after, source_schema_revision=schema)
+ assert audit_fingerprint_differences_are_compatible(before, after, contract)
+ for table in before["tables"]:
+ altered = copy.deepcopy(after)
+ if table == "audit_events":
+ altered["tables"][table]["row_count"] += 1
+ elif table == "audit_chain_heads":
+ altered["tables"][table]["row_count"] = 0
+ elif table == "operational_alerts":
+ altered["tables"][table]["row_count"] += 1
+ else:
+ altered["tables"][table]["digest"] = "0" * 64
+ assert not audit_fingerprint_differences_are_compatible(before, altered, contract), table
+
+
+@pytest.mark.parametrize("operation", ["revoke", "disable", "demote"])
+def test_last_admin_is_preserved_atomically(session: Session, operation: str) -> None:
+ issued = create(session)
+ before = service(session).list().model_dump_json()
+ events = list(session.scalars(select(AuditEvent.id)))
+ with pytest.raises(ManagedIdentityError) as refused:
+ if operation == "revoke":
+ service(session).revoke(
+ "alice",
+ ManagedIdentityRevoke(
+ expected_version=1, confirm_subject="managed:alice", reason=REASON
+ ),
+ )
+ else:
+ service(session).change(
+ "alice",
+ ManagedIdentityChange(
+ expected_version=1,
+ role="viewer" if operation == "demote" else "admin",
+ enabled=operation != "disable",
+ reason=REASON,
+ ),
+ )
+ assert refused.value.code == "LAST_MANAGED_ADMIN"
+ assert service(session).list().model_dump_json() == before
+ assert list(session.scalars(select(AuditEvent.id))) == events
+ assert authenticate_managed_operator(session, issued.one_time_key).role == PrincipalRole.ADMIN
+
+
+def test_initial_admin_and_revision_conflicts(session: Session) -> None:
+ with pytest.raises(ManagedIdentityError, match="first managed"):
+ create(session, role="viewer")
+ assert service(session).list().revision == 0
+ create(session)
+ with pytest.raises(ManagedIdentityError) as conflict:
+ service(session).create(
+ ManagedIdentityCreate(
+ slug="bob", role="viewer", expires_in_days=1, expected_revision=0, reason=REASON
+ )
+ )
+ assert conflict.value.code == "MANAGED_IDENTITY_VERSION_CONFLICT"
+ assert session.get(ManagedOperatorIdentity, "bob") is None
+
+
+def test_bounded_overlap_revoke_and_no_fallback(session: Session) -> None:
+ first = create(session)
+ rotated = service(session).rotate(
+ "alice",
+ ManagedIdentityRotate(
+ expected_version=1, expires_in_days=30, overlap_seconds=900, reason=REASON
+ ),
+ )
+ assert len(rotated.identity.credentials) == 2
+ assert authenticate_managed_operator(session, first.one_time_key).subject == "managed:alice"
+ latest = service(session).rotate(
+ "alice",
+ ManagedIdentityRotate(
+ expected_version=2, expires_in_days=30, overlap_seconds=0, reason=REASON
+ ),
+ )
+ for token in (first.one_time_key, rotated.one_time_key):
+ with pytest.raises(ManagedIdentityError):
+ authenticate_managed_operator(session, token)
+ assert authenticate_managed_operator(session, latest.one_time_key)
+ with pytest.raises(HTTPException):
+ authenticate_operator_token(
+ Settings(_env_file=None, operator_api_key=SecretStr(latest.one_time_key)),
+ latest.one_time_key,
+ )
+
+
+def test_initiator_revoked_after_authentication_is_rechecked(session: Session) -> None:
+ alice = create(session)
+ create(session, "bob")
+ stale = authenticate_managed_operator(session, alice.one_time_key)
+ session.commit()
+ service(session).revoke(
+ "alice",
+ ManagedIdentityRevoke(expected_version=1, confirm_subject="managed:alice", reason=REASON),
+ )
+ with pytest.raises(ManagedIdentityError) as denied:
+ service(session, stale).change(
+ "bob",
+ ManagedIdentityChange(expected_version=1, role="admin", enabled=True, reason=REASON),
+ )
+ assert denied.value.code == "MANAGED_ADMIN_NO_LONGER_AUTHORIZED"
+ assert session.get(ManagedOperatorIdentity, "bob").version == 1
+
+
+def test_recovery_activation_quarantine_preserves_authoritative_history(session: Session) -> None:
+ original = create(session)
+ before = _without_lab_execution(_without_model_changes(
+ _without_model_updates(
+ _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+ )
+ ))
+ assert MANAGED_ACTIVATION_TABLE in CURRENT_TRUTH_TABLES
+ session.execute(delete(ManagedOperatorCredentialActivation))
+ session.commit()
+ after = _without_lab_execution(_without_model_changes(
+ _without_model_updates(
+ _without_selected_comparisons(_without_creative(fingerprint_session(session)))
+ )
+ ))
+ contract = fingerprint_compatibility(
+ before, after, source_schema_revision=SECRET_SCHEMA
+ )
+ assert audit_fingerprint_differences_are_compatible(before, after, contract)
+ with pytest.raises(ManagedIdentityError):
+ authenticate_managed_operator(session, original.one_time_key)
+ metadata = service(session).list().identities[0].credentials[0]
+ assert not metadata.activated and not metadata.usable
+ recovered = service(session).rotate(
+ "alice",
+ ManagedIdentityRotate(
+ expected_version=1, expires_in_days=30, overlap_seconds=900, reason=REASON
+ ),
+ )
+ assert authenticate_managed_operator(session, recovered.one_time_key)
+ with pytest.raises(ManagedIdentityError):
+ authenticate_managed_operator(session, original.one_time_key)
+
+
+def test_audit_failure_rolls_back_key_and_revision(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def failed(*args, **kwargs):
+ raise RuntimeError("audit unavailable")
+
+ monkeypatch.setattr(audit_module, "_append_canonical_audit_event", failed)
+ with pytest.raises(RuntimeError):
+ create(session)
+ assert service(session).list().revision == 0
+ assert session.scalar(select(ManagedOperatorCredential)) is None
+
+
+def test_metadata_failure_preserves_previous_key_and_audit(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ issued = create(session)
+ before = list(session.scalars(select(AuditEvent.id)))
+
+ def unavailable(*args, **kwargs):
+ raise RuntimeError("metadata unavailable")
+
+ subject = service(session)
+ monkeypatch.setattr(subject, "_metadata", unavailable)
+ with pytest.raises(RuntimeError, match="metadata unavailable"):
+ subject.rotate(
+ "alice",
+ ManagedIdentityRotate(
+ expected_version=1, expires_in_days=30, overlap_seconds=0, reason=REASON
+ ),
+ )
+ assert list(session.scalars(select(AuditEvent.id))) == before
+ assert service(session).list().revision == 1
+ assert authenticate_managed_operator(session, issued.one_time_key)
+
+
+def test_issuance_returns_snapshot_without_reads_after_commit(session: Session) -> None:
+ committed = False
+
+ def mark_commit(session):
+ nonlocal committed
+ committed = True
+
+ def reject_late_read(orm_execute_state):
+ if committed:
+ raise AssertionError("database read after commit")
+
+ event.listen(session, "after_commit", mark_commit)
+ event.listen(session, "do_orm_execute", reject_late_read)
+ try:
+ issued = create(session)
+ assert issued.revision == 1 and issued.identity.credentials[0].usable
+ assert "one_time_key" in issued.model_dump_json()
+ finally:
+ event.remove(session, "after_commit", mark_commit)
+ event.remove(session, "do_orm_execute", reject_late_read)
+
+
+@pytest.mark.parametrize("alteration", ["expiry", "disabled", "revoked", "quarantined"])
+def test_unusable_credential_rejected(session: Session, alteration: str) -> None:
+ issued = create(session)
+ credential_id = uuid.UUID(issued.identity.credentials[0].id)
+ if alteration == "expiry":
+ session.execute(
+ update(ManagedOperatorCredential)
+ .where(ManagedOperatorCredential.id == credential_id)
+ .values(expires_at=datetime.now(UTC) - timedelta(seconds=1))
+ )
+ elif alteration == "revoked":
+ session.execute(
+ update(ManagedOperatorCredential)
+ .where(ManagedOperatorCredential.id == credential_id)
+ .values(revoked_at=datetime.now(UTC))
+ )
+ elif alteration == "disabled":
+ session.execute(update(ManagedOperatorIdentity).values(enabled=False))
+ else:
+ session.execute(delete(ManagedOperatorCredentialActivation))
+ session.commit()
+ with pytest.raises(ManagedIdentityError) as error:
+ authenticate_managed_operator(session, issued.one_time_key)
+ assert error.value.status_code == 401
+ assert not service(session).list().identities[0].credentials[0].usable
+
+
+def test_non_admin_management_and_exact_revoke_confirmation(session: Session) -> None:
+ create(session)
+ viewer = create(session, "viewer", "viewer")
+ principal = authenticate_managed_operator(session, viewer.one_time_key)
+ session.commit()
+ with pytest.raises(ManagedIdentityError) as denied:
+ service(session, principal).list()
+ assert denied.value.status_code == 403
+ with pytest.raises(ManagedIdentityError) as denied:
+ service(session).revoke(
+ "viewer",
+ ManagedIdentityRevoke(expected_version=1, confirm_subject="viewer", reason=REASON),
+ )
+ assert denied.value.code == "MANAGED_IDENTITY_CONFIRMATION_REQUIRED"
+ result = service(session).revoke(
+ "viewer",
+ ManagedIdentityRevoke(expected_version=1, confirm_subject="managed:viewer", reason=REASON),
+ )
+ assert result.identity.version == 2 and not result.identity.credentials[0].usable
+
+
+def test_metadata_history_is_bounded(session: Session) -> None:
+ create(session)
+ for version in range(1, 12):
+ service(session).rotate(
+ "alice",
+ ManagedIdentityRotate(
+ expected_version=version, expires_in_days=1, overlap_seconds=0, reason=REASON
+ ),
+ )
+ assert len(service(session).list().identities[0].credentials) == 10
+
+
+def test_migration_empty_downgrade_and_history_refusal() -> None:
+ path = (
+ Path(__file__).resolve().parents[1]
+ / "alembic/versions/20260908_0027_managed_operator_identities.py"
+ )
+ specification = importlib.util.spec_from_file_location("managed_identity_migration", path)
+ assert specification is not None and specification.loader is not None
+ migration = importlib.util.module_from_spec(specification)
+ specification.loader.exec_module(migration)
+ engine = create_engine("sqlite://")
+ with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
+ migration.upgrade()
+ assert connection.scalar(select(ManagedOperatorState.revision)) == 0
+ migration.downgrade()
+ migration.upgrade()
+ connection.execute(
+ ManagedOperatorIdentity.__table__.insert().values(
+ slug="retained",
+ role="admin",
+ enabled=True,
+ version=1,
+ created_at=datetime.now(UTC),
+ )
+ )
+ with pytest.raises(RuntimeError, match="history exists"):
+ migration.downgrade()
+ engine.dispose()
+
+
+@pytest.mark.parametrize(
+ "field,value",
+ [
+ ("expires_in_days", True),
+ ("expires_in_days", "30"),
+ ("expires_in_days", 366),
+ ("expected_revision", False),
+ ("reason", " "),
+ ("reason", "copied mfopm_" + "a" * 43),
+ ("slug", "managed:alice"),
+ ],
+)
+def test_strict_inputs_and_secret_safe_reason(field: str, value: object) -> None:
+ body = {"slug": "alice", "role": "admin", "expires_in_days": 30, "expected_revision": 0, "reason": REASON}
+ body[field] = value
+ with pytest.raises(ValidationError):
+ ManagedIdentityCreate.model_validate(body)
+
+
+def test_http_boundaries_no_store_and_dependency_failure(
+ session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ settings = Settings(_env_file=None, env="test", operator_api_key="anchor-secret")
+ app.dependency_overrides[get_settings] = lambda: settings
+ app.dependency_overrides[get_session] = lambda: session
+ try:
+ client = TestClient(app)
+ url = "/api/v1/operator/managed-identities"
+ headers = {
+ "X-ModelForge-Admin-Token": "anchor-secret",
+ "X-Correlation-ID": "http-managed-42",
+ }
+ assert client.post(url, content="invalid").status_code == 401
+ response = client.post(
+ url,
+ json={
+ "slug": "alice", "role": "admin", "expires_in_days": 30, "expected_revision": 0, "reason": REASON
+ },
+ headers=headers,
+ )
+ assert response.status_code == 201, response.text
+ token = response.json()["one_time_key"]
+ assert (
+ response.headers["cache-control"] == "no-store" and "set-cookie" not in response.headers
+ )
+ own = {"X-ModelForge-Admin-Token": token}
+ identity = client.get("/api/v1/operator/identity", headers=own)
+ assert identity.json()["subject"] == "managed:alice"
+ assert token not in client.get(url, headers=own).text
+ failure = client.post(
+ url,
+ json={
+ "slug": "bob",
+ "role": "admin",
+ "expires_in_days": 30,
+ "expected_revision": 1,
+ "reason": "copied " + token,
+ },
+ headers=own,
+ )
+ assert failure.status_code == 422 and token not in failure.text
+
+ def unavailable(*args, **kwargs):
+ raise OperationalError("hidden", {}, Exception("private database detail"))
+
+ monkeypatch.setattr("modelforge_api.main.authenticate_managed_operator", unavailable)
+ response = client.post(url, content="not JSON", headers=own)
+ assert response.status_code == 503
+ assert "private database detail" not in response.text
+ assert client.get("/api/v1/operator/identity", headers=headers).status_code == 200
+ finally:
+ app.dependency_overrides.clear()
diff --git a/backend/tests/test_managed_identities_postgres.py b/backend/tests/test_managed_identities_postgres.py
new file mode 100644
index 0000000..f04251a
--- /dev/null
+++ b/backend/tests/test_managed_identities_postgres.py
@@ -0,0 +1,406 @@
+"""Managed-PG qualification: 0027 upgrades, recovery quarantine and identity races."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import re
+import threading
+import uuid
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.engine import URL, Engine, make_url
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.domain.managed_identity import (
+ ManagedIdentityChange,
+ ManagedIdentityCreate,
+ ManagedIdentityRevoke,
+ ManagedIdentityRotate,
+)
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ ManagedOperatorCredentialActivation,
+ ManagedOperatorState,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery_fingerprint import (
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.services.recovery_postgres import CommandResult, PostgresEngine, PostgresTarget
+from modelforge_api.settings import Settings
+from tests.test_configuration_postgres import _database_tool
+
+PRIOR = "20260908_0026"
+CURRENT = TARGET_SCHEMA_REVISION
+EXTERNAL_ADMIN = Principal("legacy-operator", PrincipalRole.ADMIN, "legacy_admin_token")
+REASON = "Isolated managed PostgreSQL qualification"
+
+
+@dataclass
+class IdentityDatabase:
+ runtime: Engine
+ restored_runtime: Engine
+ owner: URL
+ restored_owner: URL
+
+ def migration(self, *, restored: bool = False) -> Config:
+ config = Config()
+ config.set_main_option("script_location", str(Path(__file__).resolve().parents[1] / "alembic"))
+ url = self.restored_owner if restored else self.owner
+ config.set_main_option(
+ "sqlalchemy.url", url.render_as_string(hide_password=False).replace("%", "%%")
+ )
+ return config
+
+ def archive(self, path: Path, *, restore: bool = False) -> None:
+ owner = self.restored_owner if restore else self.owner
+ host = (
+ "127.0.0.1" if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+ else owner.host or "localhost"
+ )
+ args = ["--host", host, "--port", str(owner.port or 5432), "--username",
+ owner.username or "modelforge", "--dbname", owner.database or ""]
+ _database_tool(
+ "pg_restore" if restore else "pg_dump",
+ [*args, "--exit-on-error" if restore else "--format=custom"],
+ owner.password, path,
+ )
+ assert path.stat().st_size > 0
+
+
+@pytest.fixture
+def identity_database() -> Iterator[IdentityDatabase]:
+ admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+ if not admin_url:
+ pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ assert owner_url and runtime_url, "managed qualification requires owner and runtime roles"
+ owner, runtime = make_url(owner_url), make_url(runtime_url)
+ assert owner.username == "modelforge"
+ assert runtime.username == "modelforge_runtime"
+ names = ["test_identities_" + uuid.uuid4().hex,
+ "test_identities_restore_" + uuid.uuid4().hex]
+ admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+ runtime_engine = create_engine(runtime.set(database=names[0]), connect_args={
+ "connect_timeout": 15, "options": "-c statement_timeout=30000 -c lock_timeout=15000",
+ })
+ restored_engine = create_engine(runtime.set(database=names[1]))
+ created: list[str] = []
+ try:
+ with admin.connect() as connection:
+ for name in names:
+ connection.execute(text(f'CREATE DATABASE "{name}" OWNER modelforge'))
+ created.append(name)
+ yield IdentityDatabase(runtime_engine, restored_engine,
+ owner.set(database=names[0]), owner.set(database=names[1]))
+ finally:
+ runtime_engine.dispose()
+ restored_engine.dispose()
+ with admin.connect() as connection:
+ for name in created:
+ connection.execute(text(f'DROP DATABASE "{name}" WITH (FORCE)'))
+ admin.dispose()
+
+
+def _service(session: Session, principal: Principal = EXTERNAL_ADMIN):
+ from modelforge_api.services.operator_identities import ManagedIdentityService
+
+ return ManagedIdentityService(session, principal, "managed-pg-qualification")
+
+
+def _create(engine: Engine, slug: str):
+ with Session(engine) as session:
+ revision = session.scalar(select(ManagedOperatorState.revision))
+ assert revision is not None
+ return _service(session).create(ManagedIdentityCreate(
+ slug=slug, role="admin", expires_in_days=14,
+ expected_revision=revision, reason=REASON,
+ ))
+
+
+def test_managed_identity_upgrade_prior_dump_restore_and_least_privilege(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ database = identity_database
+ alembic = importlib.import_module("alembic.command")
+ config = database.migration()
+ alembic.upgrade(config, PRIOR)
+ with Session(database.runtime) as session:
+ AuditWriter(session, "operator", "prior-release").write(
+ "qualification.baseline", "managed_identity", None, {"source_schema": PRIOR},
+ )
+ session.commit()
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "pre-managed-identities.dump"
+ database.archive(archive)
+ alembic.upgrade(config, "head")
+ with Session(database.runtime) as session:
+ assert session.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
+ assert session.scalar(select(ManagedOperatorState.revision)) == 0
+ current = semantic_fingerprint(session.connection())
+ comparison = fingerprint_compatibility(source, current, source_schema_revision=PRIOR)
+ assert comparison.compatible
+ assert audit_fingerprint_differences_are_compatible(source, current, comparison)
+ alembic.downgrade(config, PRIOR)
+ alembic.upgrade(config, "head")
+ _create(database.runtime, "qualified-admin")
+ with database.runtime.connect() as connection:
+ role = connection.execute(text(
+ "SELECT rolsuper, rolcreatedb, rolcreaterole FROM pg_roles WHERE rolname=current_user"
+ )).one()
+ assert tuple(role) == (False, False, False)
+ for statement in (
+ "ALTER TABLE managed_operator_identities ADD COLUMN forbidden integer",
+ "TRUNCATE managed_operator_credentials CASCADE",
+ "DELETE FROM managed_operator_identities",
+ "DELETE FROM audit_events",
+ ):
+ with pytest.raises(DBAPIError):
+ connection.execute(text(statement))
+ connection.rollback()
+ with pytest.raises(RuntimeError, match="history exists"):
+ alembic.downgrade(config, PRIOR)
+ with database.runtime.connect() as connection:
+ assert connection.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
+ database.archive(archive, restore=True)
+ with database.restored_runtime.connect() as connection:
+ restored = semantic_fingerprint(connection)
+ assert restored["tables"] == source["tables"]
+ assert connection.scalar(text("SELECT version_num FROM alembic_version")) == PRIOR
+ alembic.upgrade(database.migration(restored=True), "head")
+ with database.restored_runtime.connect() as connection:
+ restored = semantic_fingerprint(connection)
+ comparison = fingerprint_compatibility(source, restored, source_schema_revision=PRIOR)
+ assert comparison.compatible
+ assert audit_fingerprint_differences_are_compatible(source, restored, comparison)
+
+
+@pytest.mark.parametrize("operation", ["revoke", "demote", "disable"])
+def test_concurrent_cross_admin_changes_preserve_a_usable_admin(
+ identity_database: IdentityDatabase, operation: str,
+) -> None:
+ from modelforge_api.services.operator_identities import (
+ ManagedIdentityError,
+ authenticate_managed_operator,
+ )
+
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ alpha, beta = _create(database.runtime, "alpha"), _create(database.runtime, "beta")
+ barrier = threading.Barrier(2)
+
+ def contender(pair) -> str:
+ initiator, target = pair
+ with Session(database.runtime) as session:
+ principal = authenticate_managed_operator(session, initiator.one_time_key)
+ session.commit()
+ barrier.wait(timeout=30)
+ try:
+ service = _service(session, principal)
+ if operation == "revoke":
+ service.revoke(target.identity.slug, ManagedIdentityRevoke(
+ expected_version=target.identity.version,
+ confirm_subject=target.identity.subject, reason=REASON,
+ ))
+ else:
+ service.change(target.identity.slug, ManagedIdentityChange(
+ expected_version=target.identity.version,
+ role="viewer" if operation == "demote" else "admin",
+ enabled=operation != "disable", reason=REASON,
+ ))
+ return "applied"
+ except ManagedIdentityError as error:
+ assert error.code == "MANAGED_ADMIN_NO_LONGER_AUTHORIZED"
+ return "rejected"
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert sorted(pool.map(contender, [(alpha, beta), (beta, alpha)])) == ["applied", "rejected"]
+ usable_admins = 0
+ for issued in (alpha, beta):
+ with Session(database.runtime) as session:
+ try:
+ principal = authenticate_managed_operator(session, issued.one_time_key)
+ usable_admins += principal.role is PrincipalRole.ADMIN
+ except ManagedIdentityError:
+ pass
+ assert usable_admins == 1
+ with Session(database.runtime) as session:
+ events = list(session.scalars(select(AuditEvent).where(
+ AuditEvent.action.in_(["managed_identity.revoked", "managed_identity.changed"]),
+ )))
+ assert len(events) == 1
+ assert events[0].actor_id in {"managed:alpha", "managed:beta"}
+
+
+def test_simultaneous_rotation_uses_version_cas(identity_database: IdentityDatabase) -> None:
+ from modelforge_api.services.operator_identities import ManagedIdentityError
+
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ identity = _create(database.runtime, "rotate-admin")
+ barrier = threading.Barrier(2)
+
+ def contender(_index: int) -> str:
+ with Session(database.runtime) as session:
+ barrier.wait(timeout=30)
+ try:
+ _service(session).rotate("rotate-admin", ManagedIdentityRotate(
+ expected_version=identity.identity.version, expires_in_days=14,
+ overlap_seconds=0, reason=REASON,
+ ))
+ return "applied"
+ except ManagedIdentityError as error:
+ assert error.status_code == 409
+ assert error.code == "MANAGED_IDENTITY_VERSION_CONFLICT"
+ return "conflict"
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert sorted(pool.map(contender, [0, 1])) == ["applied", "conflict"]
+ with Session(database.runtime) as session:
+ assert len(list(session.scalars(select(AuditEvent).where(
+ AuditEvent.action == "managed_identity.rotated",
+ )))) == 1
+
+
+def test_revoked_initiator_cannot_commit_a_previously_authorized_change(
+ identity_database: IdentityDatabase,
+) -> None:
+ from modelforge_api.services.operator_identities import (
+ ManagedIdentityError,
+ authenticate_managed_operator,
+ )
+
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ alpha, beta = _create(database.runtime, "alpha"), _create(database.runtime, "beta")
+ authenticated = threading.Event()
+ revoked = threading.Event()
+
+ def stale_request() -> str:
+ with Session(database.runtime) as session:
+ principal = authenticate_managed_operator(session, alpha.one_time_key)
+ session.commit()
+ authenticated.set()
+ assert revoked.wait(timeout=30)
+ with pytest.raises(ManagedIdentityError) as rejected:
+ _service(session, principal).rotate("beta", ManagedIdentityRotate(
+ expected_version=beta.identity.version, expires_in_days=14,
+ overlap_seconds=0, reason=REASON,
+ ))
+ assert rejected.value.status_code in {401, 403}
+ return "rejected"
+
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ pending = pool.submit(stale_request)
+ assert authenticated.wait(timeout=30)
+ try:
+ with Session(database.runtime) as session:
+ _service(session).revoke("alpha", ManagedIdentityRevoke(
+ expected_version=alpha.identity.version,
+ confirm_subject=alpha.identity.subject, reason=REASON,
+ ))
+ finally:
+ revoked.set()
+ assert pending.result(timeout=30) == "rejected"
+ with Session(database.runtime) as session:
+ assert authenticate_managed_operator(session, beta.one_time_key).role is PrincipalRole.ADMIN
+
+
+class ManagedPostgresTools(PostgresEngine):
+ """Use real psql in the managed DB container, retaining the production SQL adapter."""
+
+ def _run(
+ self, argv: list[str], target: PostgresTarget, *, stdin: str | None = None,
+ ) -> CommandResult:
+ container = os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
+ if container:
+ assert re.fullmatch(r"[0-9a-f]{12,64}", container)
+ argv = list(argv)
+ argv[argv.index("--host") + 1] = "127.0.0.1"
+ argv = ["docker", "exec", "-i", "-e", "PGPASSWORD", container, *argv]
+ return super()._run(argv, target, stdin=stdin)
+
+
+def test_supported_restore_quarantines_old_managed_keys_until_fresh_issuance(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ from modelforge_api.services.operator_identities import (
+ ManagedIdentityError,
+ authenticate_managed_operator,
+ )
+ from modelforge_api.services.recovery import RecoveryError, RecoveryService
+
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ issued = _create(database.runtime, "restore-admin")
+ with Session(database.runtime) as session:
+ assert authenticate_managed_operator(session, issued.one_time_key).role is PrincipalRole.ADMIN
+ session.commit()
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "managed-identity.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ with Session(database.restored_runtime) as session:
+ assert len(list(session.scalars(select(ManagedOperatorCredentialActivation)))) == 1
+ assert semantic_fingerprint(session.connection())["tables"] == source["tables"]
+
+ # Exercise the supported recovery phase itself, with the actual restored database,
+ # audit-chain checks and marker CAS; only the tool's execution location is adapted.
+ restore_id = uuid.uuid4()
+ owner = database.restored_owner
+ with Session(database.runtime) as session:
+ recovery = RecoveryService(session, Settings(
+ _env_file=None, env="test",
+ alembic_directory=Path(__file__).resolve().parents[1] / "alembic",
+ ))
+ recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+ record = SimpleNamespace(id=restore_id, source_fingerprint=source)
+ plan = SimpleNamespace(
+ database_destination=owner.render_as_string(hide_password=False),
+ backup_set=SimpleNamespace(backup_id="managed-identity-qualification",
+ schema_revision=CURRENT, completed_at=datetime.now(UTC)),
+ validation_requirements={},
+ )
+ with pytest.raises(RecoveryError) as unready:
+ recovery._validate_restored(record, plan)
+ assert unready.value.details["gate"]["current_truth_reconciled"] is False
+ session.rollback()
+ result = recovery._reconcile_restored(record, plan)
+ session.commit()
+ assert result["cleared_current_truth"]["managed_operator_credential_activations"] == 1
+ ready = recovery._validate_restored(record, plan)
+ assert ready["gate"]["current_truth_reconciled"] is True
+ assert ready["residual_current_truth_rows"]["managed_operator_credential_activations"] == 0
+ session.commit()
+ with Session(database.restored_runtime) as session:
+ assert list(session.scalars(select(ManagedOperatorCredentialActivation))) == []
+ restored = semantic_fingerprint(session.connection())
+ for table in ("managed_operator_state", "managed_operator_identities",
+ "managed_operator_credentials"):
+ assert restored["tables"][table] == source["tables"][table]
+ with pytest.raises(ManagedIdentityError):
+ authenticate_managed_operator(session, issued.one_time_key)
+ session.rollback()
+ fresh = _service(session).rotate("restore-admin", ManagedIdentityRotate(
+ expected_version=issued.identity.version, expires_in_days=14,
+ overlap_seconds=0, reason=REASON,
+ ))
+ with Session(database.restored_runtime) as session:
+ assert authenticate_managed_operator(session, fresh.one_time_key).role is PrincipalRole.ADMIN
+ with pytest.raises(ManagedIdentityError):
+ authenticate_managed_operator(session, issued.one_time_key)
diff --git a/backend/tests/test_manifests.py b/backend/tests/test_manifests.py
index 97634f8..e005611 100644
--- a/backend/tests/test_manifests.py
+++ b/backend/tests/test_manifests.py
@@ -1,7 +1,9 @@
from pathlib import Path
import pytest
+from pydantic import ValidationError
+from modelforge_api.domain.contracts import ProjectManifest
from modelforge_api.services.manifest_registry import PROJECT_ROOT, ManifestRegistry
@@ -26,3 +28,35 @@ def test_project_referencing_missing_contract_is_rejected(tmp_path: Path) -> Non
)
with pytest.raises(ValueError, match="missing capability"):
ManifestRegistry(tmp_path).projects()
+
+
+def test_duplicate_project_manifest_identity_is_rejected(tmp_path: Path) -> None:
+ (tmp_path / "capabilities").mkdir()
+ projects = tmp_path / "projects"
+ projects.mkdir()
+ (projects / "first.yaml").write_text(
+ "project:\n id: shared\n name: First\n description: First declaration\nbindings: {}\n",
+ encoding="utf-8",
+ )
+ (projects / "second.yaml").write_text(
+ "project:\n id: shared\n name: Second\n description: Conflicting declaration\nbindings: {}\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="duplicate project manifest identity"):
+ ManifestRegistry(tmp_path).projects()
+
+
+def test_project_manifest_rejects_secret_bearing_extra_fields() -> None:
+ payload = {
+ "project": {
+ "id": "local-consumer",
+ "name": "Local Consumer",
+ "description": "A non-secret project contract.",
+ "service_token": "placeholder-only",
+ },
+ "bindings": {},
+ }
+
+ with pytest.raises(ValidationError):
+ ProjectManifest.model_validate(payload)
diff --git a/backend/tests/test_migration_engine_m13.py b/backend/tests/test_migration_engine_m13.py
index 5b7e51c..6e4a73a 100644
--- a/backend/tests/test_migration_engine_m13.py
+++ b/backend/tests/test_migration_engine_m13.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import hashlib
import uuid
+from datetime import timedelta
import pytest
from sqlalchemy import create_engine
@@ -579,3 +580,28 @@ def test_migration_events_are_append_only(session: Session) -> None:
with pytest.raises(ValueError, match="append-only"):
stored.reason = "rewritten"
session.commit()
+
+
+def test_plan_index_respects_the_requested_result_limit(session: Session) -> None:
+ service, first = planned_migration(session, target="m13-index-first")
+ source = session.get(MigrationPlan, first.id)
+ assert source is not None
+ values = {column.key: getattr(source, column.key) for column in MigrationPlan.__table__.columns}
+ values.update(
+ id=uuid.uuid4(),
+ target_shadow_target="m13-index-second",
+ idempotency_key=f"m13-index-second-{uuid.uuid4()}",
+ plan_fingerprint=digest("m13-index-second-plan"),
+ created_at=source.created_at + timedelta(seconds=1),
+ )
+ second = MigrationPlan(**values)
+ session.add(second)
+ session.commit()
+
+ limited = service.plans(limit=1)
+ complete = service.plans(limit=2)
+
+ assert len(limited) == 1
+ assert limited[0].id == second.id
+ assert len(complete) == 2
+ assert {item.id for item in complete} == {first.id, second.id}
diff --git a/backend/tests/test_migration_plan_index_api.py b/backend/tests/test_migration_plan_index_api.py
new file mode 100644
index 0000000..c6f3b73
--- /dev/null
+++ b/backend/tests/test_migration_plan_index_api.py
@@ -0,0 +1,36 @@
+from uuid import uuid4
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+
+from modelforge_api.api.routes.migrations import get_service
+from modelforge_api.main import app
+from modelforge_api.settings import Settings, get_settings
+
+_TOKEN = uuid4().hex
+_HEADERS = {"X-ModelForge-Admin-Token": _TOKEN}
+
+
+def test_migration_plan_index_passes_the_bounded_limit_and_rejects_out_of_range_values() -> None:
+ class IndexService:
+ requested_limit: int | None = None
+
+ def plans(self, limit: int) -> list[object]:
+ self.requested_limit = limit
+ return []
+
+ service = IndexService()
+ app.dependency_overrides[get_settings] = lambda: Settings(
+ _env_file=None, operator_api_key=SecretStr(_TOKEN)
+ )
+ app.dependency_overrides[get_service] = lambda: service
+ try:
+ client = TestClient(app)
+ response = client.get("/api/v1/admin/migrations/plans?limit=37", headers=_HEADERS)
+ invalid = client.get("/api/v1/admin/migrations/plans?limit=201", headers=_HEADERS)
+
+ assert response.status_code == 200
+ assert service.requested_limit == 37
+ assert invalid.status_code == 422
+ finally:
+ app.dependency_overrides.clear()
diff --git a/backend/tests/test_model_change_recovery.py b/backend/tests/test_model_change_recovery.py
new file mode 100644
index 0000000..60ddcd8
--- /dev/null
+++ b/backend/tests/test_model_change_recovery.py
@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+from copy import deepcopy
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+ CURRENT_TRUTH_TABLES,
+ MODEL_CHANGE_SCHEMA,
+ MODEL_CHANGE_TABLES,
+ PORTFOLIO_SCHEMA,
+ SERVING_GENERATION_SCHEMA,
+ _without_lab_execution,
+ _without_model_changes,
+ audit_fingerprint_differences_are_compatible,
+ classified_tables,
+ fingerprint_compatibility,
+ fingerprint_session,
+ unclassified_tables,
+)
+
+
+def _current_fingerprint() -> dict[str, object]:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ return _without_lab_execution(fingerprint_session(session))
+
+
+def test_model_change_cases_are_authoritative_history() -> None:
+ assert frozenset({"model_change_cases"}) == MODEL_CHANGE_TABLES
+ assert classified_tables() >= MODEL_CHANGE_TABLES
+ assert not MODEL_CHANGE_TABLES & CURRENT_TRUTH_TABLES
+ assert unclassified_tables() == frozenset()
+
+
+def test_schema_0035_requires_exact_model_change_history() -> None:
+ from modelforge_api.services.recovery_fingerprint import _without_portfolio
+
+ source = _without_portfolio(_current_fingerprint())
+ restored = deepcopy(source)
+
+ contract = fingerprint_compatibility(
+ source,
+ restored,
+ source_schema_revision=MODEL_CHANGE_SCHEMA,
+ )
+ assert contract.compatible is True
+ assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|MODEL_CHANGE_STRICT|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+ changed = deepcopy(restored)
+ entry = changed["tables"]["model_change_cases"] # type: ignore[index]
+ entry["digest"] = "f" * 64 # type: ignore[index]
+ assert audit_fingerprint_differences_are_compatible(source, changed, contract) is False
+
+
+def test_schema_0034_upgrade_may_introduce_only_empty_model_change_history() -> None:
+ restored = _current_fingerprint()
+ from modelforge_api.services.recovery_fingerprint import _without_portfolio
+
+ source = _without_portfolio(_without_model_changes(restored))
+
+ contract = fingerprint_compatibility(
+ source,
+ restored,
+ source_schema_revision=SERVING_GENERATION_SCHEMA,
+ )
+ assert contract.compatible is True
+ assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_UPGRADE|MODEL_CHANGE_UPGRADE|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+ populated = deepcopy(restored)
+ entry = populated["tables"]["model_change_cases"] # type: ignore[index]
+ entry["row_count"] = 1 # type: ignore[index]
+ entry["digest"] = "e" * 64 # type: ignore[index]
+ rejected = fingerprint_compatibility(
+ source,
+ populated,
+ source_schema_revision=SERVING_GENERATION_SCHEMA,
+ )
+ assert rejected.compatible is False
+
+
+def test_schema_0036_requires_exact_portfolio_history() -> None:
+ source = _current_fingerprint()
+ restored = deepcopy(source)
+
+ contract = fingerprint_compatibility(
+ source,
+ restored,
+ source_schema_revision=PORTFOLIO_SCHEMA,
+ )
+ assert contract.compatible is True
+ assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_STRICT|MODEL_CHANGE_STRICT|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+ changed = deepcopy(restored)
+ changed["tables"]["portfolio_reconciliation_runs"]["digest"] = "f" * 64 # type: ignore[index]
+ assert audit_fingerprint_differences_are_compatible(source, changed, contract) is False
+
+
+def test_schema_0035_upgrade_may_introduce_only_empty_portfolio_history() -> None:
+ from modelforge_api.services.recovery_fingerprint import _without_portfolio
+
+ restored = _current_fingerprint()
+ source = _without_portfolio(restored)
+
+ contract = fingerprint_compatibility(
+ source,
+ restored,
+ source_schema_revision=MODEL_CHANGE_SCHEMA,
+ )
+ assert contract.compatible is True
+ assert contract.mode.startswith("ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_UPGRADE|MODEL_CHANGE_STRICT|")
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract) is True
+
+ populated = deepcopy(restored)
+ populated["tables"]["portfolio_reconciliation_runs"]["row_count"] = 1 # type: ignore[index]
+ rejected = fingerprint_compatibility(
+ source,
+ populated,
+ source_schema_revision=MODEL_CHANGE_SCHEMA,
+ )
+ assert rejected.compatible is False
diff --git a/backend/tests/test_model_changes.py b/backend/tests/test_model_changes.py
new file mode 100644
index 0000000..a9115ee
--- /dev/null
+++ b/backend/tests/test_model_changes.py
@@ -0,0 +1,200 @@
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+from uuid import uuid4
+
+import pytest
+from sqlalchemy.dialects import sqlite
+
+from modelforge_api.domain.model_changes import (
+ ModelChangeCandidateLink,
+ ModelChangeEvidenceLink,
+ ModelChangeResponse,
+)
+from modelforge_api.persistence.models import (
+ CapabilityDeployment,
+ DiscoveryCandidateAssessment,
+ SelectedModelComparison,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_changes import ModelChangeError, ModelChangeService
+
+
+def _service() -> tuple[ModelChangeService, MagicMock]:
+ session = MagicMock()
+ service = ModelChangeService(session, AuditContext("operator", "test-user", "test-correlation"))
+ return service, session
+
+
+def _open_case() -> SimpleNamespace:
+ return SimpleNamespace(
+ id=uuid4(),
+ project_binding_id=uuid4(),
+ current_deployment_id=uuid4(),
+ candidate_model_id=None,
+ candidate_revision_id=None,
+ candidate_artifact_set_id=None,
+ candidate_deployment_id=None,
+ discovery_assessment_id=None,
+ download_plan_id=None,
+ artifact_job_id=None,
+ selected_comparison_id=None,
+ migration_plan_id=None,
+ lifecycle_plan_id=None,
+ status="searching",
+ status_reason="",
+ version=1,
+ closed_at=None,
+ updated_by="test-user",
+ )
+
+
+def test_list_returns_a_bounded_filtered_page_with_stable_order(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ service, session = _service()
+ binding_id = uuid4()
+ row = SimpleNamespace(id=uuid4())
+ session.scalar.return_value = 241
+ session.scalars.return_value.all.return_value = [row]
+ monkeypatch.setattr(
+ service,
+ "_response",
+ lambda item: ModelChangeResponse.model_construct(id=item.id),
+ )
+
+ page = service.list(
+ project_key="examplerag",
+ project_binding_id=binding_id,
+ open_only=True,
+ limit=100,
+ offset=100,
+ )
+
+ assert page.total == 241
+ assert [item.id for item in page.items] == [row.id]
+ statement = session.scalars.call_args.args[0]
+ sql = str(statement.compile(dialect=sqlite.dialect()))
+ assert "ORDER BY model_change_cases.updated_at DESC, model_change_cases.id ASC" in sql
+ assert "LIMIT ? OFFSET ?" in sql
+ assert statement._limit_clause.value == 100
+ assert statement._offset_clause.value == 100
+ count_sql = str(session.scalar.call_args.args[0].compile(dialect=sqlite.dialect()))
+ assert "model_change_cases.closed_at IS NULL" in count_sql
+ assert "model_change_cases.project_binding_id" in count_sql
+ assert 'projects."key" =' in count_sql
+
+
+def _binding_context(row: SimpleNamespace) -> tuple[SimpleNamespace, SimpleNamespace, SimpleNamespace, SimpleNamespace]:
+ binding = SimpleNamespace(id=row.project_binding_id)
+ project = SimpleNamespace(id=uuid4(), key="examplerag", name="ExampleRAG")
+ contract = SimpleNamespace(id=uuid4())
+ capability = SimpleNamespace(id=uuid4(), key="rag.embedding")
+ return binding, project, contract, capability
+
+
+def test_candidate_deployment_can_never_be_the_current_production_deployment(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, session = _service()
+ row = _open_case()
+ binding, project, contract, capability = _binding_context(row)
+ deployment_id = row.current_deployment_id
+ deployment = SimpleNamespace(
+ id=deployment_id,
+ capability_contract_id=contract.id,
+ artifact_set_id=uuid4(),
+ production=True,
+ channel="stable",
+ status="stable",
+ deprecated_at=None,
+ )
+ monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+ monkeypatch.setattr(service, "_binding_context", lambda _binding_id: (binding, project, contract, capability))
+ session.get.side_effect = lambda model, identifier: deployment if model is CapabilityDeployment and identifier == deployment_id else None
+
+ request = ModelChangeCandidateLink(expected_version=1, candidate_deployment_id=deployment_id)
+ with pytest.raises(ModelChangeError) as exc:
+ service.link_candidate(row.id, request)
+
+ assert exc.value.code == "MODEL_CHANGE_CANDIDATE_DEPLOYMENT_INVALID"
+ session.commit.assert_not_called()
+
+
+def test_comparison_evidence_must_match_exact_project_and_current_candidate_pair(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, session = _service()
+ row = _open_case()
+ row.candidate_deployment_id = uuid4()
+ binding, project, contract, capability = _binding_context(row)
+ comparison_id = uuid4()
+ comparison = SimpleNamespace(
+ id=comparison_id,
+ project_binding_id=uuid4(),
+ current_deployment_id=row.current_deployment_id,
+ candidate_deployment_id=row.candidate_deployment_id,
+ status="completed",
+ )
+ monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+ monkeypatch.setattr(service, "_binding_context", lambda _binding_id: (binding, project, contract, capability))
+ session.get.side_effect = lambda model, identifier: comparison if model is SelectedModelComparison and identifier == comparison_id else None
+
+ request = ModelChangeEvidenceLink(expected_version=1, selected_comparison_id=comparison_id)
+ with pytest.raises(ModelChangeError) as exc:
+ service.link_evidence(row.id, request)
+
+ assert exc.value.code == "MODEL_CHANGE_COMPARISON_MISMATCH"
+ session.commit.assert_not_called()
+
+
+def test_discovery_assessment_must_belong_to_exact_immutable_revision(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, session = _service()
+ row = _open_case()
+ binding, project, contract, capability = _binding_context(row)
+ model_id = uuid4()
+ revision_id = uuid4()
+ assessment_id = uuid4()
+ revision = SimpleNamespace(id=revision_id, model_id=model_id, resolved_commit_sha="exact-sha")
+ model = SimpleNamespace(id=model_id)
+ assessment = SimpleNamespace(
+ id=assessment_id,
+ model_id=model_id,
+ resolved_commit_sha="different-sha",
+ )
+ monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+ monkeypatch.setattr(service, "_binding_context", lambda _binding_id: (binding, project, contract, capability))
+
+ def get(model_type: object, identifier: object) -> object | None:
+ from modelforge_api.persistence.models import Model, ModelRevision
+
+ if model_type is ModelRevision and identifier == revision_id:
+ return revision
+ if model_type is Model and identifier == model_id:
+ return model
+ if model_type is DiscoveryCandidateAssessment and identifier == assessment_id:
+ return assessment
+ return None
+
+ session.get.side_effect = get
+ request = ModelChangeCandidateLink(
+ expected_version=1,
+ candidate_model_id=model_id,
+ candidate_revision_id=revision_id,
+ discovery_assessment_id=assessment_id,
+ )
+
+ with pytest.raises(ModelChangeError) as exc:
+ service.link_candidate(row.id, request)
+
+ assert exc.value.code == "MODEL_CHANGE_ASSESSMENT_MISMATCH"
+ session.commit.assert_not_called()
+
+
+def test_stale_case_version_is_rejected_before_candidate_mutation(monkeypatch: pytest.MonkeyPatch) -> None:
+ service, session = _service()
+ row = _open_case()
+ row.version = 7
+ monkeypatch.setattr(service, "_row", lambda _identifier, lock=False: row)
+
+ request = ModelChangeCandidateLink(expected_version=6, candidate_model_id=uuid4())
+ with pytest.raises(ModelChangeError) as exc:
+ service.link_candidate(row.id, request)
+
+ assert exc.value.code == "MODEL_CHANGE_VERSION_CONFLICT"
+ session.commit.assert_not_called()
diff --git a/backend/tests/test_model_update_polling.py b/backend/tests/test_model_update_polling.py
new file mode 100644
index 0000000..f0ac515
--- /dev/null
+++ b/backend/tests/test_model_update_polling.py
@@ -0,0 +1,78 @@
+from datetime import UTC, datetime, timedelta
+from unittest.mock import Mock
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, Model, ModelRevision, ModelUpdateWatch
+from modelforge_api.services.model_update_polling import ModelUpdatePollingService
+from modelforge_api.services.registry import RegistryConflict
+from modelforge_api.settings import Settings
+
+
+def test_monitor_checks_only_one_due_enabled_unclaimed_watch_and_passes_due_fence() -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ settings = Settings(_env_file=None, env="test")
+ now = datetime.now(UTC)
+ with Session(engine) as session:
+ model = Model(key="chosen", display_name="Chosen", upstream_provider="huggingface",
+ upstream_source="fixture/chosen")
+ session.add(model)
+ session.flush()
+ revision = ModelRevision(model_id=model.id, upstream_revision="main",
+ resolved_commit_sha="a" * 40)
+ session.add(revision)
+ session.flush()
+ watches = [ModelUpdateWatch(
+ model_id=model.id, baseline_revision_id=revision.id,
+ baseline_commit_sha=revision.resolved_commit_sha, upstream_ref=ref,
+ enabled=enabled, last_checked_at=checked, check_until=claimed,
+ ) for ref, enabled, checked, claimed in (
+ ("paused", False, None, None),
+ ("fresh", True, now, None),
+ ("busy", True, None, now + timedelta(minutes=5)),
+ ("due", True, now - timedelta(days=2), None),
+ ("later", True, now - timedelta(hours=25), None),
+ )]
+ session.add_all(watches)
+ session.commit()
+ due_id = watches[3].id
+ poller = ModelUpdatePollingService(settings, engine)
+ service = Mock()
+ poller.service = Mock(return_value=service)
+ poller.run_once()
+ assert service.check.call_count == 1
+ assert service.check.call_args.args == (due_id,)
+ cutoff = service.check.call_args.kwargs["if_checked_before"]
+ assert now - timedelta(days=1) <= cutoff <= datetime.now(UTC) - timedelta(days=1)
+ # A concurrent winner is a safe no-op and cannot kill the monitor.
+ service.check.side_effect = RegistryConflict("already checked")
+ poller.run_once()
+ poller.stop()
+ poller.run_once()
+ assert service.check.call_count == 2
+ engine.dispose()
+
+
+def test_disabled_monitor_never_opens_database_or_constructs_provider() -> None:
+ engine = Mock()
+ poller = ModelUpdatePollingService(
+ Settings(_env_file=None, env="test", model_update_monitor_enabled=False), engine,
+ )
+ poller.service = Mock()
+ poller.run_once()
+ assert engine.mock_calls == []
+ poller.service.assert_not_called()
+
+
+def test_unexpected_database_failure_is_redacted_and_does_not_escape(capsys) -> None:
+ # Deliberately absent tables in an isolated database; no deployment state is consulted.
+ engine = create_engine("sqlite://")
+ poller = ModelUpdatePollingService(Settings(_env_file=None, env="test"), engine)
+ poller.run_once()
+ output = capsys.readouterr().out
+ assert "model_update_monitor_check_failed" in output
+ assert "OperationalError" in output
+ assert "SELECT" not in output
+ engine.dispose()
diff --git a/backend/tests/test_model_update_recovery.py b/backend/tests/test_model_update_recovery.py
new file mode 100644
index 0000000..dfd9f90
--- /dev/null
+++ b/backend/tests/test_model_update_recovery.py
@@ -0,0 +1,118 @@
+import copy
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+ Base,
+ Model,
+ ModelRevision,
+ ModelUpdateNotice,
+ ModelUpdateWatch,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery_fingerprint import (
+ CREATIVE_TABLES,
+ MODEL_UPDATE_TABLES,
+ SECRET_SCHEMA,
+ SELECTED_COMPARISON_SCHEMA,
+ SELECTED_COMPARISON_TABLES,
+ _without_creative,
+ _without_lab_execution,
+ _without_model_changes,
+ _without_model_updates,
+ _without_portfolio,
+ _without_selected_comparisons,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ fingerprint_session as _current_fingerprint_session,
+)
+
+
+def fingerprint_session(session: Session):
+ return _without_lab_execution(
+ _without_portfolio(_without_model_changes(_current_fingerprint_session(session)))
+ )
+
+
+def test_previous_schema_allows_only_exact_empty_update_introduction() -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ restored = fingerprint_session(session)
+ source = _without_creative(_without_selected_comparisons(_without_model_updates(restored)))
+ assert not (CREATIVE_TABLES | SELECTED_COMPARISON_TABLES | MODEL_UPDATE_TABLES) & source["tables"].keys()
+ contract = fingerprint_compatibility(source, restored, source_schema_revision=SECRET_SCHEMA)
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+ for table in MODEL_UPDATE_TABLES:
+ for field, value in (("row_count", 1), ("digest", "0" * 64), ("status", "BOUNDED")):
+ changed = copy.deepcopy(restored)
+ changed["tables"][table][field] = value
+ assert not fingerprint_compatibility(source, changed, source_schema_revision=SECRET_SCHEMA).compatible
+ assert not audit_fingerprint_differences_are_compatible(source, changed, contract)
+ missing = copy.deepcopy(restored)
+ del missing["tables"][table]
+ assert not fingerprint_compatibility(source, missing, source_schema_revision=SECRET_SCHEMA).compatible
+ engine.dispose()
+
+
+def test_watch_claim_is_transient_but_baseline_and_notice_history_are_authoritative() -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ model = Model(key="watched", display_name="Watched", upstream_provider="huggingface", upstream_source="org/watched")
+ session.add(model)
+ session.flush()
+ baseline = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="a" * 40)
+ candidate = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="b" * 40)
+ session.add_all([baseline, candidate])
+ session.flush()
+ watch = ModelUpdateWatch(model_id=model.id, baseline_revision_id=baseline.id,
+ baseline_commit_sha=baseline.resolved_commit_sha, upstream_ref="main",
+ check_token=uuid4(), check_until=datetime.now(UTC) + timedelta(minutes=5))
+ session.add(watch)
+ session.flush()
+ notice = ModelUpdateNotice(watch_id=watch.id, model_id=model.id,
+ baseline_commit_sha=baseline.resolved_commit_sha, candidate_revision_id=candidate.id,
+ candidate_commit_sha=candidate.resolved_commit_sha)
+ session.add(notice)
+ session.commit()
+ before = fingerprint_session(session)
+ watch.check_token = None
+ watch.check_until = None
+ session.commit()
+ cleared = fingerprint_session(session)
+ assert before["tables"] == cleared["tables"]
+ AuditWriter(session, "operator", "restore-test").write("RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {})
+ session.commit()
+ restored = fingerprint_session(session)
+ contract = fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+ notice.status = "dismissed"
+ session.commit()
+ changed = fingerprint_session(session)
+ assert not audit_fingerprint_differences_are_compatible(before, changed, contract)
+ notice.status = "new"
+ watch.baseline_commit_sha = "c" * 40
+ session.commit()
+ assert not audit_fingerprint_differences_are_compatible(before, fingerprint_session(session), contract)
+ engine.dispose()
+
+
+@pytest.mark.parametrize("table", sorted(MODEL_UPDATE_TABLES))
+def test_current_schema_never_projects_away_missing_update_state(table: str) -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ before = fingerprint_session(session)
+ restored = copy.deepcopy(before)
+ del restored["tables"][table]
+ assert not fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA).compatible
+ engine.dispose()
diff --git a/backend/tests/test_model_updates.py b/backend/tests/test_model_updates.py
new file mode 100644
index 0000000..8fe7055
--- /dev/null
+++ b/backend/tests/test_model_updates.py
@@ -0,0 +1,345 @@
+from datetime import UTC, datetime, timedelta
+from unittest.mock import Mock
+from uuid import uuid4
+
+import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import (
+ Principal,
+ PrincipalRole,
+ authenticate_operator_principal,
+)
+from modelforge_api.api.routes.model_updates import get_model_update_service, router
+from modelforge_api.domain.model_updates import ModelWatchCreate, ModelWatchUpdate
+from modelforge_api.persistence.models import (
+ ArtifactJob,
+ AuditEvent,
+ Base,
+ CapabilityDeployment,
+ Model,
+ ModelRevision,
+ ModelUpdateWatch,
+)
+from modelforge_api.services.acquisition import AcquisitionService, ProviderBoundaryError
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.services.registry import RegistryConflict, RegistryService
+
+
+@pytest.fixture
+def setup():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ model = Model(key="selected", display_name="Selected", upstream_provider="huggingface", upstream_source="org/selected")
+ session.add(model)
+ session.flush()
+ revision = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="a" * 40)
+ session.add(revision)
+ session.commit()
+ acquisition = Mock(spec=AcquisitionService)
+ def refresh(model_id, ref):
+ candidate = session.scalar(select(ModelRevision).where(ModelRevision.resolved_commit_sha == "b" * 40))
+ if candidate is None:
+ session.add(ModelRevision(model_id=model_id, upstream_revision=ref, resolved_commit_sha="b" * 40))
+ session.commit()
+ return Mock(resolved_commit_sha="b" * 40)
+ acquisition.refresh_model.side_effect = refresh
+ service = ModelUpdateService(session, acquisition, AuditContext("operator", "test"))
+ request = ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id)
+ yield session, service, acquisition, request
+ engine.dispose()
+
+
+def test_tracking_and_notice_history_explain_registry_delete_refusal(setup):
+ session, service, _, request = setup
+ watch = service.create(request)
+ service.check(watch.id)
+ notice, = service.notices()
+ service.dismiss(notice.id)
+ service.change(watch.id, ModelWatchUpdate(
+ expected_version=watch.version,
+ expected_baseline_revision_id=watch.baseline_revision_id,
+ baseline_revision_id=watch.baseline_revision_id, enabled=False,
+ reason="Keep historical tracking while paused",
+ ))
+ registry = RegistryService(session)
+ for revision_id, relation in (
+ (watch.baseline_revision_id, "update_tracking_baseline"),
+ (notice.candidate_revision_id, "update_candidate_history"),
+ ):
+ with pytest.raises(RegistryConflict) as blocked:
+ registry.delete("model_revision", revision_id)
+ assert any(item["relation"] == relation for item in blocked.value.details["dependencies"])
+ assert session.get(ModelRevision, revision_id) is not None
+ with pytest.raises(RegistryConflict) as blocked_model:
+ registry.delete("model", request.model_id)
+ assert {"update_tracking_baseline", "update_candidate_history"}.issubset(
+ {item["relation"] for item in blocked_model.value.details["dependencies"]})
+
+
+def test_check_deduplicates_dismissed_notice_and_never_deploys(setup):
+ session, service, acquisition, request = setup
+ watch = service.create(request)
+ assert service.create(request).id == watch.id
+ for _ in range(2):
+ assert service.check(watch.id).last_error_code is None
+ notice, = service.notices()
+ assert notice.baseline_commit_sha == "a" * 40
+ assert notice.candidate_commit_sha == "b" * 40
+ service.dismiss(notice.id)
+ service.check(watch.id)
+ assert len(service.notices()) == 1
+ assert service.notices()[0].status == "dismissed"
+ assert session.scalar(select(func.count()).select_from(ArtifactJob)) == 0
+ assert session.scalar(select(func.count()).select_from(CapabilityDeployment)) == 0
+ acquisition.create_plan.assert_not_called()
+ acquisition.execute_plan.assert_not_called()
+
+
+def test_unchanged_and_provider_failure_preserve_baseline(setup):
+ session, service, acquisition, request = setup
+ watch = service.create(request)
+ acquisition.refresh_model.side_effect = None
+ acquisition.refresh_model.return_value = Mock(resolved_commit_sha="a" * 40)
+ service.check(watch.id)
+ assert service.notices() == []
+ acquisition.refresh_model.side_effect = ProviderBoundaryError("private provider details")
+ failed = service.check(watch.id)
+ assert failed.last_error_code == "upstream_unavailable"
+ assert failed.baseline_commit_sha == watch.baseline_commit_sha
+ assert failed.last_checked_at is not None
+ assert service.notices() == []
+ assert "private provider details" not in failed.model_dump_json()
+ assert session.get(ModelUpdateWatch, watch.id).check_token is None
+
+
+def test_concurrent_check_refused_and_expired_lease_recoverable(setup):
+ session, service, acquisition, request = setup
+ watch = service.create(request)
+ item = session.get(ModelUpdateWatch, watch.id)
+ item.check_until = datetime.now(UTC) + timedelta(minutes=1)
+ item.check_token = uuid4()
+ session.commit()
+ with pytest.raises(RegistryConflict):
+ service.check(watch.id)
+ acquisition.refresh_model.assert_not_called()
+ item.check_until = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ service.check(watch.id)
+ assert len(service.notices()) == 1
+
+
+def test_unexpected_failure_records_failure_and_releases_lease(setup):
+ session, service, acquisition, request = setup
+ watch = service.create(request)
+ acquisition.refresh_model.side_effect = RuntimeError("test crash")
+ with pytest.raises(RuntimeError):
+ service.check(watch.id)
+ item = session.get(ModelUpdateWatch, watch.id)
+ assert item.last_error_code == "check_failed"
+ assert item.check_token is None
+ assert item.baseline_commit_sha == watch.baseline_commit_sha
+
+
+def test_baseline_mismatch_and_stale_baseline_refused(setup):
+ session, service, _acquisition, request = setup
+ watch = service.create(request)
+ service.check(watch.id)
+ revision = session.scalar(select(ModelRevision).where(ModelRevision.resolved_commit_sha == "b" * 40))
+ with pytest.raises(RegistryConflict, match="explicit tracking update"):
+ service.create(request.model_copy(update={"baseline_revision_id": revision.id}))
+ other = Model(key="other", display_name="Other", upstream_provider="huggingface", upstream_source="org/other")
+ session.add(other)
+ session.commit()
+ with pytest.raises(RegistryConflict, match="own exact"):
+ service.create(request.model_copy(update={"model_id": other.id}))
+
+
+def test_watch_bound_and_pagination(setup):
+ _, service, _, request = setup
+ for index in range(100):
+ service.create(request.model_copy(update={"upstream_ref": f"branch-{index}"}))
+ with pytest.raises(RegistryConflict, match="100"):
+ service.create(request)
+ assert len(service.watches(limit=100)) == 100
+ assert len(service.watches(limit=10, offset=95)) == 5
+
+
+@pytest.mark.parametrize("failure", [False, True])
+def test_superseded_check_cannot_clear_successor_or_publish_notice(setup, failure):
+ session, service, acquisition, request = setup
+ watch = service.create(request)
+ successor = uuid4()
+ def supersede(model_id, ref):
+ item = session.get(ModelUpdateWatch, watch.id)
+ item.check_token = successor
+ item.check_until = datetime.now(UTC) + timedelta(minutes=5)
+ session.commit()
+ if failure:
+ raise RuntimeError("superseded failure")
+ return Mock(resolved_commit_sha="a" * 40)
+ acquisition.refresh_model.side_effect = supersede
+ with pytest.raises(RuntimeError if failure else RegistryConflict):
+ service.check(watch.id)
+ item = session.get(ModelUpdateWatch, watch.id)
+ assert item.check_token == successor
+ assert item.last_checked_at is None
+ assert service.notices() == []
+
+
+def test_metadata_audit_uses_authenticated_context(setup):
+ _, service, acquisition, _ = setup
+ assert acquisition.audit is service.audit
+ assert acquisition.audit.context.actor_id == "test"
+
+
+@pytest.mark.parametrize("role,status", [(None,401),(PrincipalRole.VIEWER,403),(PrincipalRole.OPERATOR,200),(PrincipalRole.ADMIN,200)])
+@pytest.mark.parametrize("action", ["create", "check", "dismiss", "change"])
+def test_mutation_roles(role, status, action, setup):
+ _, service, _, request = setup
+ watch = service.create(request)
+ service.check(watch.id)
+ notice = service.notices()[0]
+ app = FastAPI()
+ app.include_router(router)
+ def principal():
+ if role is None:
+ raise HTTPException(401)
+ return Principal("test", role, "test")
+ mocked = Mock()
+ mocked.create.return_value = watch
+ mocked.check.return_value = watch
+ mocked.dismiss.return_value = notice
+ mocked.change.return_value = watch
+ app.dependency_overrides[authenticate_operator_principal] = principal
+ app.dependency_overrides[get_model_update_service] = lambda: mocked
+ with TestClient(app) as client:
+ if action == "create":
+ response = client.post("/api/v1/model-updates/watches", json=request.model_dump(mode="json"))
+ elif action == "change":
+ response = client.patch(f"/api/v1/model-updates/watches/{watch.id}", json={
+ "expected_version": watch.version, "expected_baseline_revision_id": str(watch.baseline_revision_id),
+ "baseline_revision_id": str(watch.baseline_revision_id), "enabled": False, "reason": "Pause tracking",
+ })
+ else:
+ path = f"watches/{watch.id}/check" if action == "check" else f"notices/{notice.id}/dismiss"
+ response = client.post(f"/api/v1/model-updates/{path}")
+ assert response.status_code == status
+ assert getattr(mocked, action).call_count == int(status == 200)
+
+
+def test_list_bounds_require_viewer():
+ app = FastAPI()
+ app.include_router(router)
+ mocked = Mock()
+ mocked.watches.return_value = []
+ app.dependency_overrides[get_model_update_service] = lambda: mocked
+ app.dependency_overrides[authenticate_operator_principal] = lambda: Principal("viewer", PrincipalRole.VIEWER, "test")
+ with TestClient(app) as client:
+ assert client.get("/api/v1/model-updates/watches").status_code == 200
+ assert client.get("/api/v1/model-updates/watches?limit=101").status_code == 422
+ assert client.get("/api/v1/model-updates/watches?offset=-1").status_code == 422
+
+
+def tracking_change(watch, **changes):
+ return ModelWatchUpdate(expected_version=watch.version,
+ expected_baseline_revision_id=watch.baseline_revision_id,
+ baseline_revision_id=changes.get("baseline_revision_id", watch.baseline_revision_id),
+ enabled=changes.get("enabled", watch.enabled), reason="Reviewed tracking change")
+
+
+def test_pause_resume_fences_in_flight_check_and_stale_changes(setup):
+ session, service, acquisition, request = setup
+ watch = service.create(request)
+ original = acquisition.refresh_model.side_effect
+ def interrupted(model_id, ref):
+ candidate = original(model_id, ref)
+ paused = service.change(watch.id, tracking_change(watch, enabled=False))
+ assert not paused.enabled and paused.version == 2
+ assert session.get(ModelUpdateWatch, watch.id).check_token is None
+ with pytest.raises(RegistryConflict, match="changed"):
+ service.change(watch.id, tracking_change(watch, enabled=True))
+ resumed = service.change(watch.id, tracking_change(paused, enabled=True))
+ assert resumed.enabled and resumed.version == 3
+ return candidate
+ acquisition.refresh_model.side_effect = interrupted
+ with pytest.raises(RegistryConflict, match="superseded"):
+ service.check(watch.id)
+ assert service.notices() == []
+ assert service.watches()[0].last_checked_at is None
+
+
+def test_paused_watch_and_not_due_check_do_not_call_provider(setup):
+ _, service, acquisition, request = setup
+ watch = service.create(request)
+ paused = service.change(watch.id, tracking_change(watch, enabled=False))
+ with pytest.raises(RegistryConflict, match="paused"):
+ service.check(watch.id)
+ acquisition.refresh_model.assert_not_called()
+ service.change(watch.id, tracking_change(paused, enabled=True))
+ service.check(watch.id)
+ acquisition.refresh_model.reset_mock()
+ with pytest.raises(RegistryConflict, match="not due"):
+ service.check(watch.id, if_checked_before=datetime.now(UTC) - timedelta(hours=1))
+ acquisition.refresh_model.assert_not_called()
+
+
+def test_baseline_advance_is_audited_cas_and_preserves_notice_provenance(setup):
+ session, service, _, request = setup
+ watch = service.create(request)
+ service.check(watch.id)
+ notice, = service.notices()
+ advanced = service.change(watch.id, tracking_change(watch, baseline_revision_id=notice.candidate_revision_id))
+ assert advanced.baseline_commit_sha == notice.candidate_commit_sha
+ assert advanced.version == watch.version + 1
+ assert advanced.last_checked_at is None
+ assert service.notices()[0] == notice
+ audit = session.scalar(select(AuditEvent).where(AuditEvent.action == "MODEL_WATCH_TRACKING_CHANGED"))
+ assert audit.details["previous"]["baseline_revision_id"] == str(watch.baseline_revision_id)
+ assert audit.details["baseline_revision_id"] == str(notice.candidate_revision_id)
+ assert audit.details["reason"] == "Reviewed tracking change"
+ with pytest.raises(RegistryConflict, match="changed"):
+ service.change(watch.id, tracking_change(watch, enabled=False))
+ assert session.scalar(select(func.count()).select_from(CapabilityDeployment)) == 0
+
+
+def test_baseline_advance_rejects_other_models_and_active_check(setup):
+ session, service, _, request = setup
+ watch = service.create(request)
+ service.check(watch.id)
+ notice = service.notices()[0]
+ item = session.get(ModelUpdateWatch, watch.id)
+ item.check_token = uuid4()
+ item.check_until = datetime.now(UTC) + timedelta(minutes=1)
+ session.commit()
+ with pytest.raises(RegistryConflict, match="active check"):
+ service.change(watch.id, tracking_change(watch, baseline_revision_id=notice.candidate_revision_id))
+ other = Model(key="other", display_name="Other", upstream_provider="huggingface", upstream_source="org/other")
+ session.add(other)
+ session.flush()
+ other_revision = ModelRevision(model_id=other.id, upstream_revision="main", resolved_commit_sha="c" * 40)
+ session.add(other_revision)
+ session.commit()
+ with pytest.raises(RegistryConflict, match="same model"):
+ service.change(watch.id, tracking_change(watch, baseline_revision_id=other_revision.id))
+ paused = service.change(watch.id, tracking_change(watch, enabled=False))
+ updated = service.change(watch.id, tracking_change(paused, baseline_revision_id=notice.candidate_revision_id))
+ assert not updated.enabled
+ assert session.get(ModelUpdateWatch, watch.id).check_token is None
+
+
+@pytest.mark.parametrize("env,enabled,active", [("production",True,True),("production",False,False),("development",True,False)])
+def test_monitor_status_reports_actual_environment_gate(env, enabled, active):
+ from modelforge_api.settings import Settings, get_settings
+ app = FastAPI()
+ app.include_router(router)
+ app.dependency_overrides[authenticate_operator_principal] = lambda: Principal("viewer", PrincipalRole.VIEWER, "test")
+ app.dependency_overrides[get_settings] = lambda: Settings(env=env, model_update_monitor_enabled=enabled,
+ model_update_check_interval_seconds=7200)
+ with TestClient(app) as client:
+ assert client.get("/api/v1/model-updates/monitor").json() == {"enabled": active, "check_interval_seconds": 7200}
diff --git a/backend/tests/test_model_updates_postgres.py b/backend/tests/test_model_updates_postgres.py
new file mode 100644
index 0000000..e3df9e0
--- /dev/null
+++ b/backend/tests/test_model_updates_postgres.py
@@ -0,0 +1,156 @@
+"""Actual owner/runtime PostgreSQL upgrade and dump/restore qualification."""
+
+import importlib
+import os
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.model_updates import ModelWatchCreate, ModelWatchUpdate
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.persistence.models import Model, ModelRevision, ModelUpdateWatch
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.model_updates import ModelUpdateService
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+ IdentityDatabase,
+ ManagedPostgresTools,
+ identity_database,
+)
+
+__all__ = ["identity_database"]
+PRIOR = "20260909_0029"
+CURRENT = "20260910_0034"
+
+
+def test_model_watch_upgrade_prior_restore_and_runtime_permissions(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ database = identity_database
+ alembic = importlib.import_module("alembic.command")
+ alembic.upgrade(database.migration(), PRIOR)
+ with database.runtime.connect() as connection:
+ prior_tables = set(inspect(connection).get_table_names())
+ assert not {"model_update_watches", "model_update_notices"} & prior_tables
+ with Session(database.runtime) as session:
+ model = Model(key="selected-model", display_name="Selected model",
+ upstream_provider="huggingface", upstream_source="fixture/selected")
+ session.add(model)
+ session.flush()
+ revision = ModelRevision(model_id=model.id, upstream_revision="main",
+ resolved_commit_sha="a" * 40)
+ session.add(revision)
+ AuditWriter(session, "operator", "prior-release").write(
+ "qualification.baseline", "model", str(model.id), {})
+ session.commit()
+ request = ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id)
+ before = semantic_fingerprint(session.connection())
+ archive = tmp_path / "pre-model-updates.dump"
+ database.archive(archive)
+ alembic.upgrade(database.migration(), CURRENT)
+ with database.runtime.connect() as connection:
+ after = semantic_fingerprint(connection)
+ comparison = fingerprint_compatibility(before, after, source_schema_revision=PRIOR)
+ assert comparison.compatible
+ assert audit_fingerprint_differences_are_compatible(before, after, comparison)
+ # Empty tables are mechanically reversible; populated history is deliberately retained.
+ alembic.downgrade(database.migration(), PRIOR)
+ alembic.upgrade(database.migration(), CURRENT)
+ with Session(database.runtime) as session:
+ service = ModelUpdateService(session, Mock(spec=AcquisitionService),
+ AuditContext("operator", "qualified", "model-watch-pg"))
+ watch = service.create(request)
+ assert session.scalar(select(ModelUpdateWatch.baseline_commit_sha)) == "a" * 40
+ assert watch.model_id == request.model_id
+ assert watch.enabled and watch.version == 1
+ paused = service.change(watch.id, ModelWatchUpdate(
+ expected_version=watch.version, expected_baseline_revision_id=watch.baseline_revision_id,
+ baseline_revision_id=watch.baseline_revision_id, enabled=False, reason="Qualified pause",
+ ))
+ assert not paused.enabled and paused.version == 2
+ with database.runtime.connect() as connection:
+ for statement in (
+ "DELETE FROM model_update_watches",
+ "TRUNCATE model_update_notices CASCADE",
+ "ALTER TABLE model_update_watches ADD COLUMN forbidden integer",
+ ):
+ with pytest.raises(DBAPIError):
+ connection.execute(text(statement))
+ connection.rollback()
+ with pytest.raises(RuntimeError, match="history exists"):
+ alembic.downgrade(database.migration(), PRIOR)
+ database.archive(archive, restore=True)
+ alembic.upgrade(database.migration(restored=True), CURRENT)
+ with database.restored_runtime.connect() as connection:
+ restored = semantic_fingerprint(connection)
+ comparison = fingerprint_compatibility(before, restored, source_schema_revision=PRIOR)
+ assert comparison.compatible
+ assert audit_fingerprint_differences_are_compatible(before, restored, comparison)
+ assert connection.scalar(text("SELECT count(*) FROM model_update_watches")) == 0
+
+
+def test_model_watch_restore_clears_execution_claim_but_preserves_tracking(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), TARGET_SCHEMA_REVISION)
+ with Session(database.runtime) as session:
+ model = Model(key="restore-watch", display_name="Restore watch",
+ upstream_provider="huggingface", upstream_source="fixture/restore")
+ session.add(model)
+ session.flush()
+ revision = ModelRevision(model_id=model.id, upstream_revision="main",
+ resolved_commit_sha="a" * 40)
+ session.add(revision)
+ session.commit()
+ watch = ModelUpdateService(session, Mock(spec=AcquisitionService),
+ AuditContext("operator", "restore-test")).create(
+ ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id))
+ row = session.get(ModelUpdateWatch, watch.id)
+ assert row is not None
+ row.check_token = uuid.uuid4()
+ row.check_until = datetime.now(UTC) + timedelta(minutes=5)
+ session.commit()
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "model-watch.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ settings = Settings(_env_file=None, env="test",
+ alembic_directory=Path(__file__).resolve().parents[1] / "alembic")
+ with Session(database.runtime) as session:
+ recovery = RecoveryService(session, settings)
+ recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+ record = SimpleNamespace(id=uuid.uuid4(), source_fingerprint=source)
+ plan = SimpleNamespace(
+ database_destination=database.restored_owner.render_as_string(hide_password=False),
+ backup_set=SimpleNamespace(backup_id="watch-qualification", schema_revision=TARGET_SCHEMA_REVISION,
+ completed_at=datetime.now(UTC)), validation_requirements={},
+ )
+ with pytest.raises(RecoveryError):
+ recovery._validate_restored(record, plan)
+ session.rollback()
+ recovery._reconcile_restored(record, plan)
+ session.commit()
+ assert all(recovery._validate_restored(record, plan)["gate"].values())
+ session.commit()
+ with database.restored_runtime.connect() as connection:
+ assert connection.execute(text(
+ "SELECT check_token, check_until FROM model_update_watches"
+ )).one() == (None, None)
+ restored = semantic_fingerprint(connection)
+ for table in ("model_update_watches", "model_update_notices"):
+ assert restored["tables"][table] == source["tables"][table]
diff --git a/backend/tests/test_node_agent_api.py b/backend/tests/test_node_agent_api.py
index 7615ef8..88d5943 100644
--- a/backend/tests/test_node_agent_api.py
+++ b/backend/tests/test_node_agent_api.py
@@ -5,6 +5,7 @@ import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
from hardware_fakes import FakeAcceleratorCollector, FakeHostCollector
+from http_contract_fakes import isolated_http_contract # noqa: F401 - opt-in pytest fixture
from pydantic import SecretStr
from sqlalchemy import create_engine, text
from sqlalchemy.orm import Session
@@ -18,6 +19,8 @@ from modelforge_api.services.hardware_inventory import HardwareInventoryService
from modelforge_api.services.node_agent import NodeAgentService
from modelforge_api.settings import Settings, get_settings
+pytestmark = pytest.mark.usefixtures("isolated_http_contract")
+
def enrollment_payload(token: str, identity: str = "remote-a") -> dict:
return {
@@ -71,6 +74,16 @@ def test_agent_api_enrollment_secret_is_once_only_and_node_scoped() -> None:
assert enrolled.status_code == 201
assert enrolled.headers["cache-control"] == "no-store"
credential = enrolled.json()["node_credential"]
+ enrollments_after_use = client.get(
+ "/api/v1/admin/node-enrollments",
+ headers={"X-ModelForge-Admin-Token": "admin-key"},
+ )
+ assert enrollments_after_use.status_code == 200
+ used_summary = next(
+ item for item in enrollments_after_use.json() if item["id"] == created.json()["id"]
+ )
+ assert used_summary["enrolled_node_id"] == enrolled.json()["node_id"]
+ assert "enrollment_token" not in used_summary
assert (
client.post("/api/v1/agent/enroll", json=enrollment_payload(token)).status_code
== 401
@@ -189,6 +202,8 @@ def test_agent_openapi_contract_contains_versioned_surfaces_without_list_secret(
"/api/v1/agent/telemetry",
"/api/v1/admin/node-enrollments",
"/api/v1/admin/hardware/nodes/{node_id}/credential/rotate",
+ "/api/v1/admin/hardware/nodes/{node_id}/lifecycle/preview",
+ "/api/v1/admin/hardware/nodes/{node_id}/lifecycle",
"/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
"/api/v1/admin/hardware/nodes/{node_id}/decommission",
"/api/v1/agent/runtime-probes/next",
@@ -199,3 +214,4 @@ def test_agent_openapi_contract_contains_versioned_surfaces_without_list_secret(
assert path in schema["paths"]
summary = schema["components"]["schemas"]["EnrollmentTokenSummary"]
assert "enrollment_token" not in summary["properties"]
+ assert "enrolled_node_id" in summary["properties"]
diff --git a/backend/tests/test_node_auth_scope_migration.py b/backend/tests/test_node_auth_scope_migration.py
index cf4b53c..3b8abde 100644
--- a/backend/tests/test_node_auth_scope_migration.py
+++ b/backend/tests/test_node_auth_scope_migration.py
@@ -6,13 +6,13 @@ from types import ModuleType
import pytest
import sqlalchemy as sa
-from sqlalchemy import Engine
-from sqlalchemy.exc import IntegrityError
-
from alembic.config import Config
from alembic.migration import MigrationContext
from alembic.operations import Operations
from alembic.script import ScriptDirectory
+from sqlalchemy import Engine
+from sqlalchemy.exc import IntegrityError
+
from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
from modelforge_api.persistence.models import NodeCredential, NodeEnrollment
@@ -75,10 +75,23 @@ def test_0023_is_the_linear_auth_step_before_the_current_audit_head() -> None:
scripts = ScriptDirectory.from_config(config)
migration = _migration()
- assert scripts.get_heads() == ["20260830_0024"]
+ assert scripts.get_heads() == ["20260924_0039"]
assert migration.down_revision == "20260828_0022"
assert scripts.get_revision("20260830_0024").down_revision == "20260830_0023"
- assert TARGET_SCHEMA_REVISION == "20260830_0024"
+ assert scripts.get_revision("20260907_0025").down_revision == "20260830_0024"
+ assert scripts.get_revision("20260908_0026").down_revision == "20260907_0025"
+ assert scripts.get_revision("20260908_0027").down_revision == "20260908_0026"
+ assert scripts.get_revision("20260908_0028").down_revision == "20260908_0027"
+ assert scripts.get_revision("20260909_0029").down_revision == "20260908_0028"
+ assert scripts.get_revision("20260909_0030").down_revision == "20260909_0029"
+ assert scripts.get_revision("20260909_0031").down_revision == "20260909_0030"
+ assert scripts.get_revision("20260909_0032").down_revision == "20260909_0031"
+ assert scripts.get_revision("20260909_0033").down_revision == "20260909_0032"
+ assert scripts.get_revision("20260910_0034").down_revision == "20260909_0033"
+ assert scripts.get_revision("20260911_0035").down_revision == "20260910_0034"
+ assert scripts.get_revision("20260923_0036").down_revision == "20260911_0035"
+ assert TARGET_SCHEMA_REVISION == "20260924_0039"
+ assert scripts.get_revision(TARGET_SCHEMA_REVISION).down_revision == "20260924_0038"
def test_current_model_metadata_carries_both_exact_scope_constraints() -> None:
diff --git a/backend/tests/test_node_lifecycle.py b/backend/tests/test_node_lifecycle.py
new file mode 100644
index 0000000..7f85127
--- /dev/null
+++ b/backend/tests/test_node_lifecycle.py
@@ -0,0 +1,433 @@
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.agent import (
+ get_agent_service,
+ get_decommission_service,
+)
+from modelforge_api.api.routes.agent import (
+ router as agent_router,
+)
+from modelforge_api.domain.agent_protocol import (
+ AgentMetadata,
+ HeartbeatRequest,
+ NodeManagementUpdate,
+)
+from modelforge_api.domain.node_decommission import NodeDecommissionExecute
+from modelforge_api.domain.node_lifecycle import (
+ NodeLifecycleChange,
+ NodeLifecyclePreviewRequest,
+)
+from modelforge_api.main import agent_protocol_error, node_decommission_error
+from modelforge_api.persistence.models import (
+ Accelerator,
+ AuditEvent,
+ Base,
+ ComputeNode,
+ GatewayRequest,
+ GpuLease,
+ NodeCredential,
+)
+from modelforge_api.services.node_agent import (
+ AgentAuthorizationError,
+ AgentConflictError,
+ NodeAgentService,
+ secret_hash,
+)
+from modelforge_api.services.node_decommission import (
+ NodeDecommissionError,
+ NodeDecommissionService,
+)
+from modelforge_api.settings import Settings, get_settings
+
+
+@pytest.fixture
+def session() -> Session:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+
+
+def _node(session: Session, *, enabled: bool = True, liveness: str = "offline") -> ComputeNode:
+ node = ComputeNode(
+ key=f"node-{uuid.uuid4()}",
+ hostname="node-lifecycle-test",
+ display_name="Lifecycle test node",
+ enabled=enabled,
+ status="active",
+ liveness_state=liveness,
+ inventory={"source": "test"},
+ observation_source="remote_agent",
+ agent_protocol_version=1,
+ agent_capabilities=["hardware.inventory"],
+ agent_version="1.3.0-rc.1",
+ last_heartbeat_at=datetime.now(UTC),
+ )
+ session.add(node)
+ session.commit()
+ return node
+
+
+def _request(preview, *, target_enabled: bool | None = None) -> NodeLifecycleChange:
+ return NodeLifecycleChange(
+ target_enabled=preview.target_enabled if target_enabled is None else target_enabled,
+ expected_generation=preview.node_generation,
+ dependency_digest=preview.dependency_digest,
+ operator="test-operator",
+ reason="Planned node maintenance with workloads drained.",
+ confirmation=preview.persisted_identity,
+ )
+
+
+def test_disable_requires_preview_confirmation_and_audits_transition(session: Session) -> None:
+ node = _node(session, liveness="online")
+ service = NodeDecommissionService(session)
+
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ assert preview.can_apply is True
+ assert preview.current_enabled is True
+ assert preview.target_enabled is False
+ assert preview.blockers == []
+
+ result = service.apply_enabled_change(node.id, _request(preview))
+
+ session.refresh(node)
+ assert result.enabled is False
+ assert result.liveness == "disabled"
+ assert node.enabled is False
+ assert node.liveness_state == "disabled"
+ event = session.scalar(
+ select(AuditEvent).where(
+ AuditEvent.action == "NODE_DISABLED",
+ AuditEvent.resource_id == str(node.id),
+ )
+ )
+ assert event is not None
+ assert event.details["reason"] == "Planned node maintenance with workloads drained."
+
+
+def test_disable_fails_closed_for_active_gateway_request_and_gpu_lease(session: Session) -> None:
+ node = _node(session)
+ accelerator = Accelerator(
+ compute_node_id=node.id,
+ device_index=0,
+ device_uuid="GPU-node-lifecycle",
+ name="Test GPU",
+ )
+ session.add(accelerator)
+ session.flush()
+ gateway = GatewayRequest(
+ request_id=uuid.uuid4(),
+ capability_key="assistant.general",
+ capability_version=1,
+ compute_node_id=node.id,
+ status="running",
+ priority="production",
+ input_sha256="a" * 64,
+ input_count=1,
+ )
+ lease = GpuLease(
+ accelerator_id=accelerator.id,
+ deployment_id=uuid.uuid4(),
+ priority="production",
+ reserved_vram_mb=1024,
+ state="active",
+ )
+ session.add_all([gateway, lease])
+ session.commit()
+ service = NodeDecommissionService(session)
+
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ assert preview.can_apply is False
+ assert {item.code for item in preview.blockers} >= {
+ "active_gateway_request",
+ "active_gpu_lease",
+ }
+ with pytest.raises(NodeDecommissionError) as raised:
+ service.apply_enabled_change(node.id, _request(preview))
+ assert raised.value.code == "node_lifecycle_blocked"
+ session.refresh(node)
+ assert node.enabled is True
+
+
+def test_disable_rejects_preview_when_work_arrives_after_impact_review(session: Session) -> None:
+ node = _node(session)
+ service = NodeDecommissionService(session)
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ request = _request(preview)
+ session.add(
+ GatewayRequest(
+ request_id=uuid.uuid4(),
+ capability_key="assistant.general",
+ capability_version=1,
+ compute_node_id=node.id,
+ status="queued",
+ priority="production",
+ input_sha256="b" * 64,
+ input_count=1,
+ )
+ )
+ session.commit()
+
+ with pytest.raises(NodeDecommissionError) as raised:
+ service.apply_enabled_change(node.id, request)
+ assert raised.value.code == "node_lifecycle_preview_stale"
+ session.refresh(node)
+ assert node.enabled is True
+ assert node.liveness_state == "offline"
+
+
+def test_node_generation_change_invalidates_lifecycle_preview(session: Session) -> None:
+ node = _node(session)
+ service = NodeDecommissionService(session)
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ request = _request(preview)
+ node.display_name = "Renamed after preview"
+ session.commit()
+
+ with pytest.raises(NodeDecommissionError) as raised:
+ service.apply_enabled_change(node.id, request)
+ assert raised.value.code == "node_generation_conflict"
+ session.refresh(node)
+ assert node.enabled is True
+
+
+def test_enable_requires_active_remote_credential_and_preserves_unknown_readiness(
+ session: Session,
+) -> None:
+ node = _node(session, enabled=False, liveness="disabled")
+ session.add(
+ NodeCredential(
+ compute_node_id=node.id,
+ secret_hash="c" * 64,
+ scope="node.publish",
+ )
+ )
+ session.commit()
+ service = NodeDecommissionService(session)
+
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=True)
+ )
+ assert preview.can_apply is True
+ result = service.apply_enabled_change(node.id, _request(preview))
+
+ session.refresh(node)
+ assert result.enabled is True
+ assert result.liveness == "offline"
+ assert result.agent_evidence_state == "awaiting_fresh_authenticated_heartbeat"
+ assert node.enabled is True
+ assert node.liveness_state == "offline"
+ assert node.agent_protocol_version is None
+ assert node.agent_capabilities == []
+ assert node.agent_version is None
+ assert node.last_heartbeat_at is None
+ assert "awaiting fresh authenticated heartbeat" in node.last_connection_error
+
+
+def test_enable_without_active_credential_and_enable_of_tombstone_are_blocked(
+ session: Session,
+) -> None:
+ node = _node(session, enabled=False, liveness="disabled")
+ service = NodeDecommissionService(session)
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=True)
+ )
+ assert preview.can_apply is False
+ assert "active_node_credential_missing" in {item.code for item in preview.blockers}
+ with pytest.raises(NodeDecommissionError) as raised:
+ service.apply_enabled_change(node.id, _request(preview))
+ assert raised.value.code == "node_lifecycle_blocked"
+ with pytest.raises(AgentConflictError, match="impact preview and typed confirmation"):
+ NodeAgentService(session, Settings(_env_file=None)).update_node(
+ node.id, NodeManagementUpdate(enabled=True)
+ )
+
+
+def test_direct_node_patch_rejects_enabled_but_keeps_metadata_updates(session: Session) -> None:
+ node = _node(session)
+ service = NodeAgentService(session, Settings(_env_file=None))
+
+ updated = service.update_node(node.id, NodeManagementUpdate(display_name="Renamed safely"))
+ assert updated.display_name == "Renamed safely"
+ with pytest.raises(AgentConflictError, match="impact preview and typed confirmation"):
+ service.update_node(node.id, NodeManagementUpdate(enabled=False))
+ session.refresh(node)
+ assert node.enabled is True
+
+
+def test_agent_heartbeat_and_post_body_auth_refuse_node_disabled_after_prebody_auth(
+ session: Session,
+) -> None:
+ node = _node(session)
+ credential_id = uuid.uuid4()
+ raw_credential = f"mfnode_{credential_id}_race-test-secret"
+ session.add(
+ NodeCredential(
+ id=credential_id,
+ compute_node_id=node.id,
+ secret_hash=secret_hash(raw_credential),
+ scope="node.publish",
+ )
+ )
+ session.commit()
+
+ # Keep the pre-body identity cached like a request whose lifecycle changes while its body is
+ # being read. A later admin transaction disables the same row before request dispatch.
+ agent_session = Session(session.get_bind(), expire_on_commit=False)
+ agent_service = NodeAgentService(agent_session, Settings(_env_file=None))
+ (_credential, stale_node), evidence = agent_service.authenticate_with_evidence(
+ f"Bearer {raw_credential}"
+ )
+ agent_session.commit()
+ assert stale_node.enabled is True
+
+ lifecycle_session = Session(session.get_bind())
+ lifecycle = NodeDecommissionService(lifecycle_session)
+ preview = lifecycle.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ lifecycle.apply_enabled_change(node.id, _request(preview))
+ lifecycle_session.close()
+
+ with pytest.raises(AgentAuthorizationError, match="compute node is disabled"):
+ agent_service.reuse_authentication(evidence, f"Bearer {raw_credential}")
+
+ heartbeat = HeartbeatRequest(
+ identity_key=node.key,
+ metadata=AgentMetadata(
+ agent_version="1.3.0-rc.1",
+ protocol_version=1,
+ supported_capabilities=["hardware.inventory"],
+ started_at=datetime.now(UTC),
+ ),
+ observed_at=datetime.now(UTC),
+ )
+ with pytest.raises(AgentAuthorizationError, match="compute node is disabled"):
+ agent_service.heartbeat(stale_node, heartbeat)
+
+ session.refresh(node)
+ assert node.enabled is False
+ assert node.liveness_state == "disabled"
+ agent_session.close()
+
+
+def test_decommissioned_identity_cannot_be_reenabled(session: Session) -> None:
+ node = _node(session, enabled=False, liveness="disabled")
+ service = NodeDecommissionService(session)
+ retirement = service.preview(node.id)
+ service.execute(
+ node.id,
+ NodeDecommissionExecute(
+ expected_generation=retirement.node_generation,
+ preview_digest=retirement.dependency_digest,
+ idempotency_key=f"decom-{uuid.uuid4()}",
+ operator="test-operator",
+ reason="This test node has left service permanently.",
+ confirmation=retirement.persisted_identity,
+ ),
+ )
+
+ preview = service.preview_enabled_change(
+ node.id, NodeLifecyclePreviewRequest(target_enabled=True)
+ )
+ assert preview.can_apply is False
+ assert "node_decommissioned" in {item.code for item in preview.blockers}
+ with pytest.raises(NodeDecommissionError) as raised:
+ service.apply_enabled_change(node.id, _request(preview))
+ assert raised.value.code == "node_lifecycle_blocked"
+
+
+def test_admin_api_requires_preview_for_enabled_patch_but_preserves_metadata_patch(
+ session: Session,
+) -> None:
+ node = _node(session)
+ settings = Settings(_env_file=None, operator_api_key=SecretStr("admin-key"))
+ agent = NodeAgentService(session, settings)
+ lifecycle = NodeDecommissionService(session)
+ test_app = FastAPI()
+ test_app.include_router(agent_router)
+ test_app.add_exception_handler(AgentConflictError, agent_protocol_error)
+ test_app.add_exception_handler(NodeDecommissionError, node_decommission_error)
+ test_app.dependency_overrides[get_agent_service] = lambda: agent
+ test_app.dependency_overrides[get_decommission_service] = lambda: lifecycle
+ test_app.dependency_overrides[get_settings] = lambda: settings
+
+ try:
+ with TestClient(test_app) as client:
+ path = f"/api/v1/admin/hardware/nodes/{node.id}/lifecycle/preview"
+ assert client.post(path, json={"target_enabled": False}).status_code == 401
+ headers = {"X-ModelForge-Admin-Token": "admin-key"}
+ preview_response = client.post(path, headers=headers, json={"target_enabled": False})
+ assert preview_response.status_code == 200
+ preview = preview_response.json()
+ assert preview["can_apply"] is True
+
+ bypass = client.patch(
+ f"/api/v1/admin/hardware/nodes/{node.id}",
+ headers=headers,
+ json={"enabled": False},
+ )
+ assert bypass.status_code == 409
+ assert bypass.json()["error"]["code"] == "agent_conflict"
+ metadata = client.patch(
+ f"/api/v1/admin/hardware/nodes/{node.id}",
+ headers=headers,
+ json={"display_name": "Updated metadata"},
+ )
+ assert metadata.status_code == 200
+ assert node.display_name == "Updated metadata"
+
+ stale_change = {
+ "target_enabled": False,
+ "expected_generation": preview["node_generation"],
+ "dependency_digest": preview["dependency_digest"],
+ "operator": "test-operator",
+ "reason": "Planned node maintenance after draining workloads.",
+ "confirmation": preview["persisted_identity"],
+ }
+ stale = client.post(
+ f"/api/v1/admin/hardware/nodes/{node.id}/lifecycle",
+ headers=headers,
+ json=stale_change,
+ )
+ assert stale.status_code == 409
+ assert stale.json()["error"]["code"] == "node_generation_conflict"
+
+ refreshed_preview = client.post(
+ path, headers=headers, json={"target_enabled": False}
+ ).json()
+ change = {**stale_change, "expected_generation": refreshed_preview["node_generation"], "dependency_digest": refreshed_preview["dependency_digest"]}
+ applied = client.post(
+ f"/api/v1/admin/hardware/nodes/{node.id}/lifecycle",
+ headers=headers,
+ json=change,
+ )
+ assert applied.status_code == 200
+ assert applied.json()["enabled"] is False
+ assert applied.json()["liveness"] == "disabled"
+ finally:
+ test_app.dependency_overrides.clear()
diff --git a/backend/tests/test_node_lifecycle_postgres.py b/backend/tests/test_node_lifecycle_postgres.py
new file mode 100644
index 0000000..3e42fa0
--- /dev/null
+++ b/backend/tests/test_node_lifecycle_postgres.py
@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import importlib
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+pytest_plugins = ("tests.test_managed_identities_postgres",)
+
+from modelforge_api.domain.agent_protocol import AgentMetadata, HeartbeatRequest
+from modelforge_api.domain.node_lifecycle import NodeLifecycleChange, NodeLifecyclePreviewRequest
+from modelforge_api.persistence.models import ComputeNode, NodeCredential
+from modelforge_api.services.node_agent import (
+ AgentAuthorizationError,
+ NodeAgentService,
+ NodeAuthenticationEvidence,
+ secret_hash,
+)
+from modelforge_api.services.node_decommission import (
+ NodeDecommissionError,
+ NodeDecommissionService,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import IdentityDatabase
+
+
+def test_agent_request_waits_for_disable_and_rechecks_locked_node(
+ identity_database: IdentityDatabase,
+) -> None:
+ """PostgreSQL proof that a pre-body auth result cannot outlive a lifecycle disable."""
+ importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+ engine = identity_database.runtime
+ node_id = uuid.uuid4()
+ credential_id = uuid.uuid4()
+ raw = f"mfnode_{credential_id}_lifecycle-race-secret"
+ with Session(engine) as session:
+ node = ComputeNode(
+ id=node_id,
+ key=f"postgres-race-{node_id}",
+ hostname="postgres-race-node",
+ display_name="PostgreSQL lifecycle race",
+ enabled=True,
+ status="active",
+ liveness_state="offline",
+ inventory={"source": "test"},
+ observation_source="remote_agent",
+ )
+ session.add(node)
+ session.add(
+ NodeCredential(
+ id=credential_id,
+ compute_node_id=node_id,
+ secret_hash=secret_hash(raw),
+ scope="node.publish",
+ )
+ )
+ session.commit()
+
+ preview = NodeDecommissionService(session).preview_enabled_change(
+ node_id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ change = NodeLifecycleChange(
+ target_enabled=False,
+ expected_generation=preview.node_generation,
+ dependency_digest=preview.dependency_digest,
+ operator="postgres-race-test",
+ reason="Prove that disable serializes with agent authentication.",
+ confirmation=preview.persisted_identity,
+ )
+
+ request_session = Session(engine, expire_on_commit=False)
+ agent_service = NodeAgentService(request_session, Settings(_env_file=None))
+ (_credential, stale_node), evidence = agent_service.authenticate_with_evidence(
+ f"Bearer {raw}"
+ )
+ request_session.commit()
+ assert stale_node.enabled is True
+
+ lifecycle_has_locks = threading.Event()
+ allow_lifecycle_to_commit = threading.Event()
+ agent_lock_attempted = threading.Event()
+
+ with Session(engine) as lifecycle_session:
+ lifecycle_service = NodeDecommissionService(lifecycle_session)
+ lock_dependencies = lifecycle_service._lock_dependency_tables
+
+ def hold_lifecycle_locks() -> None:
+ lock_dependencies()
+ lifecycle_has_locks.set()
+ if not allow_lifecycle_to_commit.wait(timeout=20):
+ raise RuntimeError("timed out holding lifecycle locks in race test")
+
+ lifecycle_service._lock_dependency_tables = hold_lifecycle_locks # type: ignore[method-assign]
+
+ @event.listens_for(engine, "before_cursor_execute")
+ def observe_agent_lock_attempt(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ normalized = statement.lower()
+ if (
+ threading.current_thread().name.startswith("agent-request")
+ and "compute_nodes" in normalized
+ and "for update" in normalized
+ ):
+ agent_lock_attempted.set()
+
+ try:
+ with ThreadPoolExecutor(max_workers=1, thread_name_prefix="lifecycle") as lifecycle_pool:
+ lifecycle_future = lifecycle_pool.submit(
+ lifecycle_service.apply_enabled_change, node_id, change
+ )
+ assert lifecycle_has_locks.wait(timeout=10), "lifecycle never acquired its locks"
+ with ThreadPoolExecutor(max_workers=1, thread_name_prefix="agent-request") as agent_pool:
+ agent_future = agent_pool.submit(
+ _publish_authenticated_heartbeat,
+ agent_service,
+ evidence,
+ raw,
+ stale_node.key,
+ )
+ assert agent_lock_attempted.wait(timeout=10), "agent never requested its node lock"
+ with pytest.raises(FutureTimeoutError):
+ agent_future.result(timeout=0.25)
+
+ allow_lifecycle_to_commit.set()
+ lifecycle_future.result(timeout=30)
+ assert agent_future.result(timeout=30) == "rejected"
+ finally:
+ allow_lifecycle_to_commit.set()
+ event.remove(engine, "before_cursor_execute", observe_agent_lock_attempt)
+ request_session.close()
+
+ with Session(engine) as session:
+ persisted = session.scalar(select(ComputeNode).where(ComputeNode.id == node_id))
+ assert persisted is not None
+ assert persisted.enabled is False
+ assert persisted.liveness_state == "disabled"
+ assert persisted.generation == change.expected_generation + 1
+
+
+def test_heartbeat_wins_node_lock_and_invalidates_lifecycle_preview(
+ identity_database: IdentityDatabase,
+) -> None:
+ """A heartbeat serialized before disable advances the CAS generation."""
+ importlib.import_module("alembic.command").upgrade(identity_database.migration(), "head")
+ engine = identity_database.runtime
+ node_id = uuid.uuid4()
+ credential_id = uuid.uuid4()
+ raw = f"mfnode_{credential_id}_heartbeat-first-race-secret"
+ with Session(engine) as session:
+ node = ComputeNode(
+ id=node_id,
+ key=f"postgres-heartbeat-race-{node_id}",
+ hostname="postgres-heartbeat-race-node",
+ display_name="PostgreSQL heartbeat race",
+ enabled=True,
+ status="active",
+ liveness_state="offline",
+ inventory={"source": "test"},
+ observation_source="remote_agent",
+ )
+ session.add(node)
+ session.add(
+ NodeCredential(
+ id=credential_id,
+ compute_node_id=node_id,
+ secret_hash=secret_hash(raw),
+ scope="node.publish",
+ )
+ )
+ session.commit()
+ lifecycle = NodeDecommissionService(session)
+ preview = lifecycle.preview_enabled_change(
+ node_id, NodeLifecyclePreviewRequest(target_enabled=False)
+ )
+ change = NodeLifecycleChange(
+ target_enabled=False,
+ expected_generation=preview.node_generation,
+ dependency_digest=preview.dependency_digest,
+ operator="postgres-race-test",
+ reason="Prove that fresh heartbeat evidence invalidates this preview.",
+ confirmation=preview.persisted_identity,
+ )
+
+ agent_session = Session(engine, expire_on_commit=False)
+ agent_service = NodeAgentService(agent_session, Settings(_env_file=None))
+ (_credential, stale_node), evidence = agent_service.authenticate_with_evidence(
+ f"Bearer {raw}"
+ )
+ agent_session.commit()
+ heartbeat_has_node_lock = threading.Event()
+ allow_heartbeat_to_commit = threading.Event()
+ lifecycle_lock_attempted = threading.Event()
+ original_commit = agent_session.commit
+
+ def hold_heartbeat_commit() -> None:
+ heartbeat_has_node_lock.set()
+ if not allow_heartbeat_to_commit.wait(timeout=20):
+ raise RuntimeError("timed out holding the heartbeat node lock")
+ original_commit()
+
+ agent_session.commit = hold_heartbeat_commit # type: ignore[method-assign]
+
+ @event.listens_for(engine, "before_cursor_execute")
+ def observe_lifecycle_lock_attempt(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ normalized = statement.lower()
+ if (
+ threading.current_thread().name.startswith("lifecycle-after-heartbeat")
+ and "compute_nodes" in normalized
+ and "for update" in normalized
+ ):
+ lifecycle_lock_attempted.set()
+
+ request = HeartbeatRequest(
+ identity_key=stale_node.key,
+ metadata=AgentMetadata(
+ agent_version="1.3.0-rc.1",
+ protocol_version=1,
+ supported_capabilities=["hardware.inventory"],
+ started_at=datetime.now(UTC),
+ ),
+ observed_at=datetime.now(UTC),
+ )
+
+ try:
+ with ThreadPoolExecutor(max_workers=1, thread_name_prefix="heartbeat") as agent_pool:
+ heartbeat_future = agent_pool.submit(
+ _publish_and_hold_authenticated_heartbeat,
+ agent_service,
+ evidence,
+ raw,
+ request,
+ )
+ assert heartbeat_has_node_lock.wait(timeout=10), "heartbeat never acquired node lock"
+ with ThreadPoolExecutor(
+ max_workers=1, thread_name_prefix="lifecycle-after-heartbeat"
+ ) as lifecycle_pool:
+ lifecycle_future = lifecycle_pool.submit(
+ _apply_stale_lifecycle_change, engine, node_id, change
+ )
+ assert lifecycle_lock_attempted.wait(timeout=10), "lifecycle never requested node lock"
+ with pytest.raises(FutureTimeoutError):
+ lifecycle_future.result(timeout=0.25)
+ allow_heartbeat_to_commit.set()
+ heartbeat_future.result(timeout=30)
+ assert lifecycle_future.result(timeout=30) == "node_generation_conflict"
+ finally:
+ allow_heartbeat_to_commit.set()
+ event.remove(engine, "before_cursor_execute", observe_lifecycle_lock_attempt)
+ agent_session.close()
+
+ with Session(engine) as session:
+ persisted = session.scalar(select(ComputeNode).where(ComputeNode.id == node_id))
+ assert persisted is not None
+ assert persisted.enabled is True
+ assert persisted.liveness_state == "online"
+ assert persisted.last_heartbeat_at is not None
+ assert persisted.generation == change.expected_generation + 1
+
+
+def _publish_and_hold_authenticated_heartbeat(
+ service: NodeAgentService,
+ evidence: NodeAuthenticationEvidence,
+ raw_credential: str,
+ request: HeartbeatRequest,
+) -> None:
+ _credential, node = service.reuse_authentication(evidence, f"Bearer {raw_credential}")
+ service.heartbeat(node, request)
+
+
+def _apply_stale_lifecycle_change(
+ engine: Engine, node_id: uuid.UUID, change: NodeLifecycleChange
+) -> str:
+ with Session(engine) as session:
+ try:
+ NodeDecommissionService(session).apply_enabled_change(node_id, change)
+ except NodeDecommissionError as exc:
+ if exc.code != "node_generation_conflict":
+ raise
+ return exc.code
+ return "applied"
+
+
+def _publish_authenticated_heartbeat(
+ service: NodeAgentService,
+ evidence: NodeAuthenticationEvidence,
+ raw_credential: str,
+ identity_key: str,
+) -> str:
+ try:
+ _credential, node = service.reuse_authentication(
+ evidence, f"Bearer {raw_credential}"
+ )
+ service.heartbeat(
+ node,
+ HeartbeatRequest(
+ identity_key=identity_key,
+ metadata=AgentMetadata(
+ agent_version="1.3.0-rc.1",
+ protocol_version=1,
+ supported_capabilities=["hardware.inventory"],
+ started_at=datetime.now(UTC),
+ ),
+ observed_at=datetime.now(UTC),
+ ),
+ )
+ except AgentAuthorizationError:
+ return "rejected"
+ return "published"
diff --git a/backend/tests/test_observability_api_m14.py b/backend/tests/test_observability_api_m14.py
index ff080ea..b4c8733 100644
--- a/backend/tests/test_observability_api_m14.py
+++ b/backend/tests/test_observability_api_m14.py
@@ -1,9 +1,14 @@
from __future__ import annotations
+import hashlib
+import json
from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from uuid import UUID
import pytest
from fastapi.testclient import TestClient
+from pydantic import SecretStr
from sqlalchemy import create_engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
@@ -11,7 +16,16 @@ from sqlalchemy.pool import StaticPool
from modelforge_api.api.routes.observability import get_service
from modelforge_api.main import app
-from modelforge_api.persistence.models import Base
+from modelforge_api.persistence.models import (
+ Accelerator,
+ Base,
+ ComputeNode,
+ GatewayRequest,
+ OperationalIncident,
+ ProjectBinding,
+ ServiceClient,
+ ServingGpuLease,
+)
from modelforge_api.services.observability import ObservabilityService
from modelforge_api.settings import Settings, get_settings
@@ -34,6 +48,84 @@ def clean_overrides() -> Iterator[None]:
app.dependency_overrides.clear()
+def add_active_lease(
+ session: Session,
+ *,
+ lease_id: str,
+ lease_created_at: datetime,
+ expires_at: datetime,
+ state: str = "active",
+ request_status: str = "running",
+ heartbeat_at: datetime | None = None,
+ include_request: bool = True,
+ request_deployment_id: UUID | None = None,
+) -> tuple[str, str, str, str]:
+ node_id = UUID(f"00000000-0000-0000-0000-{lease_id[-12:]}")
+ accelerator_id = UUID(f"10000000-0000-0000-0000-{lease_id[-12:]}")
+ deployment_id = UUID(f"20000000-0000-0000-0000-{lease_id[-12:]}")
+ lease_uuid = UUID(f"30000000-0000-0000-0000-{lease_id[-12:]}")
+ request_uuid = UUID(f"40000000-0000-0000-0000-{lease_id[-12:]}")
+ client_uuid = UUID(f"50000000-0000-0000-0000-{lease_id[-12:]}")
+ project_binding_uuid = UUID(f"60000000-0000-0000-0000-{lease_id[-12:]}")
+ project_uuid = UUID(f"70000000-0000-0000-0000-{lease_id[-12:]}")
+ contract_uuid = UUID(f"80000000-0000-0000-0000-{lease_id[-12:]}")
+ node = ComputeNode(id=node_id, key=f"node-{lease_id}", hostname=f"{lease_id}.local")
+ accelerator = Accelerator(
+ id=accelerator_id,
+ compute_node_id=node_id,
+ device_index=0,
+ device_uuid=f"GPU-{lease_id}",
+ name="Test GPU",
+ )
+ binding = ProjectBinding(
+ id=project_binding_uuid,
+ project_id=project_uuid,
+ capability_contract_id=contract_uuid,
+ channel="stable",
+ priority="production",
+ )
+ client = ServiceClient(
+ id=client_uuid,
+ name=f"private-client-{lease_id}",
+ project_binding_id=project_binding_uuid,
+ )
+ session.add_all([node, accelerator, binding, client])
+ if include_request:
+ session.add(
+ GatewayRequest(
+ request_id=request_uuid,
+ service_client_id=client_uuid,
+ capability_key="rag.embedding",
+ capability_version=1,
+ capability_deployment_id=request_deployment_id or deployment_id,
+ compute_node_id=node_id,
+ status=request_status,
+ priority="production",
+ input_sha256="f" * 64,
+ input_count=1,
+ )
+ )
+ session.add(
+ ServingGpuLease(
+ id=lease_uuid,
+ accelerator_id=accelerator_id,
+ capability_deployment_id=deployment_id,
+ request_id=request_uuid if include_request else None,
+ reserved_vram_bytes=2_048,
+ priority="production",
+ state=state,
+ owner=f"private-owner-{lease_id}",
+ created_at=lease_created_at,
+ expires_at=expires_at,
+ acquired_at=lease_created_at,
+ heartbeat_at=heartbeat_at or lease_created_at,
+ lease_type="request_execution",
+ materialized_vram_bytes=512,
+ )
+ )
+ return str(lease_uuid), str(project_uuid), str(node_id), str(deployment_id)
+
+
def test_operations_routes_require_the_operator_credential(session: Session) -> None:
subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
subject.ensure_defaults()
@@ -56,6 +148,239 @@ def test_operations_routes_require_the_operator_credential(session: Session) ->
assert response.json()["status"] == "HEALTHY"
+def test_incident_list_keeps_legacy_shape_and_supports_bounded_pages(session: Session) -> None:
+ observed_at = datetime.now(UTC)
+ session.add_all([
+ OperationalIncident(
+ fingerprint=f"{index:064x}",
+ title=f"Incident {index}",
+ state="OPEN",
+ severity="CRITICAL",
+ root_subject_type="node",
+ root_subject_ref=f"node-{index}",
+ correlation="LIKELY_ROOT",
+ first_seen_at=observed_at + timedelta(seconds=index),
+ last_seen_at=observed_at + timedelta(seconds=index),
+ )
+ for index in range(3)
+ ])
+ session.commit()
+
+ service = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+ app.dependency_overrides[get_service] = lambda: service
+ app.dependency_overrides[get_settings] = lambda: Settings(
+ operator_api_key="operator-secret", observability_monitor_enabled=False
+ )
+ client = TestClient(app)
+ headers = {"X-ModelForge-Admin-Token": "operator-secret"}
+ path = "/api/v1/admin/operations/incidents"
+
+ legacy = client.get(path, headers=headers)
+ first_page = client.get(f"{path}?limit=2&offset=0", headers=headers)
+ second_page = client.get(f"{path}?limit=2&offset=2", headers=headers)
+ empty_page = client.get(f"{path}?limit=2&offset=3", headers=headers)
+
+ assert legacy.status_code == 200 and len(legacy.json()) == 3
+ assert first_page.status_code == 200 and len(first_page.json()) == 2
+ assert second_page.status_code == 200 and len(second_page.json()) == 1
+ assert empty_page.status_code == 200 and empty_page.json() == []
+ assert [row["title"] for row in first_page.json() + second_page.json()] == [
+ "Incident 2", "Incident 1", "Incident 0",
+ ]
+ assert set(first_page.json()[0]) == {
+ "id", "fingerprint", "title", "state", "severity", "root_subject_type",
+ "root_subject_ref", "correlation", "first_seen_at", "last_seen_at", "resolved_at",
+ }
+ assert client.get(f"{path}?limit=501&offset=0", headers=headers).status_code == 422
+ assert client.get(f"{path}?offset=2", headers=headers).status_code == 422
+
+
+def test_active_lease_inventory_is_admin_only_and_returns_safe_current_identity(
+ session: Session,
+) -> None:
+ now = datetime.now(UTC)
+ lease_id, project_id, node_id, deployment_id = add_active_lease(
+ session,
+ lease_id="lease-000000000001",
+ lease_created_at=now - timedelta(seconds=2),
+ expires_at=now + timedelta(seconds=30),
+ )
+ subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+ app.dependency_overrides[get_service] = lambda: subject
+ tokens = {role: f"mfop_{role[0] * 43}" for role in ("viewer", "operator", "admin")}
+ identities = [
+ {
+ "credential_id": f"active-leases-{role}",
+ "subject": f"active-leases-{role}",
+ "role": role,
+ "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+ "expires_at": (now + timedelta(hours=1)).isoformat(),
+ }
+ for role, token in tokens.items()
+ ]
+ app.dependency_overrides[get_settings] = lambda: Settings(
+ _env_file=None,
+ operator_api_key=None,
+ operator_identities=SecretStr(json.dumps(identities)),
+ observability_monitor_enabled=False,
+ )
+ client = TestClient(app)
+ path = "/api/v1/admin/operations/active-leases"
+
+ assert client.get(path).status_code == 401
+
+ for role in ("viewer", "operator"):
+ response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens[role]})
+ assert response.status_code == 403
+
+ response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens["admin"]})
+
+ assert response.status_code == 200
+ payload = response.json()
+ assert set(payload) == {"observed_at", "completeness", "limit", "returned_count", "leases"}
+ assert payload["completeness"] == "COMPLETE"
+ assert payload["limit"] == 100
+ assert payload["returned_count"] == 1
+ assert set(payload["leases"][0]) == {
+ "lease_id",
+ "project_id",
+ "node_id",
+ "accelerator_id",
+ "deployment_id",
+ "lease_state",
+ "workload_status",
+ "priority",
+ "lease_type",
+ "reserved_vram_bytes",
+ "materialized_vram_bytes",
+ "created_at",
+ "acquired_at",
+ "heartbeat_at",
+ "expires_at",
+ }
+ lease = payload["leases"][0]
+ assert lease["lease_id"] == lease_id
+ assert lease["project_id"] == project_id
+ assert lease["node_id"] == node_id
+ assert lease["deployment_id"] == deployment_id
+ assert lease["lease_state"] == "ACTIVE"
+ assert lease["workload_status"] == "RUNNING"
+ for forbidden in (
+ "owner",
+ "request_id",
+ "private-owner",
+ "private-client",
+ "input_sha256",
+ "payload",
+ ):
+ assert forbidden not in response.text
+ assert not session.new
+ assert not session.dirty
+ assert not session.deleted
+
+
+def test_active_lease_inventory_excludes_expired_rows_and_bounds_sorted_results(
+ session: Session,
+) -> None:
+ now = datetime.now(UTC)
+ first_id, *_ = add_active_lease(
+ session,
+ lease_id="lease-000000000011",
+ lease_created_at=now - timedelta(seconds=10),
+ expires_at=now + timedelta(seconds=60),
+ )
+ second_id, *_ = add_active_lease(
+ session,
+ lease_id="lease-000000000012",
+ lease_created_at=now - timedelta(seconds=5),
+ expires_at=now + timedelta(seconds=60),
+ )
+ expired_id, *_ = add_active_lease(
+ session,
+ lease_id="lease-000000000013",
+ lease_created_at=now - timedelta(seconds=30),
+ expires_at=now - timedelta(seconds=1),
+ )
+ terminal_id, *_ = add_active_lease(
+ session,
+ lease_id="lease-000000000014",
+ lease_created_at=now - timedelta(seconds=3),
+ expires_at=now + timedelta(seconds=60),
+ state="released",
+ )
+ app.dependency_overrides[get_service] = lambda: ObservabilityService(
+ session, Settings(observability_monitor_enabled=False)
+ )
+ app.dependency_overrides[get_settings] = lambda: Settings(
+ _env_file=None,
+ operator_api_key=SecretStr("operator-secret"),
+ observability_monitor_enabled=False,
+ )
+
+ response = TestClient(app).get(
+ "/api/v1/admin/operations/active-leases?limit=1&sort_by=created_at&order=desc",
+ headers={"X-ModelForge-Admin-Token": "operator-secret"},
+ )
+
+ assert response.status_code == 200
+ payload = response.json()
+ assert payload["completeness"] == "TRUNCATED"
+ assert payload["limit"] == payload["returned_count"] == 1
+ assert [lease["lease_id"] for lease in payload["leases"]] == [second_id]
+ assert expired_id not in response.text
+ assert terminal_id not in response.text
+ assert first_id not in response.text
+ assert TestClient(app).get(
+ "/api/v1/admin/operations/active-leases?limit=501",
+ headers={"X-ModelForge-Admin-Token": "operator-secret"},
+ ).status_code == 422
+
+
+def test_active_lease_inventory_marks_unresolved_workload_status_unknown(
+ session: Session,
+) -> None:
+ now = datetime.now(UTC)
+ lease_id, project_id, *_ = add_active_lease(
+ session,
+ lease_id="lease-000000000021",
+ lease_created_at=now - timedelta(seconds=2),
+ expires_at=now + timedelta(seconds=30),
+ request_status="completed",
+ heartbeat_at=now - timedelta(minutes=5),
+ )
+ mismatched_lease_id, *_ = add_active_lease(
+ session,
+ lease_id="lease-000000000022",
+ lease_created_at=now - timedelta(seconds=1),
+ expires_at=now + timedelta(seconds=30),
+ state="pending",
+ request_status="running",
+ )
+ app.dependency_overrides[get_service] = lambda: ObservabilityService(
+ session, Settings(observability_monitor_enabled=False)
+ )
+ app.dependency_overrides[get_settings] = lambda: Settings(
+ _env_file=None,
+ operator_api_key=SecretStr("operator-secret"),
+ observability_monitor_enabled=False,
+ )
+
+ response = TestClient(app).get(
+ "/api/v1/admin/operations/active-leases",
+ headers={"X-ModelForge-Admin-Token": "operator-secret"},
+ )
+
+ assert response.status_code == 200
+ leases = {item["lease_id"]: item for item in response.json()["leases"]}
+ lease = leases[lease_id]
+ assert lease["project_id"] == project_id
+ assert lease["workload_status"] == "UNKNOWN"
+ assert lease["lease_state"] == "ACTIVE"
+ assert lease["heartbeat_at"] == (now - timedelta(minutes=5)).isoformat().replace("+00:00", "Z")
+ assert leases[mismatched_lease_id]["lease_state"] == "PENDING"
+ assert leases[mismatched_lease_id]["workload_status"] == "UNKNOWN"
+
+
def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload(
session: Session,
) -> None:
diff --git a/backend/tests/test_operator_authorization.py b/backend/tests/test_operator_authorization.py
index fee3df8..6824c1e 100644
--- a/backend/tests/test_operator_authorization.py
+++ b/backend/tests/test_operator_authorization.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import json
import uuid
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterable, Iterator
from dataclasses import dataclass
from datetime import UTC, datetime
from unittest.mock import Mock
@@ -31,10 +31,12 @@ from modelforge_api.api.authorization import (
from modelforge_api.api.request_limits import MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS
from modelforge_api.api.routes.acquisition import get_acquisition_service
from modelforge_api.api.routes.agent import authenticated_node, get_agent_service
+from modelforge_api.api.routes.external_creative import creative_client
from modelforge_api.api.routes.hardware import get_hardware_service
from modelforge_api.api.routes.registry import get_registry_service
from modelforge_api.api.routes.runtime import get_runtime_service
from modelforge_api.api.routes.serving import (
+ authenticate_check_client,
authenticate_document_ocr_client,
authenticate_rag_embedding_client,
authenticate_rag_reranking_client,
@@ -70,6 +72,19 @@ ALLOWED_ORIGIN = application_settings.cors_origin_list[0]
DISALLOWED_ORIGIN = "https://not-allowed.invalid"
+def _api_routes(routes: Iterable[object]) -> Iterator[APIRoute]:
+ """Walk both flattened and FastAPI 0.141+ nested included-router inventories."""
+
+ for route in routes:
+ if isinstance(route, APIRoute):
+ yield route
+ continue
+ nested_owner = getattr(route, "original_router", route)
+ nested_routes = getattr(nested_owner, "routes", None)
+ if isinstance(nested_routes, list):
+ yield from _api_routes(nested_routes)
+
+
@dataclass(frozen=True, slots=True)
class MachineBoundaryHarness:
settings: Settings
@@ -308,6 +323,7 @@ def operator_client() -> Iterator[TestClient]:
TARGET_MUTATION_POLICIES = {
+ ("POST", "/api/v1/admin/operations/notifications/test"): "admin",
# Acquisition: operators prepare and run work; approval is an administrative decision.
("POST", "/api/v1/discovery/search"): "operator",
("POST", "/api/v1/models/{model_id}/refresh-upstream"): "operator",
@@ -350,6 +366,7 @@ TARGET_MUTATION_POLICIES = {
DEPENDENCY_POLICIES = {
+ authenticate_check_client: "capability_client",
require_viewer: "viewer",
require_operator: "operator",
require_admin: "admin",
@@ -359,14 +376,17 @@ DEPENDENCY_POLICIES = {
authenticate_document_ocr_client: "capability_client",
authenticate_vision_embedding_client: "capability_client",
authenticate_speech_transcription_client: "capability_client",
+ creative_client: "capability_client",
}
CAPABILITY_AUTHENTICATORS = {
+ authenticate_check_client,
authenticate_rag_embedding_client,
authenticate_rag_reranking_client,
authenticate_document_ocr_client,
authenticate_vision_embedding_client,
authenticate_speech_transcription_client,
+ creative_client,
}
CAPABILITY_AUTHENTICATOR_SCOPES = {
@@ -375,9 +395,16 @@ CAPABILITY_AUTHENTICATOR_SCOPES = {
authenticate_document_ocr_client: "document.ocr@1",
authenticate_vision_embedding_client: "vision.embedding@1",
authenticate_speech_transcription_client: "speech.transcription@1",
+ creative_client: "creative.resource@1",
}
EXPECTED_CAPABILITY_AUTHENTICATORS = {
+ ("GET", "/api/v1/selected-model-comparisons/assigned"): authenticate_rag_embedding_client,
+ **{("POST", f"/api/v1/selected-model-comparisons/{{comparison_id}}/{action}"): authenticate_rag_embedding_client
+ for action in ("claim", "progress", "result", "finish")},
+ **{("GET", f"/api/v1/capabilities/{capability}/check"): authenticate_check_client for capability in (
+ "rag.embedding@1", "rag.reranking@1", "document.ocr@1", "vision.embedding@1", "speech.transcription@1",
+ )},
("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): authenticate_rag_embedding_client,
("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): authenticate_rag_reranking_client,
("POST", "/api/v1/capabilities/document.ocr@1/invoke"): authenticate_document_ocr_client,
@@ -394,6 +421,13 @@ EXPECTED_CAPABILITY_AUTHENTICATORS = {
"/api/v1/capability-experiments/{route_key}/invoke",
): authenticate_rag_embedding_client,
("POST", "/v1/embeddings"): authenticate_rag_embedding_client,
+ ("POST", "/api/v1/capabilities/creative.resource@1/allocations"): creative_client,
+ ("GET", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}"): creative_client,
+ ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/heartbeat"): creative_client,
+ ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/attest"): creative_client,
+ ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/release"): creative_client,
+ ("POST", "/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/uncertain"): creative_client,
+ ("POST", "/api/v1/capabilities/creative.resource@1/profiles/observations"): creative_client,
}
@@ -423,9 +457,7 @@ def route_policy(route: APIRoute, method: str) -> str | None:
def test_every_route_has_one_explicit_access_policy() -> None:
unclassified: list[tuple[str, str]] = []
- for route in app.routes:
- if not isinstance(route, APIRoute):
- continue
+ for route in _api_routes(app.routes):
for method in route.methods:
if route_policy(route, method) is None:
unclassified.append((method, route.path))
@@ -434,9 +466,7 @@ def test_every_route_has_one_explicit_access_policy() -> None:
def test_every_control_plane_route_has_an_explicit_human_role_dependency() -> None:
missing: list[tuple[str, str]] = []
- for route in app.routes:
- if not isinstance(route, APIRoute):
- continue
+ for route in _api_routes(app.routes):
dependencies = {dependency.call for dependency in route.dependant.dependencies}
for method in route.methods:
if access_boundary_for_request(
@@ -451,11 +481,11 @@ def test_every_control_plane_route_has_an_explicit_human_role_dependency() -> No
def test_only_contractual_routes_bypass_the_control_plane_boundary() -> None:
boundaries = {
(method, route.path): access_boundary_for_request(method, route.path)
- for route in app.routes
- if isinstance(route, APIRoute)
+ for route in _api_routes(app.routes)
for method in route.methods
}
assert {key for key, boundary in boundaries.items() if boundary is AccessBoundary.PUBLIC} == {
+ ("POST", "/api/v1/operator/login"),
("GET", "/"),
("GET", "/api/v1/health/live"),
("GET", "/api/v1/health/ready"),
@@ -481,8 +511,14 @@ def test_only_contractual_routes_bypass_the_control_plane_boundary() -> None:
("GET", "/api/v1/agent/serving-jobs/next"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
+ ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancellation"),
+ ("POST", "/api/v1/agent/serving-jobs/{job_id}/cancelled"),
("POST", "/api/v1/agent/serving-state"),
}
+ for action in ("cancellation", "cancelled"):
+ assert access_boundary_for_request(
+ "POST", f"/api/v1/agent/serving-jobs/00000000-0000-0000-0000-000000000001/{action}"
+ ) is AccessBoundary.NODE
assert {
key for key, boundary in boundaries.items() if boundary is AccessBoundary.CAPABILITY_CLIENT
} == set(EXPECTED_CAPABILITY_AUTHENTICATORS)
@@ -490,9 +526,7 @@ def test_only_contractual_routes_bypass_the_control_plane_boundary() -> None:
def test_machine_routes_have_direct_authentication_dependency_evidence() -> None:
missing: list[tuple[str, str, str]] = []
- for route in app.routes:
- if not isinstance(route, APIRoute):
- continue
+ for route in _api_routes(app.routes):
dependency_calls = {dependency.call for dependency in route.dependant.dependencies}
for method in route.methods:
boundary = access_boundary_for_request(method, route.path)
@@ -508,8 +542,7 @@ def test_machine_routes_have_direct_authentication_dependency_evidence() -> None
def test_each_capability_route_authenticates_its_exact_required_scope() -> None:
routes = {
(method, route.path): route
- for route in app.routes
- if isinstance(route, APIRoute)
+ for route in _api_routes(app.routes)
for method in route.methods
}
for key, expected_authenticator in EXPECTED_CAPABILITY_AUTHENTICATORS.items():
@@ -586,8 +619,7 @@ def test_interactive_api_schema_is_enabled_only_for_development() -> None:
def test_audited_mutations_have_the_intended_operator_or_admin_policy() -> None:
routes = {
(method, route.path): route_policy(route, method)
- for route in app.routes
- if isinstance(route, APIRoute)
+ for route in _api_routes(app.routes)
for method in route.methods
}
assert {key: routes[key] for key in TARGET_MUTATION_POLICIES} == TARGET_MUTATION_POLICIES
diff --git a/backend/tests/test_operator_identities.py b/backend/tests/test_operator_identities.py
new file mode 100644
index 0000000..a608f59
--- /dev/null
+++ b/backend/tests/test_operator_identities.py
@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import authenticate_operator_token
+from modelforge_api.db import get_session
+from modelforge_api.main import app
+from modelforge_api.persistence.models import AuditEvent, Base
+from modelforge_api.services.startup_validation import validate_settings
+from modelforge_api.settings import Settings, get_settings
+
+
+def token(role: str) -> str:
+ return "mfop_" + role[0] * 43
+
+
+def record(credential_role: str, **overrides: object) -> dict[str, object]:
+ role = credential_role
+ return {
+ "credential_id": f"test-{role}",
+ "subject": f"person-{role}",
+ "role": role,
+ "token_sha256": hashlib.sha256(token(role).encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+ **overrides,
+ }
+
+
+def settings(records: list[dict[str, object]] | None = None) -> Settings:
+ return Settings(
+ _env_file=None, env="test", operator_api_key=None,
+ operator_identities=SecretStr(json.dumps(
+ records if records is not None else [record(role) for role in ("viewer", "operator", "admin")]
+ )),
+ )
+
+
+@pytest.fixture
+def client() -> Iterator[tuple[TestClient, Session]]:
+ engine = create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ app.dependency_overrides[get_settings] = lambda: settings()
+ app.dependency_overrides[get_session] = lambda: session
+ try:
+ yield TestClient(app), session
+ finally:
+ app.dependency_overrides.clear()
+ engine.dispose()
+
+
+@pytest.mark.parametrize("role", ["viewer", "operator", "admin"])
+def test_real_key_authentication_and_no_store_identity(client: tuple[TestClient, Session], role: str) -> None:
+ http, _ = client
+ response = http.get("/api/v1/operator/identity", headers={"X-ModelForge-Admin-Token": token(role)})
+ assert response.status_code == 200
+ assert response.headers["cache-control"] == "no-store"
+ assert response.json()["subject"] == f"person-{role}"
+ assert response.json()["role"] == role
+ assert response.json()["credential_expires_at"] is not None
+ assert token(role) not in response.text
+ assert "token_sha256" not in response.text
+ assert "set-cookie" not in response.headers
+
+
+def test_roles_and_attributable_configuration_audit(client: tuple[TestClient, Session]) -> None:
+ http, database = client
+ change = {"expected_revision": 0, "changes": {"gateway_max_batch_size": 2}}
+ viewer = {"X-ModelForge-Admin-Token": token("viewer")}
+ operator = {"X-ModelForge-Admin-Token": token("operator")}
+ admin = {"X-ModelForge-Admin-Token": token("admin"), "X-Correlation-ID": "named-admin-change"}
+ assert http.get("/api/v1/configuration/settings", headers=viewer).status_code == 200
+ assert http.post("/api/v1/configuration/preview", json=change, headers=viewer).status_code == 403
+ preview = http.post("/api/v1/configuration/preview", json=change, headers=operator)
+ assert preview.status_code == 200
+ body = change | {"preview_digest": preview.json()["preview_digest"], "confirm": True}
+ assert http.post("/api/v1/configuration/apply", json=body, headers=operator).status_code == 403
+ assert http.post("/api/v1/configuration/apply", json=body, headers=admin).status_code == 200
+ event = database.scalar(select(AuditEvent).where(AuditEvent.action == "configuration.applied"))
+ assert event is not None and event.actor_id == "person-admin"
+ assert event.correlation_id == "named-admin-change"
+ assert token("admin") not in str(event.details)
+
+
+@pytest.mark.parametrize("credential", [None, "unknown", "mfsvc_synthetic", "mfnode_synthetic", "mfenroll_synthetic", "a" * 513, "ünicode"])
+def test_other_credentials_are_refused_before_body(client: tuple[TestClient, Session], credential: str | None) -> None:
+ http, _ = client
+ response = http.post("/api/v1/configuration/apply", content="not JSON", headers={
+ "X-ModelForge-Admin-Token": credential.encode().decode("latin1"),
+ } if credential and credential.isascii() else {})
+ assert response.status_code == 401
+ with pytest.raises(HTTPException) as failure:
+ authenticate_operator_token(settings(), credential)
+ assert failure.value.status_code == 401
+
+
+def test_expiry_revocation_rotation_and_no_legacy_fallback() -> None:
+ old = record("admin")
+ rotated_token = "mfop_" + "R" * 43
+ rotated = record("admin", credential_id="rotated", token_sha256=hashlib.sha256(rotated_token.encode()).hexdigest())
+ overlap = settings([old, rotated])
+ assert authenticate_operator_token(overlap, token("admin")).subject == "person-admin"
+ assert authenticate_operator_token(overlap, rotated_token).subject == "person-admin"
+ revoked = settings([rotated])
+ expired = settings([record("admin", expires_at=datetime.now(UTC).isoformat())])
+ for snapshot in (revoked, expired):
+ snapshot.operator_api_key = SecretStr(token("admin"))
+ with pytest.raises(HTTPException) as failure:
+ authenticate_operator_token(snapshot, token("admin"))
+ assert failure.value.status_code == 401
+
+
+@pytest.mark.parametrize("records", [
+ [], [record("admin", role="owner")], [record("admin", subject="legacy-operator")],
+ [record("admin"), record("admin")],
+ [record("viewer", subject="same"), record("admin", subject="same")],
+ [record("admin", expires_at="2026-09-09T12:00:00")],
+ [record("admin", plaintext="never-reflect-this")],
+])
+def test_invalid_identity_configuration_fails_closed_without_echo(records: list[dict[str, object]]) -> None:
+ with pytest.raises(ValidationError) as failure:
+ settings(records)
+ assert "never-reflect-this" not in str(failure.value)
+ assert "token_sha256" not in str(failure.value)
+
+
+def test_duplicate_json_fields_are_refused() -> None:
+ with pytest.raises(ValidationError):
+ Settings(_env_file=None, operator_identities=SecretStr('[{"subject":"one","subject":"two"}]'))
+
+
+def test_malformed_named_namespace_never_falls_back_to_bootstrap() -> None:
+ candidate = settings()
+ candidate.operator_api_key = SecretStr("mfop_malformed")
+ with pytest.raises(HTTPException) as failure:
+ authenticate_operator_token(candidate, "mfop_malformed")
+ assert failure.value.status_code == 401
+
+
+def test_named_administrator_can_replace_bootstrap_key_in_production() -> None:
+ candidate = settings()
+ candidate.env = "production"
+ assert not any(issue.setting == "MODELFORGE_OPERATOR_API_KEY" for issue in validate_settings(candidate).problems)
+ candidate = settings([record("viewer")])
+ candidate.env = "production"
+ assert any(issue.setting == "MODELFORGE_OPERATOR_API_KEY" for issue in validate_settings(candidate).problems)
+
+
+@pytest.mark.parametrize("identities", [None, "", json.dumps([record("admin", expires_at="2020-01-01T00:00:00Z")])])
+def test_production_startup_refuses_missing_or_expired_administration(identities: str | None) -> None:
+ candidate = Settings(
+ _env_file=None, env="production", operator_api_key=SecretStr(""),
+ operator_identities=SecretStr(identities) if identities is not None else None,
+ )
+ assert any(issue.setting == "MODELFORGE_OPERATOR_API_KEY" for issue in validate_settings(candidate).problems)
diff --git a/backend/tests/test_operator_password.py b/backend/tests/test_operator_password.py
new file mode 100644
index 0000000..c282bc7
--- /dev/null
+++ b/backend/tests/test_operator_password.py
@@ -0,0 +1,311 @@
+from __future__ import annotations
+
+import time
+from unittest.mock import Mock
+
+import pytest
+import redis
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+
+from modelforge_api.api.authorization import authenticate_operator_token
+from modelforge_api.api.routes import operator_login
+from modelforge_api.main import app
+from modelforge_api.services.operator_password import (
+ hash_password,
+ issue_session,
+ parse_password_hash,
+ verify_password,
+ verify_session,
+)
+from modelforge_api.services.startup_validation import validate_settings
+from modelforge_api.settings import Settings, get_settings
+
+PASSWORD = "synthetic password fixture" # noqa: S105 - synthetic test input
+BOOTSTRAP = "synthetic-bootstrap-fixture-" + "x" * 32
+
+
+@pytest.fixture(scope="module")
+def verifier() -> str:
+ return hash_password(PASSWORD)
+
+
+@pytest.fixture
+def settings(verifier: str) -> Settings:
+ return Settings(
+ _env_file=None,
+ env="test",
+ operator_api_key=SecretStr(BOOTSTRAP),
+ operator_password_hash=SecretStr(verifier),
+ )
+
+
+@pytest.fixture
+def client(settings: Settings, monkeypatch: pytest.MonkeyPatch):
+ connection = Mock()
+ connection.__enter__ = Mock(return_value=connection)
+ connection.__exit__ = Mock(return_value=False)
+ connection.eval.return_value = 1
+ monkeypatch.setattr(operator_login.redis.Redis, "from_url", Mock(return_value=connection))
+ app.dependency_overrides[get_settings] = lambda: settings
+ yield TestClient(app), connection
+ app.dependency_overrides.pop(get_settings, None)
+
+
+def test_salted_password_and_wrong_value(verifier: str):
+ assert verify_password(PASSWORD, verifier)
+ assert not verify_password("wrong", verifier)
+ assert hash_password(PASSWORD) != verifier
+ assert len(parse_password_hash(verifier)[0]) == 16
+
+
+@pytest.mark.parametrize(
+ "encoded",
+ [
+ "",
+ "scrypt$2$8$1$a$b",
+ "x" * 10000,
+ "scrypt$999999999$8$1$a$b",
+ "scrypt$32768$8$1$" + "a" * 66,
+ ],
+)
+def test_malformed_or_weak_hash_rejected(encoded: str):
+ with pytest.raises(ValueError):
+ parse_password_hash(encoded)
+ configured = Settings(_env_file=None, operator_password_hash=SecretStr(encoded))
+ problems = validate_settings(configured).problems
+ assert (
+ any(p.setting == "MODELFORGE_OPERATOR_PASSWORD_HASH" for p in problems)
+ if encoded
+ else not any(p.setting == "MODELFORGE_OPERATOR_PASSWORD_HASH" for p in problems)
+ )
+ assert all(encoded not in p.message for p in problems) if encoded else True
+
+
+def test_sessions_signed_expiring_rotating(verifier: str):
+ token, expiry = issue_session(BOOTSTRAP, verifier, now=1000)
+ assert expiry == 1900
+ assert verify_session(token, BOOTSTRAP, verifier, now=1000)
+ assert verify_session(token, BOOTSTRAP, verifier, now=1899)
+ assert not verify_session(token, BOOTSTRAP, verifier, now=1900)
+ assert not verify_session(token, BOOTSTRAP, verifier, now=999)
+ assert not verify_session(token, BOOTSTRAP + "rotated", verifier, now=1100)
+ assert not verify_session(token, BOOTSTRAP, hash_password(PASSWORD), now=1100)
+ assert not verify_session(token.replace("1900", "9900"), BOOTSTRAP, verifier, now=1100)
+ for malformed in [
+ "mfpw_",
+ "mfpw_" + "x" * 2000,
+ token + ".",
+ "mfsvc_" + token,
+ "mfpw_1.901.😃.😃",
+ ]:
+ assert not verify_session(malformed, BOOTSTRAP, verifier, now=1100)
+
+
+def test_login_session_and_legacy_key(client, settings: Settings):
+ http, connection = client
+ response = http.post("/api/v1/operator/login", json={"password": PASSWORD})
+ assert response.status_code == 200
+ assert response.headers["cache-control"] == "no-store"
+ result = response.json()
+ assert result["token_type"] == "bearer" # noqa: S105 - protocol type
+ assert PASSWORD not in response.text and BOOTSTRAP not in response.text
+ assert result["expires_in"] == 900
+ token = result["access_token"]
+ principal = authenticate_operator_token(settings, token)
+ assert principal.authentication_method == "password_session"
+ assert principal.role == "admin"
+ assert principal.expires_at is not None
+ identity = http.get("/api/v1/operator/identity", headers={"Authorization": f"Bearer {token}"})
+ assert identity.status_code == 200
+ assert identity.json()["authentication_method"] == "password_session"
+ assert identity.json()["credential_expires_at"] == result["expires_at"]
+ assert (
+ authenticate_operator_token(settings, BOOTSTRAP).authentication_method
+ == "legacy_admin_token"
+ )
+ for headers in [{"Authorization": f"Bearer {token}"}, {"X-ModelForge-Admin-Token": token}]:
+ # Routing 404 proves the protected request passed the authentication boundary.
+ assert http.get("/api/v1/not-a-route", headers=headers).status_code == 404
+ assert (
+ http.get("/api/v1/not-a-route", headers={"X-ModelForge-Admin-Token": PASSWORD}).status_code
+ == 401
+ )
+ connection.eval.assert_called_once()
+
+
+@pytest.mark.parametrize(
+ "payload",
+ [
+ {},
+ {"password": ""},
+ {"password": None},
+ {"password": 123},
+ {"password": ["do-not-echo"]},
+ {"password": "z" * 1025},
+ {"password": PASSWORD, "do-not-echo": PASSWORD},
+ ],
+)
+def test_invalid_login_redacted(client, payload):
+ http, connection = client
+ response = http.post("/api/v1/operator/login", json=payload)
+ assert response.status_code == 422
+ assert PASSWORD not in response.text and "do-not-echo" not in response.text
+ assert response.json()["error"]["details"] == {}
+ assert response.headers["cache-control"] == "no-store"
+ connection.eval.assert_called_once()
+
+
+def test_wrong_password_and_no_config(client, settings: Settings):
+ http, _ = client
+ response = http.post("/api/v1/operator/login", json={"password": "wrong"})
+ assert response.status_code == 401 and "wrong" not in response.text
+ settings.operator_password_hash = None
+ assert http.post("/api/v1/operator/login", json={"password": PASSWORD}).status_code == 503
+
+
+def test_limiter_runs_before_parser_and_ignores_forwarded_ip(client):
+ http, connection = client
+ connection.eval.side_effect = [1, 0, 0]
+ assert http.post("/api/v1/operator/login", content="not json").status_code == 422
+ assert (
+ http.post(
+ "/api/v1/operator/login",
+ json={"password": PASSWORD},
+ headers={"X-Forwarded-For": "203.0.113.1"},
+ ).status_code
+ == 429
+ )
+ assert (
+ http.post(
+ "/api/v1/operator/login",
+ json={"password": PASSWORD},
+ headers={"X-Forwarded-For": "203.0.113.2"},
+ ).status_code
+ == 429
+ )
+ assert len({call.args[-1] for call in connection.eval.call_args_list}) == 1
+ script = connection.eval.call_args.args[0]
+ assert "redis.call('INCR', KEYS[1])" in script
+ assert "redis.call('EXPIRE', KEYS[1], 60)" in script
+ assert "global > 30" in script and "peer > 5" in script
+
+
+def test_redis_failure_fails_closed(client):
+ http, connection = client
+ connection.eval.side_effect = redis.ConnectionError("secret topology must not leak")
+ response = http.post("/api/v1/operator/login", json={"password": PASSWORD})
+ assert response.status_code == 503
+ assert "topology" not in response.text and PASSWORD not in response.text
+
+
+def test_invalid_session_never_falls_back(settings: Settings):
+ for token in ["mfpw_fake", PASSWORD, None]:
+ with pytest.raises(HTTPException) as exc:
+ authenticate_operator_token(settings, token)
+ assert exc.value.status_code == 401
+ token, _ = issue_session(
+ BOOTSTRAP, settings.operator_password_hash.get_secret_value(), now=int(time.time()) - 1000
+ )
+ with pytest.raises(HTTPException):
+ authenticate_operator_token(settings, token)
+
+
+def test_rotations_reject_existing_session(client, settings: Settings):
+ http, _ = client
+ token = http.post("/api/v1/operator/login", json={"password": PASSWORD}).json()["access_token"]
+ headers = {"Authorization": f"Bearer {token}"}
+ settings.operator_password_hash = SecretStr(hash_password(PASSWORD))
+ assert http.get("/api/v1/not-a-route", headers=headers).status_code == 401
+ token = http.post("/api/v1/operator/login", json={"password": PASSWORD}).json()["access_token"]
+ settings.operator_api_key = SecretStr(BOOTSTRAP + "rotated")
+ assert (
+ http.get("/api/v1/not-a-route", headers={"Authorization": f"Bearer {token}"}).status_code
+ == 401
+ )
+
+
+def test_login_body_bounded_before_parsing(client):
+ http, connection = client
+ response = http.post("/api/v1/operator/login", content="x" * 9000)
+ assert response.status_code == 413
+ assert response.headers["cache-control"] == "no-store"
+ assert "x" * 100 not in response.text
+ connection.eval.assert_called_once()
+
+
+def test_login_requires_strong_bootstrap(client, settings: Settings):
+ http, _ = client
+ settings.operator_api_key = SecretStr("short fixture")
+ assert http.post("/api/v1/operator/login", json={"password": PASSWORD}).status_code == 503
+
+
+def test_cli_stdin_outputs_hash_only():
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ root = Path(__file__).resolve().parents[2]
+ result = subprocess.run( # noqa: S603 - fixed local CLI and synthetic stdin
+ [sys.executable, str(root / "scripts/operator_password.py")],
+ input=PASSWORD + "\n",
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ assert result.returncode == 0 and result.stderr == ""
+ assert PASSWORD not in result.stdout
+ assert verify_password(PASSWORD, result.stdout.strip())
+ bad = subprocess.run( # noqa: S603 - fixed local CLI and synthetic stdin
+ [sys.executable, str(root / "scripts/operator_password.py")],
+ input="z" * 10000,
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ assert bad.returncode == 2 and bad.stdout == "" and "z" * 10 not in bad.stderr
+
+
+def test_empty_optional_password_hash_is_unset():
+ for value in ["", SecretStr("")]:
+ configured = Settings(_env_file=None, operator_password_hash=value)
+ assert configured.operator_password_hash is None
+ assert not any(
+ p.setting == "MODELFORGE_OPERATOR_PASSWORD_HASH"
+ for p in validate_settings(configured).problems
+ )
+
+
+def test_legacy_key_with_session_prefix_remains_accepted(settings: Settings):
+ legacy = "mfpw_" + "synthetic-key-fixture" * 3
+ settings.operator_api_key = SecretStr(legacy)
+ assert (
+ authenticate_operator_token(settings, legacy).authentication_method == "legacy_admin_token"
+ )
+
+
+def test_password_configuration_requires_signing_key_in_production(verifier: str):
+ configured = Settings(_env_file=None, env="production", operator_api_key=None,
+ operator_password_hash=SecretStr(verifier))
+ assert any(p.setting == "MODELFORGE_OPERATOR_API_KEY" and "session signing" in p.message
+ for p in validate_settings(configured).problems)
+
+
+@pytest.mark.parametrize("bootstrap", [None, "", "short", "x" * 31, "x" * 32])
+def test_named_admin_cannot_replace_password_signing_key(verifier: str, bootstrap: str | None):
+ import hashlib
+ import json
+ from datetime import UTC, datetime, timedelta
+
+ identities = SecretStr(json.dumps([{
+ "credential_id": "fixture-admin", "subject": "fixture-person", "role": "admin",
+ "token_sha256": hashlib.sha256(("mfop_" + "a" * 43).encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(),
+ }]))
+ configured = Settings(_env_file=None, env="production", operator_identities=identities,
+ operator_api_key=None if bootstrap is None else SecretStr(bootstrap),
+ operator_password_hash=SecretStr(verifier))
+ signing_problems = [p for p in validate_settings(configured).problems
+ if p.setting == "MODELFORGE_OPERATOR_API_KEY" and "session signing" in p.message]
+ assert bool(signing_problems) == (bootstrap is None or len(bootstrap) < 32)
diff --git a/backend/tests/test_packaging_m17.py b/backend/tests/test_packaging_m17.py
index d1edd91..9f44be7 100644
--- a/backend/tests/test_packaging_m17.py
+++ b/backend/tests/test_packaging_m17.py
@@ -114,7 +114,6 @@ def test_the_production_overlay_exists_and_sets_the_production_profile() -> None
"MODELFORGE_RUNTIME_DB_PASSWORD",
"MODELFORGE_MIGRATION_DATABASE_URL",
"MODELFORGE_RUNTIME_DATABASE_URL",
- "MODELFORGE_OPERATOR_API_KEY",
"MODELFORGE_BACKUP_ENCRYPTION_KEY",
"MODELFORGE_CORS_ORIGINS",
],
@@ -215,13 +214,13 @@ def test_the_release_builder_never_mutates_a_floating_latest_tag() -> None:
assert 'f"{image}:latest"' not in source
-def test_upgrade_runbook_starts_prebuilt_release_images() -> None:
+def test_upgrade_runbook_marks_its_v1_2_3_commands_as_historical() -> None:
runbook = (ROOT / "docs" / "UPGRADE.md").read_text("utf-8")
start_step = runbook.split("## 5. Start the new version", 1)[1].split("## 6. Verify", 1)[0]
- version = (ROOT / "VERSION").read_text("utf-8").strip()
- assert f"MODELFORGE_VERSION={version}" in start_step
- assert f"MODELFORGE_API_IMAGE=modelforge-api:{version}" in start_step
- assert f"MODELFORGE_WEB_IMAGE=modelforge-web:{version}" in start_step
+ assert "The remaining release commands document the previously documented v1.2.3 hotfix" in runbook
+ assert "MODELFORGE_VERSION=1.2.3" in start_step
+ assert "MODELFORGE_API_IMAGE=modelforge-api:1.2.3" in start_step
+ assert "MODELFORGE_WEB_IMAGE=modelforge-web:1.2.3" in start_step
assert "up -d --no-build api web" in start_step
assert "up -d --build api web" not in start_step
@@ -275,15 +274,27 @@ def test_the_release_package_includes_what_an_operator_needs_to_install() -> Non
module = module_from_spec(spec)
spec.loader.exec_module(module)
paths = set(module.ARTIFACT_PATHS)
+ assert "clients" in paths
+ for client in ("python/modelforge_client.py", "typescript/modelforge-client.mjs", "typescript/modelforge-client.d.mts"):
+ assert (ROOT / "clients" / client).is_file()
for required in (
"docker-compose.yml",
"docker-compose.production.yml",
+ "docker-compose.rc-ingress.yml",
+ "docker-compose.validation-tls.yml",
".env.example",
"VERSION",
"docs",
"config",
+ "deploy/validation",
"scripts/bootstrap.py",
"scripts/preflight.py",
+ "scripts/operator_identity.py",
+ "scripts/rc_isolation_preflight.py",
+ "scripts/rc_stack_retention.py",
+ "scripts/m16_chaos.py",
+ "scripts/m16_invariants.py",
+ "scripts/m16_soak.py",
):
assert required in paths, f"a release without {required} cannot be installed from"
@@ -361,7 +372,7 @@ def test_every_manifest_the_control_plane_needs_at_startup_is_tracked() -> None:
@pytest.mark.parametrize(
"directory",
- ["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs"],
+ ["config", "backend/src", "backend/tests", "node-agent/src", "runtime-worker/src", "docs", "clients"],
)
def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
"""Whatever these directories grow, none of it may be invisible to a clean clone.
@@ -374,7 +385,7 @@ def test_no_source_file_is_excluded_from_the_repository(directory: str) -> None:
"""
tracked = _tracked_files()
- suffixes = {".py", ".ts", ".tsx", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
+ suffixes = {".py", ".ts", ".tsx", ".mts", ".mjs", ".md", ".yaml", ".yml", ".json", ".inc", ".template"}
on_disk = {
str(path.relative_to(ROOT)).replace("\\", "/")
for path in (ROOT / directory).rglob("*")
diff --git a/backend/tests/test_portfolio_reconciliation.py b/backend/tests/test_portfolio_reconciliation.py
new file mode 100644
index 0000000..bd8221d
--- /dev/null
+++ b/backend/tests/test_portfolio_reconciliation.py
@@ -0,0 +1,617 @@
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import json
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, inspect, select, text
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import AccessBoundary, access_boundary_for_request
+from modelforge_api.db import get_session
+from modelforge_api.domain.portfolio import PortfolioReconcileRequest
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ Base,
+ PortfolioReconciliationRun,
+ PortfolioRepositoryObservation,
+ Project,
+ ProjectBinding,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.portfolio_registry import (
+ PortfolioRegistryError,
+ PortfolioRegistryService,
+)
+from modelforge_api.settings import Settings, get_settings
+
+ROOT = Path(__file__).resolve().parents[1]
+MIGRATION_PATH = (
+ ROOT / "alembic" / "versions" / "20260923_0036_portfolio_repository_reconciliation.py"
+)
+
+
+def _migration() -> ModuleType:
+ spec = importlib.util.spec_from_file_location("portfolio_reconciliation_0036", MIGRATION_PATH)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+@pytest.fixture
+def session() -> Session:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+def observation(
+ *,
+ repository_id: str = "all-examplerag",
+ project_key: str = "examplerag",
+ classification: str = "gateway",
+ slug: str = "ITWorx/ExampleRAG",
+ head_sha: str = "a" * 40,
+ branch: str = "master",
+ accept_head_update: bool = False,
+ runtime_evidence: list[dict[str, object]] | None = None,
+) -> dict[str, object]:
+ return {
+ "external_repository_id": repository_id,
+ "project_key": project_key,
+ "slug": slug,
+ "default_branch": branch,
+ "head_sha": head_sha,
+ "classification": classification,
+ "accept_head_update": accept_head_update,
+ "code_evidence": [
+ {
+ "path": "backend/src/client.py",
+ "signal": "capability_client",
+ "capabilities": ["rag.embedding@1"],
+ }
+ ],
+ "runtime_evidence": runtime_evidence or [],
+ }
+
+
+def request(
+ *,
+ snapshot_id: str = "gitea-001",
+ observed_at: datetime | None = None,
+ repositories: list[dict[str, object]] | None = None,
+ complete: bool = False,
+) -> PortfolioReconcileRequest:
+ return PortfolioReconcileRequest.model_validate(
+ {
+ "source_system": "gitea",
+ "snapshot_id": snapshot_id,
+ "observed_at": (observed_at or datetime(2026, 9, 23, 12, tzinfo=UTC)).isoformat(),
+ "complete": complete,
+ "repositories": [observation()] if repositories is None else repositories,
+ }
+ )
+
+
+def reconcile(
+ session: Session, item: PortfolioReconcileRequest
+):
+ return PortfolioRegistryService(session).reconcile(
+ item, AuditContext("operator", "portfolio-test", "portfolio-correlation")
+ )
+
+
+def test_reconciliation_reuses_project_and_exact_replay_is_a_noop(session: Session) -> None:
+ project = Project(
+ key="examplerag", name="ExampleRAG legacy", description="operator description", active=False
+ )
+ session.add(project)
+ session.commit()
+ item = request()
+
+ first = reconcile(session, item)
+ second = reconcile(session, item)
+
+ assert first.projects_created == 0
+ assert first.repositories_created == 1
+ assert first.replayed is False
+ assert second.replayed is True
+ assert second.snapshot_digest == first.snapshot_digest
+ assert session.query(Project).count() == 1
+ assert session.query(ProjectBinding).count() == 0
+ assert session.query(PortfolioRepositoryObservation).count() == 1
+ session.refresh(project)
+ assert (project.name, project.description, project.active) == (
+ "ExampleRAG legacy",
+ "operator description",
+ False,
+ )
+ assert session.query(PortfolioReconciliationRun).count() == 1
+ assert session.scalar(
+ select(AuditEvent).where(AuditEvent.action == "PORTFOLIO_RECONCILED")
+ ) is not None
+ assert session.query(AuditEvent).filter_by(action="PORTFOLIO_RECONCILED").count() == 1
+
+
+def test_authenticated_portfolio_artifact_creates_only_source_confirmed_projects(
+ session: Session,
+) -> None:
+ artifact = ROOT.parent / "docs/product/PORTFOLIO_RECONCILE_REQUEST_COMPLETE_2026-09-25.json"
+ item = PortfolioReconcileRequest.model_validate_json(artifact.read_text(encoding="utf-8"))
+
+ first = reconcile(session, item)
+ second = reconcile(session, item)
+
+ assert item.complete is True
+ assert first.repositories_created == 69
+ assert first.projects_created == 12
+ assert second.replayed is True
+ assert second.snapshot_digest == first.snapshot_digest
+ assert session.query(PortfolioRepositoryObservation).count() == 69
+ assert session.query(Project).count() == 12
+ assert session.query(ProjectBinding).count() == 0
+ inventory = PortfolioRegistryService(session).inventory()
+ assert len(inventory.repositories) == 69
+ assert sum(row.project_id is not None for row in inventory.repositories) == 12
+
+
+def test_non_consumer_repositories_are_recorded_without_creating_projects(
+ session: Session,
+) -> None:
+ item = request(
+ repositories=[
+ observation(
+ repository_id="all-docs-tool",
+ project_key="docs-tool",
+ classification="non_ai",
+ slug="ITWorx/Docs-Tool",
+ )
+ ]
+ )
+
+ reconcile(session, item)
+
+ row = session.scalar(select(PortfolioRepositoryObservation))
+ view = PortfolioRegistryService(session).inventory().repositories[0]
+ assert row and row.project_id is None and row.project_key == "docs-tool"
+ assert view.project_key == "docs-tool" and view.project_id is None and view.project_name is None
+ assert session.query(Project).count() == 0
+
+
+def test_operator_classification_links_an_existing_observation_only_when_relevant(
+ session: Session,
+) -> None:
+ reconcile(
+ session,
+ request(
+ repositories=[
+ observation(classification="hosted_model", project_key="examplerag")
+ ]
+ ),
+ )
+ row = session.scalar(select(PortfolioRepositoryObservation))
+ assert row and row.project_id is None
+ assert session.query(Project).count() == 0
+
+ reconcile(
+ session,
+ request(
+ snapshot_id="reviewed-local-consumer",
+ observed_at=datetime(2026, 9, 23, 12, 5, tzinfo=UTC),
+ repositories=[observation(classification="gateway", project_key="examplerag")],
+ ),
+ )
+ session.refresh(row)
+ assert row.project_id is not None
+ assert session.query(Project).count() == 1
+
+
+def test_nonlocal_to_local_head_change_stays_unlinked_until_exact_candidate_is_accepted(
+ session: Session,
+) -> None:
+ start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+ reconcile(
+ session,
+ request(
+ observed_at=start,
+ repositories=[observation(classification="hosted_model")],
+ ),
+ )
+
+ pending = reconcile(
+ session,
+ request(
+ snapshot_id="local-candidate",
+ observed_at=start + timedelta(minutes=5),
+ repositories=[
+ observation(classification="gateway", head_sha="b" * 40)
+ ],
+ ),
+ )
+ row = session.scalar(select(PortfolioRepositoryObservation))
+ view = PortfolioRegistryService(session).inventory().repositories[0]
+ assert pending.head_changes_review_required == 1
+ assert row and row.project_id is None and row.head_sha == "a" * 40
+ assert row.classification == "hosted_model"
+ assert row.pending_head_sha == "b" * 40
+ assert row.pending_classification == "gateway"
+ assert row.pending_project_key == "examplerag"
+ assert view.project_id is None and view.pending_project_key == "examplerag"
+ assert session.query(Project).count() == 0
+
+ with pytest.raises(PortfolioRegistryError) as error:
+ reconcile(
+ session,
+ request(
+ snapshot_id="accept-unreviewed-head",
+ observed_at=start + timedelta(minutes=7),
+ repositories=[
+ observation(
+ classification="gateway",
+ head_sha="c" * 40,
+ accept_head_update=True,
+ )
+ ],
+ ),
+ )
+ assert error.value.code == "PORTFOLIO_PENDING_HEAD_CONFLICT"
+ session.refresh(row)
+ assert row.project_id is None and row.pending_head_sha == "b" * 40
+ assert session.query(Project).count() == 0
+
+ accepted = reconcile(
+ session,
+ request(
+ snapshot_id="accept-local-candidate",
+ observed_at=start + timedelta(minutes=10),
+ repositories=[
+ observation(
+ classification="gateway",
+ head_sha="b" * 40,
+ accept_head_update=True,
+ )
+ ],
+ ),
+ )
+ session.refresh(row)
+ assert accepted.head_changes_accepted == 1
+ assert row.project_id is not None and row.head_sha == "b" * 40
+ assert row.classification == "gateway" and row.pending_head_sha is None
+ assert row.pending_project_key is None
+ assert session.query(Project).count() == 1
+
+
+def test_duplicate_project_key_across_repositories_is_an_explicit_conflict(
+ session: Session,
+) -> None:
+ documents = observation(
+ repository_id="all-docs-tool",
+ project_key="shared-key",
+ classification="non_ai",
+ slug="ITWorx/Docs-Tool",
+ )
+ dashboard = observation(
+ repository_id="all-dashboard",
+ project_key="shared-key",
+ classification="runtime_observer",
+ slug="ITWorx/Dashboard",
+ )
+
+ with pytest.raises(PortfolioRegistryError) as error:
+ reconcile(session, request(repositories=[documents, dashboard]))
+
+ assert error.value.status_code == 409
+ assert error.value.code == "PORTFOLIO_PROJECT_KEY_CONFLICT"
+ assert session.query(PortfolioRepositoryObservation).count() == 0
+ assert session.query(Project).count() == 0
+ assert session.query(PortfolioReconciliationRun).count() == 0
+
+
+def test_snapshot_identity_conflict_does_not_change_portfolio(session: Session) -> None:
+ first = request()
+ reconcile(session, first)
+ changed = request(
+ snapshot_id=first.snapshot_id,
+ observed_at=first.observed_at + timedelta(minutes=1),
+ repositories=[observation(head_sha="b" * 40)],
+ )
+
+ with pytest.raises(PortfolioRegistryError) as error:
+ reconcile(session, changed)
+
+ assert error.value.status_code == 409
+ row = session.scalar(select(PortfolioRepositoryObservation))
+ assert row and row.head_sha == "a" * 40
+ assert session.query(PortfolioReconciliationRun).count() == 1
+
+
+def test_head_change_requires_explicit_operator_snapshot_acceptance(session: Session) -> None:
+ first_time = datetime(2026, 9, 23, 12, tzinfo=UTC)
+ reconcile(session, request(observed_at=first_time, complete=True))
+
+ pending = reconcile(
+ session,
+ request(
+ snapshot_id="gitea-002",
+ observed_at=first_time + timedelta(minutes=5),
+ repositories=[observation(head_sha="b" * 40)],
+ ),
+ )
+ row = session.scalar(select(PortfolioRepositoryObservation))
+ assert row and row.head_sha == "a" * 40
+ assert row.pending_head_sha == "b" * 40
+ assert pending.head_changes_review_required == 1
+ assert row.source_observed_at.replace(tzinfo=UTC) == first_time
+
+ stale = reconcile(
+ session,
+ request(
+ snapshot_id="gitea-old",
+ observed_at=first_time + timedelta(minutes=3),
+ repositories=[observation(head_sha="c" * 40)],
+ ),
+ )
+ assert stale.stale_snapshot is False
+ assert stale.stale_observations == 1
+ session.refresh(row)
+ assert row.head_sha == "a" * 40
+ assert row.pending_head_sha == "b" * 40
+
+ accepted = reconcile(
+ session,
+ request(
+ snapshot_id="gitea-003",
+ observed_at=first_time + timedelta(minutes=7),
+ repositories=[observation(head_sha="b" * 40, accept_head_update=True)],
+ ),
+ )
+ session.refresh(row)
+ assert accepted.head_changes_accepted == 1
+ assert row.head_sha == "b" * 40
+ assert row.pending_head_sha is None
+
+
+def test_incomplete_snapshot_never_marks_absent_repository_stale(session: Session) -> None:
+ start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+ second = observation(
+ repository_id="all-examplevision", project_key="examplevision", slug="ITWorx/ExampleVision"
+ )
+ reconcile(session, request(observed_at=start, repositories=[observation(), second], complete=True))
+
+ partial = reconcile(
+ session,
+ request(
+ snapshot_id="partial",
+ observed_at=start + timedelta(minutes=1),
+ repositories=[observation()],
+ complete=False,
+ ),
+ )
+
+ examplevision = session.scalar(
+ select(PortfolioRepositoryObservation).where(
+ PortfolioRepositoryObservation.external_repository_id == "all-examplevision"
+ )
+ )
+ assert partial.repositories_marked_stale == 0
+ assert examplevision and examplevision.stale_at is None
+
+
+def test_newer_complete_snapshot_marks_only_missing_rows_stale_without_deleting(session: Session) -> None:
+ start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+ second = observation(
+ repository_id="all-examplevision", project_key="examplevision", slug="ITWorx/ExampleVision"
+ )
+ reconcile(session, request(observed_at=start, repositories=[observation(), second], complete=True))
+
+ result = reconcile(
+ session,
+ request(
+ snapshot_id="complete-newer",
+ observed_at=start + timedelta(minutes=2),
+ repositories=[observation()],
+ complete=True,
+ ),
+ )
+
+ examplevision = session.scalar(
+ select(PortfolioRepositoryObservation).where(
+ PortfolioRepositoryObservation.external_repository_id == "all-examplevision"
+ )
+ )
+ pokevault_project = session.scalar(select(Project).where(Project.key == "examplevision"))
+ assert result.repositories_marked_stale == 1
+ assert examplevision and examplevision.stale_at == (start + timedelta(minutes=2)).replace(tzinfo=None)
+ assert pokevault_project and pokevault_project.active is True
+ assert session.query(PortfolioRepositoryObservation).count() == 2
+ assert session.query(ProjectBinding).count() == 0
+
+
+def test_runtime_observations_are_separate_and_monotone(session: Session) -> None:
+ start = datetime(2026, 9, 23, 12, tzinfo=UTC)
+ running = {
+ "component": "backend",
+ "state": "running",
+ "observed_at": (start + timedelta(minutes=2)).isoformat(),
+ "observation_source": "container_inventory",
+ "git_revision": "1234567890ab",
+ }
+ reconcile(
+ session,
+ request(observed_at=start, repositories=[observation(runtime_evidence=[running])]),
+ )
+ # A later source-only snapshot must not erase runtime evidence.
+ reconcile(
+ session,
+ request(
+ snapshot_id="source-only",
+ observed_at=start + timedelta(minutes=3),
+ repositories=[observation()],
+ ),
+ )
+ row = session.scalar(select(PortfolioRepositoryObservation))
+ assert row and row.classification == "gateway"
+ assert row.runtime_evidence[0]["state"] == "running"
+ from modelforge_api.domain.portfolio import PortfolioRuntimeEvidence
+
+ runtime = PortfolioRuntimeEvidence.model_validate(row.runtime_evidence[0])
+ assert runtime.observed_at == start + timedelta(minutes=2)
+
+
+def test_payload_contract_rejects_secrets_and_unsafe_evidence_paths() -> None:
+ body = request().model_dump(mode="json")
+ body["api_key"] = "this-value-must-not-be-accepted"
+ with pytest.raises(ValidationError):
+ PortfolioReconcileRequest.model_validate(body)
+
+ unsafe = request().model_dump(mode="json")
+ unsafe["repositories"][0]["code_evidence"][0]["path"] = "../private-key.env"
+ with pytest.raises(ValidationError):
+ PortfolioReconcileRequest.model_validate(unsafe)
+
+ windows = request().model_dump(mode="json")
+ windows["repositories"][0]["code_evidence"][0]["path"] = "C:/private-key.env"
+ with pytest.raises(ValidationError):
+ PortfolioReconcileRequest.model_validate(windows)
+
+
+def test_inventory_and_reconcile_roles_and_validation_responses_are_safe(
+ session: Session,
+) -> None:
+ tokens = {role: f"mfop_{role[0] * 43}" for role in ("viewer", "operator", "admin")}
+ settings = Settings(
+ _env_file=None,
+ env="test",
+ operator_identities=SecretStr(
+ json.dumps(
+ [
+ {
+ "subject": role,
+ "role": role,
+ "credential_id": role,
+ "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat(),
+ }
+ for role, token in tokens.items()
+ ]
+ )
+ ),
+ )
+ app.dependency_overrides[get_settings] = lambda: settings
+ app.dependency_overrides[get_session] = lambda: session
+ try:
+ client = TestClient(app)
+ assert access_boundary_for_request("GET", "/api/v1/portfolio") is AccessBoundary.CONTROL_PLANE
+ assert client.get("/api/v1/portfolio").status_code == 401
+ assert client.get(
+ "/api/v1/portfolio", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}
+ ).status_code == 200
+
+ body = request().model_dump(mode="json")
+ assert client.post("/api/v1/portfolio/reconcile", json=body).status_code == 401
+ assert client.post(
+ "/api/v1/portfolio/reconcile",
+ headers={"X-ModelForge-Admin-Token": tokens["viewer"]},
+ json=body,
+ ).status_code == 403
+
+ invalid = dict(body)
+ invalid["clone_url"] = "https://private-user:private-password@example.invalid/repo.git"
+ invalid["api-key-secret-value-must-not-echo"] = "untrusted"
+ rejected = client.post(
+ "/api/v1/portfolio/reconcile",
+ headers={"X-ModelForge-Admin-Token": tokens["operator"]},
+ json=invalid,
+ )
+ assert rejected.status_code == 422
+ assert "private-password" not in rejected.text
+ assert "example.invalid" not in rejected.text
+ assert "api-key-secret-value-must-not-echo" not in rejected.text
+
+ accepted = client.post(
+ "/api/v1/portfolio/reconcile",
+ headers={
+ "X-ModelForge-Admin-Token": tokens["operator"],
+ "X-Correlation-ID": "portfolio-reconcile-test",
+ },
+ json=body,
+ )
+ assert accepted.status_code == 200
+ assert accepted.json()["repositories_created"] == 1
+ audit = session.scalar(
+ select(AuditEvent).where(AuditEvent.action == "PORTFOLIO_RECONCILED")
+ )
+ assert audit and audit.actor_id == "operator"
+ assert audit.correlation_id == "portfolio-reconcile-test"
+ finally:
+ app.dependency_overrides.clear()
+
+
+def test_portfolio_migration_upgrades_and_downgrades_empty_history() -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ with engine.begin() as connection:
+ connection.execute(text("CREATE TABLE projects (id VARCHAR(36) PRIMARY KEY)"))
+ migration = _migration()
+ migration.op = Operations(MigrationContext.configure(connection))
+ migration.upgrade()
+ assert inspect(connection).has_table("portfolio_repository_observations")
+ assert inspect(connection).has_table("portfolio_reconciliation_runs")
+ foreign_keys = inspect(connection).get_foreign_keys("portfolio_repository_observations")
+ assert any(item["referred_table"] == "projects" for item in foreign_keys)
+ repository_unique_constraints = {
+ item["name"]
+ for item in inspect(connection).get_unique_constraints(
+ "portfolio_repository_observations"
+ )
+ }
+ assert "uq_portfolio_source_project_key" in repository_unique_constraints
+ assert "uq_portfolio_repository_project_id" in repository_unique_constraints
+
+ migration.downgrade()
+ assert not inspect(connection).has_table("portfolio_repository_observations")
+ assert not inspect(connection).has_table("portfolio_reconciliation_runs")
+ engine.dispose()
+
+
+def test_portfolio_migration_refuses_downgrade_after_a_snapshot_is_saved() -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ with engine.begin() as connection:
+ connection.execute(text("CREATE TABLE projects (id VARCHAR(36) PRIMARY KEY)"))
+ migration = _migration()
+ migration.op = Operations(MigrationContext.configure(connection))
+ migration.upgrade()
+ connection.execute(
+ text(
+ "INSERT INTO portfolio_reconciliation_runs "
+ "(id, source_system, snapshot_id, payload_digest, observed_at, complete, "
+ "stale_snapshot, projects_created, repositories_created, repositories_updated, "
+ "stale_observations, repositories_marked_stale, runtime_observations_updated, "
+ "head_changes_review_required, head_changes_accepted) "
+ "VALUES ('00000000-0000-0000-0000-000000000001', 'gitea', 'snapshot-1', :digest, "
+ "'2026-09-23 12:00:00', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)"
+ ),
+ {"digest": "a" * 64},
+ )
+
+ with pytest.raises(RuntimeError, match="portfolio reconciliation history exists"):
+ migration.downgrade()
+ assert inspect(connection).has_table("portfolio_reconciliation_runs")
+ assert inspect(connection).has_table("portfolio_repository_observations")
+ engine.dispose()
diff --git a/backend/tests/test_portfolio_review_artifact.py b/backend/tests/test_portfolio_review_artifact.py
new file mode 100644
index 0000000..7e4e549
--- /dev/null
+++ b/backend/tests/test_portfolio_review_artifact.py
@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "backend/src"))
+
+from scripts.portfolio_gitea_snapshot import (
+ EXECUTABLE_LOCAL_MODEL_REPOSITORIES,
+ build_snapshot,
+ load_offline_heads,
+ load_repository_ids,
+ main,
+ request_digest,
+)
+
+from modelforge_api.domain.portfolio import PortfolioReconcileRequest
+from modelforge_api.services.portfolio_registry import snapshot_digest
+
+REQUEST_PATH = ROOT / "docs/product/PORTFOLIO_RECONCILE_REQUEST_2026-09-25.json"
+HEADS_PATH = ROOT / "docs/product/PORTFOLIO_DEFAULT_HEADS_2026-09-25.json"
+RUNTIME_STATUS_PATH = ROOT / "docs/product/PORTFOLIO_RUNTIME_STATUS_2026-09-25.json"
+COMPLETE_REQUEST_PATH = ROOT / "docs/product/PORTFOLIO_RECONCILE_REQUEST_COMPLETE_2026-09-25.json"
+
+
+def test_review_request_is_partial_secret_free_and_identity_stable() -> None:
+ request = json.loads(REQUEST_PATH.read_text(encoding="utf-8"))
+ manifest = json.loads(HEADS_PATH.read_text(encoding="utf-8"))
+ runtime_status = json.loads(RUNTIME_STATUS_PATH.read_text(encoding="utf-8"))
+ validated = PortfolioReconcileRequest.model_validate(request)
+
+ assert validated.complete is False
+ assert manifest["owner_inventory_verified"] is False
+ assert manifest["default_heads_stable_across_two_reads"] is True
+ assert len(validated.repositories) == manifest["repository_count"] == 69
+ assert len({row.external_repository_id for row in validated.repositories}) == 69
+
+ heads = load_offline_heads(HEADS_PATH)
+ for row in validated.repositories:
+ current = heads[row.slug.split("/", 1)[1]]
+ assert row.default_branch == current["default_branch"]
+ assert row.head_sha == current["head_sha"]
+ assert row.accept_head_update is False
+ assert row.runtime_evidence == []
+ assert all(not item.capabilities for item in row.code_evidence)
+
+ local_model_rows = [row for row in validated.repositories if row.classification == "local_model"]
+ assert {row.slug.split("/", 1)[1] for row in local_model_rows} == set(
+ EXECUTABLE_LOCAL_MODEL_REPOSITORIES
+ )
+ project_keys = [row.project_key for row in local_model_rows]
+ assert len(project_keys) == len(set(project_keys)) == 12
+
+ config_marker_rows = [
+ row for row in validated.repositories if row.classification == "unknown" and row.code_evidence
+ ]
+ assert {row.slug.split("/", 1)[1] for row in config_marker_rows} == {
+ "codex-router",
+ "ITWorx_OpenChamber_MCP",
+ }
+ assert all(row.project_key is None for row in config_marker_rows)
+
+ assert runtime_status["source_snapshot_id"] == validated.snapshot_id
+ runtime_rows = runtime_status["repositories"]
+ assert {row["external_repository_id"] for row in runtime_rows} == {
+ row.external_repository_id for row in local_model_rows
+ }
+ assert all(row["runtime_status"] == "UNKNOWN" for row in runtime_rows)
+ assert all(row["reason"] == "no_live_request_evidence_collected" for row in runtime_rows)
+
+ observed_at = datetime.fromisoformat(request["observed_at"])
+ rebuilt = build_snapshot(
+ [row.model_dump(mode="json") for row in validated.repositories],
+ observed_at,
+ complete=False,
+ expected_count=manifest["repository_count"],
+ repository_ids=load_repository_ids(),
+ )
+ assert rebuilt == request
+ assert request_digest(request) == snapshot_digest(validated)
+
+ encoded = json.dumps(request, sort_keys=True)
+ assert "MODELFORGE_GITEA_TOKEN" not in encoded
+ assert "Authorization" not in encoded
+ assert "OLLAMA_BASE_URL" not in encoded
+
+
+def test_unverified_owner_inventory_cannot_be_emitted_as_complete(tmp_path: Path) -> None:
+ with pytest.raises(SystemExit) as error:
+ main(
+ [
+ "--offline-heads",
+ str(HEADS_PATH),
+ "--checkout-root",
+ str(ROOT),
+ "--output",
+ str(tmp_path / "must-not-exist.json"),
+ ]
+ )
+
+ assert error.value.code == 2
+ assert not (tmp_path / "must-not-exist.json").exists()
+
+
+def test_authenticated_complete_request_has_exact_inventory_without_runtime_claims() -> None:
+ partial = PortfolioReconcileRequest.model_validate_json(REQUEST_PATH.read_text(encoding="utf-8"))
+ complete = PortfolioReconcileRequest.model_validate_json(
+ COMPLETE_REQUEST_PATH.read_text(encoding="utf-8")
+ )
+
+ assert complete.complete is True
+ assert complete.source_system == "gitea"
+ assert len(complete.repositories) == 69
+ assert {row.external_repository_id for row in complete.repositories} == {
+ row.external_repository_id for row in partial.repositories
+ }
+ assert {row.slug: row.head_sha for row in complete.repositories} == {
+ row.slug: row.head_sha for row in partial.repositories
+ }
+ assert sum(row.classification == "local_model" for row in complete.repositories) == 12
+ assert all(not row.accept_head_update for row in complete.repositories)
+ assert all(not row.runtime_evidence for row in complete.repositories)
+ assert all(not evidence.capabilities for row in complete.repositories for evidence in row.code_evidence)
+ assert request_digest(json.loads(COMPLETE_REQUEST_PATH.read_text(encoding="utf-8"))) == snapshot_digest(complete)
diff --git a/backend/tests/test_project_model_management.py b/backend/tests/test_project_model_management.py
new file mode 100644
index 0000000..0598c2f
--- /dev/null
+++ b/backend/tests/test_project_model_management.py
@@ -0,0 +1,56 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.projects import ProjectCreate, ProjectRequirementCreate
+from modelforge_api.persistence.models import ArtifactSet, Base, Model, ModelRevision
+from modelforge_api.services.projects import ProjectService
+from tests.test_serving import approve_and_promote, setup_m5
+
+
+def test_project_workspace_resolves_exact_model_behind_stable_capability() -> None:
+ engine = create_engine(
+ "sqlite://",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ try:
+ with Session(engine) as session:
+ serving, candidate, _node = setup_m5(session)
+ _approval, deployment = approve_and_promote(serving, candidate.id)
+ projects = ProjectService(serving)
+ projects.create(ProjectCreate(
+ key="managed-model-app",
+ name="Managed model app",
+ description="Model management contract",
+ owner="Operator",
+ ))
+ workspace = projects.bind(
+ "managed-model-app",
+ ProjectRequirementCreate(
+ capability="rag.embedding@1",
+ migration_support="reindex",
+ ),
+ )
+
+ requirement = workspace.requirements[0]
+ assert requirement.deployment_id == deployment.id
+ assert requirement.serving_model is not None
+
+ artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
+ assert artifact_set is not None
+ revision = session.get(ModelRevision, artifact_set.revision_id)
+ assert revision is not None
+ model = session.get(Model, revision.model_id)
+ assert model is not None
+
+ assert requirement.serving_model.model_id == model.id
+ assert requirement.serving_model.display_name == model.display_name
+ assert requirement.serving_model.revision_id == revision.id
+ assert requirement.serving_model.resolved_commit_sha == revision.resolved_commit_sha
+ assert requirement.serving_model.artifact_set_id == artifact_set.id
+ assert requirement.serving_model.runtime_profile_id == deployment.runtime_profile_id
+ assert requirement.serving_model.compute_node_id == deployment.compute_node_id
+ finally:
+ engine.dispose()
diff --git a/backend/tests/test_project_onboarding.py b/backend/tests/test_project_onboarding.py
new file mode 100644
index 0000000..b305c36
--- /dev/null
+++ b/backend/tests/test_project_onboarding.py
@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr, ValidationError
+from sqlalchemy import create_engine, event, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.serving import get_serving_service
+from modelforge_api.db import get_session
+from modelforge_api.domain.projects import ProjectCreate, ProjectRequirementCreate
+from modelforge_api.domain.serving import (
+ ServiceClientCreate,
+ ServiceCredentialRevoke,
+ ServiceCredentialRotate,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ Base,
+ Project,
+ ProjectOnboarding,
+ ServiceCredential,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.projects import ProjectService
+from modelforge_api.services.serving import ServingError, ServingService, _aware
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from modelforge_api.settings import Settings, get_settings
+
+
+@pytest.fixture
+def serving() -> Iterator[ServingService]:
+ engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ yield ServingService(session, Settings(_env_file=None, env="test"), ManifestRegistry(), MemoryPayloadStore())
+ engine.dispose()
+
+
+def onboard(serving: ServingService):
+ projects = ProjectService(serving)
+ projects.create(ProjectCreate(key="example-app", name="Example", description="Integration", owner="Alice"))
+ project = projects.bind("example-app", ProjectRequirementCreate(capability="rag.embedding@1", migration_support="reindex"))
+ return serving.create_client(ServiceClientCreate(
+ name="example-reader", project_key=project.key, project_binding_id=project.requirements[0].id,
+ allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Integration check",
+ credential_expires_at=datetime.now(UTC) + timedelta(days=30), workload_priority="interactive",
+ ))
+
+
+def test_project_identity_binding_safety_and_readiness_are_persistent(serving: ServingService) -> None:
+ created = onboard(serving)
+ projects = ProjectService(serving)
+ project = projects.detail("example-app")
+ assert project.owner == "Alice" and project.environment == "lab"
+ assert project.production_eligible is None and project.state == "needs_attention"
+ requirement = project.requirements[0]
+ assert requirement.credential_ready and requirement.deployment_id is None
+ assert requirement.capability_checked_at is None
+ assert len(requirement.blockers) == 3
+ serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+ assert projects.detail("example-app").requirements[0].connectivity_checked_at is not None
+ assert projects.detail("example-app").state == "needs_attention"
+ serving.session.expire_all()
+ listed = projects.list()
+ assert listed == [projects.detail("example-app")]
+ with pytest.raises(ServingError, match="already exists"):
+ projects.create(ProjectCreate(key="example-app", name="Duplicate", description="Duplicate", owner="Alice"))
+ with pytest.raises(ServingError, match="reindex"):
+ projects.bind("example-app", ProjectRequirementCreate(capability="vision.embedding@1"))
+ with pytest.raises(ValidationError):
+ ProjectRequirementCreate(capability="rag.embedding@1", model_path="/weights")
+
+
+def test_project_workspace_list_batches_base_reads_as_projects_grow(serving: ServingService) -> None:
+ now = datetime.now(UTC)
+
+ def add_projects(start: int, end: int) -> None:
+ for index in range(start, end):
+ project = Project(
+ key=f"workspace-{index:02d}",
+ name=f"Workspace {index}",
+ description="Representative project workspace",
+ )
+ serving.session.add(project)
+ serving.session.flush()
+ serving.session.add(
+ ProjectOnboarding(
+ project_id=project.id,
+ owner="Owner",
+ environment="lab",
+ labels=["representative"],
+ created_at=now,
+ )
+ )
+ serving.session.commit()
+
+ def list_with_select_count() -> tuple[int, list[object]]:
+ selects: list[str] = []
+
+ def count_selects(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ if statement.lstrip().upper().startswith("SELECT"):
+ selects.append(statement)
+
+ serving.session.expire_all()
+ engine = serving.session.get_bind()
+ event.listen(engine, "before_cursor_execute", count_selects)
+ try:
+ result = ProjectService(serving).list()
+ finally:
+ event.remove(engine, "before_cursor_execute", count_selects)
+ return len(selects), result
+
+ add_projects(1, 2)
+ one_project_selects, one_project_result = list_with_select_count()
+ assert len(one_project_result) == 1
+ assert one_project_result[0].owner == "Owner"
+ assert one_project_result[0] == ProjectService(serving).detail("workspace-01")
+
+ add_projects(2, 21)
+ twenty_project_selects, twenty_project_result = list_with_select_count()
+ assert len(twenty_project_result) == 20
+ assert [item.key for item in twenty_project_result] == [
+ f"workspace-{index:02d}" for index in range(1, 21)
+ ]
+ assert all(item.owner == "Owner" and item.environment == "lab" for item in twenty_project_result)
+ assert one_project_selects == twenty_project_selects == 3
+ assert [item.key for item in ProjectService(serving).list(limit=2, after_key="workspace-18")] == [
+ "workspace-19",
+ "workspace-20",
+ ]
+
+
+def test_explicit_binding_cannot_cross_projects_and_inactive_projects_stop_serving(serving: ServingService) -> None:
+ created = onboard(serving)
+ projects = ProjectService(serving)
+ projects.create(ProjectCreate(key="other", name="Other", description="Other", owner="Bob"))
+ with pytest.raises(ServingError, match="active binding"):
+ serving.create_client(ServiceClientCreate(
+ name="cross-project", project_key="other", project_binding_id=created.project_binding_id,
+ allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Rejected identity",
+ ))
+ project = serving.session.scalar(select(Project).where(Project.key == "example-app"))
+ assert project
+ project.active = False
+ serving.session.commit()
+ with pytest.raises(ServingError, match="inactive"):
+ serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+
+
+def test_rotation_overlap_stale_requests_expiry_and_revoke_all(serving: ServingService) -> None:
+ first = onboard(serving)
+ assert first.credential_id
+ expires = datetime.now(UTC) + timedelta(days=7)
+ request = ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=900, expires_at=expires)
+ second = serving.rotate_client_credential(first.id, request)
+ assert serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1").id == first.id
+ assert serving.authenticate(f"Bearer {second.credential}", "rag.embedding@1").id == first.id
+ assert second.credential_expires_at and _aware(second.credential_expires_at) == expires
+ with pytest.raises(ServingError, match="changed"):
+ serving.rotate_client_credential(first.id, request)
+ assert second.credential_id
+ third = serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=second.credential_id, overlap_seconds=900, expires_at=expires))
+ with pytest.raises(ServingError, match="revoked"):
+ serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1")
+ assert third.credential_id
+ with pytest.raises(ServingError, match="exact"):
+ serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation="wrong"))
+ serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation=first.name))
+ for token in (first.credential, second.credential, third.credential):
+ with pytest.raises(ServingError, match="revoked"):
+ serving.authenticate(f"Bearer {token}", "rag.embedding@1")
+ assert serving.clients()[0].credential_id is None
+ exposed = str(serving.credential_history(first.id)) + str([event.details for event in serving.session.scalars(select(AuditEvent))])
+ assert all(token not in exposed for token in (first.credential, second.credential, third.credential))
+
+
+def test_overlap_never_extends_expiry_and_expired_or_disabled_rotation_refuses(serving: ServingService) -> None:
+ first = onboard(serving)
+ current = serving.session.get(ServiceCredential, first.credential_id)
+ assert current and first.credential_id
+ deadline = datetime.now(UTC) + timedelta(seconds=1)
+ current.expires_at = deadline
+ serving.session.commit()
+ serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=86400, expires_at=datetime.now(UTC) + timedelta(days=1)))
+ serving.session.refresh(current)
+ assert _aware(current.expires_at) == deadline
+ current.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ serving.session.commit()
+ with pytest.raises(ServingError, match="expired"):
+ serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1")
+
+
+def test_http_roles_audit_no_store_and_exact_machine_scope(serving: ServingService) -> None:
+ tokens = {role: "mfop_" + role[0] * 43 for role in ("viewer", "operator", "admin")}
+ settings = Settings(_env_file=None, env="test", operator_identities=SecretStr(json.dumps([
+ {"subject": role, "role": role, "credential_id": role, "token_sha256": hashlib.sha256(token.encode()).hexdigest(), "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat()}
+ for role, token in tokens.items()
+ ])))
+ app.dependency_overrides[get_settings] = lambda: settings
+ app.dependency_overrides[get_session] = lambda: serving.session
+ app.dependency_overrides[get_serving_service] = lambda: serving
+ try:
+ http = TestClient(app)
+ body = {"key": "api-project", "name": "API", "description": "API project", "owner": "Alice"}
+ path = "/api/v1/project-workspace"
+ assert http.get(path).status_code == 401
+ assert http.get(f"{path}?limit=1").status_code == 401
+ empty_page = http.get(f"{path}?limit=1", headers={"X-ModelForge-Admin-Token": tokens["viewer"]})
+ assert empty_page.status_code == 200 and empty_page.json() == []
+ assert http.post(path, json=body).status_code == 401
+ assert http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 403
+ response = http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["operator"], "X-Correlation-ID": "project-create"})
+ assert response.status_code == 201 and response.headers["cache-control"] == "no-store"
+ event = serving.session.scalar(select(AuditEvent).where(AuditEvent.action == "PROJECT_CREATED"))
+ assert event and event.actor_id == "operator" and event.correlation_id == "project-create"
+ created = onboard(serving)
+ check = "/api/v1/capabilities/rag.embedding@1/check"
+ headers = {"Authorization": f"Bearer {created.credential}"}
+ assert http.get(check, headers=headers).json()["client_id"] == str(created.id)
+ assert http.get(check.replace("rag.embedding", "vision.embedding"), headers=headers).status_code == 403
+ assert http.get(path, headers=headers).status_code == 401
+ assert http.get(check, headers={"X-ModelForge-Admin-Token": tokens["admin"]}).status_code == 401
+ assert http.post(f"/api/v1/admin/service-clients/{created.id}/credentials/rotate", json={}, headers={"X-ModelForge-Admin-Token": tokens["operator"]}).status_code == 403
+ assert http.get("/api/v1/projects", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 200
+ viewer_headers = {"X-ModelForge-Admin-Token": tokens["viewer"]}
+ changes_path = f"{path}/changes"
+ assert http.get(changes_path).status_code == 401
+ empty_changes = http.get(
+ f"{changes_path}?project_key=example-app&project_binding_id={created.project_binding_id}"
+ "&open_only=true&limit=1&offset=0",
+ headers=viewer_headers,
+ )
+ assert empty_changes.status_code == 200
+ assert empty_changes.json() == {"items": [], "total": 0}
+ assert http.get(f"{changes_path}?limit=201", headers=viewer_headers).status_code == 422
+ assert http.get(f"{changes_path}?offset=-1", headers=viewer_headers).status_code == 422
+ legacy_index = http.get(path, headers=viewer_headers)
+ first_page = http.get(f"{path}?limit=1", headers=viewer_headers)
+ first_key = first_page.json()[0]["key"]
+ second_page = http.get(f"{path}?limit=1&after_key={first_key}", headers=viewer_headers)
+ last_key = second_page.json()[0]["key"]
+ empty_page = http.get(f"{path}?limit=1&after_key={last_key}", headers=viewer_headers)
+ assert legacy_index.status_code == 200
+ assert [item["key"] for item in legacy_index.json()] == ["api-project", "example-app"]
+ assert [item["key"] for item in first_page.json() + second_page.json()] == ["api-project", "example-app"]
+ assert set(first_page.json()[0]) == {
+ "id", "key", "name", "description", "active", "owner", "environment", "labels",
+ "source", "production_eligible", "state", "requirements", "clients",
+ }
+ assert empty_page.status_code == 200 and empty_page.json() == []
+ assert http.get(f"{path}?limit=201", headers=viewer_headers).status_code == 422
+ assert http.get(f"{path}?after_key={first_key}", headers=viewer_headers).status_code == 422
+ finally:
+ app.dependency_overrides.clear()
+
+
+def test_readiness_requires_current_credential_current_deployment_and_fresh_success(serving: ServingService) -> None:
+ from modelforge_api.persistence.models import GatewayRequest
+ from tests.test_serving import approve_and_promote, setup_m5
+
+ service, candidate, _node = setup_m5(serving.session)
+ _, deployment = approve_and_promote(service, candidate.id)
+ first = onboard(service)
+ projects = ProjectService(service)
+ service.authenticate("Bearer " + first.credential, "rag.embedding@1")
+ record = GatewayRequest(
+ request_id=uuid.uuid4(), service_client_id=first.id,
+ capability_key="rag.embedding", capability_version=1,
+ capability_deployment_id=deployment.id, status="completed", priority="interactive",
+ input_sha256="0" * 64, input_count=1,
+ decision_evidence={"service_credential_id": str(first.credential_id)}, created_at=datetime.now(UTC),
+ )
+ service.session.add(record); service.session.commit()
+ assert projects.detail("example-app").state == "ready"
+ assert first.credential_id
+ second = service.rotate_client_credential(first.id, ServiceCredentialRotate(
+ expected_credential_id=first.credential_id, overlap_seconds=900,
+ expires_at=datetime.now(UTC) + timedelta(days=1),
+ ))
+ service.authenticate("Bearer " + second.credential, "rag.embedding@1")
+ record.created_at = datetime.now(UTC)
+ service.session.commit()
+ assert projects.detail("example-app").state == "needs_attention" # Old overlap cannot qualify new key.
+ record.decision_evidence = {"service_credential_id": str(second.credential_id)}
+ service.session.commit()
+ assert projects.detail("example-app").state == "ready"
+ record.created_at = datetime.now(UTC) - timedelta(days=2)
+ service.session.commit()
+ assert projects.detail("example-app").state == "needs_attention"
diff --git a/backend/tests/test_project_onboarding_postgres.py b/backend/tests/test_project_onboarding_postgres.py
new file mode 100644
index 0000000..4001f27
--- /dev/null
+++ b/backend/tests/test_project_onboarding_postgres.py
@@ -0,0 +1,246 @@
+"""Isolated PostgreSQL 0025 upgrade/backup restore and credential race qualification."""
+from __future__ import annotations
+
+import hashlib
+import importlib
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+from alembic.config import Config
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.serving import (
+ ServiceClientCreated,
+ ServiceClientResponse,
+ ServiceCredentialRevoke,
+ ServiceCredentialRotate,
+)
+from modelforge_api.persistence.models import AuditEvent, ServiceCredential
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.recovery_fingerprint import (
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.settings import Settings
+from tests.test_configuration_postgres import _database_tool
+
+
+@pytest.mark.parametrize(
+ "competitors",
+ [("rotate", "rotate"), ("rotate", "revoke"), ("revoke", "revoke")],
+)
+def test_project_upgrade_restore_and_credential_rotation_races(
+ tmp_path: Path,
+ competitors: tuple[str, str],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
+ if not admin_url:
+ pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
+ owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
+ runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
+ assert owner_url and runtime_url
+ database = "test_onboarding_" + uuid.uuid4().hex
+ restored = "test_onboarding_restore_" + uuid.uuid4().hex
+ admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
+ owner = make_url(owner_url).set(database=database)
+ runtime = create_engine(make_url(runtime_url).set(database=database))
+ restored_runtime = create_engine(make_url(runtime_url).set(database=restored))
+ config = Config()
+ config.set_main_option("script_location", str(Path(__file__).resolve().parents[1] / "alembic"))
+ config.set_main_option("sqlalchemy.url", owner.render_as_string(hide_password=False).replace("%", "%%"))
+ alembic = importlib.import_module("alembic.command")
+ client_id, credential_id = uuid.uuid4(), uuid.uuid4()
+ secret = "mf_" + "p" * 48
+ with admin.connect() as connection:
+ for name in (database, restored):
+ connection.execute(text(f'CREATE DATABASE "{name}" OWNER modelforge'))
+ try:
+ alembic.upgrade(config, "20260907_0025")
+ with Session(runtime) as session:
+ session.execute(text("INSERT INTO service_clients(id,name,status,allowed_capabilities,requests_per_minute,max_concurrent_requests,workload_priority) VALUES (:id,'prior-client','active','[\"rag.embedding@1\"]',60,1,'interactive')"), {"id": client_id})
+ session.execute(text("INSERT INTO service_credentials(id,service_client_id,secret_hash,secret_prefix,expires_at) VALUES (:id,:client,:digest,'mf_prior',:expiry)"), {"id": credential_id, "client": client_id, "digest": hashlib.sha256(secret.encode()).hexdigest(), "expiry": datetime.now(UTC) + timedelta(days=30)})
+ AuditWriter(session, "operator", "prior-release").write("qualification.baseline", "service_client", str(client_id), {})
+ session.commit()
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "before-onboarding.dump"
+ args = ["--host", "127.0.0.1" if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER") else owner.host or "localhost", "--port", str(owner.port or 5432), "--username", owner.username or "modelforge", "--dbname", database]
+ _database_tool("pg_dump", [*args, "--format=custom"], owner.password, archive)
+ assert archive.stat().st_size > 0
+ alembic.upgrade(config, "head")
+ with Session(runtime) as session:
+ current = semantic_fingerprint(session.connection())
+ comparison = fingerprint_compatibility(source, current, source_schema_revision="20260907_0025")
+ assert comparison.compatible
+ assert audit_fingerprint_differences_are_compatible(source, current, comparison)
+ service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+ assert service.authenticate("Bearer " + secret, "rag.embedding@1").id == client_id
+ alembic.downgrade(config, "20260907_0025")
+ alembic.upgrade(config, "head")
+ with Session(runtime) as session:
+ service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+ prior_overlap = service.rotate_client_credential(
+ client_id,
+ ServiceCredentialRotate(
+ expected_credential_id=credential_id,
+ overlap_seconds=1800,
+ expires_at=datetime.now(UTC) + timedelta(days=14),
+ ),
+ )
+ assert prior_overlap.credential_id
+ prior_overlap_id = prior_overlap.credential_id
+ prior_overlap_secret = prior_overlap.credential
+ prior_overlap_expires_at = prior_overlap.credential_expires_at
+ assert prior_overlap_expires_at
+ assert service.authenticate("Bearer " + secret, "rag.embedding@1").id == client_id
+ with Session(runtime) as session:
+ existing_event_ids = set(session.scalars(select(AuditEvent.id)))
+
+ # Hold both operations after their current-credential read and before the conditional
+ # claim UPDATE, so the loser must observe the winner's committed CAS change.
+ claim_barrier = threading.Barrier(2)
+ original_claim = ServingService._claim_credential
+
+ def claim_at_cas(service: ServingService, credential: ServiceCredential) -> None:
+ claim_barrier.wait(timeout=30)
+ original_claim(service, credential)
+
+ monkeypatch.setattr(ServingService, "_claim_credential", claim_at_cas)
+ race_credential_id = uuid.UUID(str(prior_overlap_id))
+ expires = datetime.now(UTC) + timedelta(days=14)
+
+ def contender(
+ action: str,
+ ) -> tuple[str, str, ServiceClientCreated | ServiceClientResponse | None]:
+ with Session(runtime) as session:
+ service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+ try:
+ if action == "rotate":
+ result = service.rotate_client_credential(
+ client_id,
+ ServiceCredentialRotate(
+ expected_credential_id=race_credential_id,
+ overlap_seconds=900,
+ expires_at=expires,
+ ),
+ )
+ else:
+ result = service.revoke_client_credential(
+ client_id,
+ ServiceCredentialRevoke(
+ expected_credential_id=race_credential_id,
+ confirmation="prior-client",
+ ),
+ )
+ return action, "applied", result
+ except ServingError as error:
+ assert error.code == "CREDENTIAL_CHANGED"
+ return action, "conflict", None
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ outcomes = list(pool.map(contender, competitors))
+ assert sorted(outcome for _action, outcome, _result in outcomes) == [
+ "applied",
+ "conflict",
+ ]
+ winner_action, _, winner_result = next(
+ outcome for outcome in outcomes if outcome[1] == "applied"
+ )
+ with Session(runtime) as session:
+ new_events = list(
+ session.scalars(
+ select(AuditEvent).where(
+ AuditEvent.action.in_(
+ ["CLIENT_CREDENTIAL_ROTATED", "CLIENT_CREDENTIAL_REVOKED"]
+ ),
+ AuditEvent.id.not_in(existing_event_ids),
+ )
+ )
+ )
+ assert len(new_events) == 1
+ assert new_events[0].action == (
+ "CLIENT_CREDENTIAL_ROTATED"
+ if winner_action == "rotate"
+ else "CLIENT_CREDENTIAL_REVOKED"
+ )
+ credentials = list(
+ session.scalars(
+ select(ServiceCredential).where(
+ ServiceCredential.service_client_id == client_id
+ )
+ )
+ )
+ current = session.get(ServiceCredential, race_credential_id)
+ original = session.get(ServiceCredential, credential_id)
+ assert current and original
+ assert current.superseded_at is not None
+ assert original.superseded_at is not None and original.revoked_at is not None
+ active = [
+ item
+ for item in credentials
+ if item.superseded_at is None and item.revoked_at is None
+ ]
+ if winner_action == "rotate":
+ assert isinstance(winner_result, ServiceClientCreated)
+ winner_credential_id = uuid.UUID(str(winner_result.credential_id))
+ assert len(active) == 1 and active[0].id == winner_credential_id
+ assert current.revoked_at is None
+ assert current.expires_at is not None
+ assert datetime.now(UTC) < current.expires_at <= prior_overlap_expires_at
+ assert current.superseded_at is not None
+ assert current.expires_at <= current.superseded_at + timedelta(seconds=900)
+ assert new_events[0].details["previous_credential_id"] == str(
+ race_credential_id
+ )
+ assert new_events[0].details["credential_id"] == str(winner_credential_id)
+ assert new_events[0].details["overlap_seconds"] == 900
+ else:
+ assert len(active) == 0
+ assert current.revoked_at is not None
+ assert all(item.revoked_at is not None for item in credentials)
+
+ with Session(runtime) as session:
+ service = ServingService(session, Settings(_env_file=None), ManifestRegistry(), None)
+ if winner_action == "rotate":
+ assert isinstance(winner_result, ServiceClientCreated)
+ assert service.authenticate(
+ "Bearer " + prior_overlap_secret, "rag.embedding@1"
+ ).id == client_id
+ assert service.authenticate(
+ "Bearer " + winner_result.credential, "rag.embedding@1"
+ ).id == client_id
+ with pytest.raises(ServingError, match="revoked"):
+ service.authenticate("Bearer " + secret, "rag.embedding@1")
+ else:
+ for retired_secret in (secret, prior_overlap_secret):
+ with pytest.raises(ServingError, match="revoked"):
+ service.authenticate("Bearer " + retired_secret, "rag.embedding@1")
+ with pytest.raises(RuntimeError, match="history exists"):
+ alembic.downgrade(config, "20260907_0025")
+ args[-1] = restored
+ _database_tool("pg_restore", [*args, "--exit-on-error"], owner.password, archive)
+ with restored_runtime.connect() as connection:
+ recovered = semantic_fingerprint(connection)
+ assert recovered["tables"] == source["tables"]
+ config.set_main_option("sqlalchemy.url", owner.set(database=restored).render_as_string(hide_password=False).replace("%", "%%"))
+ alembic.upgrade(config, "head")
+ with restored_runtime.connect() as connection:
+ recovered = semantic_fingerprint(connection)
+ comparison = fingerprint_compatibility(source, recovered, source_schema_revision="20260907_0025")
+ assert comparison.compatible and audit_fingerprint_differences_are_compatible(source, recovered, comparison)
+ finally:
+ runtime.dispose(); restored_runtime.dispose()
+ with admin.connect() as connection:
+ for name in (database, restored):
+ connection.execute(text(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)'))
+ admin.dispose()
diff --git a/backend/tests/test_project_python_client.py b/backend/tests/test_project_python_client.py
new file mode 100644
index 0000000..6e7300b
--- /dev/null
+++ b/backend/tests/test_project_python_client.py
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.error import HTTPError
+
+import pytest
+
+spec = importlib.util.spec_from_file_location("modelforge_client", Path(__file__).resolve().parents[2] / "clients/python/modelforge_client.py")
+assert spec and spec.loader
+sdk = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(sdk)
+
+
+def test_python_client_scope_typed_errors_and_no_redirect_or_retry() -> None:
+ seen = []
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ seen.append((self.path, self.headers.get("Authorization")))
+ self.send_response(200); self.end_headers(); self.wfile.write(b'{"connected":true}')
+
+ def do_POST(self):
+ seen.append((self.path, self.headers.get("Authorization")))
+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ if body.get("redirect"):
+ self.send_response(307); self.send_header("Location", "/must-not-follow"); self.end_headers()
+ elif body.get("flat_error"):
+ self.send_response(503)
+ self.send_header("Content-Type", "application/json")
+ self.end_headers()
+ self.wfile.write(
+ b'{"code":"LEGACY_ERROR","message":"Legacy error","correlation_id":"legacy-case"}'
+ )
+ else:
+ self.send_response(503); self.send_header("Content-Type", "application/json"); self.end_headers()
+ self.wfile.write(
+ json.dumps(
+ {
+ "error": {
+ "code": "NO_ELIGIBLE_NODE",
+ "message": "No node",
+ "correlation_id": "sdk-case",
+ "details": {
+ "input": "private benchmark input that must remain redacted"
+ },
+ }
+ }
+ ).encode()
+ )
+
+ def log_message(self, *_args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ worker = threading.Thread(target=server.serve_forever, daemon=True); worker.start()
+ token = "mf_" + "k" * 48
+ try:
+ client = sdk.ModelForgeClient(f"http://127.0.0.1:{server.server_port}", token, "rag.embedding@1", allow_http=True, timeout=2)
+ assert client.health() == {"connected": True}
+ with pytest.raises(sdk.ModelForgeError) as failure:
+ client.invoke({"input": ["test"]})
+ assert failure.value.code == "NO_ELIGIBLE_NODE"
+ assert str(failure.value) == "No node"
+ assert failure.value.status == 503 and failure.value.correlation_id == "sdk-case"
+ assert "private benchmark input" not in str(failure.value)
+ with pytest.raises(sdk.ModelForgeError) as legacy:
+ client.invoke({"flat_error": True})
+ assert legacy.value.code == "LEGACY_ERROR"
+ assert str(legacy.value) == "Legacy error"
+ assert legacy.value.correlation_id == "legacy-case"
+ with pytest.raises(sdk.ModelForgeError) as redirected:
+ client.invoke({"redirect": True})
+ assert redirected.value.status == 307
+ assert len(seen) == 4 and all("must-not-follow" not in path for path, _ in seen)
+ assert all(header == f"Bearer {token}" for _, header in seen)
+ assert token not in repr(client) + repr(failure.value)
+ finally:
+ server.shutdown(); server.server_close(); worker.join(timeout=3)
+
+
+@pytest.mark.parametrize("origin", ["http://lab.invalid", "https://user:password@lab.invalid", "https://lab.invalid/path", "https://lab.invalid/?q=key"])
+def test_python_client_rejects_unsafe_origin(origin: str) -> None:
+ with pytest.raises(ValueError):
+ sdk.ModelForgeClient(origin, "mf_" + "k" * 48, "rag.embedding@1")
+
+
+def test_python_client_preserves_typed_error_when_error_body_times_out() -> None:
+ class StalledBody:
+ closed = False
+
+ def read(self, _limit):
+ raise TimeoutError("private transport details")
+
+ def close(self):
+ pass
+
+ class Opener:
+ def open(self, _request, *, timeout):
+ raise HTTPError("https://lab.invalid", 503, "Unavailable",
+ {"X-Correlation-ID": "stalled-error"}, StalledBody())
+
+ client = sdk.ModelForgeClient("https://lab.invalid", "mf_" + "k" * 48, "rag.embedding@1")
+ client._opener = Opener()
+ with pytest.raises(sdk.ModelForgeError) as failure:
+ client.health()
+ assert failure.value.code == "TRANSPORT_ERROR"
+ assert failure.value.status == 503
+ assert failure.value.correlation_id == "stalled-error"
+ assert "private transport details" not in str(failure.value)
diff --git a/backend/tests/test_project_registry.py b/backend/tests/test_project_registry.py
index 4382115..cb97acf 100644
--- a/backend/tests/test_project_registry.py
+++ b/backend/tests/test_project_registry.py
@@ -1,4 +1,5 @@
-from sqlalchemy import create_engine
+import pytest
+from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from modelforge_api.persistence.models import (
@@ -16,28 +17,120 @@ def test_project_registry_materializes_only_real_contracts_and_is_idempotent() -
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as session:
- capability = Capability(key="rag.embedding", description="test")
- session.add(capability)
- session.flush()
- session.add(
- CapabilityContract(
- capability_id=capability.id,
- version=1,
- input_schema={},
- output_schema={},
- contract={},
- upgrade_class="requires_reindex",
+ manifests = ManifestRegistry()
+ enabled_capabilities = {"rag.embedding", "rag.reranking", "vision.embedding"}
+ for manifest in manifests.capabilities():
+ if manifest.capability not in enabled_capabilities:
+ continue
+ capability = Capability(key=manifest.capability, description=manifest.description)
+ session.add(capability)
+ session.flush()
+ session.add(
+ CapabilityContract(
+ capability_id=capability.id,
+ version=manifest.version,
+ input_schema=manifest.input_schema,
+ output_schema=manifest.output_schema,
+ contract=manifest.model_dump(mode="json"),
+ upgrade_class=manifest.upgrade_class.value,
+ )
)
- )
session.commit()
- first = sync_project_registry(session, ManifestRegistry())
- second = sync_project_registry(session, ManifestRegistry())
+ first = sync_project_registry(session, manifests)
+ second = sync_project_registry(session, manifests)
assert first.projects_created == 3
- assert first.bindings_created == 1
+ assert first.bindings_created == 3
assert first.unavailable_contracts
assert second.projects_created == 0
assert second.bindings_created == 0
assert session.query(Project).count() == 3
- assert session.query(ProjectBinding).count() == 1
+ bindings = session.execute(
+ select(Project.key, Capability.key)
+ .join(ProjectBinding, ProjectBinding.project_id == Project.id)
+ .join(
+ CapabilityContract,
+ CapabilityContract.id == ProjectBinding.capability_contract_id,
+ )
+ .join(Capability, Capability.id == CapabilityContract.capability_id)
+ ).all()
+ assert set(bindings) == {
+ ("examplerag", "rag.embedding"),
+ ("examplerag", "rag.reranking"),
+ ("examplevision", "vision.embedding"),
+ }
+
+
+@pytest.mark.parametrize("declared_stability", ["planned", "blocked"])
+def test_project_registry_skips_unavailable_capabilities_and_preserves_operator_bindings(
+ declared_stability: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ engine = create_engine("sqlite+pysqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ manifests = ManifestRegistry()
+ capability_manifest = next(
+ item
+ for item in manifests.capabilities()
+ if item.capability == "assistant.general" and item.version == 1
+ )
+ project_manifest = next(
+ item for item in manifests.projects() if item.project.id == "example-ops"
+ )
+ capability_manifest = capability_manifest.model_copy(
+ update={
+ "estate": capability_manifest.estate.model_copy(
+ update={"stability": declared_stability}
+ )
+ }
+ )
+ monkeypatch.setattr(manifests, "capabilities", lambda: [capability_manifest])
+ monkeypatch.setattr(manifests, "projects", lambda: [project_manifest])
+
+ capability = Capability(
+ key=capability_manifest.capability, description=capability_manifest.description
+ )
+ session.add(capability)
+ session.flush()
+ contract = CapabilityContract(
+ capability_id=capability.id,
+ version=capability_manifest.version,
+ input_schema=capability_manifest.input_schema,
+ output_schema=capability_manifest.output_schema,
+ contract=capability_manifest.model_dump(mode="json"),
+ upgrade_class=capability_manifest.upgrade_class.value,
+ )
+ session.add(contract)
+ session.commit()
+
+ first = sync_project_registry(session, manifests)
+
+ assert first.bindings_created == 0
+ assert session.scalar(select(ProjectBinding.id)) is None
+ assert [
+ (
+ item.project_key,
+ item.capability,
+ item.contract_version,
+ item.declared_stability,
+ )
+ for item in first.unavailable_bindings
+ ] == [("example-ops", "assistant.general", 1, declared_stability)]
+
+ project = session.scalar(select(Project).where(Project.key == "example-ops"))
+ assert project is not None
+ operator_binding = ProjectBinding(
+ project_id=project.id,
+ capability_contract_id=contract.id,
+ channel="stable",
+ priority="interactive",
+ )
+ session.add(operator_binding)
+ session.commit()
+ operator_binding_id = operator_binding.id
+
+ second = sync_project_registry(session, manifests)
+
+ assert second.bindings_created == 0
+ assert session.scalar(select(ProjectBinding.id)) == operator_binding_id
diff --git a/backend/tests/test_rc_candidate_freeze.py b/backend/tests/test_rc_candidate_freeze.py
new file mode 100644
index 0000000..df00684
--- /dev/null
+++ b/backend/tests/test_rc_candidate_freeze.py
@@ -0,0 +1,216 @@
+"""Unit contracts for the transportable, exact RC candidate image set."""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import io
+import json
+import subprocess
+import sys
+import tarfile
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+RELEASE_BUILD = ROOT / "scripts" / "release_build.py"
+
+
+def _release_build_module():
+ spec = importlib.util.spec_from_file_location("rc_candidate_release_build", RELEASE_BUILD)
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def _write_public_source_manifest(root: Path, module, source_revision: str) -> dict[str, object]:
+ files = []
+ for relative_path in module.SOURCE_DOCKERFILES:
+ path = root / relative_path
+ path.parent.mkdir(parents=True, exist_ok=True)
+ payload = (
+ f'LABEL {module.OCI_SOURCE_LABEL}="{module.PUBLIC_SOURCE_REPOSITORY}"\n'
+ ).encode()
+ path.write_bytes(payload)
+ files.append(
+ {
+ "path": relative_path,
+ "bytes": len(payload),
+ "sha256": hashlib.sha256(payload).hexdigest(),
+ }
+ )
+ manifest: dict[str, object] = {
+ "schemaVersion": 1,
+ "sourceRevision": source_revision,
+ "sourceRepository": module.PUBLIC_SOURCE_REPOSITORY,
+ "files": files,
+ }
+ (root / "PUBLIC_SOURCE_MANIFEST.json").write_text(json.dumps(manifest), encoding="utf-8")
+ return manifest
+
+
+def test_release_command_failure_keeps_the_docker_error_tail(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ module = _release_build_module()
+ failure = subprocess.CompletedProcess(
+ ["docker", "build"], 73, stdout="", stderr="progress\n" * 300 + "final Docker error"
+ )
+ monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: failure)
+
+ with pytest.raises(RuntimeError) as error:
+ module.run("docker", "build")
+
+ assert "final Docker error" in str(error.value)
+ assert "failed (73)" in str(error.value)
+ assert "progress\n" * 300 not in str(error.value)
+
+
+def _add_file(archive: tarfile.TarFile, name: str, payload: bytes) -> None:
+ info = tarfile.TarInfo(name)
+ info.size = len(payload)
+ archive.addfile(info, io.BytesIO(payload))
+
+
+def _image_records(module) -> dict[str, dict[str, object]]:
+ records: dict[str, dict[str, object]] = {}
+ for index, name in enumerate(module.IMAGES):
+ records[name] = {
+ "tag": f"{name}:1.3.0-rc.1",
+ "architecture": "amd64",
+ "labels": {
+ "org.opencontainers.image.version": "1.3.0-rc.1",
+ "org.opencontainers.image.revision": "a" * 40,
+ },
+ "rootfs_layers": [f"sha256:{index + 1:064x}"],
+ }
+ return records
+
+
+def _write_bundle(path: Path, records: dict[str, dict[str, object]]) -> None:
+ manifest: list[dict[str, object]] = []
+ with tarfile.open(path, "w") as archive:
+ for index, (_name, record) in enumerate(records.items()):
+ config = f"{index:064x}.json"
+ layer = f"{index:064x}/layer.tar"
+ payload = {
+ "architecture": record["architecture"],
+ "config": {"Labels": record["labels"]},
+ "rootfs": {"diff_ids": record["rootfs_layers"]},
+ }
+ _add_file(archive, config, json.dumps(payload).encode())
+ _add_file(archive, layer, b"layer")
+ manifest.append({"Config": config, "RepoTags": [record["tag"]], "Layers": [layer]})
+ _add_file(archive, "manifest.json", json.dumps(manifest).encode())
+
+
+def test_image_bundle_binds_exact_tags_labels_and_layer_sets(tmp_path: Path) -> None:
+ module = _release_build_module()
+ records = _image_records(module)
+ bundle = tmp_path / "candidate-images.tar"
+ _write_bundle(bundle, records)
+
+ verified = module.verify_image_bundle(bundle, records)
+
+ assert set(verified) == set(module.IMAGES)
+ assert verified["modelforge-api"]["tag"] == "modelforge-api:1.3.0-rc.1"
+
+
+def test_image_bundle_rejects_a_changed_filesystem_layer_set(tmp_path: Path) -> None:
+ module = _release_build_module()
+ records = _image_records(module)
+ bundle = tmp_path / "candidate-images.tar"
+ _write_bundle(bundle, records)
+ records["modelforge-api"]["rootfs_layers"] = ["sha256:" + "f" * 64]
+
+ with pytest.raises(ValueError, match="layer set"):
+ module.verify_image_bundle(bundle, records)
+
+
+def test_parentless_export_requires_an_exact_canonical_source_manifest(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ module = _release_build_module()
+ build_commit = "b" * 40
+ canonical_commit = "a" * 40
+
+ monkeypatch.setattr(module, "ROOT", tmp_path)
+ monkeypatch.setattr(
+ module,
+ "run",
+ lambda *args, **kwargs: {
+ ("git", "rev-parse", "HEAD"): build_commit,
+ ("git", "status", "--porcelain"): "",
+ ("git", "describe", "--tags", "--always"): "candidate",
+ ("git", "remote", "get-url", "origin"): "",
+ ("git", "rev-parse", "--abbrev-ref", "HEAD"): "main",
+ }.get(args, ""),
+ )
+
+ with pytest.raises(SystemExit, match="unbound source-label override"):
+ module.source_identity(False, canonical_commit)
+
+ manifest = _write_public_source_manifest(tmp_path, module, canonical_commit)
+
+ with pytest.raises(SystemExit, match="requires --canonical-source-commit"):
+ module.source_identity(False)
+
+ identity = module.source_identity(False, canonical_commit)
+
+ assert identity["commit"] == canonical_commit
+ assert identity["build_commit"] == build_commit
+ assert identity["public_source_manifest_sha256"]
+ assert identity["source_repository"] == module.PUBLIC_SOURCE_REPOSITORY
+
+ manifest_path = tmp_path / "PUBLIC_SOURCE_MANIFEST.json"
+ manifest["sourceRepository"] = "https://attacker.invalid/forged"
+ manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+ with pytest.raises(SystemExit, match="approved sanitized identity"):
+ module.source_identity(False, canonical_commit)
+
+ manifest["sourceRepository"] = module.PUBLIC_SOURCE_REPOSITORY
+ manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+ dockerfile = tmp_path / module.SOURCE_DOCKERFILES[0]
+ dockerfile.write_text("LABEL org.opencontainers.image.source=\"https://attacker.invalid\"\n")
+ with pytest.raises(SystemExit, match="does not bind the current"):
+ module.source_identity(False, canonical_commit)
+
+
+def test_private_release_identity_uses_the_canonical_https_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ module = _release_build_module()
+ commit = "b" * 40
+ monkeypatch.setattr(module, "ROOT", ROOT)
+ monkeypatch.setattr(
+ module,
+ "run",
+ lambda *args, **kwargs: {
+ ("git", "rev-parse", "HEAD"): commit,
+ ("git", "status", "--porcelain"): "",
+ ("git", "describe", "--tags", "--always"): "candidate",
+ ("git", "remote", "get-url", "origin"): "ssh://private.example/repo.git",
+ ("git", "rev-parse", "--abbrev-ref", "HEAD"): "main",
+ }.get(args, ""),
+ )
+
+ identity = module.source_identity(False)
+
+ assert identity["source_repository"] == module.CANONICAL_SOURCE_REPOSITORY
+
+
+def test_release_builder_checks_built_image_source_labels() -> None:
+ module = _release_build_module()
+ source = module.PUBLIC_SOURCE_REPOSITORY
+ records = {
+ "modelforge-api": {"labels": {module.OCI_SOURCE_LABEL: source}},
+ "modelforge-web": {"labels": {module.OCI_SOURCE_LABEL: source}},
+ }
+
+ module._assert_image_source_identity(records, source)
+ records["modelforge-web"]["labels"][module.OCI_SOURCE_LABEL] = "https://attacker.invalid"
+ with pytest.raises(SystemExit, match="source label does not match source identity"):
+ module._assert_image_source_identity(records, source)
diff --git a/backend/tests/test_rc_failure_node.py b/backend/tests/test_rc_failure_node.py
new file mode 100644
index 0000000..c96efee
--- /dev/null
+++ b/backend/tests/test_rc_failure_node.py
@@ -0,0 +1,124 @@
+"""Deterministic node and NVML failure injection for RC-QUAL-03.
+
+These tests exercise the serving boundary without a live GPU. The isolated
+candidate still needs the corresponding real-stack fault evidence.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.serving import (
+ EmbeddingInvokeRequest,
+ PlacementPlanRequest,
+ ServiceClientCreate,
+)
+from modelforge_api.persistence.models import (
+ Base,
+ GatewayRequest,
+ ServingGpuLease,
+ ServingJob,
+)
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import approve_and_promote, setup_m5
+
+
+@pytest.fixture
+def session() -> Session:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as value:
+ yield value
+ engine.dispose()
+
+
+def _count(session: Session, model: type[GatewayRequest | ServingGpuLease | ServingJob]) -> int:
+ return session.scalar(select(func.count()).select_from(model)) or 0
+
+
+def test_stale_node_refuses_route_and_placement_without_allocating_work(
+ session: Session,
+) -> None:
+ service, candidate, node = setup_m5(session)
+ _approval, promoted = approve_and_promote(service, candidate.id)
+ client = service.create_client(
+ ServiceClientCreate(name="node-loss", allowed_capabilities=["rag.embedding@1"])
+ )
+ authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
+ counts = tuple(
+ _count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob)
+ )
+
+ node.liveness_state = "stale"
+ node.last_heartbeat_at = datetime.now(UTC) - timedelta(minutes=5)
+ session.commit()
+
+ plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
+ assert plan.verdict == "REJECT_HEALTH"
+ assert plan.reason_codes == ["NODE_UNAVAILABLE"]
+ assert plan.evictions == []
+ with pytest.raises(ServingError) as caught:
+ service.invoke(EmbeddingInvokeRequest(input="must stay local"), authenticated)
+ assert caught.value.status_code == 503
+ assert caught.value.code == "NO_ELIGIBLE_NODE"
+ assert (
+ tuple(_count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob))
+ == counts
+ )
+
+
+@pytest.mark.parametrize("telemetry_failure", ["missing", "stale"])
+def test_nvml_unknown_refuses_invoke_and_cleans_request_without_worker_fallback(
+ session: Session,
+ telemetry_failure: str,
+) -> None:
+ service, candidate, _node = setup_m5(session)
+ _approval, promoted = approve_and_promote(service, candidate.id)
+ client = service.create_client(
+ ServiceClientCreate(
+ name=f"nvml-{telemetry_failure}", allowed_capabilities=["rag.embedding@1"]
+ )
+ )
+ authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
+ jobs_before = _count(session, ServingJob)
+ leases_before = _count(session, ServingGpuLease)
+ telemetry = service.repo.telemetry(promoted.accelerator_id)
+ assert telemetry is not None
+ if telemetry_failure == "missing":
+ telemetry.used_vram_bytes = None
+ telemetry.free_vram_bytes = None
+ else:
+ telemetry.observed_at = datetime.now(UTC) - timedelta(hours=1)
+ session.commit()
+
+ budget = service.scheduler_overview()[0]
+ assert budget.attribution_confidence == "UNKNOWN"
+ assert budget.schedulable_free_vram_bytes == 0
+ plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
+ assert plan.verdict == "REJECT_HEALTH"
+ assert plan.reason_codes == ["SCHEDULER_STATE_STALE"]
+ assert plan.evictions == []
+
+ with pytest.raises(ServingError) as caught:
+ service.invoke(
+ EmbeddingInvokeRequest(input="never infer on guessed capacity"), authenticated
+ )
+ assert caught.value.status_code == 503
+ assert caught.value.code == "SCHEDULER_STATE_STALE"
+ request = session.scalar(select(GatewayRequest).order_by(GatewayRequest.created_at.desc()))
+ assert request is not None
+ assert request.status == "failed"
+ assert request.failure_code == "SCHEDULER_STATE_STALE"
+ assert request.finished_at is not None
+ assert request.compute_node_id == promoted.compute_node_id
+ assert _count(session, ServingGpuLease) == leases_before
+ assert _count(session, ServingJob) == jobs_before
diff --git a/backend/tests/test_rc_failure_storage.py b/backend/tests/test_rc_failure_storage.py
new file mode 100644
index 0000000..5647122
--- /dev/null
+++ b/backend/tests/test_rc_failure_storage.py
@@ -0,0 +1,73 @@
+"""Deterministic storage fault injection for the authenticated embedding gateway."""
+
+from __future__ import annotations
+
+import redis
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.serving import EmbeddingInvokeRequest, ServiceClientCreate
+from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+def test_redis_payload_write_outage_fails_closed_and_releases_gateway_lease() -> None:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ base, candidate, _node = setup_m5(session)
+ approve_and_promote(base, candidate.id)
+ service = InlineWorkerServingService(
+ session, base.settings, base.manifests, base.payload_store
+ )
+ created = service.create_client(
+ ServiceClientCreate(
+ name="storage-fault-client",
+ allowed_capabilities=["rag.embedding@1"],
+ requests_per_minute=10,
+ )
+ )
+ client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+ payload_store = service.payload_store
+ original_put = payload_store.put
+
+ def unavailable(*_args: object, **_kwargs: object) -> str:
+ raise redis.ConnectionError("private redis host and credential")
+
+ payload_store.put = unavailable
+ try:
+ try:
+ service.invoke(EmbeddingInvokeRequest(input="sensitive test input"), client)
+ except ServingError as error:
+ assert error.status_code == 503
+ assert error.code == "PAYLOAD_STORE_UNAVAILABLE"
+ assert "private" not in str(error)
+ else:
+ raise AssertionError("Redis outage must prevent dispatch")
+ finally:
+ payload_store.put = original_put
+
+ request = session.scalars(select(GatewayRequest)).one()
+ lease = session.scalars(select(ServingGpuLease)).one()
+ assert request.status == "failed"
+ assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+ assert request.finished_at is not None
+ assert request.decision_evidence["input_persisted"] is False
+ assert lease.state == "failed"
+ assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+ assert lease.released_at is not None
+ assert (
+ session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all() == []
+ )
+ assert payload_store.values == {}
+
+ # Recovery must use normal serving, without a stale lease or hidden fallback.
+ recovered = service.invoke(EmbeddingInvokeRequest(input="after recovery"), client)
+ assert recovered.dimension == 1024
+ assert session.scalars(select(GatewayRequest)).all()[-1].status == "completed"
diff --git a/backend/tests/test_rc_failure_storage_modalities.py b/backend/tests/test_rc_failure_storage_modalities.py
new file mode 100644
index 0000000..6ecbda8
--- /dev/null
+++ b/backend/tests/test_rc_failure_storage_modalities.py
@@ -0,0 +1,108 @@
+"""Redis write outages fail closed on non-text Gateway invocation routes."""
+
+from __future__ import annotations
+
+import pytest
+import redis
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.serving import get_serving_service
+from modelforge_api.domain.serving import CapabilityExperimentCreate, ServiceClientCreate
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob
+from modelforge_api.settings import get_settings
+from tests.test_serving import InlineWorkerServingService, setup_m5
+
+
+@pytest.mark.parametrize(
+ ("capability", "body"),
+ [
+ (
+ "rag.reranking",
+ {
+ "query": "private query",
+ "documents": [{"id": "one", "text": "private document"}],
+ "top_n": 1,
+ },
+ ),
+ ("vision.embedding", {"items": [{"text": "private visual query"}]}),
+ ],
+)
+def test_payload_write_outage_fails_closed_for_non_text_routes(
+ capability: str, body: dict[str, object], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ engine = create_engine(
+ "sqlite+pysqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ base, candidate, _node = setup_m5(session)
+ probe = base.repo.probe(candidate.runtime_probe_id)
+ assert probe is not None
+ if capability == "rag.reranking":
+ probe.inference_result = {
+ "status": "passed",
+ "output_type": "ranked_scores",
+ "count": 1,
+ "finite": True,
+ "latency_ms": 30.0,
+ }
+ probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
+ session.commit()
+ base.create_experiment(
+ candidate.id,
+ CapabilityExperimentCreate(
+ route_key=f"storage-fault-{capability.replace('.', '-')}",
+ execution_approval_id=probe.execution_approval_id,
+ capability=capability,
+ purpose="Test the exact local Gateway route during a Redis payload write outage.",
+ ),
+ )
+ service = InlineWorkerServingService(
+ session, base.settings, base.manifests, base.payload_store
+ )
+ created = service.create_client(
+ ServiceClientCreate(
+ name=f"storage-fault-{capability.replace('.', '-')}",
+ allowed_capabilities=[f"{capability}@1"],
+ requests_per_minute=10,
+ )
+ )
+
+ def unavailable(*_args: object, **_kwargs: object) -> str:
+ raise redis.ConnectionError("private redis host and credential")
+
+ monkeypatch.setattr(service.payload_store, "put", unavailable)
+ app.dependency_overrides[get_settings] = lambda: service.settings
+ app.dependency_overrides[get_serving_service] = lambda: service
+ try:
+ response = TestClient(app).post(
+ f"/api/v1/capabilities/{capability}@1/invoke",
+ headers={"Authorization": f"Bearer {created.credential}"},
+ json=body,
+ )
+ finally:
+ app.dependency_overrides.clear()
+
+ assert response.status_code == 503
+ assert response.json()["error"]["code"] == "PAYLOAD_STORE_UNAVAILABLE"
+ assert "private redis host" not in response.text
+ request = session.scalars(select(GatewayRequest)).one()
+ lease = session.scalars(select(ServingGpuLease)).one()
+ assert request.status == "failed"
+ assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+ assert request.finished_at is not None
+ assert request.decision_evidence["input_persisted"] is False
+ assert "private" not in str(request.decision_evidence)
+ assert lease.state == "failed"
+ assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+ assert lease.released_at is not None
+ assert not session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all()
+ assert service.payload_store.values == {}
+ residency = service.repo.residency(request.capability_deployment_id)
+ assert residency is not None and residency.active_requests == 0
diff --git a/backend/tests/test_rc_readiness_program.py b/backend/tests/test_rc_readiness_program.py
new file mode 100644
index 0000000..977622d
--- /dev/null
+++ b/backend/tests/test_rc_readiness_program.py
@@ -0,0 +1,138 @@
+"""The RC ledger gate must fail closed on ambiguous plans or incomplete evidence."""
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("rc_readiness", ROOT / "scripts/rc_readiness.py")
+assert SPEC is not None and SPEC.loader is not None
+readiness = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(readiness)
+
+
+@pytest.fixture
+def repository(tmp_path: Path) -> Path:
+ for relative in (readiness.CONFIG, readiness.LEDGER, readiness.ROADMAP):
+ target = tmp_path / relative
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text((ROOT / relative).read_text(encoding="utf-8"), encoding="utf-8")
+ return tmp_path
+
+
+def mutate_plan(repository: Path, change) -> None:
+ path = repository / readiness.CONFIG
+ plan = json.loads(path.read_text(encoding="utf-8"))
+ change(plan)
+ path.write_text(json.dumps(plan), encoding="utf-8")
+
+
+def test_actual_repository_is_valid_but_incomplete() -> None:
+ result = readiness.validate(ROOT)
+ plan = json.loads((ROOT / readiness.CONFIG).read_text(encoding="utf-8"))
+ assert result["packet_count"] == len(plan["packets"])
+ assert result["required_gap_count"] > result["verified_count"] > 0
+ assert result["outstanding_ids"]
+ assert result["all_required_rows_verified"] is False
+ assert result["topological_order"].index("P06") < result["topological_order"].index("P03")
+ assert "release_ready" not in result
+
+
+def test_repository_path_guard_rejects_escape_before_read(repository: Path) -> None:
+ with pytest.raises(readiness.InvalidPlan, match="escapes root"):
+ readiness._read(repository, "../outside.md")
+
+
+def test_status_documentation_is_not_parsed_as_issue_rows(repository: Path) -> None:
+ before = readiness.validate(repository)
+ path = repository / readiness.LEDGER
+ with path.open("a", encoding="utf-8") as output:
+ output.write("\n| Classification | UNKNOWN |\nProse RC-FAKE-01 is not an issue row.\n")
+ assert readiness.validate(repository) == before
+
+
+@pytest.mark.parametrize("change", [
+ lambda p: p.update(schema_version=True),
+ lambda p: p.update(schema_version=2),
+ lambda p: p.update(extra=True),
+ lambda p: p.pop("ledger"),
+ lambda p: p.update(target_version="stable"),
+ lambda p: p.update(baseline_commit="main"),
+ lambda p: p.update(ledger="../../outside.md"),
+ lambda p: p.update(roadmap="C:/outside.md"),
+ lambda p: p.update(packets=[]),
+ lambda p: p["packets"].append(p["packets"][0]),
+ lambda p: p["packets"][0].update(id="bad"),
+ lambda p: p["packets"][0]["depends_on"].append("P99"),
+ lambda p: p["packets"][0]["depends_on"].append("P15"),
+ lambda p: p["packets"][0]["depends_on"].extend(["P01", "P01"]),
+ lambda p: p["packets"][0]["gaps"].pop(),
+ lambda p: p["packets"][0]["gaps"].append("RC-UNKNOWN-01"),
+ lambda p: p["packets"][0]["gaps"].append(p["packets"][1]["gaps"][0]),
+ lambda p: p["deferred_gaps"].append(p["deferred_gaps"][0]),
+ lambda p: p["deferred_gaps"].append(p["packets"][0]["gaps"].pop()),
+ lambda p: p["packets"][0]["gaps"].append(p["deferred_gaps"].pop()),
+])
+def test_invalid_plans_are_rejected(repository: Path, change) -> None:
+ mutate_plan(repository, change)
+ with pytest.raises(readiness.InvalidPlan):
+ readiness.validate(repository)
+
+
+@pytest.mark.parametrize("text", ["{", '{"schema_version":1,"schema_version":1}',
+ '{"packets":[{"id":"P00","id":"P01"}]}'])
+def test_malformed_and_duplicate_json_fields(repository: Path, text: str) -> None:
+ (repository / readiness.CONFIG).write_text(text, encoding="utf-8")
+ with pytest.raises(ValueError):
+ readiness.validate(repository)
+
+
+@pytest.mark.parametrize("relative", [readiness.CONFIG, readiness.LEDGER, readiness.ROADMAP])
+def test_missing_inputs_fail(repository: Path, relative: str) -> None:
+ (repository / relative).unlink()
+ with pytest.raises(OSError):
+ readiness.validate(repository)
+
+
+@pytest.mark.parametrize("mode", ["duplicate", "empty", "unknown_status", "false_verified",
+ "deferral", "classification", "malformed"])
+def test_invalid_ledger_rows_fail(repository: Path, mode: str) -> None:
+ path = repository / readiness.LEDGER
+ text = path.read_text(encoding="utf-8")
+ row = next(line for line in text.splitlines() if line.startswith("| RC-"))
+ changed = {
+ "duplicate": text + "\n" + row,
+ "empty": "# No rows",
+ "unknown_status": text.replace("| VERIFIED |", "| APPROVED |", 1),
+ "false_verified": text.replace("| VERIFIED |", "| VERIFIED — pending |", 1),
+ "deferral": text.replace("| VERIFIED |", "| DEFERRED |", 1),
+ "classification": text.replace(row, row.replace(row.split("|")[4], " UNKNOWN ", 1)),
+ "malformed": text.replace(row, row.rstrip("|"), 1),
+ }[mode]
+ path.write_text(changed, encoding="utf-8")
+ with pytest.raises(readiness.InvalidPlan):
+ readiness.validate(repository)
+
+
+def test_cli_exit_codes_and_full_verification(repository: Path, monkeypatch, capsys) -> None:
+ original = readiness.validate
+ monkeypatch.setattr(readiness, "validate", lambda: original(repository))
+ assert readiness.main([]) == 0
+ assert readiness.main(["--require-verified"]) == 1
+ path = repository / readiness.LEDGER
+ lines = path.read_text(encoding="utf-8").splitlines()
+ path.write_text("\n".join(
+ line.rsplit("|", 2)[0] + "| VERIFIED |"
+ if line.startswith("| RC-") and "| POST-RC |" not in line else line
+ for line in lines
+ ), encoding="utf-8")
+ assert readiness.main(["--require-verified"]) == 0
+ result = json.loads(capsys.readouterr().out.splitlines()[-1])
+ assert result["all_required_rows_verified"] is True
+ assert result["outstanding_ids"] == []
+ assert result["verified_count"] == result["required_gap_count"]
+ path.unlink()
+ assert readiness.main([]) == 2
+ assert json.loads(capsys.readouterr().out)["all_required_rows_verified"] is False
diff --git a/backend/tests/test_recovery_api_m15.py b/backend/tests/test_recovery_api_m15.py
index bd26428..491a997 100644
--- a/backend/tests/test_recovery_api_m15.py
+++ b/backend/tests/test_recovery_api_m15.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
import uuid
from pathlib import Path
from typing import Any
@@ -33,6 +34,7 @@ RECOVERY_ROUTES = (
("GET", "/api/v1/admin/recovery/restore-operations"),
("GET", "/api/v1/admin/recovery/artifact-recoveries"),
("GET", "/api/v1/admin/recovery/fingerprint"),
+ ("GET", "/api/v1/admin/recovery/retention/preview"),
("POST", "/api/v1/admin/recovery/retention/run"),
)
@@ -105,6 +107,34 @@ def test_dashboard_reports_measured_state_and_never_invents_a_backup(recovery: A
assert payload["verified_backup_count"] == 0
+def test_retention_preview_returns_exact_ids_without_backup_or_secret_metadata(
+ recovery: Any, tmp_path: Path
+) -> None:
+ session, _service, _settings = recovery
+ _subject, record = seed_backup(
+ session, tmp_path, backup_id="m15-retention-api-preview"
+ )
+
+ response = client.get("/api/v1/admin/recovery/retention/preview", headers=ADMIN)
+
+ assert response.status_code == 200
+ payload = response.json()
+ assert payload["policy_key"] == "control-plane.database"
+ assert payload["minimum_verified_backups"] == 2
+ assert payload["retained_backup_ids"] == [record.backup_id]
+ assert payload["expiring_backup_ids"] == []
+ assert len(payload["decisions"]) == 1
+ assert payload["decisions"][0]["backup_id"] == record.backup_id
+ assert payload["decisions"][0]["current_state"] == "CREATED"
+ assert payload["decisions"][0]["decision"] == "RETAIN"
+ assert payload["decisions"][0]["reason"] == "RETENTION_NOT_REACHED"
+ assert payload["decisions"][0]["expires_at"] is not None
+ serialized = json.dumps(payload).lower()
+ assert "destination_root" not in payload
+ assert "encryption_key_id" not in payload
+ assert "secret" not in serialized
+
+
def test_policies_and_assets_expose_the_full_classification(recovery: Any) -> None:
policies = client.get("/api/v1/admin/recovery/policies", headers=ADMIN).json()
assert {item["key"] for item in policies} >= {
diff --git a/backend/tests/test_recovery_audit_integrity_rc.py b/backend/tests/test_recovery_audit_integrity_rc.py
index 19180f4..92af2ad 100644
--- a/backend/tests/test_recovery_audit_integrity_rc.py
+++ b/backend/tests/test_recovery_audit_integrity_rc.py
@@ -38,10 +38,21 @@ from modelforge_api.services.audit import (
)
from modelforge_api.services.recovery import RecoveryError, RecoveryService
from modelforge_api.services.recovery_fingerprint import (
+ ALERT_NOTIFICATION_TABLES,
+ CREATIVE_TABLES,
CURRENT_TRUTH_TABLES,
FINGERPRINT_VERSION,
+ LAB_EXECUTION_TABLES,
LEGACY_FINGERPRINT_VERSION,
+ MANAGED_ACTIVATION_TABLE,
+ MANAGED_IDENTITY_TABLES,
MAX_ROWS_PER_TABLE,
+ MODEL_CHANGE_TABLES,
+ MODEL_UPDATE_TABLES,
+ PORTFOLIO_TABLES,
+ SECRET_CURRENT_TABLES,
+ SECRET_TABLES,
+ SELECTED_COMPARISON_TABLES,
_canonical_row,
_table_digest,
audit_fingerprint_differences_are_compatible,
@@ -334,6 +345,31 @@ def test_valid_resume_checks_chain_before_returning_already_applied(
assert engine.operations.index("query_rows") < engine.operations.index("scalar")
+@pytest.mark.parametrize("table_exists", [False, True])
+def test_reconciliation_clears_only_present_model_watch_claims(
+ session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, table_exists: bool,
+) -> None:
+ subject = _service(session, tmp_path)
+ sql_seen: list[str] = []
+ def scalar(target: object, sql: str) -> str:
+ sql_seen.append(sql)
+ if "table_name='model_update_watches'" in sql:
+ return "1" if table_exists else "0"
+ if "select count(*) from model_update_watches" in sql:
+ return "2"
+ return "" if "string_agg" in sql else "0"
+ monkeypatch.setattr(subject.engine, "scalar", scalar)
+ monkeypatch.setattr(subject, "_verify_restored_audit_chain", lambda target: {})
+ monkeypatch.setattr(subject, "_append_restored_audit_marker", lambda *args, **kwargs: None)
+ record = SimpleNamespace(id=uuid.uuid4())
+ plan = SimpleNamespace(database_destination="postgresql://modelforge:test@postgres:5432/restored",
+ backup_set=SimpleNamespace(backup_id="watch-recovery"))
+ result = subject._reconcile_restored(record, plan)
+ assert ("update model_update_watches set check_token=NULL, check_until=NULL" in sql_seen) is table_exists
+ assert result["cleared_current_truth"].get("model_update_watch_claims", 0) == (2 if table_exists else 0)
+ assert not any("delete from model_update" in sql for sql in sql_seen)
+
+
def test_corruption_is_an_independent_ready_gate_before_fingerprints(
session: Session, tmp_path: Path
) -> None:
@@ -506,7 +542,8 @@ def _fingerprint(
*,
include_checkpoint: bool,
) -> dict[str, object]:
- table_names = set(classified_tables())
+ # These fixtures model schema 0022/0024, before online identities and secrets.
+ table_names = set(classified_tables()) - ALERT_NOTIFICATION_TABLES - PORTFOLIO_TABLES - MODEL_CHANGE_TABLES - CREATIVE_TABLES - SELECTED_COMPARISON_TABLES - MODEL_UPDATE_TABLES - SECRET_TABLES - MANAGED_IDENTITY_TABLES - LAB_EXECUTION_TABLES - {"configuration_state", "configuration_revisions", "project_onboarding"}
if not include_checkpoint:
table_names.remove("audit_chain_heads")
tables: dict[str, object] = {
@@ -536,7 +573,7 @@ def _fingerprint(
"tables": tables,
"digest": hashlib.sha256(version.encode()).hexdigest(),
"groups": {},
- "physical_tables": sorted(table_names | CURRENT_TRUTH_TABLES),
+ "physical_tables": sorted(table_names | (CURRENT_TRUTH_TABLES - SECRET_CURRENT_TABLES - {MANAGED_ACTIVATION_TABLE})),
"unexpected_tables": [],
}
@@ -548,7 +585,9 @@ def _with_legitimate_reconciliation_delta(
restored["version"] = FINGERPRINT_VERSION
restored["digest"] = "f" * 64
restored["unexpected_tables"] = []
- restored["physical_tables"] = sorted(classified_tables() | CURRENT_TRUTH_TABLES)
+ restored["physical_tables"] = sorted(
+ (classified_tables() - ALERT_NOTIFICATION_TABLES - PORTFOLIO_TABLES - MODEL_CHANGE_TABLES - CREATIVE_TABLES - SELECTED_COMPARISON_TABLES - MODEL_UPDATE_TABLES - SECRET_TABLES - MANAGED_IDENTITY_TABLES - LAB_EXECUTION_TABLES - {"configuration_state", "configuration_revisions", "project_onboarding"}) | (CURRENT_TRUTH_TABLES - SECRET_CURRENT_TABLES - {MANAGED_ACTIVATION_TABLE})
+ )
tables = restored["tables"]
assert isinstance(tables, dict)
tables["audit_events"] = {
diff --git a/backend/tests/test_recovery_audit_order_postgres.py b/backend/tests/test_recovery_audit_order_postgres.py
new file mode 100644
index 0000000..f7ed51e
--- /dev/null
+++ b/backend/tests/test_recovery_audit_order_postgres.py
@@ -0,0 +1,57 @@
+"""Real restored audit pagination must preserve numeric sequence ordering."""
+
+from __future__ import annotations
+
+import importlib
+import os
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.services.recovery_postgres import parse_target
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+ IdentityDatabase,
+ ManagedPostgresTools,
+ identity_database,
+)
+
+__all__ = ["identity_database"]
+
+
+def test_restored_audit_numeric_order_crosses_page_boundary(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ event_count = 512 # Production iteration reads 500 rows per page.
+ with Session(database.runtime) as session:
+ writer = AuditWriter(session, "operator", "audit-pagination-qualification")
+ expected_ids = []
+ for ordinal in range(1, event_count + 1):
+ event = writer.write(
+ "qualification.audit_pagination", "recovery", None,
+ {"ordinal": ordinal, "source": "isolated regression fixture"},
+ )
+ assert event.sequence == ordinal
+ expected_ids.append(str(event.id))
+ session.commit()
+
+ archive = tmp_path / "audit-pagination.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ target = parse_target(database.restored_owner.render_as_string(hide_password=False))
+ with Session(database.runtime) as session:
+ recovery = RecoveryService(session, Settings(_env_file=None, env="test"))
+ recovery.engine = ManagedPostgresTools(
+ base_environment=dict(os.environ), timeout_seconds=60,
+ )
+ events = list(recovery._restored_audit_events(target))
+ assert [event.sequence for event in events] == list(range(1, event_count + 1))
+ assert [event.id for event in events] == expected_ids
+ assert len({event.id for event in events}) == event_count
+ checkpoint = recovery._verify_restored_audit_chain(target)
+ assert checkpoint.event_count == checkpoint.last_sequence == event_count
+ assert checkpoint.last_event_hash == events[-1].event_hash
diff --git a/backend/tests/test_recovery_m15.py b/backend/tests/test_recovery_m15.py
index 95f7191..56c5f1a 100644
--- a/backend/tests/test_recovery_m15.py
+++ b/backend/tests/test_recovery_m15.py
@@ -6,13 +6,15 @@ import base64
import hashlib
import inspect
import json
+import os
+import stat
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
import pytest
-from sqlalchemy import create_engine, func, select
+from sqlalchemy import create_engine, event, func, select
from sqlalchemy.orm import Session
from modelforge_api.domain.recovery import (
@@ -45,6 +47,8 @@ from modelforge_api.persistence.models import (
ModelRevision,
RecoveryAssetRecord,
RecoveryPolicyRevision,
+ RestoreOperation,
+ RestorePlan,
StorageRoot,
UpstreamSnapshot,
)
@@ -328,6 +332,7 @@ def seed_backup(
encrypted: bool = False,
payload: bytes = b"PGDMP-fake-custom-dump-payload",
milestone: str | None = None,
+ legal_hold: bool = False,
) -> tuple[RecoveryService, BackupSet]:
"""Create a VERIFIED-shaped backup on disk without invoking PostgreSQL tooling."""
@@ -403,6 +408,7 @@ def seed_backup(
encryption_key_id="modelforge-backup-key-1" if encrypted else None,
verification_details={},
milestone=milestone,
+ legal_hold=legal_hold,
reason="unit test fixture",
created_by="test",
started_at=datetime.now(UTC),
@@ -554,6 +560,50 @@ def test_an_encrypted_backup_verified_with_the_wrong_key_fails_closed(
assert not any(path.name.startswith("database.dump.enc.verify-") for path in directory.iterdir())
+def test_encrypted_verification_stages_only_on_restore_volume(
+ session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ subject, record = seed_backup(
+ session, tmp_path, backup_id="m15-read-only-source", encrypted=True
+ )
+ observed: list[Path] = []
+
+ def inspect_dump(path: Path) -> int:
+ observed.append(path)
+ assert path.parent == (tmp_path / "restore").resolve()
+ assert path.read_bytes() == b"PGDMP-fake-custom-dump-payload"
+ if os.name == "posix":
+ assert stat.S_IMODE(path.stat().st_mode) == 0o600
+ return 1
+
+ monkeypatch.setattr(subject.engine, "list_dump_contents", inspect_dump)
+ result = subject.verify_backup(record.id)
+
+ assert result.state is BackupState.VERIFIED
+ assert len(observed) == 1
+ assert not observed[0].exists()
+ assert not list((tmp_path / "restore").iterdir())
+ assert sorted(path.name for path in (tmp_path / "backups" / "m15-read-only-source").iterdir()) == [
+ "database.dump.enc", "manifest.json"
+ ]
+
+
+def test_encrypted_verification_refuses_staging_inside_backup_root(
+ session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ subject, record = seed_backup(
+ session, tmp_path, backup_id="m15-overlapping-roots", encrypted=True
+ )
+ subject.settings = settings_for(tmp_path, backup_restore_root=tmp_path / "backups" / "staging")
+ monkeypatch.setattr(subject.engine, "list_dump_contents", lambda _payload: 1)
+
+ result = subject.verify_backup(record.id)
+
+ assert result.state is BackupState.FAILED
+ assert result.failure_code == RecoveryFailureCode.PATH_NOT_ALLOWED.value
+ assert not list((tmp_path / "backups" / "staging").iterdir())
+
+
def test_an_encrypted_backup_without_a_key_reports_key_unavailable(
session: Session, tmp_path: Path
) -> None:
@@ -605,6 +655,12 @@ def test_retention_never_expires_the_last_verified_backup(
record.expires_at = datetime.now(UTC) - timedelta(days=1)
session.commit()
+ preview = subject.preview_retention()
+ assert preview.retained_backup_ids == [record.backup_id]
+ assert preview.expiring_backup_ids == []
+ assert preview.decisions[0].reason == "MINIMUM_VERIFIED_BACKUPS"
+ assert subject.backup(record.id).state is BackupState.VERIFIED
+
result = subject.apply_retention()
assert result["expired"] == 0
assert subject.backup(record.id).state is BackupState.VERIFIED
@@ -637,6 +693,178 @@ def test_retention_expires_surplus_backups_but_keeps_the_policy_minimum(
assert states["m15-retention-0"] is BackupState.EXPIRED
+def test_retention_preview_matches_apply_and_does_not_commit_or_write_audit(
+ session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ observed_at = datetime(2026, 9, 23, 12, 0, tzinfo=UTC)
+ records: list[BackupSet] = []
+ for backup_id, milestone, legal_hold in (
+ ("m15-retention-expire", None, False),
+ ("m15-retention-hold", None, True),
+ ("m15-retention-milestone", "rc-baseline", False),
+ ("m15-retention-minimum-a", None, False),
+ ("m15-retention-minimum-b", None, False),
+ ("m15-retention-created", None, False),
+ ):
+ subject, record = seed_backup(
+ session,
+ tmp_path,
+ backup_id=backup_id,
+ milestone=milestone,
+ legal_hold=legal_hold,
+ )
+ records.append(record)
+ monkeypatch.setattr(subject.engine, "list_dump_contents", lambda _payload: 1)
+ for index, record in enumerate(records[:-1]):
+ subject.verify_backup(record.id)
+ record.verified_at = observed_at - timedelta(days=len(records) - index)
+ record.expires_at = observed_at - timedelta(days=1)
+ records[-1].expires_at = observed_at - timedelta(days=1)
+ session.commit()
+
+ states_before = {item.backup_id: item.state for item in records}
+ audit_count_before = session.scalar(select(func.count()).select_from(AuditEvent))
+ pending_policy = RecoveryPolicyRevision(
+ key="control-plane.preview-pending-test",
+ revision=1,
+ name="Pending preview policy",
+ asset_class="AUTHORITATIVE",
+ backup_method="POSTGRES_LOGICAL_CUSTOM",
+ retention_days=1,
+ minimum_verified_backups=1,
+ rpo_seconds=60,
+ rto_target_seconds=60,
+ restore_verification="FULL_RESTORE",
+ encryption_required=True,
+ external_dependency=False,
+ rehydration_allowed=False,
+ secret_class=None,
+ rationale="A pending row detects accidental autoflush in the read-only preview.",
+ active=True,
+ fingerprint="c" * 64,
+ created_by="test",
+ )
+ session.add(pending_policy)
+ commits: list[None] = []
+
+ def after_commit(_session: Session) -> None:
+ commits.append(None)
+
+ event.listen(session, "after_commit", after_commit)
+ try:
+ preview = subject.preview_retention(observed_at)
+ finally:
+ event.remove(session, "after_commit", after_commit)
+
+ assert preview.observed_at == observed_at
+ assert preview.minimum_verified_backups == 2
+ assert preview.retained_backup_ids == [
+ "m15-retention-hold",
+ "m15-retention-milestone",
+ "m15-retention-minimum-a",
+ "m15-retention-minimum-b",
+ ]
+ assert preview.expiring_backup_ids == ["m15-retention-created", "m15-retention-expire"]
+ reasons = {item.backup_id: item.reason for item in preview.decisions}
+ assert reasons["m15-retention-hold"] == "LEGAL_HOLD"
+ assert reasons["m15-retention-milestone"] == "MILESTONE"
+ assert reasons["m15-retention-minimum-a"] == "MINIMUM_VERIFIED_BACKUPS"
+ assert {item.backup_id: item.state for item in records} == states_before
+ assert pending_policy in session.new
+ session.expunge(pending_policy)
+ assert session.scalar(select(func.count()).select_from(AuditEvent)) == audit_count_before
+ assert commits == []
+ assert not session.new and not session.dirty and not session.deleted
+
+ applied = subject.apply_retention(observed_at)
+ assert applied == {"expired": 2, "protected": 4, "skipped_active_restore": 0}
+ expired = session.scalars(
+ select(AuditEvent).where(AuditEvent.action == "BACKUP_EXPIRED")
+ )
+ assert {item.details["backup_id"] for item in expired} == set(preview.expiring_backup_ids)
+ assert {
+ item.backup_id for item in records if item.state == BackupState.EXPIRED.value
+ } == set(preview.expiring_backup_ids)
+
+
+def test_retention_preview_reports_active_restore_and_apply_skips_without_policy_seeding(
+ session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ subject, record = seed_backup(session, tmp_path, backup_id="m15-retention-active")
+ monkeypatch.setattr(subject.engine, "list_dump_contents", lambda _payload: 1)
+ subject.verify_backup(record.id)
+ record.expires_at = datetime.now(UTC) - timedelta(days=1)
+
+ plan = RestorePlan(
+ backup_set_id=record.id,
+ mode="VALIDATION",
+ state="PREFLIGHT_PASSED",
+ target_environment="ISOLATED",
+ target_label="active-retention-preview",
+ database_destination="postgresql+psycopg://modelforge:secret@postgres:5432/mf_restore",
+ artifact_strategy="NONE",
+ secret_strategy="MANUAL", # noqa: S106 - recovery strategy label is not a secret
+ node_strategy="NONE",
+ preflight={},
+ validation_requirements={},
+ fingerprint="a" * 64,
+ reason="active restore retention guard test",
+ created_by="test",
+ )
+ session.add(plan)
+ session.flush()
+ session.add(
+ RestoreOperation(
+ plan_id=plan.id,
+ backup_set_id=record.id,
+ state="RESTORING_DATABASE",
+ attempt=1,
+ idempotency_key="b" * 64,
+ preflight_result={},
+ phase_durations={},
+ source_fingerprint={},
+ restored_fingerprint={},
+ fingerprint_diff={},
+ validation_result={},
+ )
+ )
+ session.commit()
+
+ preview = subject.preview_retention()
+ assert preview.active_restore_in_progress is True
+ assert preview.retained_backup_ids == [record.backup_id]
+ assert preview.expiring_backup_ids == []
+ assert preview.decisions[0].reason == "ACTIVE_RESTORE_IN_PROGRESS"
+
+ policy = session.scalar(
+ select(RecoveryPolicyRevision).where(
+ RecoveryPolicyRevision.key == "control-plane.database",
+ RecoveryPolicyRevision.active.is_(True),
+ )
+ )
+ assert policy is not None
+ policy.active = False
+ session.commit()
+
+ with pytest.raises(RecoveryError) as error:
+ subject.preview_retention()
+ assert error.value.code == "recovery_policy_missing"
+
+ # Historically the active-restore guard returned before looking up or seeding a policy.
+ assert subject.apply_retention() == {
+ "expired": 0,
+ "protected": 1,
+ "skipped_active_restore": 1,
+ }
+ assert session.scalar(
+ select(func.count()).select_from(RecoveryPolicyRevision).where(
+ RecoveryPolicyRevision.key == "control-plane.database",
+ RecoveryPolicyRevision.active.is_(True),
+ )
+ ) == 0
+ assert record.state == BackupState.VERIFIED.value
+
+
def test_milestone_and_legal_hold_backups_are_never_expired(
session: Session, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -1057,7 +1285,7 @@ def test_the_alembic_tree_is_found_from_the_repository_layout() -> None:
assert root is not None and (root / "versions").is_dir()
revisions = _known_alembic_revisions()
assert revisions[0] == "20260824_0001"
- assert revisions[-1] == "20260830_0024"
+ assert revisions[-1] == "20260924_0039"
assert len(revisions) == len(set(revisions))
diff --git a/backend/tests/test_registry.py b/backend/tests/test_registry.py
index 227e886..83b91ed 100644
--- a/backend/tests/test_registry.py
+++ b/backend/tests/test_registry.py
@@ -6,6 +6,7 @@ from pathlib import Path
import pytest
from fastapi.testclient import TestClient
+from http_contract_fakes import isolated_http_contract # noqa: F401 - opt-in pytest fixture
from pydantic import SecretStr
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
@@ -33,6 +34,8 @@ from modelforge_api.services.registry import (
)
from modelforge_api.settings import Settings, get_settings
+pytestmark = pytest.mark.usefixtures("isolated_http_contract")
+
def _test_operator_credential() -> str:
return "registry-test-operator"
diff --git a/backend/tests/test_release_contract_m17.py b/backend/tests/test_release_contract_m17.py
index b380cb2..7f0d1b1 100644
--- a/backend/tests/test_release_contract_m17.py
+++ b/backend/tests/test_release_contract_m17.py
@@ -28,8 +28,10 @@ from modelforge_api.domain.release import (
TARGET_SCHEMA_REVISION,
Compatibility,
SemanticVersion,
+ _canonicalize_packaged_version,
agent_protocol_compatibility,
build_identity,
+ release_channel_for,
schema_compatibility,
upgrade_required,
)
@@ -44,9 +46,6 @@ from modelforge_api.settings import Settings
ROOT = Path(__file__).resolve().parents[2]
-# --------------------------------------------------------------------------- one version
-
-
def test_the_version_file_is_the_single_source_of_truth() -> None:
declared = (ROOT / "VERSION").read_text("utf-8").strip()
assert declared == PRODUCT_VERSION
@@ -58,8 +57,6 @@ def test_the_version_file_is_the_single_source_of_truth() -> None:
["backend/pyproject.toml", "node-agent/pyproject.toml", "runtime-worker/pyproject.toml"],
)
def test_every_python_manifest_matches_the_version_file(manifest: str) -> None:
- """Four independent copies of the version is three chances to publish a wrong one."""
-
data = tomllib.loads((ROOT / manifest).read_text("utf-8"))
assert data["project"]["version"] == PRODUCT_VERSION, (
f"{manifest} declares {data['project']['version']!r}, VERSION says {PRODUCT_VERSION!r}"
@@ -85,40 +82,76 @@ def test_every_component_module_matches_the_version_file(module: str) -> None:
assert match.group(1) == PRODUCT_VERSION
-def test_the_release_version_is_semantic_and_not_a_prerelease() -> None:
+def test_the_release_version_is_the_explicit_rc_prerelease() -> None:
parsed = SemanticVersion.parse(PRODUCT_VERSION)
- assert not parsed.is_prerelease
+ assert parsed.is_prerelease
+ assert parsed.prerelease == "rc.1"
+ assert RELEASE_CHANNEL == "rc"
assert parsed.major >= 1, "a v1 release cannot carry a 0.x major"
-@pytest.mark.parametrize("bad", ["1.0", "1.0.0.0", "v1.0.0", "one.0.0", ""])
+def test_release_channel_is_derived_from_a_semantic_version() -> None:
+ assert release_channel_for("1.3.0") == "stable"
+ assert release_channel_for("1.3.0-rc.2") == "rc"
+ assert release_channel_for("1.3.0-rc.preview") == "prerelease"
+ assert release_channel_for("1.3.0-preview.2") == "prerelease"
+
+
+@pytest.mark.parametrize(
+ ("metadata_version", "expected"),
+ [
+ ("1.3.0", "1.3.0"),
+ ("1.3.0rc1", "1.3.0-rc.1"),
+ ("1.3.0rc01", "1.3.0rc01"),
+ ("1.3.0a1", "1.3.0a1"),
+ ],
+)
+def test_installed_package_metadata_only_canonicalizes_numbered_rc_semver(
+ metadata_version: str, expected: str
+) -> None:
+ assert _canonicalize_packaged_version(metadata_version) == expected
+
+
+@pytest.mark.parametrize("bad", ["1.0", "1.0.0.0", "v1.0.0", "one.0.0", "1.0.0-", ""])
def test_a_malformed_version_is_refused(bad: str) -> None:
with pytest.raises(ValueError):
SemanticVersion.parse(bad)
-# --------------------------------------------------------------------------- compatibility
-
-
def test_the_target_schema_revision_is_supported() -> None:
- assert TARGET_SCHEMA_REVISION == "20260830_0024"
+ assert TARGET_SCHEMA_REVISION == "20260924_0039"
assert TARGET_SCHEMA_REVISION in SUPPORTED_SCHEMA_REVISIONS
assert schema_compatibility(TARGET_SCHEMA_REVISION) is Compatibility.COMPATIBLE
assert "20260827_0021" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
assert "20260828_0022" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
+ assert "20260923_0037" in SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
assert "20260827_0021" not in SUPPORTED_SCHEMA_REVISIONS
+ assert "20260923_0037" not in SUPPORTED_SCHEMA_REVISIONS
def test_the_rc_schema_has_one_explicit_direct_upgrade_contract() -> None:
- # The version bump is a later release tranche. This schema tranche must already describe every
- # accepted source honestly so an intermediate protected merge cannot migrate the wrong target.
- assert PRODUCT_VERSION == "1.2.2"
+ assert PRODUCT_VERSION == "1.3.0-rc.1"
assert MINIMUM_UPGRADE_SOURCE == "v1.0.0"
assert SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS == (
"20260827_0021",
"20260828_0022",
"20260830_0023",
"20260830_0024",
+ "20260907_0025",
+ "20260908_0026",
+ "20260908_0027",
+ "20260908_0028",
+ "20260909_0029",
+ "20260909_0030",
+ "20260909_0031",
+ "20260909_0032",
+ "20260909_0033",
+ "20260910_0034",
+ "20260911_0035",
+ "20260923_0036",
+ "20260923_0037",
+ "20260924_0038",
+ "20260924_0039",
)
@@ -137,7 +170,19 @@ def test_v1_1_documents_the_destructive_downgrade_boundary_truthfully() -> None:
@pytest.mark.parametrize(
("revision", "expected"),
[
- ("20260830_0024", Compatibility.COMPATIBLE),
+ ("20260923_0036", Compatibility.TOO_OLD),
+ ("20260923_0037", Compatibility.TOO_OLD),
+ ("20260924_0038", Compatibility.TOO_OLD),
+ ("20260924_0039", Compatibility.COMPATIBLE),
+ ("20260911_0035", Compatibility.TOO_OLD),
+ ("20260910_0034", Compatibility.TOO_OLD),
+ ("20260909_0033", Compatibility.TOO_OLD),
+ ("20260909_0032", Compatibility.TOO_OLD),
+ ("20260908_0028", Compatibility.TOO_OLD),
+ ("20260908_0027", Compatibility.TOO_OLD),
+ ("20260908_0026", Compatibility.TOO_OLD),
+ ("20260907_0025", Compatibility.TOO_OLD),
+ ("20260830_0024", Compatibility.TOO_OLD),
("20260830_0023", Compatibility.TOO_OLD),
("20260828_0022", Compatibility.TOO_OLD),
("20260827_0021", Compatibility.TOO_OLD),
@@ -182,9 +227,6 @@ def test_the_declared_protocol_matches_the_wire_constant() -> None:
assert AGENT_PROTOCOL_VERSION == CURRENT_AGENT_PROTOCOL_VERSION
-# --------------------------------------------------------------------------- build identity
-
-
def test_build_identity_reports_absent_values_as_null_rather_than_guessing() -> None:
identity = build_identity()
assert identity.version == PRODUCT_VERSION
@@ -197,8 +239,6 @@ def test_build_identity_reports_absent_values_as_null_rather_than_guessing() ->
@pytest.mark.parametrize("placeholder", ["", " ", "unknown", "UNKNOWN", "none", "null"])
def test_an_unsubstituted_build_argument_is_not_reported_as_a_fact(placeholder: str) -> None:
- """A Dockerfile ARG that was never passed must not become a claimed commit."""
-
identity = build_identity(source_commit=placeholder, image_digest=placeholder)
assert identity.source_commit is None
assert identity.image_digest is None
@@ -211,9 +251,6 @@ def test_build_identity_carries_a_real_commit_when_given_one() -> None:
assert "source_commit" in identity.as_dict()
-# --------------------------------------------------------------------------- configuration
-
-
def _production(**overrides: object) -> Settings:
base: dict[str, object] = {
"env": "production",
@@ -342,8 +379,6 @@ def test_production_refuses_to_start_while_development_defaults_survive(tmp_path
def test_development_reports_the_same_problems_without_refusing_to_start(tmp_path: Path) -> None:
- """A developer with no backup key should still be able to run the API — and be told."""
-
settings = Settings(
env="development",
operator_api_key=None,
@@ -352,7 +387,7 @@ def test_development_reports_the_same_problems_without_refusing_to_start(tmp_pat
backup_root=tmp_path,
)
report = enforce_startup(settings)
- assert report.ok or report.problems is not None # never raises outside production
+ assert report.ok or report.problems is not None
def test_the_minimum_postgres_major_is_stated() -> None:
@@ -360,12 +395,6 @@ def test_the_minimum_postgres_major_is_stated() -> None:
def test_the_node_agent_pins_the_control_plane_at_the_product_version() -> None:
- """They ship as one product, so a stale pin makes the agent image unbuildable at release time.
-
- Found exactly that way: bumping the product version to 1.0.0 left this at 0.1.0 and the Node
- Agent image failed to resolve its dependencies during the fresh-install rehearsal.
- """
-
data = tomllib.loads((ROOT / "node-agent" / "pyproject.toml").read_text("utf-8"))
pins = [
dependency
diff --git a/backend/tests/test_release_provenance_v121.py b/backend/tests/test_release_provenance_v121.py
index ad2ccdf..520bc78 100644
--- a/backend/tests/test_release_provenance_v121.py
+++ b/backend/tests/test_release_provenance_v121.py
@@ -36,8 +36,9 @@ RELEASE_BUILD = ROOT / "scripts" / "release_build.py"
FRONTEND_DOCKERFILE = ROOT / "frontend" / "Dockerfile"
NODE_AGENT_DOCKERFILE = ROOT / "node-agent" / "Dockerfile"
-#: Every image the release builder publishes. The Runtime Worker is a first-class private runtime
-#: artifact even though it exposes no public listener.
+#: Every image the release builder publishes. The Runtime Worker is a first-class release image:
+#: the remote GPU projection may still build locally for development, but an RC installs its exact
+#: immutable tag rather than reconstructing it from source.
RELEASE_IMAGES = (
"modelforge-api",
"modelforge-web",
@@ -149,7 +150,7 @@ def test_the_console_csp_is_derived_from_the_same_origin_argument() -> None:
dockerfile = FRONTEND_DOCKERFILE.read_text(encoding="utf-8")
assert 'API_ORIGIN=$(printf \'%s\' "${VITE_API_BASE_URL}" | cut -d/ -f1-3)' in dockerfile
# The build fails rather than shipping a policy with an unsubstituted placeholder.
- assert 'grep -q "connect-src \'self\' ${API_ORIGIN};"' in dockerfile
+ assert 'grep -q "connect-src ${API_ORIGIN};"' in dockerfile
def test_the_console_csp_is_never_widened_to_a_wildcard() -> None:
@@ -157,7 +158,7 @@ def test_the_console_csp_is_never_widened_to_a_wildcard() -> None:
connect = [line for line in template.splitlines() if "connect-src" in line]
assert connect, "the template must define connect-src"
for line in connect:
- assert "connect-src 'self' __API_ORIGIN__" in line
+ assert "connect-src __API_ORIGIN__" in line
assert "*" not in line.split("connect-src", 1)[1].split(";")[0]
@@ -228,6 +229,16 @@ def test_the_node_agent_compose_build_passes_the_release_identity() -> None:
assert args["MODELFORGE_BUILT_AT"] == "${MODELFORGE_BUILT_AT:-}"
+def test_the_runtime_worker_compose_projection_accepts_an_exact_release_image() -> None:
+ worker = _compose("docker-compose.runtime-worker.yml")["services"]["runtime-worker"]
+ assert "MODELFORGE_RUNTIME_WORKER_IMAGE" in worker["image"]
+ assert "latest" not in worker["image"]
+ args = worker["build"]["args"]
+ assert args["MODELFORGE_VERSION"] == "${MODELFORGE_VERSION:-0.0.0}"
+ assert args["MODELFORGE_COMMIT"] == "${MODELFORGE_COMMIT:-}"
+ assert args["MODELFORGE_BUILT_AT"] == "${MODELFORGE_BUILT_AT:-}"
+
+
def test_every_composed_release_image_passes_the_release_identity() -> None:
"""Whatever builds a published image must stamp it; otherwise the tag is the only identity."""
@@ -252,14 +263,6 @@ def test_the_node_agent_image_reference_is_versioned_and_never_floating() -> Non
assert ":latest" not in reference
-def test_the_runtime_worker_image_reference_is_versioned_and_never_floating() -> None:
- service = _compose("docker-compose.runtime-worker.yml")["services"]["runtime-worker"]
- reference = service["image"]
- assert "${MODELFORGE_RUNTIME_WORKER_IMAGE:-" in reference
- assert "${MODELFORGE_VERSION:-local}" in reference
- assert ":latest" not in reference
-
-
def test_the_agent_version_derives_from_the_repository_version_file() -> None:
"""Single-source versioning: VERSION drives every packaged manifest.
diff --git a/backend/tests/test_secret_slots.py b/backend/tests/test_secret_slots.py
new file mode 100644
index 0000000..d78bb1c
--- /dev/null
+++ b/backend/tests/test_secret_slots.py
@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import base64
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import httpx
+import pytest
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, delete, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.db import get_session
+from modelforge_api.domain.secret_slots import (
+ SLOT,
+ SecretSlotRevoke,
+ SecretSlotSet,
+ SecretSlotVersionRequest,
+)
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ Base,
+ SecretSlotActivation,
+ SecretSlotTest,
+ SecretSlotVersion,
+)
+from modelforge_api.services import secret_slots as module
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.configuration_changes import ConfigurationService
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN = Principal("recovery-admin", PrincipalRole.ADMIN, "legacy_admin_token")
+TOKEN = "hf_disposable_unit_test_only" # noqa: S105 - synthetic offline test data
+REASON = "Reviewed credential lifecycle"
+
+
+@pytest.fixture
+def session():
+ engine = create_engine("sqlite://", poolclass=StaticPool,
+ connect_args={"check_same_thread": False})
+ Base.metadata.create_all(engine)
+ with Session(engine) as connection:
+ yield connection
+ engine.dispose()
+
+
+@pytest.fixture
+def settings():
+ return Settings(_env_file=None, env="test", hf_token="hf_deployment_test_only", # noqa: S106
+ secret_slot_root_key=base64.b64encode(bytes(range(32))).decode())
+
+
+def service(session, settings, principal=ADMIN):
+ return module.SecretSlotService(session, settings, principal, "slot-test-42")
+
+
+def candidate(subject, value=TOKEN):
+ return subject.set(SecretSlotSet(expected_revision=subject.metadata().revision,
+ reason=REASON, value=value))
+
+
+def request(metadata):
+ return SecretSlotVersionRequest(expected_revision=metadata.revision,
+ version=metadata.candidate_version, reason=REASON)
+
+
+def test_full_lifecycle_encryption_and_no_fallback(session, settings, monkeypatch):
+ subject = service(session, settings)
+ assert module.resolve_huggingface_token(session, settings) == settings.hf_token.get_secret_value()
+ first = candidate(subject)
+ assert first.state == "pending" and first.active_version is None
+ row = session.get(SecretSlotVersion, 1)
+ assert TOKEN not in row.secret_ciphertext
+ with pytest.raises(module.SecretSlotError, match="explicitly activated"):
+ module.resolve_huggingface_token(session, settings)
+ with pytest.raises(module.SecretSlotError, match="successful test"):
+ subject.activate(request(first))
+ monkeypatch.setattr(module, "test_huggingface_token", lambda token: (token == TOKEN, "HF_AUTHENTICATED"))
+ tested = subject.test(request(first))
+ active = subject.activate(request(tested))
+ assert active.state == "active"
+ assert module.resolve_huggingface_token(session, settings) == TOKEN
+ rotation = candidate(subject, "hf_replacement_test_only")
+ assert module.resolve_huggingface_token(session, settings) == TOKEN
+ monkeypatch.setattr(module, "test_huggingface_token", lambda token: (False, "HF_AUTHENTICATION_REJECTED"))
+ failed = subject.test(request(rotation))
+ assert failed.versions[0].test_status == "failed" and failed.active_version == 1
+ revoked = subject.revoke(SecretSlotRevoke(expected_revision=failed.revision,
+ confirm_slot=SLOT, reason=REASON))
+ assert revoked.state == "revoked"
+ with pytest.raises(module.SecretSlotError):
+ module.resolve_huggingface_token(session, settings)
+ exported = ConfigurationService(session, settings, AuditContext("operator", "test", "c")).export()
+ evidence = subject.catalog().model_dump_json() + exported.model_dump_json()
+ evidence += str(list(session.scalars(select(AuditEvent.details))))
+ assert SLOT in exported.secret_slots
+ assert TOKEN not in evidence and row.secret_ciphertext not in evidence
+
+
+@pytest.mark.parametrize("role", [PrincipalRole.VIEWER, PrincipalRole.OPERATOR])
+def test_non_admin_cannot_modify(session, settings, role):
+ subject = service(session, settings, Principal("person", role, "named_operator_key"))
+ assert subject.catalog().slot.source == "deployment"
+ with pytest.raises(module.SecretSlotError) as denied:
+ candidate(subject)
+ assert denied.value.status_code == 403
+ assert subject.metadata().revision == 0
+
+
+def test_test_result_cannot_bless_concurrent_replacement(session, settings, monkeypatch):
+ subject = service(session, settings)
+ first = candidate(subject)
+ def changed(token):
+ candidate(subject, "hf_newer_candidate")
+ return True, "HF_AUTHENTICATED"
+ monkeypatch.setattr(module, "test_huggingface_token", changed)
+ with pytest.raises(module.SecretSlotError) as conflict:
+ subject.test(request(first))
+ assert conflict.value.code == "SECRET_SLOT_VERSION_CONFLICT"
+ assert subject.metadata().versions[0].test_status == "untested"
+
+
+def test_root_key_binding_and_recovery_quarantine(session, settings, monkeypatch):
+ subject = service(session, settings)
+ first = candidate(subject)
+ monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+ active = subject.activate(request(subject.test(request(first))))
+ for replacement in (None, SecretStr(base64.b64encode(b"x" * 32).decode())):
+ changed = settings.model_copy(update={"secret_slot_root_key": replacement})
+ with pytest.raises(module.SecretSlotError):
+ module.resolve_huggingface_token(session, changed)
+ wrong_id = settings.model_copy(update={"secret_slot_root_key_id": "different-key"})
+ with pytest.raises(module.SecretSlotError):
+ module.resolve_huggingface_token(session, wrong_id)
+ # Supported recovery clears these exact current-truth tables; real restore is PG-qualified.
+ session.execute(delete(SecretSlotActivation))
+ session.execute(delete(SecretSlotTest))
+ session.commit()
+ assert subject.metadata().state == "quarantined"
+ with pytest.raises(module.SecretSlotError):
+ module.resolve_huggingface_token(session, settings)
+ with pytest.raises(module.SecretSlotError):
+ subject.activate(request(active))
+ refreshed = subject.test(request(subject.metadata()))
+ subject.activate(request(refreshed))
+ assert module.resolve_huggingface_token(session, settings) == TOKEN
+
+
+def test_expired_evidence_and_audit_failure_do_not_activate(session, settings, monkeypatch):
+ subject = service(session, settings)
+ first = candidate(subject)
+ monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+ tested = subject.test(request(first))
+ session.get(SecretSlotTest, 1).tested_at = datetime.now(UTC) - timedelta(minutes=6)
+ session.commit()
+ with pytest.raises(module.SecretSlotError):
+ subject.activate(request(tested))
+ tested = subject.test(request(tested))
+ def unavailable(*args, **kwargs):
+ raise RuntimeError("audit unavailable")
+ monkeypatch.setattr(subject.audit, "write", unavailable)
+ with pytest.raises(RuntimeError):
+ subject.activate(request(tested))
+ assert subject.metadata().active_version is None
+ assert subject.metadata().revision == tested.revision
+
+
+class Body(httpx.SyncByteStream):
+ def __init__(self, chunks):
+ self.chunks = chunks
+ def __iter__(self):
+ yield from self.chunks
+
+
+@pytest.mark.parametrize("status,headers,chunks,expected", [
+ (200, {}, [b'{"name":"local-test"}'], (True, "HF_AUTHENTICATED")),
+ (401, {}, [], (False, "HF_AUTHENTICATION_REJECTED")),
+ (302, {"location": "https://untrusted.invalid"}, [], (False, "HF_PROVIDER_UNAVAILABLE")),
+ (200, {"content-encoding": "gzip"}, [], (False, "HF_RESPONSE_INVALID")),
+ (200, {}, [b"x" * 65537], (False, "HF_RESPONSE_INVALID")),
+ (200, {}, [b'{}'], (False, "HF_RESPONSE_INVALID")),
+ (200, {}, [b"[" * 2000 + b"0" + b"]" * 2000], (False, "HF_RESPONSE_INVALID")),
+])
+def test_provider_check_is_fixed_bounded_and_redacted(monkeypatch, status, headers, chunks, expected):
+ actual_client = httpx.Client
+ calls = []
+ def send(request):
+ calls.append(request)
+ return httpx.Response(status, headers=headers, stream=Body(chunks))
+ def factory(**kwargs):
+ assert kwargs["follow_redirects"] is False and kwargs["trust_env"] is False
+ return actual_client(transport=httpx.MockTransport(send), **kwargs)
+ monkeypatch.setattr(module.httpx, "Client", factory)
+ assert module.test_huggingface_token(TOKEN) == expected
+ assert len(calls) == 1 and str(calls[0].url) == "https://huggingface.co/api/whoami-v2"
+ assert calls[0].headers["authorization"] == f"Bearer {TOKEN}"
+
+
+def test_provider_stream_has_total_deadline(monkeypatch):
+ actual_client = httpx.Client
+ def send(request):
+ return httpx.Response(200, stream=Body([b'{"name":"local-test"}']))
+ monkeypatch.setattr(module.httpx, "Client", lambda **kwargs: actual_client(
+ transport=httpx.MockTransport(send), **kwargs))
+ moments = iter([0.0, 11.0])
+ monkeypatch.setattr(module.time, "monotonic", lambda: next(moments))
+ assert module.test_huggingface_token(TOKEN) == (False, "HF_PROVIDER_TIMEOUT")
+
+
+def test_http_write_only_validation_and_no_store(session, settings):
+ settings = settings.model_copy(update={"operator_api_key": SecretStr("local-anchor")})
+ prior = dict(app.dependency_overrides)
+ app.dependency_overrides[get_settings] = lambda: settings
+ app.dependency_overrides[get_session] = lambda: session
+ try:
+ client = TestClient(app)
+ url = "/api/v1/configuration/secret-slots"
+ assert client.get(url).status_code == 401
+ headers = {"X-ModelForge-Admin-Token": "local-anchor"}
+ response = client.post(url + "/huggingface_control_plane/set", headers=headers,
+ json={"expected_revision": 0, "reason": REASON, "value": TOKEN})
+ assert response.status_code == 200, response.text
+ assert response.headers["cache-control"] == "no-store"
+ assert TOKEN not in response.text
+ invalid = client.post(url + "/huggingface_control_plane/set", headers=headers,
+ json={"expected_revision": 1, "reason": TOKEN, "value": TOKEN + " "})
+ assert invalid.status_code == 422 and TOKEN not in invalid.text
+ metadata = client.get(url, headers=headers)
+ assert metadata.status_code == 200 and TOKEN not in metadata.text
+ finally:
+ app.dependency_overrides.clear()
+ app.dependency_overrides.update(prior)
+
+
+def test_both_control_plane_consumers_resolve_active_snapshot(session, settings, monkeypatch):
+ from modelforge_api.api.routes.acquisition import get_acquisition_service
+ from modelforge_api.api.routes.agent import get_agent_acquisition_service
+ from modelforge_api.services.registry import RegistryNotFound
+
+ subject = service(session, settings)
+ first = candidate(subject)
+ for consumer in (get_acquisition_service, get_agent_acquisition_service):
+ local_service = consumer(session, settings)
+ # Read-only persisted history and node progress must not require upstream credentials.
+ with pytest.raises(RegistryNotFound, match="model revision not found"):
+ local_service.artifact_sets(uuid.uuid4())
+ with pytest.raises(module.SecretSlotError):
+ local_service.provider.resolve()
+ monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+ subject.activate(request(subject.test(request(first))))
+ for consumer in (get_acquisition_service, get_agent_acquisition_service):
+ resolved = consumer(session, settings)
+ assert resolved.provider.resolve().token == TOKEN
+ assert resolved.credential_configured is True
+
+
+def test_revoked_managed_admin_cannot_start_external_test(session, settings, monkeypatch):
+ from modelforge_api.domain.managed_identity import ManagedIdentityCreate, ManagedIdentityRevoke
+ from modelforge_api.services.operator_identities import (
+ ManagedIdentityService,
+ authenticate_managed_operator,
+ )
+
+ manager = ManagedIdentityService(session, ADMIN, "identity-check")
+ issued = manager.create(ManagedIdentityCreate(slug="alpha", role="admin",
+ expires_in_days=1, expected_revision=0, reason=REASON))
+ manager.create(ManagedIdentityCreate(slug="beta", role="admin",
+ expires_in_days=1, expected_revision=1, reason=REASON))
+ principal = authenticate_managed_operator(session, issued.one_time_key)
+ session.commit()
+ first = candidate(service(session, settings))
+ manager.revoke("alpha", ManagedIdentityRevoke(expected_version=1,
+ confirm_subject="managed:alpha", reason=REASON))
+ def forbidden(token):
+ pytest.fail("Revoked administrator must not trigger provider I/O")
+ monkeypatch.setattr(module, "test_huggingface_token", forbidden)
+ with pytest.raises(module.SecretSlotError) as rejected:
+ service(session, settings, principal).test(request(first))
+ assert rejected.value.status_code == 403
diff --git a/backend/tests/test_secret_slots_persistence.py b/backend/tests/test_secret_slots_persistence.py
new file mode 100644
index 0000000..b41230a
--- /dev/null
+++ b/backend/tests/test_secret_slots_persistence.py
@@ -0,0 +1,101 @@
+# ruff: noqa: S106 - isolated synthetic persistence fixtures
+"""Secret history fingerprint and migration contracts without external credentials."""
+import copy
+import importlib.util
+from datetime import UTC, datetime
+from pathlib import Path
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, SecretSlotState, SecretSlotVersion
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery_fingerprint import (
+ SECRET_SCHEMA,
+ SECRET_TABLES,
+ FingerprintCompatibility,
+ _without_creative,
+ _without_lab_execution,
+ _without_model_changes,
+ _without_model_updates,
+ _without_secrets,
+ _without_selected_comparisons,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ unclassified_tables,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ fingerprint_session as _current_fingerprint_session,
+)
+
+
+def fingerprint_session(session: Session):
+ return _without_lab_execution(_without_model_changes(_current_fingerprint_session(session)))
+
+
+def test_secret_history_is_strict_except_verified_recovery_marker() -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ assert not unclassified_tables()
+ session.add(SecretSlotVersion(version=1, key_id="qualification", secret_ciphertext="opaque",
+ secret_nonce="nonce", created_at=datetime.now(UTC)))
+ session.commit()
+ before = _without_creative(_without_model_updates(_without_selected_comparisons(fingerprint_session(session))))
+ assert before["tables"]["platform_secret_versions"]["redacted_columns"] == ["secret_ciphertext", "secret_nonce"]
+ AuditWriter(session, "operator", "qualification").write("RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {"restored": True})
+ session.commit()
+ after = _without_creative(_without_model_updates(_without_selected_comparisons(fingerprint_session(session))))
+ contract = fingerprint_compatibility(before, after, source_schema_revision=SECRET_SCHEMA)
+ assert contract.mode == "ALERT_NOTIFICATION_UPGRADE|PORTFOLIO_UPGRADE|SECRET_STRICT"
+ assert audit_fingerprint_differences_are_compatible(before, after, contract)
+ unrelated_marker = FingerprintCompatibility(
+ True,
+ f"UNRELATED|{contract.mode}",
+ contract.allow_audit_checkpoint_introduction,
+ contract.reason,
+ )
+ assert not audit_fingerprint_differences_are_compatible(before, after, unrelated_marker)
+ for table in SECRET_TABLES:
+ altered = copy.deepcopy(after)
+ altered["tables"][table]["digest"] = "0" * 64
+ assert not audit_fingerprint_differences_are_compatible(before, altered, contract)
+ engine.dispose()
+
+
+def test_prior_schema_requires_exact_empty_secret_introduction() -> None:
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ after = _without_creative(_without_model_updates(_without_selected_comparisons(fingerprint_session(session))))
+ before = _without_secrets(after)
+ contract = fingerprint_compatibility(before, after, source_schema_revision="20260908_0027")
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(before, after, contract)
+ for table in SECRET_TABLES:
+ altered = copy.deepcopy(after)
+ altered["tables"][table]["digest"] = "0" * 64
+ assert not fingerprint_compatibility(before, altered, source_schema_revision="20260908_0027").compatible
+ engine.dispose()
+
+
+def test_secret_migration_seed_empty_downgrade_and_history_refusal() -> None:
+ path = Path(__file__).resolve().parents[1] / "alembic/versions/20260908_0028_platform_secret_slots.py"
+ spec = importlib.util.spec_from_file_location("secret_migration", path)
+ assert spec and spec.loader
+ migration = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(migration)
+ engine = create_engine("sqlite://")
+ with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
+ migration.upgrade()
+ assert connection.scalar(select(SecretSlotState.revision)) == 0
+ migration.downgrade()
+ migration.upgrade()
+ connection.execute(SecretSlotVersion.__table__.insert().values(version=1, key_id="qualification",
+ secret_ciphertext="opaque", secret_nonce="nonce", created_at=datetime.now(UTC)))
+ with pytest.raises(RuntimeError, match="history exists"):
+ migration.downgrade()
+ engine.dispose()
diff --git a/backend/tests/test_secret_slots_postgres.py b/backend/tests/test_secret_slots_postgres.py
new file mode 100644
index 0000000..4bddcbc
--- /dev/null
+++ b/backend/tests/test_secret_slots_postgres.py
@@ -0,0 +1,195 @@
+# ruff: noqa: S106 - isolated synthetic persistence fixtures
+"""Managed PostgreSQL qualification for encrypted secret lifecycle and restore admission."""
+import base64
+import importlib
+import os
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from pydantic import SecretStr
+from sqlalchemy import select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.domain.secret_slots import SecretSlotSet, SecretSlotVersionRequest
+from modelforge_api.persistence.models import SecretSlotState
+from modelforge_api.services import secret_slots
+from modelforge_api.services.recovery_fingerprint import (
+ SECRET_TABLES,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+ EXTERNAL_ADMIN,
+ IdentityDatabase,
+ ManagedPostgresTools,
+ identity_database,
+)
+
+__all__ = ["identity_database"]
+PRIOR = "20260908_0027"
+CURRENT = TARGET_SCHEMA_REVISION
+REASON = "Isolated secret lifecycle qualification"
+
+
+def _settings() -> Settings:
+ return Settings(_env_file=None, env="test", secret_slot_root_key=SecretStr(base64.b64encode(b"q" * 32).decode()),
+ secret_slot_root_key_id="qualification", alembic_directory=Path(__file__).resolve().parents[1] / "alembic")
+
+
+def _service(session: Session) -> secret_slots.SecretSlotService:
+ return secret_slots.SecretSlotService(session, _settings(), EXTERNAL_ADMIN, "secret-pg-qualification")
+
+
+def _set(session: Session, revision: int = 0):
+ return _service(session).set(SecretSlotSet(expected_revision=revision, reason=REASON, value=SecretStr("hf_qualification_only")))
+
+
+def test_secret_upgrade_dump_restore_and_immutable_ciphertext_grants(identity_database: IdentityDatabase, tmp_path: Path) -> None:
+ database = identity_database
+ alembic = importlib.import_module("alembic.command")
+ alembic.upgrade(database.migration(), PRIOR)
+ with database.runtime.connect() as connection:
+ source = semantic_fingerprint(connection)
+ archive = tmp_path / "pre-secret.dump"
+ database.archive(archive)
+ alembic.upgrade(database.migration(), "head")
+ with database.runtime.connect() as connection:
+ assert connection.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
+ restored = semantic_fingerprint(connection)
+ contract = fingerprint_compatibility(source, restored, source_schema_revision=PRIOR)
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+ alembic.downgrade(database.migration(), PRIOR)
+ alembic.upgrade(database.migration(), "head")
+ with Session(database.runtime) as session:
+ _set(session)
+ with database.runtime.connect() as connection:
+ for sql in (
+ "UPDATE platform_secret_versions SET secret_ciphertext='changed'",
+ "UPDATE platform_secret_versions SET secret_nonce='changed'",
+ "UPDATE platform_secret_versions SET key_id='changed'",
+ "UPDATE platform_secret_versions SET version=2",
+ "UPDATE platform_secret_versions SET created_at=now()",
+ "DELETE FROM platform_secret_versions", "TRUNCATE platform_secret_versions CASCADE",
+ "DELETE FROM platform_secret_state", "ALTER TABLE platform_secret_versions ADD COLUMN forbidden integer",
+ ):
+ with pytest.raises(DBAPIError):
+ connection.execute(text(sql))
+ connection.rollback()
+ connection.execute(text("UPDATE platform_secret_versions SET revoked_at=now()"))
+ connection.commit()
+ with pytest.raises(RuntimeError, match="history exists"):
+ alembic.downgrade(database.migration(), PRIOR)
+ database.archive(archive, restore=True)
+ alembic.upgrade(database.migration(restored=True), "head")
+ with database.restored_runtime.connect() as connection:
+ restored = semantic_fingerprint(connection)
+ assert audit_fingerprint_differences_are_compatible(source, restored, fingerprint_compatibility(source, restored, source_schema_revision=PRIOR))
+
+
+def test_secret_concurrent_set_has_one_cas_winner(identity_database: IdentityDatabase) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ barrier = threading.Barrier(2)
+ def attempt() -> str:
+ with Session(database.runtime) as session:
+ barrier.wait(timeout=10)
+ try:
+ _set(session)
+ return "success"
+ except secret_slots.SecretSlotError as error:
+ return error.code
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(pool.map(lambda _: attempt(), range(2)))
+ assert results.count("success") == 1
+ assert len(set(results)) == 2
+ with database.runtime.connect() as connection:
+ assert connection.scalar(select(SecretSlotState.revision)) == 1
+ assert connection.scalar(text("SELECT count(*) FROM platform_secret_versions")) == 1
+
+
+def test_supported_restore_clears_secret_test_and_activation_until_retest(identity_database: IdentityDatabase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ from modelforge_api.services.recovery import RecoveryError, RecoveryService
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ monkeypatch.setattr(secret_slots, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
+ with Session(database.runtime) as session:
+ _set(session)
+ _service(session).test(SecretSlotVersionRequest(version=1, expected_revision=1, reason=REASON))
+ _service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=2, reason=REASON))
+ assert secret_slots.resolve_huggingface_token(session, _settings()) == "hf_qualification_only"
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "secret.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ with Session(database.runtime) as session:
+ recovery = RecoveryService(session, _settings())
+ recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+ record = SimpleNamespace(id=uuid.uuid4(), source_fingerprint=source)
+ plan = SimpleNamespace(database_destination=database.restored_owner.render_as_string(hide_password=False),
+ backup_set=SimpleNamespace(backup_id="secret-qualification", schema_revision=CURRENT, completed_at=datetime.now(UTC)), validation_requirements={})
+ with pytest.raises(RecoveryError) as refused:
+ recovery._validate_restored(record, plan)
+ assert refused.value.details["gate"]["current_truth_reconciled"] is False
+ session.rollback()
+ result = recovery._reconcile_restored(record, plan)
+ session.commit()
+ for table in ("platform_secret_tests", "platform_secret_activations"):
+ assert result["cleared_current_truth"][table] == 1
+ ready = recovery._validate_restored(record, plan)
+ assert ready["gate"]["current_truth_reconciled"] is True
+ session.commit()
+ with Session(database.restored_runtime) as session:
+ restored = semantic_fingerprint(session.connection())
+ for table in SECRET_TABLES:
+ assert restored["tables"][table] == source["tables"][table]
+ with pytest.raises(secret_slots.SecretSlotError):
+ secret_slots.resolve_huggingface_token(session, _settings())
+ session.rollback()
+ with pytest.raises(secret_slots.SecretSlotError) as refused_activation:
+ _service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=3, reason=REASON))
+ assert refused_activation.value.code == "SECRET_SLOT_TEST_REQUIRED"
+ _service(session).test(SecretSlotVersionRequest(version=1, expected_revision=3, reason=REASON))
+ _service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=4, reason=REASON))
+ assert secret_slots.resolve_huggingface_token(session, _settings()) == "hf_qualification_only"
+
+def test_secret_network_result_cannot_bless_concurrently_replaced_candidate(
+ identity_database: IdentityDatabase, monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), "head")
+ with Session(database.runtime) as session:
+ _set(session)
+ entered, resume = threading.Event(), threading.Event()
+ def provider(_token: str) -> tuple[bool, str]:
+ entered.set()
+ assert resume.wait(timeout=20)
+ return True, "HF_AUTHENTICATED"
+ monkeypatch.setattr(secret_slots, "test_huggingface_token", provider)
+ def attempt() -> str:
+ with Session(database.runtime) as session:
+ try:
+ _service(session).test(SecretSlotVersionRequest(version=1, expected_revision=1, reason=REASON))
+ return "success"
+ except secret_slots.SecretSlotError as error:
+ return error.code
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ future = pool.submit(attempt)
+ try:
+ assert entered.wait(timeout=20)
+ with Session(database.runtime) as session:
+ _set(session, 1)
+ finally:
+ resume.set()
+ assert future.result(timeout=20) != "success"
+ with database.runtime.connect() as connection:
+ assert connection.scalar(text("SELECT count(*) FROM platform_secret_tests WHERE passed")) == 0
+ assert connection.scalar(select(SecretSlotState.revision)) == 2
diff --git a/backend/tests/test_security_gate_rc_metadata.py b/backend/tests/test_security_gate_rc_metadata.py
new file mode 100644
index 0000000..68f37c9
--- /dev/null
+++ b/backend/tests/test_security_gate_rc_metadata.py
@@ -0,0 +1,32 @@
+"""Guard the release security gate's exact RC metadata and runbook URL rules."""
+
+from __future__ import annotations
+
+import runpy
+from pathlib import Path
+
+import pytest
+
+GATE = runpy.run_path(str(Path(__file__).resolve().parents[2] / "scripts" / "security_gate.py"))
+
+
+def test_first_party_rc_wheel_version_matches_only_the_same_release() -> None:
+ matches = GATE["_matches_first_party_version"]
+
+ assert matches("1.3.0-rc.1", "1.3.0-rc.1")
+ assert matches("1.3.0rc1", "1.3.0-rc.1")
+ assert not matches("1.3.0rc2", "1.3.0-rc.1")
+ assert not matches("1.3.1rc1", "1.3.0-rc.1")
+ assert not matches(None, "1.3.0-rc.1")
+
+
+def test_web_bundle_allows_only_repository_runbook_context() -> None:
+ validate = GATE["_validate_non_api_origin_contexts"]
+ input_bytes = (
+ b"https://git.example.com/example/modelforge/"
+ b"src/branch/master/docs/operations/RUNBOOK_NODE_OFFLINE.md"
+ )
+
+ validate(input_bytes)
+ with pytest.raises(GATE["GateInputError"]):
+ validate(b"https://git.example.com/api/v1/tokens")
diff --git a/backend/tests/test_selected_comparison_api.py b/backend/tests/test_selected_comparison_api.py
new file mode 100644
index 0000000..4c1aa52
--- /dev/null
+++ b/backend/tests/test_selected_comparison_api.py
@@ -0,0 +1,129 @@
+"""Exercise credential separation and secret-safe validation through real middleware."""
+
+from uuid import uuid4
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import text
+
+from modelforge_api.api.authorization import AccessBoundary, access_boundary_for_request
+from modelforge_api.db import get_session
+from modelforge_api.domain.serving import ServiceClientCreate
+from modelforge_api.main import app
+from tests.test_operator_authorization import (
+ ADMIN,
+ ALLOWED_ORIGIN,
+ MachineBoundaryHarness,
+ machine_boundary_harness,
+)
+from tests.test_selected_comparison_postgres import _valid_selection
+
+__all__ = ["machine_boundary_harness"]
+PREFIX = "/api/v1/selected-model-comparisons"
+
+
+@pytest.mark.parametrize("outcome", ["failed", "cancelled"])
+def test_authenticated_operator_and_designated_consumer_journey(
+ machine_boundary_harness: MachineBoundaryHarness, outcome: str,
+) -> None:
+ serving = machine_boundary_harness.capability_service
+ session = serving.session
+ session.execute(text("PRAGMA foreign_keys=ON"))
+ assert session.scalar(text("PRAGMA foreign_keys")) == 1
+ request, _unused_client = _valid_selection(session)
+ issued = serving.create_client(ServiceClientCreate(
+ name="comparison-http-consumer", allowed_capabilities=["rag.embedding@1"],
+ project_key="selected-comparison-fixture", project_binding_id=request.project_binding_id,
+ integration_environment="production", purpose="Offline authenticated comparison qualification",
+ ))
+ request = request.model_copy(update={"consumer_client_id": issued.id})
+ app.dependency_overrides[get_session] = lambda: session
+ operator = {**ADMIN, "Origin": ALLOWED_ORIGIN}
+ consumer = {"Authorization": f"Bearer {issued.credential}", "Origin": ALLOWED_ORIGIN}
+ client = TestClient(app)
+ created = client.post(PREFIX, headers=operator, json=request.model_dump(mode="json"))
+ assert created.status_code == 201, created.text
+ comparison_id = created.json()["id"]
+ path = f"{PREFIX}/{comparison_id}"
+ assert created.json()["status"] == "pending"
+ assigned = client.get(f"{PREFIX}/assigned", headers=consumer)
+ assert assigned.status_code == 200, assigned.text
+ assert [item["id"] for item in assigned.json()["items"]] == [comparison_id]
+ claimed = client.post(f"{path}/claim", headers=consumer)
+ assert claimed.status_code == 200, claimed.text
+ assert claimed.headers["Cache-Control"] == "no-store"
+ assert claimed.json()["comparison"]["status"] == "running"
+ assert len(claimed.json()["cases"]) == 1
+ token = claimed.json()["claim_token"]
+ progress = client.post(f"{path}/progress", headers=consumer, json={
+ "claim_token": token, "current_completed_cases": 1, "candidate_completed_cases": 0,
+ })
+ assert progress.status_code == 200, progress.text
+ assert progress.json()["current_completed_cases"] == 1
+ if outcome == "cancelled":
+ cancelled = client.post(f"{path}/cancel", headers=operator,
+ json={"expected_version": progress.json()["version"]})
+ assert cancelled.status_code == 200, cancelled.text
+ assert cancelled.json()["status"] == "cancel_requested"
+ finished = client.post(f"{path}/finish", headers=consumer, json={
+ "claim_token": token, "outcome": outcome,
+ "error_code": "FIXTURE_CONSUMER_FAILURE" if outcome == "failed" else None,
+ })
+ assert finished.status_code == 200, finished.text
+ assert finished.json()["status"] == outcome
+ assert finished.json()["production_promotion_performed"] is False
+ assert finished.json()["finished_at"] is not None
+ history = client.get(PREFIX, headers=operator)
+ assert history.status_code == 200
+ assert history.json()["items"][0]["status"] == outcome
+ detail = client.get(path, headers=operator)
+ assert detail.status_code == 200
+ assert detail.json()["consumer_client_id"] == str(issued.id)
+ assert token not in history.text and token not in detail.text
+ assert issued.credential not in history.text
+ assert client.get(f"{PREFIX}/assigned", headers=consumer).json()["items"] == []
+ assert session.execute(text("PRAGMA foreign_key_check")).all() == []
+
+
+@pytest.mark.parametrize("action", ["claim", "progress", "result", "finish"])
+def test_operator_key_cannot_become_comparison_consumer(
+ machine_boundary_harness: MachineBoundaryHarness, action: str,
+) -> None:
+ response = TestClient(app).post(f"{PREFIX}/{uuid4()}/{action}", headers={
+ **ADMIN, "Origin": ALLOWED_ORIGIN,
+ }, json={"claim_token": "private-test-claim-that-must-not-echo"})
+ assert response.status_code == 401
+ assert "private-test-claim" not in response.text
+
+
+@pytest.mark.parametrize("action", ["progress", "result", "finish"])
+def test_consumer_validation_does_not_echo_secret_or_extra_values(
+ machine_boundary_harness: MachineBoundaryHarness, action: str,
+) -> None:
+ canary = "private-claim-and-payload-canary-123456789"
+ response = TestClient(app).post(f"{PREFIX}/{uuid4()}/{action}", headers={
+ "Authorization": f"Bearer {machine_boundary_harness.capability_credential}",
+ "Origin": ALLOWED_ORIGIN,
+ }, json={"claim_token": canary, "unexpected": {canary: canary}})
+ assert response.status_code == 422
+ assert canary not in response.text
+ errors = response.json()["error"]["details"]["errors"]
+ assert errors and all(set(error) == {"type", "msg"} for error in errors)
+
+
+@pytest.mark.parametrize("path", [PREFIX, f"{PREFIX}/{uuid4()}/cancel"])
+def test_project_credential_cannot_plan_or_cancel_operator_comparison(
+ machine_boundary_harness: MachineBoundaryHarness, path: str,
+) -> None:
+ response = TestClient(app).post(path, headers={
+ "Authorization": f"Bearer {machine_boundary_harness.capability_credential}",
+ "Origin": ALLOWED_ORIGIN,
+ }, json={})
+ assert response.status_code == 401
+
+
+def test_only_exact_assigned_read_is_a_consumer_route() -> None:
+ assert access_boundary_for_request("GET", f"{PREFIX}/assigned") is AccessBoundary.CAPABILITY_CLIENT
+ assert access_boundary_for_request("GET", PREFIX) is AccessBoundary.CONTROL_PLANE
+ assert access_boundary_for_request("GET", f"{PREFIX}/{uuid4()}") is AccessBoundary.CONTROL_PLANE
+ assert access_boundary_for_request("POST", f"{PREFIX}/{uuid4()}/cancel") is AccessBoundary.CONTROL_PLANE
diff --git a/backend/tests/test_selected_comparison_postgres.py b/backend/tests/test_selected_comparison_postgres.py
new file mode 100644
index 0000000..c55ec6b
--- /dev/null
+++ b/backend/tests/test_selected_comparison_postgres.py
@@ -0,0 +1,261 @@
+"""Owner/runtime PostgreSQL qualification using a complete offline foreign-key graph."""
+from __future__ import annotations
+
+import importlib
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import create_engine, event, inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+ EvaluationCaseCreate,
+ EvaluationRevisionCreate,
+ EvaluationSuiteCreate,
+)
+from modelforge_api.domain.selected_model_comparison import ComparisonCreate
+from modelforge_api.domain.serving import CapabilityExperimentCreate
+from modelforge_api.persistence.models import (
+ AuditEvent,
+ Base,
+ CapabilityDeployment,
+ Project,
+ ProjectBinding,
+ SelectedModelComparison,
+ ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+ SELECTED_COMPARISON_RESTORE_ERROR,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+ semantic_fingerprint,
+)
+from modelforge_api.services.selected_model_comparison import (
+ SelectedModelComparisonService,
+ _digest,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+ IdentityDatabase,
+ ManagedPostgresTools,
+ identity_database,
+)
+from tests.test_serving import approve_and_promote, setup_m5
+
+__all__ = ["identity_database"]
+PRIOR = "20260909_0032"
+CURRENT = "20260911_0035"
+
+
+def _service(session: Session) -> SelectedModelComparisonService:
+ return SelectedModelComparisonService(session, AuditContext("operator", "comparison-pg-fixture"))
+
+
+def _valid_selection(session: Session) -> tuple[ComparisonCreate, UUID]:
+ # These helpers persist explicit offline runtime evidence; they never contact a provider or GPU.
+ serving, candidate, _node = setup_m5(session)
+ _approval, stable = approve_and_promote(serving, candidate.id)
+ probe = serving.repo.probe(candidate.runtime_probe_id)
+ assert probe is not None
+ experiment = serving.create_experiment(candidate.id, CapabilityExperimentCreate(
+ route_key="selected-comparison-fixture", execution_approval_id=probe.execution_approval_id,
+ purpose="Isolated PostgreSQL graph qualification; synthetic evidence only",
+ ))
+ stable_row = session.get(CapabilityDeployment, stable.id)
+ assert stable_row is not None
+ project = Project(key="selected-comparison-fixture", name="Comparison qualification", description="Offline fixture")
+ session.add(project)
+ session.flush()
+ binding = ProjectBinding(project_id=project.id, capability_contract_id=stable_row.capability_contract_id,
+ channel="stable", priority="production")
+ session.add(binding)
+ session.flush()
+ client = ServiceClient(name="selected-comparison-fixture", project_binding_id=binding.id,
+ allowed_capabilities=["rag.embedding@1"])
+ session.add(client)
+ session.commit()
+ suite = EvaluationService(session).create_suite(EvaluationSuiteCreate(
+ project_id=project.id, key="selected-comparison-fixture", name="Reviewed fixture",
+ description="Synthetic owner-reviewed case; no consumer certification claim",
+ revision=EvaluationRevisionCreate(revision="v1", dataset_revision="fixture-v1",
+ retrieval_settings={"top_k": 10}, cases=[EvaluationCaseCreate(
+ case_key="fixture-question", query="Where is the fixture answer?", relevant_chunk_ids=[uuid4()],
+ label_provenance={"source": "owner_review", "corpus_revision": "a" * 64},
+ critical=True, review_status="reviewed",
+ )]),
+ ))
+ assert suite.latest_revision_id is not None
+ return ComparisonCreate(project_binding_id=binding.id, consumer_client_id=client.id,
+ suite_revision_id=suite.latest_revision_id, current_deployment_id=stable.id,
+ candidate_deployment_id=experiment.capability_deployment_id, candidate_route_id=experiment.id,
+ corpus_sha256="a" * 64, retrieval_config_sha256=_digest({"top_k": 10}),
+ current_index_ref="fixture-current-exact", candidate_index_ref="fixture-candidate-exact",
+ idempotency_key="fixture-one"), client.id
+
+
+def test_offline_selection_fixture_has_complete_foreign_keys() -> None:
+ engine = create_engine("sqlite://")
+ @event.listens_for(engine, "connect")
+ def foreign_keys(connection, _record):
+ connection.execute("PRAGMA foreign_keys=ON")
+ try:
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ assert session.scalar(text("PRAGMA foreign_keys")) == 1
+ request, client_id = _valid_selection(session)
+ created = _service(session).create(request)
+ client = session.get(ServiceClient, client_id)
+ assert client is not None
+ assert _service(session).claim(created.id, client).comparison.status == "running"
+ assert session.execute(text("PRAGMA foreign_key_check")).all() == []
+ finally:
+ engine.dispose()
+
+
+def test_0033_upgrade_prior_dump_restore_and_runtime_permissions(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ database = identity_database
+ alembic = importlib.import_module("alembic.command")
+ alembic.upgrade(database.migration(), PRIOR)
+ with Session(database.runtime) as session:
+ assert not inspect(session.connection()).has_table("selected_model_comparisons")
+ AuditWriter(session, "operator", "prior-release").write("qualification.baseline", "comparison", None, {})
+ session.commit()
+ before = semantic_fingerprint(session.connection())
+ archive = tmp_path / "prior-comparisons.dump"
+ database.archive(archive)
+ alembic.upgrade(database.migration(), CURRENT)
+ with database.runtime.connect() as connection:
+ assert inspect(connection).has_table("selected_model_comparisons")
+ assert connection.scalar(text("SELECT count(*) FROM selected_model_comparisons")) == 0
+ after = semantic_fingerprint(connection)
+ contract = fingerprint_compatibility(before, after, source_schema_revision=PRIOR)
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(before, after, contract)
+ alembic.downgrade(database.migration(), PRIOR)
+ alembic.upgrade(database.migration(), CURRENT)
+ with Session(database.runtime) as session:
+ request, _client_id = _valid_selection(session)
+ _service(session).create(request)
+ constraints = inspect(session.connection()).get_foreign_keys("selected_model_comparisons")
+ assert {item["constrained_columns"][0] for item in constraints} >= {
+ "project_binding_id", "consumer_client_id", "suite_revision_id", "current_deployment_id",
+ "candidate_deployment_id", "candidate_route_id",
+ }
+ assert session.scalar(text("SELECT count(*) FROM pg_constraint WHERE conrelid='selected_model_comparisons'::regclass AND contype='f' AND NOT convalidated")) == 0
+ with database.runtime.connect() as connection:
+ for statement in (
+ "DELETE FROM selected_model_comparisons",
+ "TRUNCATE selected_model_comparisons CASCADE",
+ "ALTER TABLE selected_model_comparisons ADD COLUMN forbidden integer",
+ ):
+ with pytest.raises(DBAPIError):
+ connection.execute(text(statement))
+ connection.rollback()
+ with pytest.raises(RuntimeError, match="history exists"):
+ alembic.downgrade(database.migration(), PRIOR)
+ database.archive(archive, restore=True)
+ alembic.upgrade(database.migration(restored=True), CURRENT)
+ with database.restored_runtime.connect() as connection:
+ restored = semantic_fingerprint(connection)
+ contract = fingerprint_compatibility(before, restored, source_schema_revision=PRIOR)
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+
+
+def test_two_postgres_sessions_claim_exactly_once_with_one_audit_event(identity_database: IdentityDatabase) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT)
+ with Session(database.runtime) as session:
+ request, client_id = _valid_selection(session)
+ comparison_id = _service(session).create(request).id
+ barrier = threading.Barrier(2)
+ def contender(_number: int) -> str:
+ with Session(database.runtime) as session:
+ client = session.get(ServiceClient, client_id)
+ assert client is not None
+ session.commit()
+ barrier.wait(timeout=30)
+ try:
+ claimed = _service(session).claim(comparison_id, client)
+ assert claimed.comparison.status == "running"
+ return "claimed"
+ except EvaluationError as error:
+ assert error.status_code == 409
+ return "conflict"
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert sorted(pool.map(contender, [0, 1])) == ["claimed", "conflict"]
+ with Session(database.runtime) as session:
+ row = session.get(SelectedModelComparison, comparison_id)
+ assert row is not None and row.status == "running" and row.version == 2
+ assert row.claim_token_hash and row.claim_expires_at
+ events = list(session.scalars(select(AuditEvent).where(AuditEvent.action == "SELECTED_MODEL_COMPARISON_CLAIMED")))
+ assert len(events) == 1
+
+
+def test_populated_postgres_restore_interrupts_claim_and_rejects_resurrection(
+ identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+ database = identity_database
+ importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT)
+ with Session(database.runtime) as session:
+ request, client_id = _valid_selection(session)
+ comparison_id = _service(session).create(request).id
+ client = session.get(ServiceClient, client_id)
+ assert client is not None
+ claimed = _service(session).claim(comparison_id, client)
+ source = semantic_fingerprint(session.connection())
+ archive = tmp_path / "active-comparison.dump"
+ database.archive(archive)
+ database.archive(archive, restore=True)
+ # The real restore migrates the destination before reconciliation. Keep the
+ # backup at the selected-comparison revision, then exercise recovery at head.
+ importlib.import_module("alembic.command").upgrade(
+ database.migration(restored=True), "head"
+ )
+ owner = database.restored_owner
+ with Session(database.runtime) as session:
+ recovery = RecoveryService(session, Settings(_env_file=None, env="test",
+ alembic_directory=Path(__file__).resolve().parents[1] / "alembic"))
+ recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+ record = SimpleNamespace(id=uuid4(), source_fingerprint=source)
+ plan = SimpleNamespace(database_destination=owner.render_as_string(hide_password=False),
+ backup_set=SimpleNamespace(backup_id="selected-comparison-qualification", schema_revision=CURRENT,
+ completed_at=datetime.now(UTC)), validation_requirements={})
+ with pytest.raises(RecoveryError) as unready:
+ recovery._validate_restored(record, plan)
+ assert "gate" in unready.value.details, (unready.value.code, str(unready.value), unready.value.details)
+ assert unready.value.details["gate"]["current_truth_reconciled"] is False
+ session.rollback()
+ result = recovery._reconcile_restored(record, plan)
+ session.commit()
+ assert result["interrupted_selected_comparisons"] == 1
+ validated = recovery._validate_restored(record, plan)
+ assert all(validated["gate"].values())
+ session.commit()
+ assert recovery._reconcile_restored(record, plan)["reconciliation"] == "ALREADY_APPLIED"
+ with Session(database.restored_runtime) as session:
+ row = session.get(SelectedModelComparison, comparison_id)
+ assert row is not None and row.status == "interrupted"
+ assert row.version == claimed.comparison.version + 1
+ assert row.error_code == SELECTED_COMPARISON_RESTORE_ERROR
+ assert row.claim_token_hash is None and row.claim_expires_at is None
+ assert row.finished_at is None
+ restored = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(source, restored, source_schema_revision=CURRENT)
+ assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+ client = session.get(ServiceClient, client_id)
+ assert client is not None
+ with pytest.raises(EvaluationError):
+ _service(session).claim(comparison_id, client)
diff --git a/backend/tests/test_selected_comparison_recovery.py b/backend/tests/test_selected_comparison_recovery.py
new file mode 100644
index 0000000..280a1ea
--- /dev/null
+++ b/backend/tests/test_selected_comparison_recovery.py
@@ -0,0 +1,170 @@
+"""Comparison history remains exact while restored claims become unusable."""
+from __future__ import annotations
+
+import copy
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, SelectedModelComparison
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+ SELECTED_COMPARISON_RESTORE_ERROR,
+ SELECTED_COMPARISON_SCHEMA,
+ _without_lab_execution,
+ _without_model_changes,
+ _without_portfolio,
+ _without_selected_comparisons,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ semantic_fingerprint as _current_semantic_fingerprint,
+)
+from modelforge_api.services.recovery_postgres import PostgresTarget
+from modelforge_api.settings import Settings
+
+TARGET = PostgresTarget("localhost", 5432, "fixture", "fixture", None)
+
+
+def semantic_fingerprint(connection):
+ return _without_lab_execution(
+ _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
+ )
+
+
+class SQLiteRecoveryTarget:
+ """Execute the actual bounded reconciliation SQL; emulate only PostgreSQL table discovery."""
+ def __init__(self, session: Session):
+ self.session = session
+
+ def scalar(self, _target: PostgresTarget, sql: str) -> str:
+ if "information_schema.tables" in sql:
+ return "1"
+ result = self.session.execute(text(sql))
+ return str(result.scalar() or 0) if result.returns_rows else ""
+
+
+@pytest.fixture
+def fixture():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ AuditWriter(session, "operator", "test").write("fixture.created", "test", None, {})
+ session.commit()
+ service = RecoveryService(session, Settings(_env_file=None))
+ service.engine = SQLiteRecoveryTarget(session) # type: ignore[assignment]
+ yield session, service
+ engine.dispose()
+
+
+def comparison(session: Session, status: str) -> SelectedModelComparison:
+ moment = datetime(2026, 9, 9, 12, tzinfo=UTC)
+ item = SelectedModelComparison(
+ project_binding_id=uuid4(), consumer_client_id=uuid4(), suite_revision_id=uuid4(),
+ current_deployment_id=uuid4(), candidate_deployment_id=uuid4(), candidate_route_id=uuid4(),
+ idempotency_key=str(uuid4()), request_digest="a" * 64, actor_id="fixture-operator",
+ pins={"baseline": "a" * 40, "candidate": "b" * 40, "case_ids": ["one", "two"]},
+ status=status, version=4, expected_cases=2, current_completed_cases=1,
+ candidate_completed_cases=0, claim_token_hash="c" * 64,
+ claim_expires_at=moment + timedelta(minutes=5), claimed_at=moment,
+ deadline_at=moment + timedelta(minutes=10), cancel_requested_at=moment if status == "cancel_requested" else None,
+ finished_at=moment if status in {"completed", "failed", "cancelled", "interrupted"} else None,
+ baseline_run_id=uuid4(), candidate_run_id=uuid4(), evaluation_comparison_id=uuid4(),
+ result_digest="d" * 64, error_code="prior-evidence", created_at=moment, updated_at=moment,
+ )
+ session.add(item)
+ session.commit()
+ return item
+
+
+@pytest.mark.parametrize("status", ["pending", "running", "cancel_requested", "completed", "failed", "cancelled", "interrupted"])
+def test_restore_projects_only_active_state_and_never_replays_claims(fixture, status):
+ session, service = fixture
+ item = comparison(session, status)
+ before = semantic_fingerprint(session.connection())
+ previous = {name: getattr(item, name) for name in (
+ "pins", "current_completed_cases", "candidate_completed_cases", "baseline_run_id",
+ "candidate_run_id", "evaluation_comparison_id", "result_digest", "created_at", "updated_at",
+ "claimed_at", "finished_at", "deadline_at", "cancel_requested_at",
+ )}
+ active = status in {"pending", "running", "cancel_requested"}
+ assert service._selected_comparison_residuals(TARGET) == 1
+ assert service._reconcile_selected_comparisons(TARGET) == {"interrupted": int(active), "claims_cleared": 1}
+ session.expire_all()
+ assert item.status == ("interrupted" if active else status)
+ assert item.version == (5 if active else 4)
+ assert item.error_code == (SELECTED_COMPARISON_RESTORE_ERROR if active else "prior-evidence")
+ assert item.claim_token_hash is None and item.claim_expires_at is None
+ assert {name: getattr(item, name) for name in previous} == previous
+ restored = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+ assert service._selected_comparison_residuals(TARGET) == 0
+ assert service._reconcile_selected_comparisons(TARGET) == {"interrupted": 0, "claims_cleared": 0}
+ assert semantic_fingerprint(session.connection()) == restored
+
+
+def test_active_rows_cannot_pass_restore_proof_without_reconciliation(fixture):
+ session, _ = fixture
+ comparison(session, "running")
+ before = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(before, before, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+ assert contract.compatible
+ assert not audit_fingerprint_differences_are_compatible(before, before, contract)
+
+
+def test_claim_secrets_and_expiry_are_excluded_but_claim_history_is_not(fixture):
+ session, _ = fixture
+ item = comparison(session, "running")
+ before = semantic_fingerprint(session.connection())
+ entry = before["tables"]["selected_model_comparisons"]
+ assert entry["redacted_columns"] == ["claim_token_hash"]
+ session.execute(text("update selected_model_comparisons set claim_token_hash=:token,claim_expires_at=NULL"), {"token": "e" * 64})
+ assert semantic_fingerprint(session.connection()) == before
+ session.execute(text("update selected_model_comparisons set claimed_at=NULL"))
+ assert semantic_fingerprint(session.connection())["tables"]["selected_model_comparisons"]["digest"] != entry["digest"]
+ assert item.id is not None
+
+
+@pytest.mark.parametrize("mutation", ["pins", "result", "delete", "terminal_status", "progress"])
+def test_restore_rejects_changed_pins_results_missing_history_or_progress(fixture, mutation):
+ session, service = fixture
+ comparison(session, "completed")
+ before = semantic_fingerprint(session.connection())
+ service._reconcile_selected_comparisons(TARGET)
+ queries = {
+ "pins": "update selected_model_comparisons set pins='{}'",
+ "result": "update selected_model_comparisons set result_digest=NULL",
+ "delete": "delete from selected_model_comparisons",
+ "terminal_status": "update selected_model_comparisons set status='failed'",
+ "progress": "update selected_model_comparisons set current_completed_cases=0",
+ }
+ session.execute(text(queries[mutation]))
+ after = semantic_fingerprint(session.connection())
+ contract = fingerprint_compatibility(before, after, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+ assert not audit_fingerprint_differences_are_compatible(before, after, contract)
+
+
+def test_0032_upgrade_allows_only_empty_comparison_table_and_requires_current_history(fixture):
+ session, _ = fixture
+ empty = semantic_fingerprint(session.connection())
+ prior = _without_selected_comparisons(empty)
+ contract = fingerprint_compatibility(prior, empty, source_schema_revision="20260909_0032")
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(prior, empty, contract)
+ comparison(session, "completed")
+ populated = semantic_fingerprint(session.connection())
+ assert not fingerprint_compatibility(prior, populated, source_schema_revision="20260909_0032").compatible
+ missing = copy.deepcopy(populated)
+ missing["tables"].pop("selected_model_comparisons")
+ assert not fingerprint_compatibility(populated, missing, source_schema_revision=SELECTED_COMPARISON_SCHEMA).compatible
+ missing_projection = copy.deepcopy(populated)
+ missing_projection["tables"]["selected_model_comparisons"].pop("restore_projection")
+ contract = fingerprint_compatibility(populated, missing_projection, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
+ assert not audit_fingerprint_differences_are_compatible(populated, missing_projection, contract)
diff --git a/backend/tests/test_selected_model_comparison.py b/backend/tests/test_selected_model_comparison.py
new file mode 100644
index 0000000..730e40e
--- /dev/null
+++ b/backend/tests/test_selected_model_comparison.py
@@ -0,0 +1,739 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from unittest.mock import patch
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+ EvaluationCaseCreate,
+ EvaluationCaseResultCreate,
+ EvaluationRevisionCreate,
+ EvaluationRunComplete,
+ EvaluationSuiteCreate,
+ RankedResult,
+)
+from modelforge_api.domain.selected_model_comparison import (
+ ComparisonCancel,
+ ComparisonCreate,
+ ComparisonFinish,
+ ComparisonProgress,
+ ComparisonResult,
+)
+from modelforge_api.persistence.models import (
+ ArtifactSet,
+ AuditEvent,
+ Base,
+ Capability,
+ CapabilityContract,
+ CapabilityDeployment,
+ CapabilityExperimentRoute,
+ DeploymentCandidate,
+ EmbeddingSpace,
+ EvaluationCase,
+ EvaluationComparison,
+ EvaluationRun,
+ GatewayRequest,
+ Model,
+ ModelRevision,
+ Project,
+ ProjectBinding,
+ RuntimeProfile,
+ SelectedModelComparison,
+ ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from modelforge_api.services.selected_model_comparison import (
+ SelectedModelComparisonService,
+ _digest,
+)
+
+
+@pytest.fixture
+def setup():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+
+ def add(value):
+ session.add(value)
+ session.flush()
+ return value
+
+ project = add(Project(key="owner", name="Owner", description="fixture"))
+ capability = add(Capability(key="rag.embedding", description="Embeddings"))
+ contract = add(
+ CapabilityContract(
+ capability_id=capability.id,
+ version=1,
+ input_schema={},
+ output_schema={},
+ contract={},
+ upgrade_class="requires_reindex",
+ )
+ )
+ binding = add(
+ ProjectBinding(
+ project_id=project.id,
+ capability_contract_id=contract.id,
+ channel="stable",
+ priority="production",
+ )
+ )
+ client = add(
+ ServiceClient(
+ name="designated",
+ allowed_capabilities=["rag.embedding@1"],
+ project_binding_id=binding.id,
+ )
+ )
+ model = add(
+ Model(
+ key="chosen",
+ display_name="Chosen",
+ upstream_provider="huggingface",
+ upstream_source="org/chosen",
+ )
+ )
+ deployments = []
+ for index, variant in enumerate(("stable", "experiment", "experiment-current")):
+ channel = "stable" if index == 0 else "experiment"
+ revision = add(
+ ModelRevision(
+ model_id=model.id,
+ upstream_revision=variant,
+ resolved_commit_sha=str(index + 1) * 40,
+ )
+ )
+ artifact = add(
+ ArtifactSet(
+ revision_id=revision.id,
+ snapshot_id=uuid4(),
+ variant_key=variant,
+ label=variant,
+ selection_reason="fixture",
+ immutable_at=datetime.now(UTC),
+ )
+ )
+ profile = add(
+ RuntimeProfile(
+ name=variant,
+ artifact_set_id=artifact.id,
+ runtime_type="sentence_transformers",
+ runtime_version="fixture",
+ artifact_sha256=str(index + 1) * 64,
+ fingerprint=str(index + 3) * 64,
+ immutable_at=datetime.now(UTC),
+ )
+ )
+ candidate = add(
+ DeploymentCandidate(
+ artifact_set_id=artifact.id,
+ runtime_profile_id=profile.id,
+ compute_node_id=uuid4(),
+ compatibility_assessment_id=uuid4(),
+ runtime_probe_id=uuid4(),
+ )
+ )
+ space = add(
+ EmbeddingSpace(
+ capability_contract_id=contract.id,
+ artifact_set_id=artifact.id,
+ runtime_profile_id=profile.id,
+ identity_digest=str(index + 5) * 64,
+ dimension=4,
+ normalized=True,
+ identity_facts={},
+ )
+ )
+ deployments.append(
+ add(
+ CapabilityDeployment(
+ capability_contract_id=contract.id,
+ deployment_candidate_id=candidate.id,
+ embedding_space_id=space.id,
+ artifact_set_id=artifact.id,
+ runtime_profile_id=profile.id,
+ compute_node_id=candidate.compute_node_id,
+ accelerator_id=uuid4(),
+ channel=channel,
+ status="stable" if index == 0 else "candidate",
+ production=index == 0,
+ config_fingerprint=str(index + 7) * 64,
+ provenance={},
+ )
+ )
+ )
+ route = add(
+ CapabilityExperimentRoute(
+ route_key="selected-candidate",
+ capability_contract_id=contract.id,
+ capability_deployment_id=deployments[1].id,
+ purpose="fixture",
+ evidence={},
+ )
+ )
+ add(
+ CapabilityExperimentRoute(
+ route_key="selected-current-experiment",
+ capability_contract_id=contract.id,
+ capability_deployment_id=deployments[2].id,
+ purpose="fixture current experiment",
+ evidence={},
+ )
+ )
+ session.commit()
+ relevant = uuid4()
+ suite = EvaluationService(session).create_suite(
+ EvaluationSuiteCreate(
+ project_id=project.id,
+ key="reviewed",
+ name="Reviewed",
+ description="fixture",
+ revision=EvaluationRevisionCreate(
+ revision="v1",
+ dataset_revision="fixture-v1",
+ retrieval_settings={"top_k": 10},
+ cases=[
+ EvaluationCaseCreate(
+ case_key="one",
+ query="Where is the answer?",
+ relevant_chunk_ids=[relevant],
+ label_provenance={
+ "source": "owner_review",
+ "corpus_revision": "a" * 64,
+ },
+ critical=True,
+ review_status="reviewed",
+ )
+ ],
+ ),
+ )
+ )
+ service = SelectedModelComparisonService(session, AuditContext("operator", "fixture"))
+ request = ComparisonCreate(
+ project_binding_id=binding.id,
+ consumer_client_id=client.id,
+ suite_revision_id=suite.latest_revision_id,
+ current_deployment_id=deployments[0].id,
+ candidate_deployment_id=deployments[1].id,
+ candidate_route_id=route.id,
+ corpus_sha256="a" * 64,
+ retrieval_config_sha256=_digest({"top_k": 10}),
+ current_index_ref="current-exact",
+ candidate_index_ref="candidate-exact",
+ idempotency_key="one",
+ )
+ yield session, service, request, client, relevant
+ engine.dispose()
+
+
+def _claimed(setup, selected_request=None):
+ session, service, request, client, relevant = setup
+ if selected_request is not None:
+ request = selected_request
+ created = service.create(request)
+ claim = service.claim(created.id, client)
+ case = session.scalar(
+ select(EvaluationCase).where(EvaluationCase.suite_revision_id == request.suite_revision_id)
+ )
+ gateways = []
+ for deployment_id in (request.current_deployment_id, request.candidate_deployment_id):
+ gateway = GatewayRequest(
+ request_id=uuid4(),
+ service_client_id=client.id,
+ capability_key="rag.embedding",
+ capability_version=1,
+ capability_deployment_id=deployment_id,
+ status="completed",
+ priority="production",
+ input_sha256=_digest([case.query]),
+ input_count=1,
+ created_at=datetime.now(UTC),
+ finished_at=datetime.now(UTC),
+ decision_evidence={},
+ )
+ session.add(gateway)
+ gateways.append(gateway)
+ session.commit()
+ leg = EvaluationRunComplete(
+ results=[
+ EvaluationCaseResultCreate(
+ case_id=case.id,
+ ranked_results=[RankedResult(chunk_id=relevant, score=1)],
+ latency_ms=2,
+ )
+ ]
+ )
+ result = ComparisonResult(
+ claim_token=claim.claim_token,
+ pins_digest=claim.comparison.pins_digest,
+ consumer_revision_sha256="c" * 64,
+ current=leg,
+ candidate=leg,
+ current_gateway_requests={case.id: gateways[0].request_id},
+ candidate_gateway_requests={case.id: gateways[1].request_id},
+ )
+ return claim, result, gateways
+
+
+def test_real_pair_recomputes_metrics_and_retries_without_duplicate_evidence(setup):
+ session, service, request, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ assert (
+ claim.comparison.pins["current"]["dimension"]
+ == claim.comparison.pins["candidate"]["dimension"]
+ )
+ assert claim.comparison.pins["current"]["dimension"] > 0
+ completed = service.result(claim.comparison.id, result, client)
+ assert completed.status == "completed"
+ comparison = session.get(EvaluationComparison, completed.evaluation_comparison_id)
+ assert comparison.comparability == "comparable"
+ assert comparison.promotion_eligibility == "eligible"
+ assert completed.evaluation_summary.eligibility == "eligible"
+ assert completed.evaluation_summary.critical_regressions == 0
+ assert comparison.eligibility_evidence["promotion_performed"] is False
+ assert service.result(completed.id, result, client).id == completed.id
+ assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
+ assert session.scalar(select(func.count()).select_from(EvaluationComparison)) == 1
+ assert session.get(CapabilityDeployment, request.candidate_deployment_id).production is False
+ history = service.get(completed.id).model_dump_json()
+ assert claim.claim_token not in history and "claim_token_hash" not in history
+ assert all(
+ claim.claim_token not in str(event.details) for event in session.scalars(select(AuditEvent))
+ )
+
+
+@pytest.mark.parametrize(
+ "field,value", [("corpus_sha256", "d" * 64), ("current_index_ref", "candidate-exact")]
+)
+def test_rejects_mismatched_corpus_and_shared_vector_index(setup, field, value):
+ _, service, request, _, _ = setup
+ with pytest.raises(EvaluationError):
+ service.create(request.model_copy(update={field: value}))
+
+
+def test_idempotency_active_bound_and_exact_consumer(setup):
+ session, service, request, client, _ = setup
+ created = service.create(request)
+ legacy_digest = _digest(request.model_dump(mode="json", exclude={"current_route_id"}))
+ assert session.get(SelectedModelComparison, created.id).request_digest == legacy_digest
+ assert service.create(request).id == created.id
+ with pytest.raises(EvaluationError):
+ service.create(request.model_copy(update={"candidate_index_ref": "different"}))
+ with pytest.raises(EvaluationError):
+ service.create(request.model_copy(update={"idempotency_key": "second"}))
+ stranger = ServiceClient(
+ name="other",
+ project_binding_id=client.project_binding_id,
+ allowed_capabilities=["rag.embedding@1"],
+ )
+ session.add(stranger)
+ session.commit()
+ assert service.assigned(stranger).items == []
+ with pytest.raises(EvaluationError):
+ service.claim(created.id, stranger)
+ assert service.assigned(client).items[0].id == created.id
+
+
+def test_selected_experiment_pair_requires_both_exact_routes_and_is_idempotent(setup):
+ session, service, request, client, _ = setup
+ current_route = session.scalar(
+ select(CapabilityExperimentRoute).where(
+ CapabilityExperimentRoute.route_key == "selected-current-experiment"
+ )
+ )
+ selected = request.model_copy(
+ update={
+ "current_deployment_id": current_route.capability_deployment_id,
+ "current_route_id": current_route.id,
+ "idempotency_key": "two-experiments",
+ }
+ )
+
+ created = service.create(selected)
+ assert service.create(selected).id == created.id
+ with pytest.raises(EvaluationError, match="comparison idempotency conflict"):
+ service.create(selected.model_copy(update={"current_route_id": request.candidate_route_id}))
+ assert created.pins["current_route_id"] == str(current_route.id)
+ assert created.pins["current_route_key"] == current_route.route_key
+ assert created.pins["candidate_route_id"] == str(request.candidate_route_id)
+ identity_pins = {
+ "config_fingerprint",
+ "artifact_set_id",
+ "revision_id",
+ "upstream_commit_sha",
+ "runtime_profile_id",
+ "runtime_fingerprint",
+ "artifact_sha256",
+ "embedding_space_id",
+ "embedding_space_digest",
+ "dimension",
+ "compute_node_id",
+ "accelerator_id",
+ }
+ assert identity_pins <= set(created.pins["current"])
+ assert identity_pins <= set(created.pins["candidate"])
+ assert (
+ created.pins["current"]["embedding_space_id"]
+ != created.pins["candidate"]["embedding_space_id"]
+ )
+ claim, result, _ = _claimed(setup, selected)
+ completed = service.result(created.id, result, client)
+ assert claim.comparison.id == created.id
+ assert completed.status == "completed"
+ assert completed.pins["current_route_id"] == str(current_route.id)
+ current_route.status = "inactive"
+ session.commit()
+ assert service.result(created.id, result, client).id == completed.id
+
+
+@pytest.mark.parametrize(
+ "mutation",
+ ["production_current", "wrong_current_route", "inactive_current_route", "same_route"],
+)
+def test_selected_experiment_pair_rejects_non_experiment_or_mismatched_current_route(
+ setup, mutation
+):
+ session, service, request, _, _ = setup
+ current_route = session.scalar(
+ select(CapabilityExperimentRoute).where(
+ CapabilityExperimentRoute.route_key == "selected-current-experiment"
+ )
+ )
+ selected = request.model_copy(
+ update={
+ "current_deployment_id": current_route.capability_deployment_id,
+ "current_route_id": current_route.id,
+ "idempotency_key": f"invalid-{mutation}",
+ }
+ )
+ if mutation == "production_current":
+ selected = selected.model_copy(
+ update={"current_deployment_id": request.current_deployment_id}
+ )
+ elif mutation == "wrong_current_route":
+ selected = selected.model_copy(update={"current_route_id": request.candidate_route_id})
+ elif mutation == "inactive_current_route":
+ current_route.status = "inactive"
+ session.commit()
+ else:
+ selected = selected.model_copy(update={"candidate_route_id": current_route.id})
+
+ with pytest.raises(EvaluationError):
+ service.create(selected)
+ assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
+
+
+def test_current_experiment_route_key_is_revalidated_before_claim_and_result(setup):
+ session, service, request, client, _ = setup
+ current_route = session.scalar(
+ select(CapabilityExperimentRoute).where(
+ CapabilityExperimentRoute.route_key == "selected-current-experiment"
+ )
+ )
+ selected = request.model_copy(
+ update={
+ "current_deployment_id": current_route.capability_deployment_id,
+ "current_route_id": current_route.id,
+ "idempotency_key": "route-revalidation",
+ }
+ )
+ created = service.create(selected)
+ current_route.route_key = "changed-before-claim"
+ session.commit()
+ with pytest.raises(EvaluationError, match="comparison pins changed"):
+ service.claim(created.id, client)
+
+ current_route.route_key = "selected-current-experiment"
+ session.commit()
+ claim, result, _ = _claimed(setup, selected)
+ assert claim.comparison.id == created.id
+ current_route.route_key = "changed-before-result"
+ session.commit()
+ with pytest.raises(EvaluationError, match="comparison pins changed"):
+ service.result(created.id, result, client)
+
+
+@pytest.mark.parametrize("phase", ["claim", "result"])
+@pytest.mark.parametrize("mutation", ["inactive", "remapped"])
+def test_current_experiment_route_deactivation_or_remap_fails_closed(setup, phase, mutation):
+ session, service, request, client, _ = setup
+ current_route = session.scalar(
+ select(CapabilityExperimentRoute).where(
+ CapabilityExperimentRoute.route_key == "selected-current-experiment"
+ )
+ )
+ selected = request.model_copy(
+ update={
+ "current_deployment_id": current_route.capability_deployment_id,
+ "current_route_id": current_route.id,
+ "idempotency_key": f"route-{phase}-{mutation}",
+ }
+ )
+
+ if phase == "claim":
+ comparison = service.create(selected)
+ claim = None
+ result = None
+ else:
+ claim, result, _ = _claimed(setup, selected)
+ comparison = claim.comparison
+
+ if mutation == "inactive":
+ current_route.status = "inactive"
+ else:
+ current_route.capability_deployment_id = selected.candidate_deployment_id
+ session.commit()
+
+ with pytest.raises(EvaluationError):
+ if phase == "claim":
+ service.claim(comparison.id, client)
+ else:
+ assert claim is not None and result is not None
+ service.result(comparison.id, result, client)
+ assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_expiry_never_renews_or_reclaims(setup):
+ session, service, _, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ row = session.get(SelectedModelComparison, claim.comparison.id)
+ row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.result(row.id, result, client)
+ assert service.get(row.id).status == "interrupted"
+ assert row.claim_token_hash is None
+ with pytest.raises(EvaluationError):
+ service.claim(row.id, client)
+
+
+def test_cancel_is_request_until_consumer_acknowledges(setup):
+ _, service, _, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ response = service.cancel(
+ claim.comparison.id, ComparisonCancel(expected_version=claim.comparison.version)
+ )
+ assert response.status == "cancel_requested" and response.finished_at is None
+ with pytest.raises(EvaluationError):
+ service.result(response.id, result, client)
+ completed = service.finish(
+ response.id, ComparisonFinish(claim_token=claim.claim_token, outcome="cancelled"), client
+ )
+ assert completed.status == "cancelled"
+
+
+@pytest.mark.parametrize(
+ "mutation",
+ ["historical", "wrong_target", "wrong_client", "wrong_query", "not_completed", "duplicate"],
+)
+def test_gateway_provenance_rejects_unrelated_or_reused_requests(setup, mutation):
+ session, service, request, client, _ = setup
+ claim, result, gateways = _claimed(setup)
+ if mutation == "historical":
+ gateways[0].created_at = datetime.now(UTC) - timedelta(hours=1)
+ elif mutation == "wrong_target":
+ gateways[0].capability_deployment_id = request.candidate_deployment_id
+ elif mutation == "wrong_client":
+ gateways[0].service_client_id = uuid4()
+ elif mutation == "wrong_query":
+ gateways[0].input_sha256 = "d" * 64
+ elif mutation == "not_completed":
+ gateways[0].status = "failed"
+ else:
+ result.candidate_gateway_requests = result.current_gateway_requests
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.result(claim.comparison.id, result, client)
+ assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_result_failure_rolls_back_both_runs_and_keeps_claim(setup):
+ session, service, _, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ with (
+ patch.object(EvaluationService, "compare", side_effect=RuntimeError("test interruption")),
+ pytest.raises(RuntimeError),
+ ):
+ service.result(claim.comparison.id, result, client)
+ assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+ assert service.get(claim.comparison.id).status == "running"
+ assert service.result(claim.comparison.id, result, client).status == "completed"
+
+
+def test_progress_is_monotonic_and_claim_cannot_be_stolen(setup):
+ _, service, _, client, _ = setup
+ claim, _, _ = _claimed(setup)
+ with pytest.raises(EvaluationError):
+ service.claim(claim.comparison.id, client)
+ progress = ComparisonProgress(
+ claim_token=claim.claim_token, current_completed_cases=1, candidate_completed_cases=0
+ )
+ service.progress(claim.comparison.id, progress, client)
+ with pytest.raises(EvaluationError):
+ service.progress(
+ claim.comparison.id, progress.model_copy(update={"current_completed_cases": 0}), client
+ )
+ with pytest.raises(EvaluationError):
+ service.progress(
+ claim.comparison.id,
+ progress.model_copy(update={"candidate_completed_cases": 2}),
+ client,
+ )
+
+
+@pytest.mark.parametrize(
+ "mutation",
+ [
+ "wrong_project",
+ "unreviewed",
+ "missing_corpus",
+ "wrong_contract",
+ "same_target",
+ "inactive_route",
+ "space_mismatch",
+ ],
+)
+def test_create_rejects_incoherent_selected_project_provenance(setup, mutation):
+ session, service, request, _, _ = setup
+ if mutation == "wrong_project":
+ from modelforge_api.persistence.models import EvaluationSuite, EvaluationSuiteRevision
+
+ revision = session.get(EvaluationSuiteRevision, request.suite_revision_id)
+ session.get(EvaluationSuite, revision.suite_id).project_id = uuid4()
+ elif mutation in {"unreviewed", "missing_corpus"}:
+ case = session.scalar(select(EvaluationCase))
+ # Simulate imported corrupt provenance, bypassing the normal immutable ORM guard.
+ if mutation == "unreviewed":
+ session.execute(
+ update(EvaluationCase)
+ .where(EvaluationCase.id == case.id)
+ .values(review_status="draft")
+ )
+ else:
+ session.execute(
+ update(EvaluationCase)
+ .where(EvaluationCase.id == case.id)
+ .values(label_provenance={"source": "no reviewed corpus"})
+ )
+ elif mutation == "same_target":
+ request = request.model_copy(
+ update={"candidate_deployment_id": request.current_deployment_id}
+ )
+ elif mutation == "inactive_route":
+ session.get(CapabilityExperimentRoute, request.candidate_route_id).status = "inactive"
+ else:
+ deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
+ if mutation == "wrong_contract":
+ deployment.capability_contract_id = uuid4()
+ else:
+ session.execute(
+ update(EmbeddingSpace)
+ .where(EmbeddingSpace.id == deployment.embedding_space_id)
+ .values(runtime_profile_id=uuid4())
+ )
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.create(request)
+ assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
+
+
+def test_claim_and_result_revalidate_pins_and_client_revocation(setup):
+ session, service, request, client, _ = setup
+ created = service.create(request)
+ deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
+ original = deployment.config_fingerprint
+ deployment.config_fingerprint = "d" * 64
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.claim(created.id, client)
+ deployment.config_fingerprint = original
+ session.commit()
+ claim, result, _ = _claimed(setup)
+ client.disabled_at = datetime.now(UTC)
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.result(claim.comparison.id, result, client)
+ client.disabled_at = None
+ deployment.config_fingerprint = "d" * 64
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.result(claim.comparison.id, result, client)
+
+
+def test_failed_case_cannot_qualify_and_does_not_require_gateway_success(setup):
+ session, service, _, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ failed = result.current.results[0].model_copy(
+ update={"error_code": "UNAVAILABLE", "ranked_results": []}
+ )
+ result.current = EvaluationRunComplete(results=[failed])
+ result.current_gateway_requests = {}
+ completed = service.result(claim.comparison.id, result, client)
+ assert (
+ session.get(EvaluationComparison, completed.evaluation_comparison_id).promotion_eligibility
+ == "not_eligible"
+ )
+
+
+def test_missing_case_or_mapping_and_changed_retry_cannot_publish(setup):
+ session, service, _, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ original = result.current_gateway_requests
+ result.current_gateway_requests = {}
+ with pytest.raises(EvaluationError):
+ service.result(claim.comparison.id, result, client)
+ result.current_gateway_requests = original
+ completed = service.result(claim.comparison.id, result, client)
+ result.consumer_revision_sha256 = "d" * 64
+ with pytest.raises(EvaluationError):
+ service.result(completed.id, result, client)
+ assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
+
+
+@pytest.mark.parametrize("reader", ["list", "assigned", "create"])
+def test_expiry_is_reconciled_by_history_assignment_and_next_intent_without_early_commit(
+ setup, reader
+):
+ session, service, request, client, _ = setup
+ claim, _, _ = _claimed(setup)
+ row = session.get(SelectedModelComparison, claim.comparison.id)
+ row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ session.commit()
+ with patch.object(session, "commit", wraps=session.commit) as commits:
+ if reader == "list":
+ assert service.list().items[0].status == "interrupted"
+ elif reader == "assigned":
+ assert service.assigned(client).items[0].status == "interrupted"
+ else:
+ following = service.create(
+ request.model_copy(update={"idempotency_key": "explicit-new-intent"})
+ )
+ assert following.status == "pending" and following.id != row.id
+ assert commits.call_count == 1
+ assert session.get(SelectedModelComparison, row.id).status == "interrupted"
+
+
+def test_candidate_must_use_the_gateway_experiment_channel(setup):
+ session, service, request, _, _ = setup
+ candidate = session.get(CapabilityDeployment, request.candidate_deployment_id)
+ candidate.channel = "lab"
+ session.commit()
+ with pytest.raises(EvaluationError, match="comparison selection invalid"):
+ service.create(request)
+ assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
+
+
+def test_summary_fails_closed_when_linked_comparison_is_missing(setup):
+ session, service, _, client, _ = setup
+ claim, result, _ = _claimed(setup)
+ completed = service.result(claim.comparison.id, result, client)
+ session.get(SelectedModelComparison, completed.id).evaluation_comparison_id = uuid4()
+ session.commit()
+ with pytest.raises(EvaluationError):
+ service.get(completed.id)
diff --git a/backend/tests/test_selected_replacement.py b/backend/tests/test_selected_replacement.py
new file mode 100644
index 0000000..9878f5a
--- /dev/null
+++ b/backend/tests/test_selected_replacement.py
@@ -0,0 +1,145 @@
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import event, select
+
+from modelforge_api.persistence.models import (
+ CapabilityContract,
+ ProductionExecutionApproval,
+ Project,
+ ProjectBinding,
+ SelectedModelComparison,
+)
+from modelforge_api.services.lifecycle import LifecycleError
+from modelforge_api.services.selected_replacement import SelectedReplacementService
+from tests.test_lifecycle_serving import serving_pair
+
+__all__ = ["serving_pair"]
+
+
+def _preview(pair):
+ return SelectedReplacementService(pair[0]).preview(next(iter(pair[4].comparison_ids.values())))
+
+
+def test_preview_derives_exact_selection_without_writes_or_satisfied_claim(serving_pair):
+ session, _, current, candidate, selection, _ = serving_pair
+ statements = []
+
+ def observe(_conn, _cursor, statement, _parameters, _context, _many):
+ statements.append(statement.lstrip().split(None, 1)[0].upper())
+
+ event.listen(session.get_bind(), "before_cursor_execute", observe)
+ try:
+ result = _preview(serving_pair)
+ finally:
+ event.remove(session.get_bind(), "before_cursor_execute", observe)
+ assert result.selection == selection
+ assert result.route_generation == session.get(CapabilityContract, current.capability_contract_id).serving_generation
+ assert result.same_embedding_space and result.affected_bindings[0].eligible
+ assert result.production_approvals[0].id == selection.production_approval_id
+ assert [item.code for item in result.blockers] == ["PRODUCTION_VALIDATION_REVIEW_REQUIRED"]
+ assert result.production_promotion_performed is False
+ assert result.model_dump().get("production_validation") is None
+ assert not set(statements) & {"INSERT", "UPDATE", "DELETE", "TRUNCATE"}
+ assert current.status == "stable" and candidate.production is False
+ assert not session.new and not session.dirty and not session.deleted
+
+
+def test_preview_lists_other_affected_project_as_blocker(serving_pair):
+ session, _, current, _, _, _ = serving_pair
+ project = Project(key="unreviewed", name="Unreviewed", description="test")
+ session.add(project)
+ session.flush()
+ binding = ProjectBinding(project_id=project.id, capability_contract_id=current.capability_contract_id,
+ channel="stable", priority="production")
+ session.add(binding)
+ session.commit()
+ result = _preview(serving_pair)
+ assert len(result.affected_bindings) == 2
+ assert result.selection is None
+ assert any(item.code == "BINDING_COMPARISON_REQUIRED" and item.project_binding_id == binding.id for item in result.blockers)
+
+
+@pytest.mark.parametrize("mutation,expected", [
+ ("incomplete", "COMPARISON_NOT_COMPLETED"),
+ ("different-space", "SERVING_REINDEX_REQUIRED"),
+ ("revoked", "PRODUCTION_APPROVAL_REQUIRED"),
+ ("changed-route", "SERVING_SELECTION_INVALID"),
+])
+def test_preview_blocks_missing_execution_identity_or_authority(serving_pair, mutation, expected):
+ session, serving, current, candidate, selection, _ = serving_pair
+ if mutation == "incomplete":
+ session.get(SelectedModelComparison, next(iter(selection.comparison_ids.values()))).status = "interrupted"
+ elif mutation == "different-space":
+ candidate.embedding_space_id = None
+ elif mutation == "revoked":
+ serving.repo.approval(selection.production_approval_id).status = "revoked"
+ else:
+ current.status = "draining"
+ session.commit()
+ result = _preview(serving_pair)
+ assert result.selection is None
+ assert expected in {item.code for item in result.blockers}
+
+
+def test_missing_comparison_is_not_found(serving_pair):
+ with pytest.raises(LifecycleError) as error:
+ SelectedReplacementService(serving_pair[0]).preview(uuid4())
+ assert error.value.status_code == 404
+
+
+def test_latest_negative_project_fit_remains_visible_blocker(serving_pair):
+ from tests.test_lifecycle_serving import _fit
+
+ _fit(serving_pair, recommendation="KEEP_LAB", production_validation="DEFERRED_EXTERNAL_VALIDATION")
+ result = _preview(serving_pair)
+ assert result.selection is None
+ assert any(item.code == "SERVING_PROJECT_FIT_BLOCKED" for item in result.blockers)
+ assert serving_pair[0].scalar(select(CapabilityContract.serving_generation)) == result.route_generation
+
+
+def test_stale_historical_approval_does_not_hide_valid_choice(serving_pair):
+ session, _, _, _, selection, _ = serving_pair
+ valid = session.get(ProductionExecutionApproval, selection.production_approval_id)
+ stale = ProductionExecutionApproval(**{
+ column.name: getattr(valid, column.name)
+ for column in ProductionExecutionApproval.__table__.columns
+ if column.name not in {"id", "evidence_fingerprint"}
+ }, evidence_fingerprint="f" * 64)
+ session.add(stale)
+ session.commit()
+ result = _preview(serving_pair)
+ assert result.selection == selection
+ assert [item.id for item in result.production_approvals] == [valid.id]
+ assert [item.code for item in result.blockers] == ["PRODUCTION_VALIDATION_REVIEW_REQUIRED"]
+
+
+def test_actual_app_preview_requires_operator_auth_and_disables_cache(serving_pair, monkeypatch):
+ from fastapi.testclient import TestClient
+ from pydantic import SecretStr
+
+ from modelforge_api.db import get_session
+ from modelforge_api.main import app
+ from modelforge_api.settings import Settings, get_settings
+
+ token = "preview-offline-operator-test" # noqa: S105 - isolated test credential
+ configured = Settings(_env_file=None, operator_api_key=SecretStr(token))
+ previous = dict(app.dependency_overrides)
+ # SQLite fixture connections are thread-bound; qualify the real service above,
+ # then exercise middleware, dependencies and response serialization in TestClient.
+ result = _preview(serving_pair)
+ monkeypatch.setattr(SelectedReplacementService, "preview", lambda self, comparison_id: result)
+ app.dependency_overrides[get_session] = lambda: serving_pair[0]
+ app.dependency_overrides[get_settings] = lambda: configured
+ path = f"/api/v1/selected-model-comparisons/{next(iter(serving_pair[4].comparison_ids.values()))}/replacement-preview"
+ try:
+ client = TestClient(app)
+ assert client.get(path).status_code == 401
+ response = client.get(path, headers={"X-ModelForge-Admin-Token": token})
+ assert response.status_code == 200, response.text
+ assert response.headers["Cache-Control"] == "no-store"
+ assert response.json()["selection"]["candidate_deployment_id"] == str(serving_pair[3].id)
+ assert response.json()["production_promotion_performed"] is False
+ finally:
+ app.dependency_overrides.clear()
+ app.dependency_overrides.update(previous)
diff --git a/backend/tests/test_self_check.py b/backend/tests/test_self_check.py
new file mode 100644
index 0000000..bc53637
--- /dev/null
+++ b/backend/tests/test_self_check.py
@@ -0,0 +1,616 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+from sqlalchemy import create_engine, event, text
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.db import get_session
+from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
+from modelforge_api.domain.self_check import SelfCheckStatus
+from modelforge_api.main import app
+from modelforge_api.persistence.models import (
+ Accelerator,
+ Base,
+ ComputeNode,
+ ManagedOperatorCredential,
+ ManagedOperatorCredentialActivation,
+ ManagedOperatorIdentity,
+)
+from modelforge_api.services.manifest_registry import get_manifest_registry
+from modelforge_api.services.self_check import (
+ _configuration_check,
+ _credentials_check,
+ _database_check,
+ _manifest_check,
+ _node_check,
+ _redis_check,
+ _scheduler_check,
+ _storage_check,
+ run_self_check,
+)
+from modelforge_api.settings import Settings, get_settings
+
+OPERATOR_TOKEN = "self-check-test-operator-key-long-enough" # noqa: S105 - synthetic test credential
+DATABASE_SECRET = "postgresql+psycopg://runtime:db-do-not-leak@db:5432/modelforge" # noqa: S105
+REDIS_SECRET = "redis-do-not-leak" # noqa: S105 - synthetic test credential
+
+
+class FakeCandidates:
+ def __init__(self) -> None:
+ self.candidates = [object()]
+
+
+class FakePolicies:
+ def model_dump(self) -> dict[str, object]:
+ return {
+ "scheduling": object(),
+ "security": object(),
+ "promotion": object(),
+ "retention": object(),
+ }
+
+
+class FakeRegistry:
+ def capabilities(self) -> dict[str, object]:
+ return {"a": object()}
+
+ def projects(self) -> dict[str, object]:
+ return {"p": object()}
+
+ def candidates(self) -> FakeCandidates:
+ return FakeCandidates()
+
+ def benchmarks(self) -> dict[str, object]:
+ return {"b": object()}
+
+ def policies(self) -> FakePolicies:
+ return FakePolicies()
+
+
+def _settings(tmp_path: Path, *, env: str = "test") -> Settings:
+ artifact = tmp_path / "artifacts"
+ quarantine = tmp_path / "quarantine"
+ backup = tmp_path / "backups"
+ for path in (artifact, quarantine, backup):
+ path.mkdir(exist_ok=True)
+ return Settings(
+ _env_file=None,
+ env=env,
+ operator_api_key=SecretStr(OPERATOR_TOKEN),
+ database_url=DATABASE_SECRET,
+ redis_url=f"redis://cache:{REDIS_SECRET}@redis.internal:6379/0",
+ artifact_root=str(artifact),
+ quarantine_root=str(quarantine),
+ backup_root=backup,
+ )
+
+
+def _session() -> Session:
+ engine = create_engine("sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool)
+ Base.metadata.create_all(engine)
+ with engine.begin() as connection:
+ connection.execute(text("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"))
+ connection.execute(
+ text("INSERT INTO alembic_version(version_num) VALUES (:revision)"),
+ {"revision": TARGET_SCHEMA_REVISION},
+ )
+ return Session(engine)
+
+
+class RedisOk:
+ def ping(self) -> bool:
+ return True
+
+ def close(self) -> None:
+ return None
+
+
+class RedisDown:
+ def ping(self) -> bool:
+ raise ConnectionError("synthetic Redis outage")
+
+ def close(self) -> None:
+ return None
+
+
+def test_storage_probe_is_passive_and_reports_capacity(tmp_path: Path) -> None:
+ settings = _settings(tmp_path)
+ before = {
+ path: set(Path(path).iterdir())
+ for path in (settings.artifact_root, settings.quarantine_root, settings.backup_root)
+ }
+
+ result = _storage_check(settings)
+
+ assert result.status is SelfCheckStatus.PASS
+ assert all(
+ item["writability"] == "not_probed_read_only" for item in result.evidence.values()
+ )
+ assert all(item["free_bytes"] >= 0 for item in result.evidence.values())
+ after = {path: set(Path(path).iterdir()) for path in before}
+ assert after == before
+
+
+def test_storage_capacity_read_failure_remains_unknown(
+ tmp_path: Path, monkeypatch: Any
+) -> None:
+ settings = _settings(tmp_path)
+
+ def unavailable(_path: Path) -> object:
+ raise PermissionError("mount details must not be returned")
+
+ monkeypatch.setattr("modelforge_api.services.self_check.shutil.disk_usage", unavailable)
+ result = _storage_check(settings)
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert "mount details must not be returned" not in result.model_dump_json()
+
+
+def test_storage_stat_permission_failure_remains_unknown(tmp_path: Path, monkeypatch: Any) -> None:
+ settings = _settings(tmp_path)
+
+ class UnreadablePath:
+ def __init__(self, value: object) -> None:
+ self.value = value
+
+ def stat(self) -> object:
+ raise PermissionError("private storage path details must not be returned")
+
+ def __str__(self) -> str:
+ return str(self.value)
+
+ monkeypatch.setattr("modelforge_api.services.self_check.Path", UnreadablePath)
+ result = _storage_check(settings)
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert "private storage path details must not be returned" not in result.model_dump_json()
+
+
+def test_database_check_fails_closed_on_unknown_schema() -> None:
+ session = _session()
+ try:
+ session.execute(text("UPDATE alembic_version SET version_num='20991231_9999'"))
+ session.commit()
+ result = _database_check(session)
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.FAIL
+ assert result.evidence["compatibility"] == "TOO_NEW"
+
+
+def test_database_check_keeps_unreadable_state_unknown() -> None:
+ class UnavailableSession:
+ def execute(self, _statement: object) -> object:
+ raise ConnectionError("database URL must not be returned")
+
+ def rollback(self) -> None:
+ return None
+
+ result = _database_check(UnavailableSession()) # type: ignore[arg-type]
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert result.detail == "ConnectionError"
+ assert "database URL must not be returned" not in result.model_dump_json()
+
+
+def test_manifest_access_failure_is_unknown_but_missing_manifest_is_failure() -> None:
+ class UnreadableRegistry:
+ def candidates(self) -> object:
+ raise PermissionError("private manifest path details must not be returned")
+
+ class MissingRegistry:
+ def candidates(self) -> object:
+ raise FileNotFoundError("missing manifest")
+
+ unreadable = _manifest_check(UnreadableRegistry()) # type: ignore[arg-type]
+ missing = _manifest_check(MissingRegistry()) # type: ignore[arg-type]
+
+ assert unreadable.status is SelfCheckStatus.UNKNOWN
+ assert "private manifest path details must not be returned" not in unreadable.model_dump_json()
+ assert missing.status is SelfCheckStatus.FAIL
+
+
+def test_scheduler_inputs_remain_unknown_when_database_cannot_be_read() -> None:
+ class UnavailableSession:
+ def scalar(self, _statement: object) -> object:
+ raise ConnectionError("scheduler database details must not be returned")
+
+ def rollback(self) -> None:
+ return None
+
+ settings = Settings(_env_file=None)
+ result = _scheduler_check(
+ UnavailableSession(), settings, datetime.now(UTC) # type: ignore[arg-type]
+ )
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert result.detail == "ConnectionError"
+ assert "scheduler database details must not be returned" not in result.model_dump_json()
+
+
+def test_scheduler_inputs_remain_unknown_without_fresh_accelerator_telemetry(
+ tmp_path: Path,
+) -> None:
+ settings = _settings(tmp_path)
+ session = _session()
+ now = datetime(2026, 9, 6, tzinfo=UTC)
+ try:
+ node = ComputeNode(
+ key="scheduler-node",
+ hostname="scheduler-node",
+ display_name="Scheduler node",
+ identity_source="test",
+ )
+ session.add(node)
+ session.flush()
+ session.add(
+ Accelerator(
+ compute_node_id=node.id,
+ device_index=0,
+ device_uuid="GPU-scheduler-test",
+ name="Test accelerator",
+ total_vram_bytes=8_000_000_000,
+ status="active",
+ )
+ )
+ session.commit()
+
+ result = _scheduler_check(session, settings, now)
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert result.evidence["active_accelerators"] == 1
+ assert result.evidence["telemetry_missing"] == 1
+
+
+def test_redis_probe_is_typed_and_never_echoes_the_connection_url(
+ tmp_path: Path, monkeypatch: Any
+) -> None:
+ settings = _settings(tmp_path)
+ monkeypatch.setattr(
+ "modelforge_api.services.self_check.redis.Redis.from_url",
+ lambda *_args, **_kwargs: RedisDown(),
+ )
+
+ result = _redis_check(settings)
+
+ assert result.status is SelfCheckStatus.FAIL
+ assert REDIS_SECRET not in result.model_dump_json()
+ assert result.detail == "ConnectionError"
+
+
+def test_node_freshness_uses_existing_stale_and_offline_thresholds(tmp_path: Path) -> None:
+ settings = _settings(tmp_path)
+ session = _session()
+ now = datetime(2026, 9, 6, tzinfo=UTC)
+ try:
+ fresh = ComputeNode(
+ key="fresh",
+ hostname="fresh",
+ display_name="Fresh",
+ identity_source="test",
+ last_heartbeat_at=now
+ - timedelta(seconds=max(1, settings.node_stale_after_seconds // 2)),
+ )
+ stale = ComputeNode(
+ key="stale",
+ hostname="stale",
+ display_name="Stale",
+ identity_source="test",
+ last_heartbeat_at=now - timedelta(seconds=settings.node_stale_after_seconds + 1),
+ )
+ session.add_all([fresh, stale])
+ session.commit()
+
+ result = _node_check(session, settings, now)
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.WARN
+ assert result.evidence["stale_nodes"] == ["Stale"]
+ assert result.evidence["offline_nodes"] == []
+
+
+def test_node_without_any_heartbeat_evidence_is_unknown(tmp_path: Path) -> None:
+ settings = _settings(tmp_path)
+ session = _session()
+ try:
+ session.add(
+ ComputeNode(
+ key="unobserved",
+ hostname="unobserved",
+ display_name="Unobserved",
+ identity_source="test",
+ )
+ )
+ session.commit()
+ result = _node_check(session, settings, datetime(2026, 9, 6, tzinfo=UTC))
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert result.evidence["heartbeat_unknown"] == ["Unobserved"]
+
+
+def test_configuration_check_reports_missing_names_not_secret_values(tmp_path: Path) -> None:
+ settings = _settings(tmp_path, env="production")
+
+ result = _configuration_check(settings)
+
+ assert result.status is SelfCheckStatus.FAIL
+ serialized = result.model_dump_json()
+ assert OPERATOR_TOKEN not in serialized
+ assert DATABASE_SECRET not in serialized
+ assert REDIS_SECRET not in serialized
+ assert "missing_settings" in result.evidence
+
+
+def test_credentials_check_reports_only_safe_metadata(tmp_path: Path) -> None:
+ settings = _settings(tmp_path)
+ settings.operator_api_key = None
+ session = _session()
+ now = datetime(2026, 9, 6, tzinfo=UTC)
+ credential_id = uuid.uuid4()
+ try:
+ session.add(
+ ManagedOperatorIdentity(
+ slug="admin",
+ role="admin",
+ enabled=True,
+ version=1,
+ created_at=now,
+ )
+ )
+ session.add(
+ ManagedOperatorCredential(
+ id=credential_id,
+ identity_slug="admin",
+ token_sha256="a" * 64,
+ expires_at=now + timedelta(days=30),
+ created_at=now,
+ )
+ )
+ session.add(
+ ManagedOperatorCredentialActivation(
+ credential_id=credential_id,
+ activated_at=now,
+ )
+ )
+ session.commit()
+ result = _credentials_check(session, settings, now)
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.PASS
+ assert result.evidence["managed_usable_admin_credentials"] == 1
+ assert result.evidence["administrator_metadata_present"] is True
+ serialized = result.model_dump_json()
+ assert "a" * 64 not in serialized
+ assert "token_sha256" not in serialized
+ assert OPERATOR_TOKEN not in serialized
+ assert DATABASE_SECRET not in serialized
+
+
+def test_credentials_check_warns_for_expiring_managed_metadata(tmp_path: Path) -> None:
+ settings = _settings(tmp_path)
+ session = _session()
+ now = datetime(2026, 9, 6, tzinfo=UTC)
+ try:
+ session.add(
+ ManagedOperatorIdentity(
+ slug="admin",
+ role="admin",
+ enabled=True,
+ version=1,
+ created_at=now,
+ )
+ )
+ session.add(
+ ManagedOperatorCredential(
+ id=uuid.uuid4(),
+ identity_slug="admin",
+ token_sha256="b" * 64,
+ expires_at=now + timedelta(days=3),
+ created_at=now,
+ )
+ )
+ session.commit()
+ result = _credentials_check(session, settings, now)
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.WARN
+ assert result.evidence["managed_expiring_within_7_days"] == 1
+ assert "b" * 64 not in result.model_dump_json()
+
+
+def test_credential_metadata_read_failure_remains_unknown(tmp_path: Path) -> None:
+ class UnavailableSession:
+ def execute(self, _statement: object) -> object:
+ raise ConnectionError("credential database details must not be returned")
+
+ def rollback(self) -> None:
+ return None
+
+ settings = _settings(tmp_path)
+ result = _credentials_check(
+ UnavailableSession(), settings, datetime.now(UTC) # type: ignore[arg-type]
+ )
+
+ assert result.status is SelfCheckStatus.UNKNOWN
+ assert result.detail == "ConnectionError"
+ serialized = result.model_dump_json()
+ assert "credential database details must not be returned" not in serialized
+ assert OPERATOR_TOKEN not in serialized
+ assert DATABASE_SECRET not in serialized
+
+
+def test_credential_metadata_without_an_admin_source_fails_closed(tmp_path: Path) -> None:
+ settings = _settings(tmp_path)
+ settings.operator_api_key = None
+ viewer_token = "mfop_" + "v" * 43
+ operator_token = "mfop_" + "o" * 43
+ settings.operator_identities = SecretStr(
+ json.dumps(
+ [
+ {
+ "credential_id": role,
+ "subject": role,
+ "role": role,
+ "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(days=30)).isoformat(),
+ }
+ for role, token in (
+ ("viewer", viewer_token),
+ ("operator", operator_token),
+ )
+ ]
+ )
+ )
+ session = _session()
+ try:
+ result = _credentials_check(session, settings, datetime.now(UTC))
+ finally:
+ session.close()
+
+ assert result.status is SelfCheckStatus.FAIL
+ assert result.evidence["administrator_metadata_present"] is False
+ serialized = result.model_dump_json()
+ assert viewer_token not in serialized
+ assert operator_token not in serialized
+
+
+def test_self_check_endpoint_is_admin_authenticated_read_only_and_secret_safe(
+ tmp_path: Path, monkeypatch: Any
+) -> None:
+ settings = _settings(tmp_path)
+ session = _session()
+ monkeypatch.setattr(
+ "modelforge_api.services.self_check.redis.Redis.from_url",
+ lambda *_args, **_kwargs: RedisOk(),
+ )
+ app.dependency_overrides[get_settings] = lambda: settings
+ app.dependency_overrides[get_session] = lambda: session
+ app.dependency_overrides[get_manifest_registry] = lambda: FakeRegistry()
+ engine = session.get_bind()
+ mutations: list[str] = []
+
+ def reject_mutation(
+ _connection: Any,
+ _cursor: Any,
+ statement: str,
+ _parameters: Any,
+ _context: Any,
+ _executemany: bool,
+ ) -> None:
+ operation = statement.lstrip().split(maxsplit=1)[0].upper()
+ if operation in {"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "DROP", "ALTER"}:
+ mutations.append(operation)
+
+ event.listen(engine, "before_cursor_execute", reject_mutation)
+ client = TestClient(app)
+ try:
+ anonymous = client.get("/api/v1/admin/operations/self-check")
+ assert anonymous.status_code == 401
+
+ response = client.get(
+ "/api/v1/admin/operations/self-check",
+ headers={"X-ModelForge-Admin-Token": OPERATOR_TOKEN},
+ )
+ assert response.status_code == 200
+ payload = response.json()
+ assert payload["read_only"] is True
+ assert payload["overall"] == "degraded"
+ assert {check["key"] for check in payload["checks"]} == {
+ "api",
+ "database",
+ "redis",
+ "manifests",
+ "storage",
+ "nodes",
+ "scheduler",
+ "deployments",
+ "backup",
+ "credentials",
+ "configuration",
+ "release",
+ }
+ assert all(isinstance(check["next_action"], dict) for check in payload["checks"])
+ assert all(check["next_action"]["code"] for check in payload["checks"])
+ assert (
+ next(check for check in payload["checks"] if check["key"] == "scheduler")["status"]
+ == "unknown"
+ )
+ api_check = next(check for check in payload["checks"] if check["key"] == "api")
+ assert api_check["status"] == "pass"
+ assert api_check["next_action"]["code"] == "none_required"
+ serialized = response.text
+ assert OPERATOR_TOKEN not in serialized
+ assert DATABASE_SECRET not in serialized
+ assert REDIS_SECRET not in serialized
+
+ settings.operator_identities = SecretStr(
+ json.dumps(
+ [
+ {
+ "credential_id": f"self-check-{role}",
+ "subject": f"self-check-{role}",
+ "role": role,
+ "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(days=30)).isoformat(),
+ }
+ for role, token in (
+ ("viewer", "mfop_" + "v" * 43),
+ ("operator", "mfop_" + "o" * 43),
+ )
+ ]
+ )
+ )
+ for token in ("mfop_" + "v" * 43, "mfop_" + "o" * 43):
+ denied = client.get(
+ "/api/v1/admin/operations/self-check",
+ headers={"X-ModelForge-Admin-Token": token},
+ )
+ assert denied.status_code == 403
+ for token in ("mfsvc_project-scoped-token", "mfnode_node-agent-token"):
+ denied = client.get(
+ "/api/v1/admin/operations/self-check",
+ headers={"X-ModelForge-Admin-Token": token},
+ )
+ assert denied.status_code == 401
+ finally:
+ client.close()
+ app.dependency_overrides.clear()
+ event.remove(engine, "before_cursor_execute", reject_mutation)
+ session.close()
+ assert mutations == []
+
+
+def test_aggregate_becomes_unhealthy_when_a_required_dependency_fails(
+ tmp_path: Path, monkeypatch: Any
+) -> None:
+ settings = _settings(tmp_path)
+ session = _session()
+ monkeypatch.setattr(
+ "modelforge_api.services.self_check.redis.Redis.from_url",
+ lambda *_args, **_kwargs: RedisDown(),
+ )
+ try:
+ report = run_self_check(session, settings, FakeRegistry()) # type: ignore[arg-type]
+ finally:
+ session.close()
+
+ assert report.overall == "unhealthy"
+ redis_check = next(check for check in report.checks if check.key == "redis")
+ assert redis_check.status is SelfCheckStatus.FAIL
diff --git a/backend/tests/test_serving.py b/backend/tests/test_serving.py
index 81fdb5d..2d4a8e8 100644
--- a/backend/tests/test_serving.py
+++ b/backend/tests/test_serving.py
@@ -9,7 +9,7 @@ from unittest.mock import Mock
import pytest
from fastapi.testclient import TestClient
-from sqlalchemy import create_engine, select, update
+from sqlalchemy import create_engine, event, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from sqlalchemy.pool import StaticPool
@@ -19,6 +19,7 @@ from modelforge_api.domain.runtime import AgentRuntimeProbeComplete
from modelforge_api.domain.serving import (
AgentResidentState,
AgentServingJobComplete,
+ AgentServingJobFailure,
AgentServingStateReport,
CapabilityExperimentCreate,
CapabilityPromotionCreate,
@@ -41,6 +42,7 @@ from modelforge_api.persistence.models import (
ComputeNode,
EmbeddingSpace,
GatewayRequest,
+ SchedulerAcceleratorState,
ServiceCredential,
ServingGpuLease,
ServingJob,
@@ -543,7 +545,111 @@ def test_scheduler_accounts_external_usage_reserve_and_workstation_rejection(
assert "NODE_NOT_PRODUCTION_ELIGIBLE" in rejected["reasons"]
+@pytest.mark.parametrize(
+ "pressure_boundary",
+ [False, True],
+ ids=["first-read", "recovery-stable-window"],
+)
+def test_scheduler_get_projects_expiry_and_pressure_without_dml_or_commit(
+ session: Session,
+ pressure_boundary: bool,
+) -> None:
+ base, candidate, _node = setup_m5(session)
+ _approval, deployment = approve_and_promote(base, candidate.id)
+ baseline = base.scheduler_overview(commit=False, update_pressure=False)[0]
+
+ now = datetime.now(UTC)
+ since = now - timedelta(seconds=base.settings.scheduler_pressure_stable_seconds + 1)
+ pressure = (
+ SchedulerAcceleratorState(
+ accelerator_id=deployment.accelerator_id,
+ pressure_state="HIGH",
+ recovery_candidate="NORMAL",
+ recovery_candidate_since=since,
+ pressure_changed_at=since - timedelta(minutes=1),
+ last_observed_at=since,
+ )
+ if pressure_boundary
+ else None
+ )
+ expired_lease = ServingGpuLease(
+ accelerator_id=deployment.accelerator_id,
+ capability_deployment_id=deployment.id,
+ request_id=uuid.uuid4(),
+ reserved_vram_bytes=4 * 1024**3,
+ priority="production",
+ state="active",
+ owner="scheduler-read-only-test",
+ expires_at=now - timedelta(milliseconds=1),
+ )
+ if pressure is not None:
+ session.add(pressure)
+ session.add(expired_lease)
+ session.commit()
+
+ dml_statements: list[str] = []
+ commits: list[bool] = []
+
+ def record_statement(_conn, _cursor, statement, _parameters, _context, _many) -> None:
+ operation = statement.lstrip().split(maxsplit=1)[0].upper()
+ if operation in {"INSERT", "UPDATE", "DELETE", "REPLACE", "MERGE", "TRUNCATE"}:
+ dml_statements.append(statement)
+
+ def record_commit(_session: Session) -> None:
+ commits.append(True)
+
+ engine = session.get_bind()
+ event.listen(engine, "before_cursor_execute", record_statement)
+ event.listen(session, "after_commit", record_commit)
+ request_settings = Settings(
+ _env_file=None,
+ operator_api_key="scheduler-read-only-test-key",
+ )
+ app.dependency_overrides[get_settings] = lambda: request_settings
+ app.dependency_overrides[get_serving_service] = lambda: base
+ try:
+ response = TestClient(app).get(
+ "/api/v1/scheduler",
+ headers={"X-ModelForge-Admin-Token": "scheduler-read-only-test-key"},
+ )
+ finally:
+ app.dependency_overrides.clear()
+ event.remove(engine, "before_cursor_execute", record_statement)
+ event.remove(session, "after_commit", record_commit)
+
+ assert response.status_code == 200, response.text
+ budget = response.json()[0]
+ assert budget["pressure_state"] == "NORMAL"
+ assert budget["leased_vram_bytes"] == baseline.leased_vram_bytes
+ assert dml_statements == []
+ assert commits == []
+
+ persisted_lease = session.get(ServingGpuLease, expired_lease.id)
+ pressure_query = select(SchedulerAcceleratorState).where(
+ SchedulerAcceleratorState.accelerator_id == deployment.accelerator_id
+ )
+ persisted_pressure = session.scalar(pressure_query)
+ if pressure_boundary:
+ assert persisted_pressure is not None
+ assert persisted_pressure.pressure_state == "HIGH"
+ assert persisted_pressure.recovery_candidate == "NORMAL"
+ assert persisted_pressure.recovery_candidate_since is not None
+ assert persisted_pressure.recovery_candidate_since.replace(tzinfo=UTC) == since
+ else:
+ assert persisted_pressure is None
+ assert persisted_lease is not None and persisted_lease.state == "active"
+
+ # The default service path still stabilizes and commits pressure for write/admission callers.
+ base.scheduler_overview()
+ persisted_pressure = session.scalar(pressure_query)
+ assert persisted_pressure is not None
+ assert persisted_pressure.pressure_state == "NORMAL"
+ assert persisted_pressure.recovery_candidate is None
+
+
class InlineWorkerServingService(ServingService):
+ synthetic_worker_metrics: dict[str, object]
+
def _wait_job(self, job_id: uuid.UUID, deadline: float):
del deadline
job = self.repo.job(job_id)
@@ -577,7 +683,10 @@ class InlineWorkerServingService(ServingService):
"normalized": True,
"prompt_tokens": len(lease.input) * 4,
}
- metrics = {"inference_time_ms": 25.0}
+ metrics = {
+ "inference_time_ms": 25.0,
+ **getattr(self, "synthetic_worker_metrics", {}),
+ }
else:
assert lease.rerank_query is not None
assert lease.rerank_documents is not None
@@ -667,6 +776,67 @@ def test_real_gateway_path_uses_scheduler_cold_load_then_warm_reuse(session: Ses
assert history[0].latency_breakdown["worker_total"] >= 0
+def test_runtime_failure_fails_gateway_and_releases_request_resources(session: Session) -> None:
+ base, candidate, _node = setup_m5(session)
+ _approval, deployment = approve_and_promote(base, candidate.id)
+
+ class FailingRuntimeService(InlineWorkerServingService):
+ def _wait_job(self, job_id: uuid.UUID, deadline: float):
+ job = self.repo.job(job_id)
+ assert job is not None
+ if job.operation != "invoke":
+ return super()._wait_job(job_id, deadline)
+ node = self.repo.node(job.compute_node_id)
+ assert node is not None
+ lease = self.claim_next(node)
+ assert lease is not None and lease.job_id == job_id
+ self.fail_job(
+ job_id,
+ node,
+ AgentServingJobFailure(
+ lease_token=lease.lease_token,
+ worker_instance_id="failed-runtime-worker",
+ failure_code="RUNTIME_UNAVAILABLE",
+ failure_message="synthetic worker transport failure",
+ ),
+ )
+ return ServingService._wait_job(self, job_id, deadline)
+
+ service = FailingRuntimeService(session, base.settings, base.manifests, base.payload_store)
+ created = service.create_client(
+ ServiceClientCreate(name="runtime-failure-client", allowed_capabilities=["rag.embedding@1"])
+ )
+ app.dependency_overrides[get_settings] = lambda: service.settings
+ app.dependency_overrides[get_serving_service] = lambda: service
+ try:
+ response = TestClient(app).post(
+ "/api/v1/capabilities/rag.embedding@1/invoke",
+ headers={"Authorization": f"Bearer {created.credential}"},
+ json={"input": "must not be inferred elsewhere"},
+ )
+ finally:
+ app.dependency_overrides.clear()
+
+ assert response.status_code == 503
+ assert response.json()["error"]["code"] == "RUNTIME_UNAVAILABLE"
+ requests = list(session.scalars(select(GatewayRequest)))
+ assert len(requests) == 1
+ assert requests[0].status == "failed"
+ assert requests[0].failure_code == "RUNTIME_UNAVAILABLE"
+ assert requests[0].capability_deployment_id == deployment.id
+ jobs = list(session.scalars(select(ServingJob)))
+ assert [job.operation for job in jobs] == ["load", "invoke"]
+ assert jobs[-1].status == "failed" and jobs[-1].failure_code == "RUNTIME_UNAVAILABLE"
+ assert jobs[-1].payload_reference is None
+ assert service.payload_store.values == {}
+ leases = list(session.scalars(select(ServingGpuLease)))
+ assert len(leases) == 1
+ assert leases[0].state == "failed" and leases[0].released_at is not None
+ residency = service.repo.residency(deployment.id)
+ assert residency is not None and residency.active_requests == 0
+ assert residency.state == "cold"
+
+
def test_lab_experiment_has_opaque_route_own_space_and_no_production_cutover(
session: Session,
) -> None:
diff --git a/backend/tests/test_serving_external_budget.py b/backend/tests/test_serving_external_budget.py
new file mode 100644
index 0000000..ab32aff
--- /dev/null
+++ b/backend/tests/test_serving_external_budget.py
@@ -0,0 +1,89 @@
+"""Native admission honors external reservations until their explicit release."""
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+ Base,
+ ExternalCreativeAllocation,
+ ProjectBinding,
+ ServiceClient,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingService
+from modelforge_api.settings import Settings
+from tests.test_selected_comparison_postgres import _valid_selection
+from tests.test_serving import MemoryPayloadStore
+
+
+@pytest.fixture
+def budget_fixture():
+ engine = create_engine("sqlite://")
+ event.listen(engine, "connect", lambda connection, _: connection.execute("PRAGMA foreign_keys=ON"))
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ selection, client_id = _valid_selection(session)
+ service = ServingService(session, Settings(_env_file=None, env="test"),
+ ManifestRegistry(), MemoryPayloadStore())
+ deployment = service.repo.deployment(selection.current_deployment_id)
+ client = session.get(ServiceClient, client_id)
+ assert deployment is not None and client is not None
+ binding = session.get(ProjectBinding, client.project_binding_id)
+ assert binding is not None
+ yield session, service, deployment, (client.id, binding.project_id)
+ engine.dispose()
+
+
+def allocation(deployment, client, *, state, required):
+ return ExternalCreativeAllocation(
+ service_client_id=client[0], project_id=client[1],
+ accelerator_id=deployment.accelerator_id, request_id=str(uuid4()),
+ idempotency_key=str(uuid4()), request_sha256="a" * 64, workload="image",
+ resource_ref="fixture", provider_resource_id="fixture", required_vram_bytes=required,
+ priority="background", ttl_seconds=30, state=state, decision_code="FIXTURE",
+ decision_evidence={}, required_actions=[],
+ expires_at=datetime.now(UTC) - timedelta(seconds=1),
+ )
+
+
+@pytest.mark.parametrize(("state", "reserved"), [
+ ("granted", True), ("uncertain", True), ("queued", False), ("released", False),
+])
+def test_actual_native_budget_retains_expired_grants_and_uncertain_reservations(
+ budget_fixture, state, reserved,
+):
+ session, service, deployment, client = budget_fixture
+ before = service._budget(deployment)
+ required = 512 * 1024**2
+ session.add(allocation(deployment, client, state=state, required=required))
+ session.flush()
+ after = service._budget(deployment)
+ delta = required if reserved else 0
+ assert after.leased_vram_bytes == before.leased_vram_bytes + delta
+ assert after.schedulable_free_vram_bytes == before.schedulable_free_vram_bytes - delta
+ assert after.invariant_delta_bytes == before.invariant_delta_bytes == 0
+
+
+def test_native_placement_cannot_spend_external_reservation(budget_fixture):
+ session, service, deployment, client = budget_fixture
+ before = service._budget(deployment)
+ session.add(allocation(deployment, client, state="uncertain",
+ required=before.schedulable_free_vram_bytes - 1))
+ session.flush()
+ plan = service._placement_plan(deployment, priority="production", dry_run=True, commit=False)
+ assert plan.headroom_before_bytes == 1
+ assert plan.verdict not in {"ADMIT", "ADMIT_AFTER_EVICTION"}
+
+
+def test_overview_does_not_commit_owning_admission_transaction(budget_fixture):
+ session, service, _deployment, _client = budget_fixture
+ commits = []
+ event.listen(session, "after_commit", lambda _: commits.append(True))
+ assert service.scheduler_overview(commit=False)
+ assert commits == []
+ assert session.in_transaction()
+ service.scheduler_overview()
+ assert commits == [True]
diff --git a/backend/tests/test_serving_generation.py b/backend/tests/test_serving_generation.py
new file mode 100644
index 0000000..b2d0d3b
--- /dev/null
+++ b/backend/tests/test_serving_generation.py
@@ -0,0 +1,27 @@
+"""Legacy route entry points participate in the durable generation fence."""
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.serving import CapabilityPromotionCreate
+from modelforge_api.persistence.models import CapabilityContract
+from tests.test_serving import approve_and_promote, session, setup_m5
+
+__all__ = ["session"]
+
+
+def test_first_promotion_and_repeated_drain_do_not_allow_generation_reuse(session: Session):
+ service, candidate, _ = setup_m5(session)
+ approval, deployment = approve_and_promote(service, candidate.id)
+ contract = session.get(CapabilityContract, approval.capability_contract_id)
+ assert contract.serving_generation == deployment.route_generation == 1
+ replay = service.promote(candidate.id, CapabilityPromotionCreate(
+ production_approval_id=approval.id, keep_warm_seconds=900))
+ assert replay.id == deployment.id and contract.serving_generation == 1
+ service.request_unload(deployment.id)
+ assert contract.serving_generation == 1
+ service.request_unload(deployment.id, drain=True)
+ assert contract.serving_generation == 2
+ service.request_unload(deployment.id, drain=True)
+ assert contract.serving_generation == 2
+ replacement = service.promote(candidate.id, CapabilityPromotionCreate(
+ production_approval_id=approval.id, keep_warm_seconds=901))
+ assert replacement.id != deployment.id and contract.serving_generation == replacement.route_generation == 3
diff --git a/backend/tests/test_serving_generation_recovery.py b/backend/tests/test_serving_generation_recovery.py
new file mode 100644
index 0000000..003b6b9
--- /dev/null
+++ b/backend/tests/test_serving_generation_recovery.py
@@ -0,0 +1,168 @@
+"""Fresh source projections preserve route generation through schema migration."""
+from __future__ import annotations
+
+import copy
+import importlib.util
+import re
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import MetaData, Table, create_engine, inspect, select, text
+from sqlalchemy.engine import URL
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base, Capability, CapabilityContract
+from modelforge_api.services.recovery_fingerprint import (
+ SERVING_GENERATION_SCHEMA,
+ _without_model_changes,
+ _without_portfolio,
+ audit_fingerprint_differences_are_compatible,
+ fingerprint_compatibility,
+)
+from modelforge_api.services.recovery_fingerprint import (
+ semantic_fingerprint as _current_semantic_fingerprint,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+
+def semantic_fingerprint(connection):
+ return _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
+
+
+def _remove_lab_serving_job_link(connection) -> None:
+ create_sql = connection.scalar(
+ text("SELECT sql FROM sqlite_master WHERE type='table' AND name='serving_jobs'")
+ )
+ assert isinstance(create_sql, str)
+ index_sql = connection.execute(
+ text("SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name='serving_jobs' AND sql IS NOT NULL")
+ ).scalars().all()
+ columns = [
+ row[1]
+ for row in connection.execute(text("PRAGMA table_info(serving_jobs)")).all()
+ if row[1] != "lab_execution_run_id"
+ ]
+ legacy_sql = re.sub(
+ r"^\s*lab_execution_run_id\b[^\n]*\n", "", create_sql, flags=re.MULTILINE
+ )
+ legacy_sql = re.sub(
+ r",\s*\n\s*FOREIGN KEY\(lab_execution_run_id\)[^\n]*", "", legacy_sql
+ ).replace("CREATE TABLE serving_jobs", "CREATE TABLE serving_jobs_legacy", 1)
+ connection.exec_driver_sql(legacy_sql)
+ legacy_table = Table("serving_jobs_legacy", MetaData(), autoload_with=connection)
+ source_table = Table("serving_jobs", MetaData(), autoload_with=connection)
+ connection.execute(
+ legacy_table.insert().from_select(
+ columns, select(*(source_table.c[column] for column in columns))
+ )
+ )
+ connection.exec_driver_sql("DROP TABLE serving_jobs")
+ connection.exec_driver_sql("ALTER TABLE serving_jobs_legacy RENAME TO serving_jobs")
+ for statement in index_sql:
+ if "lab_execution_run_id" not in statement:
+ connection.exec_driver_sql(statement)
+ connection.exec_driver_sql("DROP TABLE lab_execution_runs")
+
+
+def migration():
+ path = Path(__file__).parents[1] / "alembic/versions/20260910_0034_approved_serving_transition.py"
+ spec = importlib.util.spec_from_file_location("serving_generation", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+@pytest.fixture
+def upgraded():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ with engine.begin() as connection:
+ # Keep this historical 0033 fixture physically faithful to its declared schema.
+ connection.execute(text('DROP TABLE "alert_notification_deliveries"'))
+ recreate_0038_benchmark_runs(connection)
+ _remove_lab_serving_job_link(connection)
+ with Session(engine) as session:
+ capability = Capability(key="generation.fixture", description="Isolated fixture")
+ session.add(capability)
+ session.flush()
+ session.add(CapabilityContract(capability_id=capability.id, version=1,
+ input_schema={}, output_schema={}, contract={}, upgrade_class="transparent"))
+ session.commit()
+ module = migration()
+ with engine.begin() as connection:
+ module.op = Operations(MigrationContext.configure(connection))
+ module.downgrade()
+ connection.execute(text("CREATE TABLE alembic_version(version_num VARCHAR(32))"))
+ connection.execute(text("INSERT INTO alembic_version VALUES ('20260909_0033')"))
+ before = semantic_fingerprint(connection)
+ module.upgrade()
+ connection.execute(text("UPDATE alembic_version SET version_num='20260910_0034'"))
+ yield connection, before
+ engine.dispose()
+
+
+def test_populated_prior_contracts_gain_only_zero_generation(upgraded):
+ connection, before = upgraded
+ after = semantic_fingerprint(connection)
+ assert connection.scalar(text("SELECT serving_generation FROM capability_contracts")) == 0
+ assert "serving_generation_projection" in before["tables"]["capability_contracts"]
+ contract = fingerprint_compatibility(before, after, source_schema_revision="20260909_0033")
+ assert contract.compatible
+ assert audit_fingerprint_differences_are_compatible(before, after, contract)
+ connection.execute(text("UPDATE capability_contracts SET serving_generation=1"))
+ assert not audit_fingerprint_differences_are_compatible(before, semantic_fingerprint(connection), contract)
+
+
+def test_opaque_prior_digest_cannot_prove_default_generation(upgraded):
+ connection, before = upgraded
+ del before["tables"]["capability_contracts"]["serving_generation_projection"]
+ after = semantic_fingerprint(connection)
+ contract = fingerprint_compatibility(before, after, source_schema_revision="20260909_0033")
+ assert not audit_fingerprint_differences_are_compatible(before, after, contract)
+
+
+def test_current_generation_is_preserved_and_cannot_be_projected_away(upgraded):
+ connection, _ = upgraded
+ connection.execute(text("UPDATE capability_contracts SET serving_generation=7"))
+ before = semantic_fingerprint(connection)
+ after = copy.deepcopy(before)
+ contract = fingerprint_compatibility(before, after, source_schema_revision=SERVING_GENERATION_SCHEMA)
+ assert contract.compatible and audit_fingerprint_differences_are_compatible(before, after, contract)
+ forged = copy.deepcopy(before)
+ forged["tables"]["capability_contracts"]["serving_generation_projection"] = after["tables"]["capability_contracts"]
+ assert not fingerprint_compatibility(forged, after, source_schema_revision=SERVING_GENERATION_SCHEMA).compatible
+ connection.execute(text("UPDATE capability_contracts SET serving_generation=0"))
+ assert not audit_fingerprint_differences_are_compatible(before, semantic_fingerprint(connection), contract)
+ module = migration()
+ module.op = Operations(MigrationContext.configure(connection))
+ connection.execute(text("UPDATE capability_contracts SET serving_generation=7"))
+ with pytest.raises(RuntimeError, match="history exists"):
+ module.downgrade()
+
+
+@contextmanager
+def legacy_generation_fixture(owner: URL) -> Iterator[None]:
+ """Seed isolated historical schemas using today's fixture code, then restore exact DDL.
+
+ Current ORM selects and serving fixture calls require this future column. Only the
+ fixture setup temporarily exposes it; fingerprints, dumps and migrations see the
+ actual historical schema. The runtime role receives no DDL privilege.
+ """
+ engine = create_engine(owner)
+ introduced = False
+ try:
+ with engine.begin() as connection:
+ if "serving_generation" not in {c["name"] for c in inspect(connection).get_columns("capability_contracts")}:
+ connection.execute(text("ALTER TABLE capability_contracts ADD COLUMN serving_generation BIGINT NOT NULL DEFAULT 0"))
+ introduced = True
+ yield
+ finally:
+ if introduced:
+ with engine.begin() as connection:
+ connection.execute(text("ALTER TABLE capability_contracts DROP COLUMN serving_generation"))
+ assert "serving_generation" not in {c["name"] for c in inspect(connection).get_columns("capability_contracts")}
+ engine.dispose()
diff --git a/backend/tests/test_serving_reconciliation_loop.py b/backend/tests/test_serving_reconciliation_loop.py
new file mode 100644
index 0000000..ec500b3
--- /dev/null
+++ b/backend/tests/test_serving_reconciliation_loop.py
@@ -0,0 +1,58 @@
+"""Transient infrastructure errors remain visible and do not stop reconciliation."""
+
+import uuid
+from unittest.mock import MagicMock
+
+import pytest
+import redis
+from sqlalchemy.exc import OperationalError
+
+from modelforge_api.services import serving_reconciliation as module
+from modelforge_api.settings import Settings
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("error", [
+ OperationalError("private SQL", {}, Exception("private connection data")),
+ redis.ConnectionError("private Redis address"),
+ RuntimeError("private runtime details"),
+])
+async def test_transient_failure_logged_safely_and_next_cycle_runs(monkeypatch, error):
+ settings = Settings(_env_file=None, env="test")
+ loop = module.ServingReconciliationService(settings, MagicMock())
+ session = MagicMock()
+ monkeypatch.setattr(module, "Session", session)
+ monkeypatch.setattr(module, "ManifestRegistry", MagicMock())
+ monkeypatch.setattr(module, "RedisPayloadStore", MagicMock())
+ logger = MagicMock()
+ monkeypatch.setattr(module, "logger", logger)
+ calls = 0
+
+ def reconcile():
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ raise error
+ loop.stop()
+
+ worker = MagicMock()
+ worker.return_value.reconcile.side_effect = reconcile
+ monkeypatch.setattr(module, "ServingService", worker)
+ # Skip the timer without changing stop semantics or creating a real Redis connection.
+ async def wait_for(awaitable, timeout):
+ del timeout
+ if loop._stop.is_set():
+ return await awaitable
+ awaitable.close()
+ raise TimeoutError
+
+ monkeypatch.setattr(module.asyncio, "wait_for", wait_for)
+ await loop.run()
+ assert calls == 2
+ logger.error.assert_called_once()
+ args, details = logger.error.call_args
+ assert args == ("serving_reconciliation_failed",)
+ assert details["error_type"] == type(error).__name__
+ assert uuid.UUID(details["correlation_id"])
+ assert "private" not in repr(logger.error.call_args)
+ assert session.return_value.__exit__.call_count == 2
diff --git a/backend/tests/test_supply_chain_m16.py b/backend/tests/test_supply_chain_m16.py
index c5812d7..e8db024 100644
--- a/backend/tests/test_supply_chain_m16.py
+++ b/backend/tests/test_supply_chain_m16.py
@@ -264,6 +264,7 @@ _PUBLISHED_PORT = re.compile(r'^\s*-\s*"(?P[^"]*:\d+)"\s*$')
# The API is published deliberately: every admin route is operator-authenticated and the console has
# to reach it. Everything else defaults to loopback.
_DELIBERATELY_PUBLISHED = ("MODELFORGE_API_BIND",)
+_VALIDATION_TLS_PUBLIC_MAPPING = "0.0.0.0:18444:8443"
def _published_mappings(path: Path) -> list[str]:
@@ -331,7 +332,16 @@ def test_every_compose_projection_binds_its_ports_to_loopback(compose: Path) ->
written for cannot find the next occurrence, so this one reads every projection.
"""
- for mapping in _published_mappings(compose):
+ mappings = _published_mappings(compose)
+ if compose.name == "docker-compose.validation-tls.yml":
+ # This overlay is the one explicitly isolated candidate ingress. Its rendered service,
+ # project and network are independently checked by rc_isolation_preflight.py.
+ assert mappings.count(_VALIDATION_TLS_PUBLIC_MAPPING) == 1
+ assert "\n tls:\n" in compose.read_text("utf-8")
+
+ for mapping in mappings:
+ if compose.name == "docker-compose.validation-tls.yml" and mapping == _VALIDATION_TLS_PUBLIC_MAPPING:
+ continue
if any(name in mapping for name in _DELIBERATELY_PUBLISHED):
continue
assert _binds_to_loopback(mapping), (
@@ -379,6 +389,11 @@ def test_the_console_sets_its_security_headers_in_every_location() -> None:
):
assert header in headers
+ assert "location = /.well-known/modelforge-build-provenance.json" in config
+ assert config.index("location = /.well-known/modelforge-build-provenance.json") < config.index(
+ "location ~ /\\."
+ )
+
def test_no_dependency_is_declared_as_a_floating_latest() -> None:
"""`latest` makes a build unrepeatable and pulls a compromised release automatically."""
@@ -395,9 +410,21 @@ def test_no_dependency_is_declared_as_a_floating_latest() -> None:
assert floating == [], f"floating dependency specifiers: {floating}"
-def test_the_release_images_upgrade_their_installer() -> None:
- for dockerfile in (ROOT / "backend" / "Dockerfile", ROOT / "node-agent" / "Dockerfile"):
- assert "--upgrade pip" in dockerfile.read_text("utf-8"), dockerfile
+def test_python_release_images_use_the_hash_locked_build_toolchain() -> None:
+ for dockerfile in (
+ ROOT / "backend" / "Dockerfile",
+ ROOT / "node-agent" / "Dockerfile",
+ ROOT / "runtime-worker" / "Dockerfile",
+ ):
+ text = dockerfile.read_text("utf-8")
+ assert "COPY config/security/build-toolchain.lock /tmp/modelforge-build-toolchain.lock" in text
+ assert "--require-hashes" in text
+ assert "--only-binary=:all:" in text
+ assert "--index-url https://pypi.org/simple" in text
+ assert "--no-build-isolation" in text
+ assert "--upgrade pip" not in text
+ if dockerfile.parent.name == "node-agent":
+ assert "--ignore-installed" in text
def test_every_release_base_image_is_digest_pinned_and_security_updated() -> None:
@@ -410,29 +437,49 @@ def test_every_release_base_image_is_digest_pinned_and_security_updated() -> Non
text = dockerfile.read_text("utf-8")
from_lines = [line for line in text.splitlines() if line.startswith("FROM ")]
assert from_lines
- assert all("@sha256:" in line for line in from_lines), dockerfile
- assert (
- "apk upgrade --no-cache" in text
- or "apt-get upgrade --yes" in text
+ stages = {
+ line.split(" AS ", 1)[1].strip()
+ for line in from_lines
+ if " AS " in line
+ }
+ assert all(
+ "@sha256:" in line or line.split()[1] in stages
+ for line in from_lines
), dockerfile
+ if dockerfile.parent.name == "node-agent":
+ # The Node Agent final stage is Canonical's glibc-compatible chiseled runtime. It has
+ # no package manager by design; patch currency is supplied by the exact immutable
+ # runtime digest and the managed exact-image Trivy gate. Its builder installs the
+ # hash-locked toolchain and is excluded from the shipped image.
+ assert "ubuntu/python:3.12-24.04@sha256:" in text, dockerfile
+ else:
+ assert (
+ "apk upgrade --no-cache" in text
+ or "apt-get upgrade --yes" in text
+ ), dockerfile
for dockerfile in (dockerfiles[0], dockerfiles[2]):
text = dockerfile.read_text("utf-8")
- assert '"setuptools>=78.1.1"' in text, dockerfile
- assert '"msgpack>=1.2.1"' in text, dockerfile
+ assert "--require-hashes" in text, dockerfile
+ assert "--no-build-isolation" in text, dockerfile
assert "pip check" in text, dockerfile
- assert "python -m pip uninstall --yes pip setuptools" in text, dockerfile
+ assert "-m pip uninstall --yes pip setuptools wheel" in text, dockerfile
def test_node_agent_runtime_is_glibc_multistage_and_keeps_nvidia_hardening() -> None:
dockerfile = (ROOT / "node-agent" / "Dockerfile").read_text("utf-8")
compose = (ROOT / "docker-compose.node-agent.yml").read_text("utf-8")
- assert dockerfile.count("FROM python:3.12-slim-trixie@sha256:") == 2
+ assert dockerfile.count("FROM python:3.12-slim-trixie@sha256:") == 1
+ assert "FROM ubuntu/python:3.12-24.04@sha256:" in dockerfile
+ assert " AS runtime-base" in dockerfile
assert " AS builder" in dockerfile
assert "alpine" not in dockerfile.lower()
- assert "USER modelforge-agent" in dockerfile
- assert "--uid 100" in dockerfile and "--gid 101" in dockerfile
+ assert "USER 100:101" in dockerfile
+ assert "modelforge-agent:x:100:101" in dockerfile
+ assert "modelforge-agent:x:101:" in dockerfile
+ assert "COPY --from=builder /runtime/etc/passwd /etc/passwd" in dockerfile
+ assert "COPY --from=builder /runtime/etc/group /etc/group" in dockerfile
assert (
"MODELFORGE_AGENT_ACCELERATOR_MODE: ${MODELFORGE_AGENT_ACCELERATOR_MODE:-nvidia}"
in compose
@@ -468,6 +515,8 @@ def test_release_builder_uses_reproducible_image_exports() -> None:
script = (ROOT / "scripts" / "release_build.py").read_text("utf-8")
assert '"--provenance=false"' in script
assert '"--sbom=false"' in script
+ assert '"--pull"' in script
+ assert '"--no-cache"' in script
assert "SOURCE_DATE_EPOCH" in script
diff --git a/backend/tests/test_update_preflight.py b/backend/tests/test_update_preflight.py
new file mode 100644
index 0000000..27cc04c
--- /dev/null
+++ b/backend/tests/test_update_preflight.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime, timedelta
+
+from fastapi.testclient import TestClient
+from pydantic import SecretStr
+
+from modelforge_api.domain.release import PRODUCT_VERSION, TARGET_SCHEMA_REVISION
+from modelforge_api.main import app
+from modelforge_api.settings import Settings, get_settings
+
+ADMIN_TOKEN = "update-preflight-admin-test-token-long-enough" # noqa: S105 - synthetic test credential
+VIEWER_TOKEN = "mfop_" + "v" * 43
+OPERATOR_TOKEN = "mfop_" + "o" * 43
+
+
+def test_update_preflight_is_read_only_admin_only_and_marks_missing_approval_unknown() -> None:
+ identities = [
+ {
+ "credential_id": role,
+ "subject": role,
+ "role": role,
+ "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+ "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat(),
+ }
+ for role, token in (("viewer", VIEWER_TOKEN), ("operator", OPERATOR_TOKEN))
+ ]
+ settings = Settings(
+ _env_file=None,
+ operator_api_key=SecretStr(ADMIN_TOKEN),
+ operator_identities=SecretStr(json.dumps(identities)),
+ )
+ app.dependency_overrides[get_settings] = lambda: settings
+ client = TestClient(app)
+ try:
+ path = "/api/v1/admin/updates/preflight"
+ assert client.get(path).status_code == 401
+ for token in (VIEWER_TOKEN, OPERATOR_TOKEN):
+ denied = client.get(path, headers={"X-ModelForge-Admin-Token": token})
+ assert denied.status_code == 403
+
+ response = client.get(path, headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN})
+ assert response.status_code == 200
+ payload = response.json()
+
+ assert payload["read_only"] is True
+ assert payload["status"] == "INCOMPLETE"
+ assert payload["current"]["version"] == PRODUCT_VERSION
+ assert payload["current"]["schema_revision"] == TARGET_SCHEMA_REVISION
+ assert payload["approved"]["version"] is None
+ assert payload["approved"]["status"] == "UNKNOWN"
+ assert payload["approved"]["notes"]["status"] == "UNKNOWN"
+ assert payload["approved"]["notes"]["value"] is None
+ assert payload["impact"]["schema"]["status"] == "UNKNOWN"
+ assert payload["impact"]["schema"]["value"] is None
+ assert payload["impact"]["breaking"]["status"] == "UNKNOWN"
+ assert payload["impact"]["breaking"]["value"] is None
+ assert payload["backup"]["required"]["status"] == "UNKNOWN"
+ assert payload["backup"]["required"]["value"] is None
+ assert payload["provenance"]["current"]["status"] == "UNKNOWN"
+ assert payload["provenance"]["approved"]["status"] == "UNKNOWN"
+ assert payload["execution"] == {
+ "available": False,
+ "control": "HUMAN_CONTROLLED",
+ }
+
+ assert client.post(path, headers={"X-ModelForge-Admin-Token": ADMIN_TOKEN}).status_code == 405
+ finally:
+ client.close()
+ app.dependency_overrides.clear()
diff --git a/clients/README.md b/clients/README.md
new file mode 100644
index 0000000..48828e5
--- /dev/null
+++ b/clients/README.md
@@ -0,0 +1,54 @@
+# Lightweight ModelForge clients
+
+These dependency-free clients are included in the installable release archive. They invoke an exact
+capability, never a model, node or worker. They use explicit timeouts, bounded responses and typed
+errors carrying the HTTP status and correlation ID. They do not automatically retry inference or
+follow redirects with credentials. They are server-side consumer clients, not browser session SDKs.
+
+For Python 3.11+, copy `python/modelforge_client.py` into your application's import path. For Node.js
+20+ JavaScript/TypeScript, copy `typescript/modelforge-client.mjs` and its adjacent `.d.mts` declaration
+file into your source directory. No package registry or dependency installation is required.
+
+Configure the consumer process:
+
+```text
+MODELFORGE_URL=https://modelforge.example.internal
+MODELFORGE_TOKEN_FILE=/run/secrets/modelforge-token
+MODELFORGE_CAPABILITY=rag.embedding@1
+```
+
+Store only the issued service key in that protected file. HTTP is rejected by default; set
+`MODELFORGE_ALLOW_HTTP=true` only for an explicitly trusted private lab. Origins cannot contain
+userinfo, paths, query strings or fragments. TLS certificate verification stays enabled.
+
+```python
+from modelforge_client import ModelForgeClient, ModelForgeError
+
+client = ModelForgeClient.from_env(timeout=60)
+try:
+ print(client.health()) # Verifies identity and scope; does not start inference.
+ response = client.invoke({"input": ["A non-sensitive integration check"]})
+except ModelForgeError as error:
+ print(error.code, error.status, error.correlation_id)
+```
+
+```typescript
+import { ModelForgeClient, ModelForgeError } from "./modelforge-client.mjs";
+
+const client = await ModelForgeClient.fromEnv({ timeoutMs: 60000 });
+try {
+ console.log(await client.health());
+ const response = await client.invoke({ input: ["A non-sensitive integration check"] });
+} catch (error) {
+ if (error instanceof ModelForgeError)
+ console.error(error.code, error.status, error.correlationId);
+ else throw error;
+}
+```
+
+Use the payload defined by your selected capability. `health()` proves connectivity, not model
+health or production eligibility. Do not automatically retry timed-out inference: the server may
+have admitted the original request. Retain correlation/request IDs for diagnosis.
+
+Validation: backend `test_project_python_client.py` and
+`node --test clients/typescript/modelforge-client.test.mjs`, both included in managed full validation.
diff --git a/clients/python/modelforge_client.py b/clients/python/modelforge_client.py
new file mode 100644
index 0000000..c8cecfb
--- /dev/null
+++ b/clients/python/modelforge_client.py
@@ -0,0 +1,108 @@
+"""Dependency-free ModelForge capability client. Python 3.11+, no automatic inference retries."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+from http.client import HTTPException
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote, urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+MAX_RESPONSE_BYTES = 8 * 1024 * 1024
+
+
+class ModelForgeError(Exception):
+ def __init__(self, code: str, message: str, status: int = 0, correlation_id: str | None = None):
+ super().__init__(message)
+ self.code = code
+ self.status = status
+ self.correlation_id = correlation_id
+
+
+class _NoRedirect(HTTPRedirectHandler):
+ def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None:
+ return None
+
+
+class ModelForgeClient:
+ def __init__(self, base_url: str, token: str, capability: str, *, timeout: float = 60, allow_http: bool = False):
+ url = urlsplit(base_url)
+ if (url.scheme not in ({"http", "https"} if allow_http else {"https"})
+ or not url.hostname or url.username or url.password or url.query or url.fragment
+ or url.path not in {"", "/"}):
+ raise ValueError("Use a gateway origin without userinfo, path, query or fragment; HTTP requires allow_http=True")
+ if not re.fullmatch(r"[a-z][a-z0-9_.]*@[1-9][0-9]*", capability):
+ raise ValueError("Use an exact versioned capability key")
+ if not re.fullmatch(r"mf_[A-Za-z0-9_-]{48}", token):
+ raise ValueError("Use a scoped ModelForge service credential")
+ if not math.isfinite(timeout) or not 0 < timeout <= 300:
+ raise ValueError("timeout must be between 0 and 300 seconds")
+ self.base_url = base_url.rstrip("/")
+ self.capability = capability
+ self.timeout = timeout
+ self._token = token
+ self._opener = build_opener(_NoRedirect())
+
+ @classmethod
+ def from_env(cls, *, timeout: float = 60) -> ModelForgeClient:
+ with Path(os.environ["MODELFORGE_TOKEN_FILE"]).open("r", encoding="ascii") as handle:
+ token = handle.read(513).strip()
+ return cls(os.environ["MODELFORGE_URL"], token, os.environ["MODELFORGE_CAPABILITY"],
+ timeout=timeout, allow_http=os.getenv("MODELFORGE_ALLOW_HTTP") == "true")
+
+ def _request(self, suffix: str, body: dict[str, Any] | None = None) -> Any:
+ path = f"/api/v1/capabilities/{quote(self.capability, safe='')}/{suffix}"
+ request = Request(self.base_url + path, data=json.dumps(body).encode() if body is not None else None,
+ headers={"Authorization": f"Bearer {self._token}", "Content-Type": "application/json"},
+ method="POST" if body is not None else "GET")
+ try:
+ response = self._opener.open(request, timeout=self.timeout)
+ except HTTPError as error:
+ correlation = error.headers.get("X-Correlation-ID")
+ try:
+ with error:
+ raw = error.read(MAX_RESPONSE_BYTES + 1)
+ except (TimeoutError, OSError, HTTPException):
+ raise ModelForgeError("TRANSPORT_ERROR", "Gateway response failed or timed out",
+ error.code, correlation) from None
+ try:
+ payload = json.loads(raw) if len(raw) <= MAX_RESPONSE_BYTES else {}
+ except (ValueError, UnicodeError):
+ payload = {}
+ if not isinstance(payload, dict):
+ payload = {}
+ nested_error = payload.get("error")
+ error_payload = nested_error if isinstance(nested_error, dict) else payload
+ raise ModelForgeError(
+ str(error_payload.get("code", "HTTP_ERROR")),
+ str(error_payload.get("message", "Gateway request refused")),
+ error.code,
+ error_payload.get("correlation_id") or correlation,
+ ) from None
+ except (URLError, TimeoutError, OSError, HTTPException):
+ raise ModelForgeError("TRANSPORT_ERROR", "Gateway connection failed or timed out") from None
+ try:
+ with response:
+ raw = response.read(MAX_RESPONSE_BYTES + 1)
+ correlation = response.headers.get("X-Correlation-ID")
+ except (TimeoutError, OSError, HTTPException):
+ raise ModelForgeError("TRANSPORT_ERROR", "Gateway response failed or timed out") from None
+ if len(raw) > MAX_RESPONSE_BYTES:
+ raise ModelForgeError("RESPONSE_TOO_LARGE", "Gateway response exceeds the client limit", correlation_id=correlation)
+ try:
+ return json.loads(raw)
+ except (ValueError, UnicodeError):
+ raise ModelForgeError("INVALID_RESPONSE", "Gateway response is not JSON", correlation_id=correlation) from None
+
+ def health(self) -> dict[str, Any]:
+ """Authenticate the exact scope without executing inference."""
+ return self._request("check")
+
+ def invoke(self, payload: dict[str, Any]) -> Any:
+ """Execute once. The client never changes capability, deployment or scheduling priority."""
+ return self._request("invoke", payload)
diff --git a/clients/typescript/modelforge-client.d.mts b/clients/typescript/modelforge-client.d.mts
new file mode 100644
index 0000000..4469599
--- /dev/null
+++ b/clients/typescript/modelforge-client.d.mts
@@ -0,0 +1,17 @@
+export interface ClientOptions { timeoutMs?: number; allowHttp?: boolean; }
+export interface Connectivity { client_id: string; project_key: string | null; capability: string; connected: true; correlation_id: string; }
+export class ModelForgeError extends Error {
+ readonly code: string;
+ readonly status: number;
+ readonly correlationId: string | null;
+ constructor(code: string, message: string, status?: number, correlationId?: string | null);
+}
+export class ModelForgeClient {
+ readonly baseUrl: string;
+ readonly capability: string;
+ readonly timeoutMs: number;
+ constructor(baseUrl: string, token: string, capability: string, options?: ClientOptions);
+ static fromEnv(options?: ClientOptions): Promise;
+ health(): Promise;
+ invoke(payload: Record): Promise;
+}
diff --git a/clients/typescript/modelforge-client.mjs b/clients/typescript/modelforge-client.mjs
new file mode 100644
index 0000000..da17106
--- /dev/null
+++ b/clients/typescript/modelforge-client.mjs
@@ -0,0 +1,69 @@
+import { open } from "node:fs/promises";
+
+export class ModelForgeError extends Error {
+ constructor(code, message, status = 0, correlationId = null) {
+ super(message); this.name = "ModelForgeError"; this.code = code;
+ this.status = status; this.correlationId = correlationId;
+ }
+}
+
+/** Node.js 20+. Capability-only, bounded requests, no redirects or inference retries. */
+export class ModelForgeClient {
+ #token;
+ constructor(baseUrl, token, capability, { timeoutMs = 60000, allowHttp = false } = {}) {
+ const url = new URL(baseUrl);
+ if (!((allowHttp && url.protocol === "http:") || url.protocol === "https:") || url.username || url.password || url.search || url.hash || url.pathname !== "/") throw new TypeError("Use a gateway origin; HTTP requires allowHttp: true");
+ if (!/^[a-z][a-z0-9_.]*@[1-9][0-9]*$/.test(capability)) throw new TypeError("Use an exact versioned capability key");
+ if (!/^mf_[A-Za-z0-9_-]{48}$/.test(token)) throw new TypeError("Use a scoped ModelForge service credential");
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || timeoutMs > 300000) throw new TypeError("timeoutMs must be between 0 and 300000");
+ this.baseUrl = url.origin; this.capability = capability; this.timeoutMs = timeoutMs; this.#token = token;
+ }
+ static async fromEnv(options = {}) {
+ const file = await open(process.env.MODELFORGE_TOKEN_FILE, "r");
+ let token;
+ try { const buffer = Buffer.alloc(513); const { bytesRead } = await file.read(buffer); token = buffer.subarray(0, bytesRead).toString("utf8").trim(); }
+ finally { await file.close(); }
+ return new ModelForgeClient(process.env.MODELFORGE_URL, token, process.env.MODELFORGE_CAPABILITY, { allowHttp: process.env.MODELFORGE_ALLOW_HTTP === "true", ...options });
+ }
+ async #request(suffix, payload) {
+ let response;
+ let raw;
+ try {
+ response = await fetch(`${this.baseUrl}/api/v1/capabilities/${encodeURIComponent(this.capability)}/${suffix}`, {
+ method: payload === undefined ? "GET" : "POST", redirect: "error", credentials: "omit",
+ signal: AbortSignal.timeout(this.timeoutMs),
+ headers: { Authorization: `Bearer ${this.#token}`, "Content-Type": "application/json" },
+ body: payload === undefined ? undefined : JSON.stringify(payload),
+ });
+ const reader = response.body?.getReader();
+ const chunks = []; let size = 0;
+ if (reader) { try { while (true) {
+ const { done, value } = await reader.read(); if (done) break;
+ size += value.length;
+ if (size > 8 * 1024 * 1024) { await reader.cancel(); throw new ModelForgeError("RESPONSE_TOO_LARGE", "Gateway response exceeds the client limit"); }
+ chunks.push(value);
+ } } finally { reader.releaseLock(); } }
+ raw = Buffer.concat(chunks).toString("utf8");
+ } catch (error) {
+ if (error instanceof ModelForgeError) throw error;
+ throw new ModelForgeError("TRANSPORT_ERROR", "Gateway connection failed or timed out");
+ }
+ let data;
+ try { data = JSON.parse(raw); } catch { throw new ModelForgeError("INVALID_RESPONSE", "Gateway response is not JSON", response.status, response.headers.get("X-Correlation-ID")); }
+ if (!response.ok) {
+ const nestedError = data?.error;
+ const error = nestedError !== null && typeof nestedError === "object" ? nestedError : data;
+ const code = typeof error?.code === "string" ? error.code : "HTTP_ERROR";
+ const message = typeof error?.message === "string" ? error.message : "Gateway request refused";
+ throw new ModelForgeError(
+ code,
+ message,
+ response.status,
+ error?.correlation_id ?? response.headers.get("X-Correlation-ID"),
+ );
+ }
+ return data;
+ }
+ health() { return this.#request("check"); }
+ invoke(payload) { return this.#request("invoke", payload); }
+}
diff --git a/clients/typescript/modelforge-client.test.mjs b/clients/typescript/modelforge-client.test.mjs
new file mode 100644
index 0000000..c6ac84c
--- /dev/null
+++ b/clients/typescript/modelforge-client.test.mjs
@@ -0,0 +1,54 @@
+import { strict as assert } from "node:assert";
+import { createServer } from "node:http";
+import { once } from "node:events";
+import { test } from "node:test";
+import { ModelForgeClient, ModelForgeError } from "./modelforge-client.mjs";
+
+test("capability scope, typed errors, no retries, no redirect credential forwarding", async () => {
+ const seen = [];
+ const server = createServer(async (req, res) => {
+ seen.push({ path: req.url, authorization: req.headers.authorization });
+ if (req.method === "GET") { res.end('{"connected":true}'); return; }
+ const chunks = []; for await (const chunk of req) chunks.push(chunk);
+ const body = JSON.parse(Buffer.concat(chunks));
+ if (body.redirect) { res.writeHead(307, { Location: "/must-not-follow" }); res.end(); return; }
+ res.writeHead(503, { "Content-Type": "application/json" });
+ if (body.flat_error) {
+ res.end('{"code":"LEGACY_ERROR","message":"Legacy error","correlation_id":"legacy-case"}');
+ return;
+ }
+ res.end(JSON.stringify({ error: {
+ code: "NO_ELIGIBLE_NODE", message: "No node", correlation_id: "sdk-case",
+ details: { input: "private benchmark input that must remain redacted" },
+ } }));
+ });
+ server.listen(0, "127.0.0.1"); await once(server, "listening");
+ const token = "mf_" + "k".repeat(48);
+ try {
+ const client = new ModelForgeClient(`http://127.0.0.1:${server.address().port}`, token, "rag.embedding@1", { allowHttp: true, timeoutMs: 2000 });
+ assert.deepEqual(await client.health(), { connected: true });
+ await assert.rejects(client.invoke({ input: ["test"] }), (error) => {
+ assert.ok(error instanceof ModelForgeError);
+ assert.equal(error.code, "NO_ELIGIBLE_NODE");
+ assert.equal(error.message, "No node");
+ assert.equal(error.status, 503);
+ assert.equal(error.correlationId, "sdk-case");
+ assert.ok(!error.message.includes("private benchmark input"));
+ return true;
+ });
+ await assert.rejects(client.invoke({ flat_error: true }), (error) =>
+ error instanceof ModelForgeError
+ && error.code === "LEGACY_ERROR"
+ && error.message === "Legacy error"
+ && error.correlationId === "legacy-case");
+ await assert.rejects(client.invoke({ redirect: true }), (error) => error instanceof ModelForgeError && error.code === "TRANSPORT_ERROR");
+ assert.equal(seen.length, 4);
+ assert.ok(seen.every(({ path, authorization }) => path.startsWith("/api/v1/capabilities/rag.embedding%401/") && authorization === `Bearer ${token}`));
+ assert.ok(!JSON.stringify(client).includes(token));
+ } finally { server.closeAllConnections(); await new Promise((resolve) => server.close(resolve)); }
+});
+
+test("unsafe origins and credential classes fail before network access", () => {
+ for (const origin of ["http://lab.invalid", "https://user:password@lab.invalid", "https://lab.invalid/path", "https://lab.invalid/?q=key"]) assert.throws(() => new ModelForgeClient(origin, "mf_" + "k".repeat(48), "rag.embedding@1"));
+ assert.throws(() => new ModelForgeClient("https://lab.invalid", "mfop_" + "k".repeat(43), "rag.embedding@1"));
+});
diff --git a/config/capabilities/creative.resource.v1.yaml b/config/capabilities/creative.resource.v1.yaml
new file mode 100644
index 0000000..eb8022e
--- /dev/null
+++ b/config/capabilities/creative.resource.v1.yaml
@@ -0,0 +1,17 @@
+capability: creative.resource
+version: 1
+description: Cross-project capacity coordination for creative workloads executed by registered external providers.
+input_schema: {type: object}
+output_schema: {type: object}
+modalities: {input: [text], output: [json]}
+languages: [nl, en]
+streaming: false
+structured_output: true
+quality_metrics: [admission_correctness, allocation_reconciliation]
+upgrade_class: behavioral
+fallback: {allowed: false, mode: hard_fail}
+privacy: {classification: confidential, allow_persistence: false, allow_logging_payloads: false, allow_network_egress: false}
+resources: {accelerator_required: true, cpu_fallback_allowed: false}
+production_priority: interactive
+default_residency: load_on_demand
+estate: {category: GENERATION, purpose: Cross-project GPU coordination, stability: experimental, resource_class: EXCLUSIVE_GPU, evaluation_type: generation, consumers: [privatemuse], payload_limits: {max_bytes: 16384, max_batch_count: 1}}
diff --git a/config/public-candidate-unfixed-vulnerabilities.json b/config/public-candidate-unfixed-vulnerabilities.json
index aea4365..d4b5bae 100644
--- a/config/public-candidate-unfixed-vulnerabilities.json
+++ b/config/public-candidate-unfixed-vulnerabilities.json
@@ -1,23 +1,6 @@
{
"schema_version": 1,
- "reviewed_at": "2026-09-01",
- "review_reference": "docs/quality/V1_2_1_RELEASE_VERIFICATION.md",
- "vulnerabilities": [
- {"vulnerability_id": "CVE-2026-41992", "package": "gzip", "installed_version": "1.13-1", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-54369", "package": "libacl1", "installed_version": "2.3.2-2+b1", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2025-69720", "package": "libncursesw6", "installed_version": "6.5+20250216-2", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-11822", "package": "libsqlite3-0", "installed_version": "3.46.1-7+deb13u1", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-11824", "package": "libsqlite3-0", "installed_version": "3.46.1-7+deb13u1", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2025-69720", "package": "libtinfo6", "installed_version": "6.5+20250216-2", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2025-69720", "package": "ncurses-base", "installed_version": "6.5+20250216-2", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2025-69720", "package": "ncurses-bin", "installed_version": "6.5+20250216-2", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-13221", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "CRITICAL"},
- {"vulnerability_id": "CVE-2026-42496", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "CRITICAL"},
- {"vulnerability_id": "CVE-2026-8376", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "CRITICAL"},
- {"vulnerability_id": "CVE-2026-42497", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-48962", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-57432", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-57433", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "HIGH"},
- {"vulnerability_id": "CVE-2026-9538", "package": "perl-base", "installed_version": "5.40.1-6", "severity": "HIGH"}
- ]
+ "reviewed_at": "2026-09-12",
+ "review_reference": "docs/quality/RC_NODE_IMAGE_REMEDIATION.md",
+ "vulnerabilities": []
}
diff --git a/config/release/rc-program.json b/config/release/rc-program.json
new file mode 100644
index 0000000..533511f
--- /dev/null
+++ b/config/release/rc-program.json
@@ -0,0 +1,27 @@
+{
+ "schema_version": 1,
+ "target_version": "1.3.0-rc.1",
+ "baseline_commit": "9b61f23ea3ed102ec58c9966c027b50ebd6a22b6",
+ "ledger": "docs/quality/RC_GAP_MATRIX.md",
+ "roadmap": "docs/product/RC_EXECUTION_ROADMAP.md",
+ "packets": [
+ {"id":"P00","depends_on":[],"gaps":["RC-SEC-01","RC-SEC-02","RC-DEL-01","RC-DB-01"]},
+ {"id":"P01","depends_on":["P00"],"gaps":["RC-CFG-01","RC-CFG-02","RC-CFG-03","RC-AUTH-01","RC-AUTH-02"]},
+ {"id":"P02","depends_on":["P00","P01"],"gaps":["RC-CAP-01","RC-SUP-01"]},
+ {"id":"P03","depends_on":["P02","P06"],"gaps":["RC-LAB-01"]},
+ {"id":"P04","depends_on":["P03"],"gaps":["RC-LAB-02","RC-LAB-03"]},
+ {"id":"P05","depends_on":["P04"],"gaps":["RC-LIF-01","RC-LIF-02"]},
+ {"id":"P06","depends_on":["P00","P01"],"gaps":["RC-NOD-01","RC-NOD-02","RC-NOD-03","RC-SCH-01","RC-SCH-02"]},
+ {"id":"P07","depends_on":["P00","P01","P03","P06"],"gaps":["RC-OPS-01","RC-OPS-02","RC-OPS-03","RC-OPS-04","RC-OPS-05"]},
+ {"id":"P08","depends_on":["P01","P07"],"gaps":["RC-REC-01","RC-UPD-01","RC-UPD-02"]},
+ {"id":"P09","depends_on":["P01","P02","P03","P04","P05","P06","P07","P08"],"gaps":["RC-SEA-01","RC-UX-01","RC-UX-02","RC-UX-03","RC-UX-04"]},
+ {"id":"P10","depends_on":["P01","P02","P03","P04","P05","P06","P07","P08"],"gaps":["RC-API-01","RC-API-02"]},
+ {"id":"P11","depends_on":["P09","P10"],"gaps":["RC-SEC-03","RC-DEL-02"]},
+ {"id":"P15A","depends_on":["P11"],"gaps":["RC-REL-01","RC-REL-04"]},
+ {"id":"P12","depends_on":["P00","P01","P06","P11","P15A"],"gaps":["RC-RC-01"]},
+ {"id":"P13","depends_on":["P02","P03","P04","P05","P06","P12"],"gaps":["RC-PRJ-01","RC-PRJ-02","RC-PRJ-03","RC-CON-01","RC-CON-02"]},
+ {"id":"P14","depends_on":["P07","P08","P09","P10","P11","P12","P13"],"gaps":["RC-QUAL-01","RC-QUAL-02","RC-QUAL-03","RC-QUAL-04","RC-QUAL-05","RC-QUAL-06"]},
+ {"id":"P15","depends_on":["P11","P12","P13","P14","P15A"],"gaps":["RC-REL-02","RC-REL-03","RC-DOC-01"]}
+ ],
+ "deferred_gaps": ["RC-POST-01","RC-POST-02","RC-POST-03","RC-POST-04","RC-POST-05"]
+}
diff --git a/config/security/accepted-risks.json b/config/security/accepted-risks.json
new file mode 100644
index 0000000..9746727
--- /dev/null
+++ b/config/security/accepted-risks.json
@@ -0,0 +1,4 @@
+{
+ "schema_version": 1,
+ "accepted_risks": []
+}
diff --git a/config/security/build-toolchain.lock b/config/security/build-toolchain.lock
new file mode 100644
index 0000000..49019d3
--- /dev/null
+++ b/config/security/build-toolchain.lock
@@ -0,0 +1,13 @@
+# Hash-locked release and CI build toolchain, verified against the official PyPI JSON API.
+# msgpack hashes cover CPython 3.11-3.14 glibc x86_64/aarch64 and CPython 3.12-3.14 Alpine musl x86_64/aarch64.
+editables==0.5 --hash=sha256:61e5ffa82629e0d8bfe09bc44a07db3c1ab8ed1ce78a6980732870f19b5e7d4c
+hatchling==1.32.4 --hash=sha256:08ecf7548fb48205e7f213d70c71e67b8271b7242093dc3f1da578b42c734a2c
+msgpack==1.2.2 --hash=sha256:f11e09f10210a91c169e39c7a5a1f9090eaa73ad75555fafad5023c3053c47ba --hash=sha256:b9b0c1f2aa7b0026b4bd50718100e8b04175e4f36e160aa852502377b5e572e7 --hash=sha256:77c2e018417dc1d66f235e383877ee885b60ade9d29e494dd581e08af2cb1923 --hash=sha256:90986cc9aab9d7d1d8f38bcbf65d3f7ac83bdd90c35765db7d691b4829698cba --hash=sha256:d24b38a825bcca41bb956de50eb98451ef291304a8607fad99e619043d3e79b9 --hash=sha256:1814f92306ae7862908e9ece7cfd90e0dc87ded3e89b6ae7ffdd1175d6376fdc --hash=sha256:4b554d8164ebb526892194f71dcd96ef1fefe0c250087498785d3ffc04a80be3 --hash=sha256:3e915d390d7068b257ca8b62f3fc59fad135c8631d1017ab03b0b924b07c5367 --hash=sha256:4d9a562aec0a92fe536da2e533d313b3d2a6b929157b1dec7ff623446dc0a8ab --hash=sha256:682804bf31e43d46e51a9a33bd575b51e839d715ce6bd5612c055f7b28ad637b --hash=sha256:73b0e05c32c3cfc3cd84994908e57430c0ebc6813abf905d3f18ff115d54df3f --hash=sha256:8dc4487097571f7311188c3eca2a3e86cd1f1db4c37c7a017bcc3fd38486cbfe --hash=sha256:b8dd6c71d20c28d2d0eb0c51e7cccf3584afde3b1364f6629596186c9025bd54 --hash=sha256:ccfd880988f8438d1c91c77d7edc58e70f4d2012e999167bc154c64c6f06ea6b
+packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
+pathspec==1.1.1 --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189
+pip==26.2.1 --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
+pluggy==1.6.0 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746
+setuptools==84.0.0 --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670
+tomlkit==0.14.0 --hash=sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680
+trove-classifiers==2026.9.21.13 --hash=sha256:8b1ff4f9c191b1040b71c37f1e445ab99732911e3cd91de52838453a854d7a17
+wheel==0.48.0 --hash=sha256:3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab
diff --git a/config/security/gitleaks-false-positives.json b/config/security/gitleaks-false-positives.json
new file mode 100644
index 0000000..ce111bb
--- /dev/null
+++ b/config/security/gitleaks-false-positives.json
@@ -0,0 +1,24 @@
+[
+ {
+ "scanner": "gitleaks",
+ "rule_id": "generic-api-key",
+ "fingerprint_sha256": "b25524088aadca143f5b33de32f1af64497427a4b465a5caf0ce1e49bf7c16d2",
+ "classification": "FALSE_POSITIVE",
+ "owner": "ModelForge security owner",
+ "classified_on": "2026-09-30",
+ "review_on": "2026-10-30",
+ "tracking": "RC-DEL-02 supply-chain gate review",
+ "reason": "Historical unit-test-only AES fixture; the decoded bytes are synthetic test material and were never a credential or production backup key."
+ },
+ {
+ "scanner": "gitleaks",
+ "rule_id": "generic-api-key",
+ "fingerprint_sha256": "6114f9e29b47c068fb0913c6465915044ff9fef7dcf53d4a60ff1776c8404ddb",
+ "classification": "FALSE_POSITIVE",
+ "owner": "ModelForge security owner",
+ "classified_on": "2026-09-30",
+ "review_on": "2026-10-30",
+ "tracking": "RC-DEL-02 supply-chain gate review",
+ "reason": "Historical unit-test-only AES fixture; the decoded bytes are synthetic test material and were never a credential or production backup key."
+ }
+]
diff --git a/config/security/pip-audit-toolchain.lock b/config/security/pip-audit-toolchain.lock
new file mode 100644
index 0000000..2e5ee71
--- /dev/null
+++ b/config/security/pip-audit-toolchain.lock
@@ -0,0 +1,34 @@
+# Fully resolved pip-audit 2.10.1 scanner toolchain for CPython 3.12 / Linux x86_64.
+# Generated only from https://pypi.org/simple with:
+# python -m pip download --only-binary=:all: --platform manylinux2014_x86_64 \
+# --python-version 312 --implementation cp --abi cp312 pip-audit==2.10.1
+# Every selected official wheel is pinned and hash-checked again by pip at managed-gate install.
+boolean.py==5.0 --hash=sha256:ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9
+cachecontrol==0.14.4 --hash=sha256:b7ac014ff72ee199b5f8af1de29d60239954f223e948196fa3d84adaffc71d2b
+certifi==2026.7.22 --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775
+charset-normalizer==3.5.1 --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08
+cyclonedx-python-lib==11.12.0 --hash=sha256:0e807521a921a5c3cb8ce1153f8a61d29eedfe76a46aac2796b7c6b573391a54
+defusedxml==0.7.1 --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61
+filelock==3.32.4 --hash=sha256:22e58ca3b1ae3b98993b762d7338367ae64fe50252bf78d59da3bfebcdf1cedd
+idna==3.19 --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4
+license-expression==30.4.4 --hash=sha256:421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4
+markdown-it-py==4.2.0 --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a
+mdurl==0.1.2 --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8
+msgpack==1.2.2 --hash=sha256:77c2e018417dc1d66f235e383877ee885b60ade9d29e494dd581e08af2cb1923
+packageurl-python==0.17.6 --hash=sha256:31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9
+packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
+pip==26.2.1 --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e
+pip-api==0.0.34 --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb
+pip-audit==2.10.1 --hash=sha256:99ef3f600a317c1945f1e89e227ef26e1c2d618429b8bd3fa6f4f7c440c4611a
+pip-requirements-parser==32.0.1 --hash=sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526
+platformdirs==4.11.5 --hash=sha256:89f8d42695853b89c7170bd49bc3dc593f98a71e695ede88e06a3b247bc4563b
+py-serializable==2.1.0 --hash=sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304
+pygments==2.21.0 --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9
+pyparsing==3.3.2 --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d
+requests==2.34.2 --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0
+rich==15.0.0 --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb
+sortedcontainers==2.4.0 --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0
+tomli==2.4.1 --hash=sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5
+tomli-w==1.2.0 --hash=sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90
+typing-extensions==4.16.0 --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8
+urllib3==2.7.0 --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
diff --git a/deploy/postgres/provision-existing-1.2.1.sql b/deploy/postgres/provision-existing-1.2.1.sql
index b207268..5a431a1 100644
--- a/deploy/postgres/provision-existing-1.2.1.sql
+++ b/deploy/postgres/provision-existing-1.2.1.sql
@@ -9,6 +9,26 @@
\getenv runtime_password MODELFORGE_RUNTIME_DB_PASSWORD
\getenv database_name MODELFORGE_POSTGRES_DB
+-- PostgreSQL permanently protects the cluster bootstrap superuser (OID 10): even another
+-- superuser cannot remove that role's SUPERUSER attribute. v1.2.1 installations initialized with
+-- POSTGRES_USER=modelforge therefore need a fresh cluster plus logical restore. Refuse before
+-- creating roles or changing privileges; direct pg_authid mutation is not a supported shortcut.
+do $$
+begin
+ if exists (
+ select 1
+ from pg_catalog.pg_roles
+ where rolname = 'modelforge' and oid = 10
+ ) then
+ raise exception using
+ errcode = '55000',
+ message = 'modelforge is the protected PostgreSQL bootstrap superuser (OID 10)',
+ detail = 'PostgreSQL cannot demote the bootstrap superuser.',
+ hint = 'Follow docs/UPGRADE.md: create a fresh cluster under the retained admin, logically restore with --no-owner --no-privileges, then run migration 0024.';
+ end if;
+end
+$$;
+
select pg_catalog.format(
'create role %I login password %L superuser createdb createrole inherit replication bypassrls',
:'admin_role', :'admin_password'
diff --git a/deploy/validation/Caddyfile b/deploy/validation/Caddyfile
new file mode 100644
index 0000000..2a541a3
--- /dev/null
+++ b/deploy/validation/Caddyfile
@@ -0,0 +1,11 @@
+https://modelforge.example.com:8443 {
+ tls /run/tls/fullchain.pem /run/tls/privkey.pem
+
+ @api path /api/* /v1/*
+ handle @api {
+ reverse_proxy api:8000
+ }
+ handle {
+ reverse_proxy web:3000
+ }
+}
diff --git a/docker-compose.backup.yml b/docker-compose.backup.yml
index 2e24617..bac766f 100644
--- a/docker-compose.backup.yml
+++ b/docker-compose.backup.yml
@@ -26,6 +26,7 @@ services:
done
environment:
MODELFORGE_BASE_URL: ${MODELFORGE_BACKUP_BASE_URL:-http://api:8000}
+ MODELFORGE_BACKUP_ADMIN_API_KEY: ${MODELFORGE_BACKUP_ADMIN_API_KEY:-}
MODELFORGE_OPERATOR_API_KEY: ${MODELFORGE_OPERATOR_API_KEY:-}
MODELFORGE_BACKUP_ID_PREFIX: ${MODELFORGE_BACKUP_ID_PREFIX:-scheduled}
MODELFORGE_BACKUP_INTERVAL_SECONDS: ${MODELFORGE_BACKUP_INTERVAL_SECONDS:-86400}
diff --git a/docker-compose.dr.yml b/docker-compose.dr.yml
index 2d748ce..59cd620 100644
--- a/docker-compose.dr.yml
+++ b/docker-compose.dr.yml
@@ -32,7 +32,8 @@ services:
dr-api:
build:
- context: ./backend
+ context: .
+ dockerfile: backend/Dockerfile
env_file:
- path: .env
required: false
diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index 02d4c9c..2bb3eea 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -8,8 +8,8 @@
# working tree. It still requires separate database secrets; this overlay additionally turns on
# every production startup rule and requires immutable release identities.
#
-# Every secret here uses the ${VAR:?message} form, so Compose refuses to render the project at all
-# when one is missing. Failing before the first container starts is the cheapest possible failure.
+# Database, backup and origin requirements fail Compose rendering when absent. Human authentication
+# accepts either a bootstrap key or a named administrator registry, validated before API startup.
services:
postgres:
environment:
@@ -37,7 +37,8 @@ services:
api:
image: ${MODELFORGE_API_IMAGE:-modelforge-api:${MODELFORGE_VERSION:?set an exact release version, for example 1.2.1}}
build:
- context: ./backend
+ context: .
+ dockerfile: backend/Dockerfile
args:
MODELFORGE_VERSION: ${MODELFORGE_VERSION:-0.0.0}
MODELFORGE_COMMIT: ${MODELFORGE_COMMIT:-}
@@ -47,7 +48,8 @@ services:
# production deployment silently runs under development semantics.
MODELFORGE_ENV: production
MODELFORGE_DATABASE_URL: ${MODELFORGE_RUNTIME_DATABASE_URL:?set a URL for only the modelforge_runtime role}
- MODELFORGE_OPERATOR_API_KEY: ${MODELFORGE_OPERATOR_API_KEY:?generate an operator API key; see docs/CONFIGURATION.md}
+ # Startup validates a legacy key OR an unexpired named administrator.
+ MODELFORGE_OPERATOR_API_KEY: ${MODELFORGE_OPERATOR_API_KEY:-}
MODELFORGE_BACKUP_ENCRYPTION_KEY: ${MODELFORGE_BACKUP_ENCRYPTION_KEY:?generate a backup encryption key; without it no backup can be produced}
MODELFORGE_CORS_ORIGINS: ${MODELFORGE_CORS_ORIGINS:?set the exact console origin; a wildcard is refused in production}
# Remote model code is never executed in production. Stated here as well as enforced at
diff --git a/docker-compose.rc-ingress.yml b/docker-compose.rc-ingress.yml
new file mode 100644
index 0000000..4db1034
--- /dev/null
+++ b/docker-compose.rc-ingress.yml
@@ -0,0 +1,26 @@
+# Isolated candidate ingress. Use only with a modelforge-validation-* Compose project and the
+# production overlay after scripts/rc_isolation_preflight.py approves the rendered plan.
+# The backend network is project-private; an existing reverse proxy may be attached
+# only to the project-scoped ingress network after the stack passes its preflight.
+services:
+ postgres:
+ networks:
+ - backend
+ redis:
+ networks:
+ - backend
+ migrate:
+ networks:
+ - backend
+ api:
+ networks:
+ - backend
+ - ingress
+ web:
+ networks:
+ - ingress
+
+networks:
+ backend:
+ internal: true
+ ingress: {}
diff --git a/docker-compose.validation-tls.yml b/docker-compose.validation-tls.yml
new file mode 100644
index 0000000..19a2059
--- /dev/null
+++ b/docker-compose.validation-tls.yml
@@ -0,0 +1,33 @@
+# Dedicated candidate HTTPS entry point. Render with the base, production, and
+# candidate ingress overlays, then run the isolation preflight before startup.
+# The key files must be copies confined to the candidate root, never an NPM bind.
+services:
+ tls:
+ image: caddy:2.10-alpine@sha256:4c6e91c6ed0e2fa03efd5b44747b625fec79bc9cd06ac5235a779726618e530d
+ command: ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]
+ environment:
+ XDG_CONFIG_HOME: /config
+ XDG_DATA_HOME: /data
+ volumes:
+ - ./deploy/validation/Caddyfile:/etc/caddy/Caddyfile:ro
+ - ${MODELFORGE_CANDIDATE_TLS_DIR:?set a certificate copy directory inside the candidate root}/fullchain.pem:/run/tls/fullchain.pem:ro
+ - ${MODELFORGE_CANDIDATE_TLS_DIR:?set a certificate copy directory inside the candidate root}/privkey.pem:/run/tls/privkey.pem:ro
+ networks:
+ - ingress
+ ports:
+ - "0.0.0.0:18444:8443"
+ read_only: true
+ tmpfs:
+ - /config:size=4m,mode=0700
+ - /data:size=4m,mode=0700
+ - /tmp:size=4m,mode=1777
+ cap_drop:
+ - ALL
+ security_opt:
+ - no-new-privileges:true
+ depends_on:
+ api:
+ condition: service_healthy
+ web:
+ condition: service_started
+ restart: unless-stopped
diff --git a/docker-compose.yml b/docker-compose.yml
index 6c3c77d..6526938 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,6 +5,10 @@ services:
POSTGRES_DB: ${MODELFORGE_POSTGRES_DB:-modelforge}
POSTGRES_USER: ${MODELFORGE_POSTGRES_ADMIN_USER:-postgres}
POSTGRES_PASSWORD: ${MODELFORGE_POSTGRES_ADMIN_PASSWORD:?set a bootstrap-only PostgreSQL admin password}
+ # Keep the ModelForge names available to the explicit existing-cluster provisioning script.
+ # POSTGRES_USER/PASSWORD are consumed by the upstream image; psql's \getenv reads these.
+ MODELFORGE_POSTGRES_ADMIN_USER: ${MODELFORGE_POSTGRES_ADMIN_USER:-postgres}
+ MODELFORGE_POSTGRES_ADMIN_PASSWORD: ${MODELFORGE_POSTGRES_ADMIN_PASSWORD:?set a bootstrap-only PostgreSQL admin password}
MODELFORGE_MIGRATION_DB_PASSWORD: ${MODELFORGE_MIGRATION_DB_PASSWORD:?set the non-superuser migration role password}
MODELFORGE_RUNTIME_DB_PASSWORD: ${MODELFORGE_RUNTIME_DB_PASSWORD:?set the non-owner runtime role password}
volumes:
@@ -37,7 +41,8 @@ services:
api:
build:
- context: ./backend
+ context: .
+ dockerfile: backend/Dockerfile
args:
# Build identity, so any image built from this file can say where it came from. An argument
# that is never passed stays empty and is reported as null rather than guessed.
@@ -57,7 +62,11 @@ services:
MODELFORGE_NODE_LIVENESS_MONITOR_ENABLED: "true"
MODELFORGE_REGISTRY_SEED_ON_STARTUP: "true"
MODELFORGE_OPERATOR_API_KEY: ${MODELFORGE_OPERATOR_API_KEY:-}
+ MODELFORGE_OPERATOR_IDENTITIES: ${MODELFORGE_OPERATOR_IDENTITIES:-}
+ MODELFORGE_OPERATOR_PASSWORD_HASH: ${MODELFORGE_OPERATOR_PASSWORD_HASH:-}
MODELFORGE_HF_TOKEN: ${MODELFORGE_HF_TOKEN:-}
+ MODELFORGE_SECRET_SLOT_ROOT_KEY: ${MODELFORGE_SECRET_SLOT_ROOT_KEY:-}
+ MODELFORGE_SECRET_SLOT_ROOT_KEY_ID: ${MODELFORGE_SECRET_SLOT_ROOT_KEY_ID:-hf-control-plane-1}
MODELFORGE_NODE_STALE_AFTER_SECONDS: ${MODELFORGE_NODE_STALE_AFTER_SECONDS:-30}
MODELFORGE_NODE_OFFLINE_AFTER_SECONDS: ${MODELFORGE_NODE_OFFLINE_AFTER_SECONDS:-90}
MODELFORGE_LIVENESS_POLL_INTERVAL_SECONDS: ${MODELFORGE_LIVENESS_POLL_INTERVAL_SECONDS:-5}
@@ -65,6 +74,17 @@ services:
MODELFORGE_LIFECYCLE_RECONCILIATION_ENABLED: ${MODELFORGE_LIFECYCLE_RECONCILIATION_ENABLED:-true}
MODELFORGE_MIGRATION_RECONCILIATION_ENABLED: ${MODELFORGE_MIGRATION_RECONCILIATION_ENABLED:-true}
MODELFORGE_OBSERVABILITY_MONITOR_ENABLED: ${MODELFORGE_OBSERVABILITY_MONITOR_ENABLED:-true}
+ MODELFORGE_ALERT_NOTIFICATIONS_ENABLED: ${MODELFORGE_ALERT_NOTIFICATIONS_ENABLED:-false}
+ MODELFORGE_ALERT_WEBHOOK_URL: ${MODELFORGE_ALERT_WEBHOOK_URL:-}
+ MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS: ${MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS:-}
+ MODELFORGE_ALERT_WEBHOOK_MIN_SEVERITY: ${MODELFORGE_ALERT_WEBHOOK_MIN_SEVERITY:-WARNING}
+ MODELFORGE_ALERT_NOTIFICATION_TIMEOUT_SECONDS: ${MODELFORGE_ALERT_NOTIFICATION_TIMEOUT_SECONDS:-5.0}
+ MODELFORGE_ALERT_NOTIFICATION_MAX_ATTEMPTS: ${MODELFORGE_ALERT_NOTIFICATION_MAX_ATTEMPTS:-8}
+ MODELFORGE_ALERT_NOTIFICATION_BATCH_SIZE: ${MODELFORGE_ALERT_NOTIFICATION_BATCH_SIZE:-20}
+ MODELFORGE_ALERT_NOTIFICATION_LEASE_SECONDS: ${MODELFORGE_ALERT_NOTIFICATION_LEASE_SECONDS:-60}
+ MODELFORGE_ALERT_NOTIFICATION_RETRY_BASE_SECONDS: ${MODELFORGE_ALERT_NOTIFICATION_RETRY_BASE_SECONDS:-15}
+ MODELFORGE_MODEL_UPDATE_MONITOR_ENABLED: ${MODELFORGE_MODEL_UPDATE_MONITOR_ENABLED:-true}
+ MODELFORGE_MODEL_UPDATE_CHECK_INTERVAL_SECONDS: ${MODELFORGE_MODEL_UPDATE_CHECK_INTERVAL_SECONDS:-86400}
MODELFORGE_RECOVERY_RECONCILIATION_ENABLED: ${MODELFORGE_RECOVERY_RECONCILIATION_ENABLED:-true}
MODELFORGE_SOURCE_COMMIT: ${MODELFORGE_SOURCE_COMMIT:-}
MODELFORGE_SOURCE_REFERENCE: ${MODELFORGE_SOURCE_REFERENCE:-}
@@ -105,7 +125,8 @@ services:
migrate:
build:
- context: ./backend
+ context: .
+ dockerfile: backend/Dockerfile
args:
MODELFORGE_VERSION: ${MODELFORGE_VERSION:-0.0.0}
MODELFORGE_COMMIT: ${MODELFORGE_COMMIT:-}
diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md
index f5604bb..6546745 100644
--- a/docs/COMPATIBILITY.md
+++ b/docs/COMPATIBILITY.md
@@ -1,32 +1,31 @@
-# Compatibility matrix — v1.2.1 plus unreleased RC hardening
+# Compatibility matrix — v1.2.2 source/tag contract
Nothing here is implied. The executable contract lives in
`backend/src/modelforge_api/domain/release.py`; startup and release tests enforce it.
-> RC development note: the unreleased product branch now targets additive schema
-> `20260830_0024`, accepts `20260828_0022` and auth-only `20260830_0023` as upgrade sources, and refuses to serve until the
-> audit-chain migration completes. Deployed stable v1.2.1 remains frozen on `20260828_0022`; the
-> released matrix below continues to describe that production artifact until the RC is tagged.
-> Selecting and stamping the post-0024 product version is an explicit pending release tranche; the
-> temporary repository `1.2.1` constant must not be presented as a released schema-0024 artifact.
+> Publication and deployment note: this file records the v1.2.2 source/tag compatibility contract.
+> At the 2026-09-25 reconciliation checkpoint, Gitea's newest published Release object was v1.2.1;
+> v1.2.2 existed as a tag. The observed production record was v1.2.3/schema-0032. The RC source
+> is version 1.3.0-rc.1 with schema target 20260924_0039; it must not be treated as deployed until
+> its own gates pass.
## Product and database
| Contract | Value |
| --- | --- |
-| Version | `1.2.1` |
+| Version | `1.2.2` |
| Channel | `stable` |
| Version source | repository `VERSION` |
-| Runtime schema | `20260830_0023` |
-| Direct upgrade-source schema | `20260827_0021`, `20260828_0022` |
+| Runtime schema | `20260830_0024` |
+| Direct upgrade-source schema | `20260827_0021`, `20260828_0022`, `20260830_0023` |
| Minimum direct upgrade | `v1.0.0` |
| PostgreSQL | 16+; Compose ships 17 |
| SQLite | unit tests only; **NOT SUPPORTED FOR PRODUCTION** |
-The hardened candidate serves only on schema 0023. The upgrade tool accepts 0021 or the production
-0022 head, applies the intervening revisions, validates all existing node credential scopes, and
-reaches 0023 before the API starts. PostgreSQL is the only production database contract; SQLite is
-used only to exercise DDL mechanics in isolated tests.
+v1.2.2 serves only on schema 0024. The upgrade tool accepts 0021, the production 0022 head or the
+intermediate auth-only 0023 head, applies the intervening revisions, validates existing node
+credential scopes and establishes the PostgreSQL-owned audit boundary before the API starts.
+PostgreSQL is the only production database contract; SQLite is used only for isolated tests.
Compatibility answers remain typed and fail closed:
@@ -45,19 +44,22 @@ Compatibility answers remain typed and fail closed:
| Accepted protocols | 1 |
The Node Agent and Runtime Worker remain protocol-compatible with v1.0.0. Release images should
-still be upgraded together so their product and provenance identity is v1.2.1.
+still be upgraded together so their product and provenance identity is v1.2.2. Because the Runtime
+Worker dependency update is behavioral, existing profiles require fresh LAB evidence before
+production promotion.
## Upgrade matrix
| From | Schema | To | Supported |
| --- | --- | --- | --- |
-| `v1.0.0` | `20260827_0021` | current candidate / `20260830_0023` | yes; backup and rehearsal required |
-| `v1.1.0` | `20260828_0022` | current candidate / `20260830_0023` | yes; scope validation migration |
-| `v1.1.1` | `20260828_0022` | current candidate / `20260830_0023` | yes; scope validation migration |
-| `v1.2.0` | `20260828_0022` | current candidate / `20260830_0023` | yes; scope validation migration |
-| `v1.2.1` | `20260828_0022` | current candidate / `20260830_0023` | yes; scope validation migration |
-| current candidate | `20260830_0023` | current candidate | yes; already current |
-| older than `v1.0.0` | any | `v1.2.1` | no direct upgrade |
+| `v1.0.0` | `20260827_0021` | `v1.2.2` / `20260830_0024` | yes; backup and rehearsal required |
+| `v1.1.0` | `20260828_0022` | `v1.2.2` / `20260830_0024` | yes; scope and audit migration |
+| `v1.1.1` | `20260828_0022` | `v1.2.2` / `20260830_0024` | yes; scope and audit migration |
+| `v1.2.0` | `20260828_0022` | `v1.2.2` / `20260830_0024` | yes; scope and audit migration |
+| `v1.2.1` | `20260828_0022` | `v1.2.2` / `20260830_0024` | yes; scope and audit migration |
+| intermediate RC | `20260830_0023` | `v1.2.2` / `20260830_0024` | yes; audit-boundary migration |
+| `v1.2.2` | `20260830_0024` | `v1.2.2` | yes; already current |
+| older than `v1.0.0` | any | `v1.2.2` | no direct upgrade |
Because 0022 introduces terminal node and decommission-operation state, application-only rollback
to v1.0.0 is unsupported. Restore the verified pre-upgrade backup. Alembic downgrade may be used in
@@ -95,6 +97,13 @@ PATCH under SemVer.
The console image is the one artifact whose *behaviour* differs, because the previous images could
not reach a non-localhost API at all. That is a defect being repaired, not a capability being added.
+## Why v1.2.2 is a PATCH release
+
+v1.2.2 hardens public packaging, dependency and database-authority boundaries without changing the
+capability API or agent protocol. The Runtime Worker dependency set changes and is therefore
+classified `behavioral` operationally: it remains a patch release, but production promotion is
+blocked until exact LAB probes prove compatibility, output shape and unload/reclaim behavior.
+
## Why v1.2.0 is a MINOR release
v1.2.0 adds substantive, backward-compatible product capability at the console layer: a command
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index 31f6e88..3264af4 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -1,6 +1,6 @@
# Configuration reference
-Generated from the typed settings by `scripts/generate_configuration_docs.py` for ITWorx ModelForge 1.2.1. Every setting the control plane reads appears here; a setting added without documentation fails the build.
+Generated from the typed settings by `scripts/generate_configuration_docs.py` for ITWorx ModelForge 1.3.0-rc.1. Every setting the control plane reads appears here; a setting added without documentation fails the build.
All settings are environment variables with the `MODELFORGE_` prefix, read from the process environment or from `.env`.
@@ -10,7 +10,6 @@ All settings are environment variables with the `MODELFORGE_` prefix, read from
- `MODELFORGE_DATABASE_URL` — SQLAlchemy URL for the API's non-owner modelforge_runtime role. It may read the audit trail and execute the canonical append function, but cannot mutate audit tables directly.
- `MODELFORGE_REDIS_URL` — Redis URL for transient request payloads and queues.
-- `MODELFORGE_OPERATOR_API_KEY` — Operator API key guarding every admin route. Generate at least 32 random characters; ModelForge never mints one for you.
- `MODELFORGE_BACKUP_ENCRYPTION_KEY` — Base64 AES-256 key for backup encryption. Without it no backup can be produced, and without the same key no backup can be restored — store it outside this deployment.
Production additionally refuses: a well-known development database password, a wildcard CORS origin, remote model code execution, an unwritable storage root, an unsupported schema revision, a PostgreSQL major below 16, and three policy combinations that cannot all hold at once.
@@ -34,13 +33,17 @@ Losing the backup encryption key makes every existing backup unrecoverable. It i
## Sensitivity
-5 settings are credentials. They are never logged, never written to a release artefact and never echoed in an error response:
+9 settings are credentials. They are never logged, never written to a release artefact and never echoed in an error response:
- `MODELFORGE_DATABASE_URL`
- `MODELFORGE_MIGRATION_DATABASE_URL`
+- `MODELFORGE_OPERATOR_IDENTITIES`
+- `MODELFORGE_OPERATOR_PASSWORD_HASH`
- `MODELFORGE_OPERATOR_API_KEY`
- `MODELFORGE_HF_TOKEN`
+- `MODELFORGE_SECRET_SLOT_ROOT_KEY`
- `MODELFORGE_BACKUP_ENCRYPTION_KEY`
+- `MODELFORGE_ALERT_WEBHOOK_URL`
## Every setting
@@ -56,8 +59,12 @@ Losing the backup encryption key makes every existing backup unrecoverable. It i
| `MODELFORGE_MIGRATION_DATABASE_URL` | secret, optional | — | no | secret | SQLAlchemy URL for the non-superuser modelforge schema-owner role. Set only in the one-shot migration process; production API startup refuses when this secret is present. |
| `MODELFORGE_REDIS_URL` | string | `redis://localhost:6379/0` | yes | internal | Redis URL for transient request payloads and queues. |
| `MODELFORGE_CORS_ORIGINS` | string | `http://localhost:3000` | no | public | Comma-separated exact origins allowed to call the API from a browser. A wildcard is refused in production because requests are credentialed. |
-| `MODELFORGE_OPERATOR_API_KEY` | secret, optional | — | yes | secret | Operator API key guarding every admin route. Generate at least 32 random characters; ModelForge never mints one for you. |
-| `MODELFORGE_HF_TOKEN` | secret, optional | — | no | secret | Optional Hugging Face token, used only for acquiring gated repositories. It is never passed to a runtime and never leaves the control plane. |
+| `MODELFORGE_OPERATOR_IDENTITIES` | secret, optional | — | no | secret | Optional JSON array of named human identities with role, credential ID, SHA-256 key digest and timezone-aware expiry. Provision with scripts/operator_identity.py. Replace this deployment setting and recreate the API to rotate or revoke access. |
+| `MODELFORGE_OPERATOR_PASSWORD_HASH` | secret, optional | — | no | secret | Optional salted scrypt verifier generated by scripts/operator_password.py. Enables password login with 15-minute sessions; requires the separate strong operator API key. Changing either value invalidates password sessions. |
+| `MODELFORGE_OPERATOR_API_KEY` | secret, optional | — | no | secret | Legacy bootstrap administrator key. Generate at least 32 random characters. Production requires this key or an unexpired named administrator in operator_identities. |
+| `MODELFORGE_HF_TOKEN` | secret, optional | — | no | secret | Optional deployment Hugging Face token for control-plane metadata. Compose may also provision it independently to node-agent downloads; API secret activation does not update nodes. |
+| `MODELFORGE_SECRET_SLOT_ROOT_KEY` | secret, optional | — | no | secret | External base64 32-byte AES-256 key for the managed Hugging Face control-plane slot. Keep independent of backup and operator keys; retain it externally for historical restore. |
+| `MODELFORGE_SECRET_SLOT_ROOT_KEY_ID` | string | `hf-control-plane-1` | no | public | External key identifier bound to encrypted HF slot versions. Replacing a key does not rewrap history. |
| `MODELFORGE_BACKUP_ENCRYPTION_KEY` | secret, optional | — | yes | secret | Base64 AES-256 key for backup encryption. Without it no backup can be produced, and without the same key no backup can be restored — store it outside this deployment. |
| `MODELFORGE_BACKUP_ENCRYPTION_KEY_ID` | string | `modelforge-backup-key-1` | no | public | Identifier recorded in each backup manifest so a restore can name the key it needs. |
| `MODELFORGE_HF_HOME` | string | `/data/hf-cache` | no | public | Hugging Face cache root inside the container. |
@@ -108,6 +115,17 @@ Losing the backup encryption key makes every existing backup unrecoverable. It i
| `MODELFORGE_MIGRATION_RECONCILIATION_ENABLED` | boolean | `false` | no | public | Report interrupted migration cutovers at startup. They are never auto-resolved: external alias truth cannot be inferred after a crash. |
| `MODELFORGE_OBSERVABILITY_MONITOR_ENABLED` | boolean | `false` | no | public | Run SLO and alert evaluation in this process. |
| `MODELFORGE_OBSERVABILITY_POLL_INTERVAL_SECONDS` | integer | `60` | no | public | Interval between observability evaluation passes. |
+| `MODELFORGE_ALERT_NOTIFICATIONS_ENABLED` | boolean | `false` | no | public | Queue and dispatch the deployment-managed operational alert webhook. |
+| `MODELFORGE_ALERT_WEBHOOK_URL` | secret, optional | — | no | secret | Deployment-managed HTTPS notification destination. Its exact hostname must also appear in alert_webhook_allowed_hosts; the URL is never returned by the API. |
+| `MODELFORGE_ALERT_WEBHOOK_ALLOWED_HOSTS` | string | — | no | public | Comma-separated exact destination host allowlist for the alert webhook. Wildcards and redirects are not accepted. |
+| `MODELFORGE_ALERT_WEBHOOK_MIN_SEVERITY` | `INFO`, `WARNING`, `CRITICAL` | `WARNING` | no | public | Minimum alert severity sent to the single webhook route: INFO, WARNING or CRITICAL. |
+| `MODELFORGE_ALERT_NOTIFICATION_TIMEOUT_SECONDS` | number | `5.0` | no | public | Total timeout for one webhook delivery. |
+| `MODELFORGE_ALERT_NOTIFICATION_MAX_ATTEMPTS` | integer | `8` | no | public | Maximum webhook attempts before a delivery fails permanently. |
+| `MODELFORGE_ALERT_NOTIFICATION_BATCH_SIZE` | integer | `20` | no | public | Maximum outbox rows claimed by one notification dispatch pass. |
+| `MODELFORGE_ALERT_NOTIFICATION_LEASE_SECONDS` | integer | `60` | no | public | Lease duration before an interrupted delivery can be reclaimed. |
+| `MODELFORGE_ALERT_NOTIFICATION_RETRY_BASE_SECONDS` | integer | `15` | no | public | Base delay for bounded exponential webhook retries. |
+| `MODELFORGE_MODEL_UPDATE_MONITOR_ENABLED` | boolean | `true` | no | public | In production, check explicitly followed model references for changed revisions. Metadata only; never downloads, tests or promotes a model. |
+| `MODELFORGE_MODEL_UPDATE_CHECK_INTERVAL_SECONDS` | integer | `86400` | no | public | Minimum interval between automatic checks of each followed model reference; default one day. Each API process checks at most one due reference per minute. |
| `MODELFORGE_RECOVERY_RECONCILIATION_ENABLED` | boolean | `false` | no | public | Reconcile interrupted backups and restores at startup. |
| `MODELFORGE_BACKUP_PG_DUMP_PATH` | string | `pg_dump` | no | public | pg_dump executable. Must match the server major version. |
| `MODELFORGE_BACKUP_PG_RESTORE_PATH` | string | `pg_restore` | no | public | pg_restore executable. |
@@ -172,4 +190,4 @@ Read by Compose, the Node Agent and the Runtime Worker rather than by the contro
## Restarts
-Every setting is read at process start. Changing any of them requires restarting the control plane; none is re-read from the environment while the process is running.
+Deployment environment settings are read at process start; changing an environment value requires a restart. The Console also supports a closed set of numeric serving overrides which take effect for new requests after preview and explicit apply. See the [configuration workflow](operations/CONFIGURATION_WORKFLOW.md) for scope, acknowledgement, rollback and secret-safe transfer. Other settings remain deployment-managed.
diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md
index adbd0b3..4a6fa00 100644
--- a/docs/INSTALLATION.md
+++ b/docs/INSTALLATION.md
@@ -1,7 +1,17 @@
-# Installing ITWorx ModelForge v1.2.1
+# Installing ITWorx ModelForge — historical v1.2.3 procedure
-This is the whole installation. If you have to do something that is not written here, that is a
-defect — please report it rather than working around it.
+At the 2026-09-23 reconciliation checkpoint, the observed production installation ran v1.2.3 at
+schema `20260909_0032` and no v1.2.3 Gitea Release object existed. The current `1.3.0-rc.1` source targets
+`20260924_0039`; its qualification is established by an exact published release record, not this
+historical guide. The steps below preserve the earlier bounded fresh-install
+procedure; do not treat them as validation of the current RC archive, production upgrade, or
+rollback. Select exact source and image identities and complete the [RC acceptance gates](operations/RC_ACCEPTANCE.md)
+before using a candidate package. See the [current-state reconciliation](quality/CURRENT_STATE_RECONCILIATION_2026-09-24.md)
+and [upgrade boundary](UPGRADE.md#source-and-schema-boundary).
+
+
+The final RC installation procedure must be checked against its exact archive and isolated
+clean-install rehearsal; unresolved steps are release blockers.
The steps below are release-gated against a fresh project with isolated volumes, newly generated
test secrets and an empty PostgreSQL database.
@@ -74,14 +84,25 @@ MODELFORGE_OPERATOR_API_KEY=
MODELFORGE_BACKUP_ENCRYPTION_KEY=
MODELFORGE_CORS_ORIGINS=https://modelforge.example.internal
VITE_API_BASE_URL=https://modelforge.example.internal:8000
-MODELFORGE_VERSION=1.2.1
-MODELFORGE_API_IMAGE=modelforge-api:1.2.1
-MODELFORGE_WEB_IMAGE=modelforge-web:1.2.1
+MODELFORGE_VERSION=1.2.3
+MODELFORGE_API_IMAGE=modelforge-api:1.2.3
+MODELFORGE_WEB_IMAGE=modelforge-web:1.2.3
```
Every setting is documented in [CONFIGURATION.md](CONFIGURATION.md), which is generated from the
typed settings — it cannot drift from what the code actually reads.
+For browser password sign-in, run `python scripts/operator_password.py` and enter your chosen
+password at the hidden prompt. Put only the generated hash in `.env` as
+`MODELFORGE_OPERATOR_PASSWORD_HASH=''`. Keep the single quotes: they prevent
+Compose from interpreting the dollar signs in the hash. Keep the separate generated operator
+API key above for session signing and recovery. Never use your password as that API key.
+Recreate the API after changing either setting. Password sign-in returns a temporary session
+valid for at most 15 minutes; locking or reloading the console clears it. Rotating the password
+hash or API key invalidates existing password sessions. Login attempts require Redis and are
+limited across API processes to five per socket peer and thirty globally per minute.
+The console defaults to **Password**; **Advanced: API key recovery** retains direct key access.
+
To move a port, set `MODELFORGE_API_PUBLISHED_PORT`, `MODELFORGE_WEB_PORT`,
`MODELFORGE_POSTGRES_PORT` or `MODELFORGE_REDIS_PORT`. You never have to edit a Compose file.
@@ -119,7 +140,7 @@ path, but never derives a deployable image from a floating `latest` tag.
To stamp build identity into the images so a running container can say where it came from:
```bash
-MODELFORGE_VERSION=1.2.1 \
+MODELFORGE_VERSION=1.2.3 \
MODELFORGE_COMMIT=$(git rev-parse HEAD) \
MODELFORGE_BUILT_AT=$(date -u +%FT%TZ) \
docker compose -f docker-compose.yml -f docker-compose.production.yml up -d --build
diff --git a/docs/NODE_AGENT.md b/docs/NODE_AGENT.md
index 8163a10..827bf61 100644
--- a/docs/NODE_AGENT.md
+++ b/docs/NODE_AGENT.md
@@ -19,6 +19,34 @@ different site work without inbound exposure.
## Enrolling
+For the v1.3.0 RC, an administrator can use **Console → Hardware → Enroll node**. The wizard offers
+5-minute, 15-minute, 1-hour, and 24-hour token expiry, and locks the node's API origin to the
+Console's current API origin. RC enrollment always sets `production_eligible=false`. The one-time
+token is held only in the open wizard's component memory; closing the wizard, checking readiness,
+revoking the unused token, or reaching its expiry clears the reveal. The generated `.env` contains
+only non-secret settings. Paste the start command into a Bash shell from the node-agent repository;
+it disables shell tracing, reads the token without echo, uses a temporary mode-0600 env file, and
+removes that file on exit, including error and signal paths.
+
+Use **Check node readiness** after starting the agent. Readiness is tied to the exact node ID returned
+by the enrollment record; it checks connection, inventory age, protocol compatibility, agent health,
+and telemetry age and includes diagnostic next steps. After enrollment is consumed, use the wizard's
+container recreation command to remove the one-time token from the running container environment
+while preserving its persisted identity and credential. A node is not ready merely because it
+connected.
+
+The API flow below remains available for automation and existing installations.
+
+The RC wizard defaults to the local four-image release bundle. Verify the release `SHA256SUMS`, load
+`modelforge-1.3.0-rc.1-images.tar` on the node with `docker image load`, and paste
+`images.modelforge-node-agent` from the matching release manifest. The wizard requires the exact
+`modelforge-node-agent:1.3.0-rc.1` tag, RootFS layer list, architecture, and all recorded OCI labels.
+It ignores the build-host image/config ID because Docker may rewrite it during `docker load`; before
+Compose starts it captures the local image ID and verifies the created container uses that same ID.
+Start and cleanup use `--no-build --pull never`, and the generated configuration sets the local tag
+explicitly. A registry image pinned to `@sha256:` remains available as an alternate mode.
+The source-build Compose workflow below is separate and intended for development.
+
1. An operator mints a single-use token:
```bash
@@ -79,9 +107,13 @@ bounded diagnostic check that never enrolls is available as:
```bash
docker run --rm --gpus all --network none \
-e MODELFORGE_AGENT_ACCELERATOR_MODE=nvidia \
- modelforge-node-agent:1.1.1 modelforge-node-agent --preflight
+ modelforge-node-agent:1.1.1 python3 -m modelforge_node_agent.main --preflight
```
+The chiselled runtime intentionally contains no shell or packaging-script directory. Invoke the
+reviewed Python module directly for this bounded diagnostic, just as the image's default command
+does.
+
## Identity
A node's identity is persisted and survives restarts. If a node reappears as a *new* identity, its
diff --git a/docs/PROJECT_INTEGRATION.md b/docs/PROJECT_INTEGRATION.md
index b591b14..53e1e34 100644
--- a/docs/PROJECT_INTEGRATION.md
+++ b/docs/PROJECT_INTEGRATION.md
@@ -18,9 +18,12 @@ capability key rag.embedding@1
If your integration needs a model name, a revision, a node hostname or an artifact path, something
has gone wrong — those are ModelForge's business, not yours.
+For guided setup, use the [Projects workflow](operations/PROJECT_ONBOARDING.md). Official lightweight
+[Python and TypeScript clients](../clients/README.md) ship in the release archive.
+
## Getting a credential
-An operator creates a service client scoped to your project and the capabilities you may call:
+An administrator creates a service client scoped to your project and the capabilities you may call:
```bash
curl -s -X POST -H "X-ModelForge-Admin-Token: $KEY" -H "Content-Type: application/json" \
diff --git a/docs/PUBLICATION.md b/docs/PUBLICATION.md
index 125100f..fbb8af4 100644
--- a/docs/PUBLICATION.md
+++ b/docs/PUBLICATION.md
@@ -29,7 +29,19 @@ manifest. The canonical repository and all of its Git history remain private.
automatically on persistent private runners. The exporter therefore removes the automatic
`pull_request` trigger from the public copy of `managed-validation.yml` and leaves only explicit
owner dispatch. Keep public branch protection independent of private-runner status checks until
- a public-safe CI runner is configured.
+ a public-safe CI runner is configured on a network segment that cannot reach private
+ infrastructure.
+
+## Current publication state
+
+The parentless public repository is live at `https://git.example.com/Jens/ModelForge-Public`. Its
+protected `main` branch started with one root commit, and annotated tag `v1.2.2` points to that public
+root. Subsequent source snapshots are published from the private canonical branch only after the
+exact public candidate passes server acceptance, manifest validation and secret scanning. The public
+workflow remains owner-dispatch-only: the available server runners share the
+private LAN, so container isolation alone is insufficient for executing arbitrary fork code. A
+future automatic PR runner must also have enforced private-network egress denial; it must never use
+the private validation runner, deploy runner or host Docker socket.
## License decision
diff --git a/docs/RELEASE_NOTES_v1.2.2.md b/docs/RELEASE_NOTES_v1.2.2.md
index 5129e5e..f012ae4 100644
--- a/docs/RELEASE_NOTES_v1.2.2.md
+++ b/docs/RELEASE_NOTES_v1.2.2.md
@@ -9,6 +9,9 @@ evidence before promotion; no production capability is changed automatically.
- The Runtime Worker now upgrades its digest-pinned Ubuntu base during the image build.
- Build-only Linux kernel headers are removed after dependency installation instead of remaining in
the running Runtime Worker image.
+- Explicit shipping-image validation rebuilds all four images without Docker layer cache, refreshes
+ the Node Agent's digest-pinned Python base, upgrades fixed Runtime Worker dependencies, and removes
+ pip/setuptools/wheel after the final dependency check.
- Pillow, protobuf, SentencePiece, Sentence Transformers, Transformers and inherited Python
security dependencies are pinned to reviewed fixed versions.
- The public-candidate acceptance gate builds and scans all four images, binds each Trivy report to
@@ -44,3 +47,23 @@ The control plane and Node Agent change is `transparent`. The Runtime Worker inf
update is `behavioral`: exact LAB probes, output-shape checks and unload/reclaim evidence are
required before an existing runtime profile can be promoted. Embedding-space identities and project
indexes are not reused or changed automatically.
+
+## Post-publication operational verification
+
+The exact Runtime Worker built from `b8755394117affad0178a79a627cf5574d3c8eef` passed a fresh
+GPU Node LAB probe on 2026-09-01, including offline load, finite normalized `[1, 1024]` output and exact
+VRAM return to baseline after unload. Production remained on v1.2.1 pending the separate backup,
+migration and explicit deployment gates. See
+[`quality/V1_2_2_LAB_VERIFICATION.md`](quality/V1_2_2_LAB_VERIFICATION.md).
+
+## Known limitations
+
+- Production upgrade from schema 0022 to 0024 requires downtime, a verified recovery point and
+ separate migration-owner/runtime-role provisioning; publication never performs it automatically.
+- Public pull-request CI remains disabled until an isolated runner can enforce denial of private
+ network egress for untrusted fork code.
+- ExampleRAG's ModelForge shadow candidate is not promotion-eligible after two critical retrieval
+ regressions; the existing production alias and vector space remain unchanged.
+- ExampleVision owner-photo and ASR natural-speech qualification still require owner-provided ground
+ truth. Catalog images and synthetic/library audio are not accepted as substitutes.
+- Point-in-time database recovery remains `NOT_SUPPORTED`.
diff --git a/docs/RELEASE_NOTES_v1.2.3.md b/docs/RELEASE_NOTES_v1.2.3.md
new file mode 100644
index 0000000..c3f3902
--- /dev/null
+++ b/docs/RELEASE_NOTES_v1.2.3.md
@@ -0,0 +1,36 @@
+# ModelForge v1.2.3 — operator password sign-in
+
+The console now opens with a Password field. A successful password exchange creates a signed,
+memory-only session lasting at most 15 minutes. Passwords are verified against a salted scrypt
+hash; the existing generated operator API key remains separate for signing and recovery.
+
+Existing API-key clients remain compatible. Set `MODELFORGE_OPERATOR_PASSWORD_HASH` using
+`python scripts/operator_password.py`, single-quote the hash in the Compose environment file,
+and recreate the API. See [installation](INSTALLATION.md). An unset hash preserves existing
+API-key access. Password login fails closed when Redis is unavailable. Attempts are limited
+across API processes, and forwarded headers do not change the peer used by the limiter.
+
+These release notes describe the published hotfix built from v1.2.2. Database schema `20260830_0024`, agent protocol 1, runtime
+dependencies, model artifacts, capability bindings and promotion evidence are unchanged.
+Upgrade classification: transparent. No model or inference runtime promotion is included.
+
+Validation before release: targeted backend authorization, password and release-contract tests;
+frontend full test suite and production build; independent code review; isolated Redis 7 atomic
+parallel attempt, global limit and TTL checks. Deployment and exact-image acceptance must be
+recorded separately; source validation is not evidence of a successful production login.
+
+## Known limitations
+
+Password login requires configured Redis and a password hash; there is no self-service password
+reset. Reloading the browser ends its session. Peers behind one reverse proxy share its attempt
+budget. Runtime and GPU qualification from v1.2.2 remains applicable only to unchanged exact
+runtime images; this hotfix does not qualify or promote new inference-worker images.
+
+
+## RC integration boundary
+
+The development branch also incorporates this hotfix while retaining its newer schema
+`20260908_0028`, named/managed identities and RC features. That source is not the published
+schema-0024 hotfix image and is not qualified by its release evidence. Password sessions on the
+RC branch pass the same identity, role and expiry checks as other operator sessions. Consult
+[the upgrade source boundary](UPGRADE.md#source-and-schema-boundary) before deploying either build.
diff --git a/docs/RELEASE_NOTES_v1.3.0-rc.1.md b/docs/RELEASE_NOTES_v1.3.0-rc.1.md
new file mode 100644
index 0000000..36003ef
--- /dev/null
+++ b/docs/RELEASE_NOTES_v1.3.0-rc.1.md
@@ -0,0 +1,28 @@
+# ModelForge v1.3.0-rc.1 — qualification candidate
+
+`v1.3.0-rc.1` identifies the release-candidate source and artifact set. This file belongs in its
+archive; the annotated tag, Gitea prerelease, exact image identities and retained qualification
+evidence must be verified from the published release record. A prerelease does not authorize a
+production deployment.
+
+The current source targets Alembic schema `20260924_0039` and Node Agent protocol 1. The final
+candidate must contain four first-class images: API, Web, Node Agent and Runtime Worker. P15A must
+bind their exact OCI labels, content-addressable RootFS layer sets, SBOM, provenance, source/archive
+hashes and exact-image security reports into one retained transport bundle. P12–P14 must load that
+bundle without rebuilding it.
+
+The user-visible ModelForge changes are the already merged project-first model management,
+onboarding, model library, immutable dimension pinning and guarded replacement/rollback flows. The
+candidate reports the `rc` release channel so its API health and system metadata cannot be confused
+with the stable production line.
+
+## Known limitations
+
+- The `v1.3.0-rc.1` tag and Gitea prerelease are permitted only after every mandatory readiness
+ row is verified against the exact artifact set; the version string does not prove that gate.
+- The release record must retain proof of the isolated RC stack, real ExampleRAG consumer path,
+ project replacement/rollback, PostgreSQL 0039 recovery, fault journeys and 60-minute soak.
+- A real GPU inference gate requires an available RTX 4080 SUPER window; unmanaged workloads are
+ never stopped or changed by this candidate process.
+- Observed production remains v1.2.3/schema-0032 unless a separate stable promotion procedure
+ explicitly authorizes a change.
diff --git a/docs/SECURITY.md b/docs/SECURITY.md
index 936a1f1..eca7814 100644
--- a/docs/SECURITY.md
+++ b/docs/SECURITY.md
@@ -57,8 +57,9 @@ nginx — never a development server.
Content-Security-Policy with `default-src 'none'`, `script-src 'self'` and no `unsafe-inline` or
`unsafe-eval`; inline style is permitted only as an attribute via `style-src-attr`, which covers the
dynamic width bars the console uses while still blocking an injected `