Initial public ModelForge release

backend/src/modelforge_api/__init__.py

@@ -0,0 +1,11 @@
+"""ITWorx ModelForge control plane.
+
+The version is derived from the repository's VERSION file rather than declared again here. Four
+independent copies of the product version is three chances to publish a release that misdescribes
+itself, which is exactly the class of mistake a release gate is supposed to make impossible.
+"""
+
+from modelforge_api.domain.release import PRODUCT_VERSION
+
+__all__ = ["__version__"]
+__version__ = PRODUCT_VERSION

backend/src/modelforge_api/api/authorization.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import hmac
+import re
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Annotated
+
+from fastapi import Depends, Header, HTTPException, Request
+
+from modelforge_api.settings import Settings, get_settings
+
+
+class PrincipalRole(StrEnum):
+    """Closed operator-console roles, ordered from read-only to security administration."""
+
+    VIEWER = "viewer"
+    OPERATOR = "operator"
+    ADMIN = "admin"
+
+
+class AccessBoundary(StrEnum):
+    """Mutually exclusive request authentication boundaries.
+
+    The control-plane boundary is deliberately the default. New endpoints therefore fail closed
+    behind the human operator credential until their narrower machine/public policy is explicitly
+    recorded here and in the route inventory tests.
+    """
+
+    PUBLIC = "public"
+    SINGLE_USE_ENROLLMENT = "single_use_enrollment"
+    NODE = "node"
+    CAPABILITY_CLIENT = "capability_client"
+    CONTROL_PLANE = "control_plane"
+
+
+_PUBLIC_REQUESTS = {
+    ("GET", "/"),
+    ("GET", "/api/v1/health/live"),
+    ("GET", "/api/v1/health/ready"),
+    ("GET", "/api/v1/version"),
+}
+_INTERACTIVE_API_PATHS = {"/docs", "/docs/oauth2-redirect", "/redoc", "/openapi.json"}
+_NODE_REQUESTS = {
+    ("POST", "/api/v1/agent/heartbeat"),
+    ("PUT", "/api/v1/agent/inventory"),
+    ("PUT", "/api/v1/agent/telemetry"),
+    ("GET", "/api/v1/agent/artifact-jobs/next"),
+    ("POST", "/api/v1/agent/artifact-jobs/{job_id}/progress"),
+    ("POST", "/api/v1/agent/artifact-jobs/{job_id}/complete"),
+    ("POST", "/api/v1/agent/artifact-jobs/{job_id}/fail"),
+    ("GET", "/api/v1/agent/runtime-probes/next"),
+    ("POST", "/api/v1/agent/runtime-probes/{probe_id}/progress"),
+    ("POST", "/api/v1/agent/runtime-probes/{probe_id}/complete"),
+    ("POST", "/api/v1/agent/runtime-probes/{probe_id}/fail"),
+    ("GET", "/api/v1/agent/serving-jobs/next"),
+    ("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
+    ("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
+    ("POST", "/api/v1/agent/serving-state"),
+}
+_NODE_PARAMETERIZED_REQUESTS = (
+    ("POST", re.compile(r"^/api/v1/agent/artifact-jobs/[^/]+/(?:progress|complete|fail)$")),
+    ("POST", re.compile(r"^/api/v1/agent/runtime-probes/[^/]+/(?:progress|complete|fail)$")),
+    ("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail)$")),
+)
+_CAPABILITY_REQUIREMENTS = {
+    ("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): "rag.embedding@1",
+    ("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): "rag.reranking@1",
+    ("POST", "/api/v1/capabilities/document.ocr@1/invoke"): "document.ocr@1",
+    ("POST", "/api/v1/capabilities/vision.embedding@1/invoke"): "vision.embedding@1",
+    ("POST", "/api/v1/capabilities/speech.transcription@1/invoke"): "speech.transcription@1",
+    ("POST", "/api/v1/capability-experiments/{route_key}/invoke"): "rag.embedding@1",
+    ("POST", "/v1/embeddings"): "rag.embedding@1",
+}
+_CAPABILITY_CLIENT_REQUESTS = set(_CAPABILITY_REQUIREMENTS)
+_CAPABILITY_EXPERIMENT_REQUEST = re.compile(r"^/api/v1/capability-experiments/[^/]+/invoke$")
+
+
+def access_boundary_for_request(
+    method: str,
+    path: str,
+) -> AccessBoundary:
+    """Classify an HTTP request without reading its body or accepting credential aliases."""
+
+    normalized_method = method.upper()
+    if (normalized_method, path) in _PUBLIC_REQUESTS:
+        return AccessBoundary.PUBLIC
+    if path in _INTERACTIVE_API_PATHS:
+        # These routes exist only when the FastAPI app is explicitly constructed for development.
+        # In test/production they must reach routing unauthenticated so the disabled surface is a
+        # genuine 404 rather than an operator-auth challenge that reveals a hidden endpoint.
+        return AccessBoundary.PUBLIC
+    if normalized_method == "POST" and path == "/api/v1/agent/enroll":
+        return AccessBoundary.SINGLE_USE_ENROLLMENT
+    if (normalized_method, path) in _NODE_REQUESTS or any(
+        normalized_method == rule_method and pattern.fullmatch(path)
+        for rule_method, pattern in _NODE_PARAMETERIZED_REQUESTS
+    ):
+        return AccessBoundary.NODE
+    if (normalized_method, path) in _CAPABILITY_CLIENT_REQUESTS or (
+        normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
+    ):
+        return AccessBoundary.CAPABILITY_CLIENT
+    return AccessBoundary.CONTROL_PLANE
+
+
+def required_capability_for_request(method: str, path: str) -> str | None:
+    """Return the exact capability scope for a registered project-facing route."""
+
+    normalized_method = method.upper()
+    requirement = _CAPABILITY_REQUIREMENTS.get((normalized_method, path))
+    if requirement is not None:
+        return requirement
+    if normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path):
+        return "rag.embedding@1"
+    return None
+
+
+def request_body_limit_bytes(settings: Settings, boundary: AccessBoundary) -> int:
+    """Select an explicit pre-parser body limit for every request boundary."""
+
+    if boundary is AccessBoundary.CAPABILITY_CLIENT:
+        return settings.gateway_max_payload_bytes
+    if boundary in {AccessBoundary.NODE, AccessBoundary.SINGLE_USE_ENROLLMENT}:
+        return settings.node_agent_max_payload_bytes
+    return settings.control_plane_max_payload_bytes
+
+
+_ROLE_RANK = {
+    PrincipalRole.VIEWER: 0,
+    PrincipalRole.OPERATOR: 1,
+    PrincipalRole.ADMIN: 2,
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Principal:
+    """An authenticated human/control-plane principal.
+
+    Node and capability credentials deliberately never become a ``Principal``. Their separate
+    dependencies remain the only way into node-agent and inference surfaces, preventing credential
+    confusion at the type and dependency boundaries.
+    """
+
+    subject: str
+    role: PrincipalRole
+    authentication_method: str
+
+    def permits(self, required: PrincipalRole) -> bool:
+        return _ROLE_RANK[self.role] >= _ROLE_RANK[required]
+
+
+def authenticate_operator_token(settings: Settings, token: str | None) -> Principal:
+    """Authenticate the backward-compatible operator key as an admin principal.
+
+    The legacy key is the sole human credential in this release. OIDC/session authentication can
+    add another principal producer later without changing route authorization policy.
+    """
+
+    configured = settings.operator_api_key
+    if configured is None or not configured.get_secret_value():
+        raise HTTPException(status_code=503, detail="operator API authentication is not configured")
+    if token is None or not hmac.compare_digest(token, configured.get_secret_value()):
+        raise HTTPException(status_code=401, detail="invalid operator credential")
+    return Principal(
+        subject="legacy-operator",
+        role=PrincipalRole.ADMIN,
+        authentication_method="legacy_admin_token",
+    )
+
+
+def authenticate_operator_principal(
+    request: Request,
+    settings: Annotated[Settings, Depends(get_settings)],
+    token: Annotated[str | None, Header(alias="X-ModelForge-Admin-Token")] = None,
+) -> Principal:
+    """Reuse the pre-body principal and retain a safe direct-dependency fallback."""
+
+    principal = getattr(request.state, "principal", None)
+    if isinstance(principal, Principal):
+        return principal
+    return authenticate_operator_token(settings, token)
+
+
+AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_principal)]
+
+
+def _require_role(principal: Principal, required: PrincipalRole) -> Principal:
+    if not principal.permits(required):
+        raise HTTPException(status_code=403, detail="operator role is not authorized")
+    return principal
+
+
+def require_viewer(principal: AuthenticatedPrincipal) -> Principal:
+    return _require_role(principal, PrincipalRole.VIEWER)
+
+
+def require_operator(principal: AuthenticatedPrincipal) -> Principal:
+    return _require_role(principal, PrincipalRole.OPERATOR)
+
+
+def require_admin(principal: AuthenticatedPrincipal) -> Principal:
+    return _require_role(principal, PrincipalRole.ADMIN)
+
+
+Viewer = Annotated[Principal, Depends(require_viewer)]
+Operator = Annotated[Principal, Depends(require_operator)]
+Admin = Annotated[Principal, Depends(require_admin)]

backend/src/modelforge_api/api/request_limits.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from starlette.responses import JSONResponse
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS = 32
+MAX_TOTAL_EMPTY_REQUEST_EVENTS = 128
+MAX_REQUEST_BODY_EVENTS = 4096
+
+
+@dataclass(frozen=True, slots=True)
+class RequestBodyTooLarge(Exception):
+    limit_bytes: int
+
+
+@dataclass(frozen=True, slots=True)
+class RequestBodyProgressExhausted(Exception):
+    received_events: int
+    empty_events: int
+
+
+@dataclass(frozen=True, slots=True)
+class InvalidContentLength(Exception):
+    message: str
+
+
+class RequestBodyLimitMiddleware:
+    """Reject declared or streamed oversized bodies before application parsing.
+
+    The receive wrapper forwards chunks only while the cumulative size is within the configured
+    boundary. It raises as soon as the next chunk crosses the limit and never drains or buffers the
+    remaining request body. It also rejects a body stream that exceeds the fixed request-event or
+    empty-progress budgets, preventing an immediately-ready sequence of empty ASGI frames from
+    spinning indefinitely without yielding useful input.
+    """
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    @staticmethod
+    def _declared_length(scope: Scope) -> int | None:
+        values = [
+            value for name, value in scope.get("headers", []) if name.lower() == b"content-length"
+        ]
+        if not values:
+            return None
+        if len(values) != 1:
+            raise ValueError("multiple content-length headers are not accepted")
+        try:
+            rendered = values[0].decode("ascii")
+            if not rendered.isdecimal():
+                raise ValueError("content-length must be an unsigned decimal integer")
+            return int(rendered)
+        except UnicodeDecodeError as exc:
+            raise ValueError("content-length must be ASCII") from exc
+
+    @staticmethod
+    async def _error_response(
+        scope: Scope,
+        receive: Receive,
+        send: Send,
+        *,
+        status_code: int,
+        code: str,
+        message: str,
+        details: dict[str, int] | None = None,
+    ) -> None:
+        state = scope.get("state", {})
+        correlation_id = state.get("correlation_id", "unknown")
+        response = JSONResponse(
+            status_code=status_code,
+            content={
+                "error": {
+                    "code": code,
+                    "message": message,
+                    "correlation_id": correlation_id,
+                    "details": details or {},
+                }
+            },
+            headers={"X-Correlation-ID": str(correlation_id)},
+        )
+        await response(scope, receive, send)
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        state = scope.setdefault("state", {})
+        configured_limit: int | None = None
+        headers_checked = False
+        received_bytes = 0
+        received_events = 0
+        empty_events = 0
+        consecutive_empty_events = 0
+        request_error: (
+            InvalidContentLength | RequestBodyTooLarge | RequestBodyProgressExhausted | None
+        ) = None
+
+        async def limited_receive() -> Message:
+            nonlocal configured_limit, headers_checked
+            nonlocal consecutive_empty_events, empty_events, received_bytes
+            nonlocal received_events, request_error
+            if not headers_checked:
+                candidate_limit = state.get("request_body_limit_bytes")
+                if not isinstance(candidate_limit, int) or candidate_limit < 0:
+                    raise RuntimeError("request body limit was not established before routing")
+                configured_limit = candidate_limit
+                headers_checked = True
+                try:
+                    declared_length = self._declared_length(scope)
+                except ValueError as exc:
+                    request_error = InvalidContentLength(str(exc))
+                    state["request_body_error_status_code"] = 400
+                    raise request_error from exc
+                if declared_length is not None and declared_length > configured_limit:
+                    request_error = RequestBodyTooLarge(configured_limit)
+                    state["request_body_error_status_code"] = 413
+                    raise request_error
+            message = await receive()
+            if message["type"] == "http.request":
+                if configured_limit is None:  # pragma: no cover - guarded before receive
+                    raise RuntimeError("request body limit was not established before routing")
+                received_events += 1
+                body = message.get("body", b"")
+                received_bytes += len(body)
+                if received_bytes > configured_limit:
+                    request_error = RequestBodyTooLarge(configured_limit)
+                    state["request_body_error_status_code"] = 413
+                    raise request_error
+                if body:
+                    consecutive_empty_events = 0
+                elif message.get("more_body", False):
+                    empty_events += 1
+                    consecutive_empty_events += 1
+                else:
+                    consecutive_empty_events = 0
+                if (
+                    received_events > MAX_REQUEST_BODY_EVENTS
+                    or empty_events > MAX_TOTAL_EMPTY_REQUEST_EVENTS
+                    or consecutive_empty_events > MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS
+                ):
+                    request_error = RequestBodyProgressExhausted(
+                        received_events=received_events,
+                        empty_events=empty_events,
+                    )
+                    state["request_body_error_status_code"] = 400
+                    raise request_error
+            return message
+
+        response_started = False
+
+        async def tracked_send(message: Message) -> None:
+            nonlocal response_started
+            # FastAPI deliberately converts arbitrary request-body receive failures to a generic
+            # 400. Once this middleware has observed a boundary violation, suppress that parser
+            # response and emit the boundary's typed response after the inner app unwinds.
+            if request_error is not None:
+                return
+            if message["type"] == "http.response.start":
+                response_started = True
+            await send(message)
+
+        try:
+            await self.app(scope, limited_receive, tracked_send)
+        except (InvalidContentLength, RequestBodyProgressExhausted, RequestBodyTooLarge) as exc:
+            request_error = exc
+        if request_error is not None:
+            if response_started:
+                raise request_error
+            if isinstance(request_error, InvalidContentLength):
+                await self._error_response(
+                    scope,
+                    receive,
+                    send,
+                    status_code=400,
+                    code="invalid_content_length",
+                    message=request_error.message,
+                )
+            elif isinstance(request_error, RequestBodyTooLarge):
+                await self._error_response(
+                    scope,
+                    receive,
+                    send,
+                    status_code=413,
+                    code="request_body_too_large",
+                    message="Request body exceeds the permitted boundary",
+                    details={"limit_bytes": request_error.limit_bytes},
+                )
+            else:
+                await self._error_response(
+                    scope,
+                    receive,
+                    send,
+                    status_code=400,
+                    code="request_body_progress_exhausted",
+                    message="Request body made insufficient bounded progress",
+                    details={
+                        "max_consecutive_empty_events": MAX_CONSECUTIVE_EMPTY_REQUEST_EVENTS,
+                        "max_empty_events": MAX_TOTAL_EMPTY_REQUEST_EVENTS,
+                        "max_request_events": MAX_REQUEST_BODY_EVENTS,
+                    },
+                )

backend/src/modelforge_api/api/routes/acquisition.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_admin, require_operator, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.acquisition import (
+    ArtifactJobResponse,
+    ArtifactSetResponse,
+    DiscoveryCandidate,
+    DiscoverySearchRequest,
+    DownloadPlanCreate,
+    DownloadPlanResponse,
+    UpstreamRefreshRequest,
+    UpstreamSnapshotResponse,
+)
+from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+    prefix="/api/v1",
+    tags=["artifact-acquisition"],
+    dependencies=[Depends(require_viewer)],
+)
+
+
+def get_acquisition_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> AcquisitionService:
+    token = settings.hf_token.get_secret_value() if settings.hf_token else None
+    provider = OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds)
+    return AcquisitionService(session, settings, provider)
+
+
+Service = Annotated[AcquisitionService, Depends(get_acquisition_service)]
+
+
+@router.post(
+    "/discovery/search",
+    response_model=list[DiscoveryCandidate],
+    dependencies=[Depends(require_operator)],
+)
+def search(request: DiscoverySearchRequest, service: Service) -> list[DiscoveryCandidate]:
+    return service.search(request)
+
+
+@router.post(
+    "/models/{model_id}/refresh-upstream",
+    response_model=UpstreamSnapshotResponse,
+    dependencies=[Depends(require_operator)],
+)
+def refresh_upstream(
+    model_id: uuid.UUID, request: UpstreamRefreshRequest, service: Service
+) -> UpstreamSnapshotResponse:
+    return service.refresh_model(model_id, request.revision)
+
+
+@router.get("/models/{model_id}/upstream", response_model=UpstreamSnapshotResponse)
+def model_upstream(model_id: uuid.UUID, service: Service) -> UpstreamSnapshotResponse:
+    return service.latest_snapshot(model_id)
+
+
+@router.get("/revisions/{revision_id}/artifact-sets", response_model=list[ArtifactSetResponse])
+def artifact_sets(revision_id: uuid.UUID, service: Service) -> list[ArtifactSetResponse]:
+    return service.artifact_sets(revision_id)
+
+
+@router.post(
+    "/download-plans",
+    response_model=DownloadPlanResponse,
+    status_code=201,
+    dependencies=[Depends(require_operator)],
+)
+def create_download_plan(request: DownloadPlanCreate, service: Service) -> DownloadPlanResponse:
+    return service.create_plan(request)
+
+
+@router.get("/download-plans/{plan_id}", response_model=DownloadPlanResponse)
+def download_plan(plan_id: uuid.UUID, service: Service) -> DownloadPlanResponse:
+    return service.plan_response(plan_id)
+
+
+@router.post(
+    "/download-plans/{plan_id}/approve",
+    response_model=DownloadPlanResponse,
+    dependencies=[Depends(require_admin)],
+)
+def approve_download_plan(plan_id: uuid.UUID, service: Service) -> DownloadPlanResponse:
+    return service.approve_plan(plan_id)
+
+
+@router.post(
+    "/download-plans/{plan_id}/execute",
+    response_model=ArtifactJobResponse,
+    dependencies=[Depends(require_operator)],
+)
+def execute_download_plan(plan_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
+    return service.execute_plan(plan_id)
+
+
+@router.get("/artifact-jobs", response_model=list[ArtifactJobResponse])
+def artifact_jobs(
+    service: Service, limit: Annotated[int, Query(ge=1, le=100)] = 100
+) -> list[ArtifactJobResponse]:
+    return service.jobs()[:limit]
+
+
+@router.get("/artifact-jobs/{job_id}", response_model=ArtifactJobResponse)
+def artifact_job(job_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
+    return service.job(job_id)
+
+
+@router.post(
+    "/artifact-jobs/{job_id}/cancel",
+    response_model=ArtifactJobResponse,
+    dependencies=[Depends(require_operator)],
+)
+def cancel_artifact_job(job_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
+    return service.cancel(job_id)
+
+
+@router.post(
+    "/artifact-jobs/{job_id}/retry",
+    response_model=ArtifactJobResponse,
+    dependencies=[Depends(require_operator)],
+)
+def retry_artifact_job(job_id: uuid.UUID, service: Service) -> ArtifactJobResponse:
+    return service.retry(job_id)

backend/src/modelforge_api/api/routes/agent.py

@@ -0,0 +1,371 @@
+from __future__ import annotations
+
+import time
+import uuid
+from collections import defaultdict, deque
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin
+from modelforge_api.db import get_session
+from modelforge_api.domain.acquisition import (
+    AgentArtifactJobLease,
+    AgentJobComplete,
+    AgentJobControl,
+    AgentJobFailure,
+    AgentJobProgress,
+    ArtifactJobResponse,
+)
+from modelforge_api.domain.agent_protocol import (
+    EnrollmentRequest,
+    EnrollmentResponse,
+    EnrollmentTokenCreate,
+    EnrollmentTokenCreated,
+    EnrollmentTokenSummary,
+    HeartbeatRequest,
+    InventoryReport,
+    NodeCredentialCreated,
+    NodeManagementUpdate,
+    ObservationAck,
+    TelemetryReport,
+)
+from modelforge_api.domain.node_decommission import (
+    NodeDecommissionExecute,
+    NodeDecommissionPreview,
+    NodeDecommissionResult,
+)
+from modelforge_api.domain.runtime import (
+    AgentRuntimeProbeComplete,
+    AgentRuntimeProbeControl,
+    AgentRuntimeProbeFailure,
+    AgentRuntimeProbeLease,
+    AgentRuntimeProbeProgress,
+    RuntimeProbeResponse,
+)
+from modelforge_api.persistence.models import ComputeNode, NodeCredential
+from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
+from modelforge_api.services.acquisition import AcquisitionService
+from modelforge_api.services.node_agent import NodeAgentService, NodeAuthenticationEvidence
+from modelforge_api.services.node_decommission import NodeDecommissionService
+from modelforge_api.services.runtime import RuntimeService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(tags=["node-agent"])
+
+
+class ActionResponse(BaseModel):
+    status: str = "ok"
+
+
+class AttemptLimiter:
+    def __init__(self, limit: int = 10, window_seconds: int = 60) -> None:
+        self.limit = limit
+        self.window_seconds = window_seconds
+        self.attempts: dict[str, deque[float]] = defaultdict(deque)
+
+    def check(self, key: str) -> None:
+        now = time.monotonic()
+        bucket = self.attempts[key]
+        while bucket and bucket[0] <= now - self.window_seconds:
+            bucket.popleft()
+        if len(bucket) >= self.limit:
+            raise HTTPException(status_code=429, detail="enrollment rate limit exceeded")
+        bucket.append(now)
+
+
+enrollment_limiter = AttemptLimiter()
+
+
+def get_agent_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> NodeAgentService:
+    return NodeAgentService(session, settings)
+
+
+Service = Annotated[NodeAgentService, Depends(get_agent_service)]
+
+
+def get_decommission_service(
+    session: Annotated[Session, Depends(get_session)],
+) -> NodeDecommissionService:
+    return NodeDecommissionService(session)
+
+
+DecommissionService = Annotated[NodeDecommissionService, Depends(get_decommission_service)]
+
+
+@router.post(
+    "/api/v1/admin/node-enrollments",
+    response_model=EnrollmentTokenCreated,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_enrollment(
+    request: EnrollmentTokenCreate, service: Service, _admin: Admin, response: Response
+) -> EnrollmentTokenCreated:
+    response.headers["Cache-Control"] = "no-store"
+    return service.create_enrollment(request)
+
+
+@router.get("/api/v1/admin/node-enrollments", response_model=list[EnrollmentTokenSummary])
+def list_enrollments(service: Service, _admin: Admin) -> list[EnrollmentTokenSummary]:
+    return [
+        EnrollmentTokenSummary(
+            id=str(item.id),
+            created_at=item.created_at,
+            expires_at=item.expires_at,
+            used_at=item.used_at,
+            revoked_at=item.revoked_at,
+        )
+        for item in service.repository.enrollments()
+    ]
+
+
+@router.delete("/api/v1/admin/node-enrollments/{enrollment_id}", response_model=ActionResponse)
+def revoke_enrollment(enrollment_id: uuid.UUID, service: Service, _admin: Admin) -> ActionResponse:
+    service.revoke_enrollment(enrollment_id)
+    return ActionResponse()
+
+
+@router.patch("/api/v1/admin/hardware/nodes/{node_id}", response_model=ActionResponse)
+def update_node(
+    node_id: uuid.UUID,
+    request: NodeManagementUpdate,
+    service: Service,
+    _admin: Admin,
+) -> ActionResponse:
+    service.update_node(node_id, request)
+    return ActionResponse()
+
+
+@router.post(
+    "/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
+    response_model=NodeDecommissionPreview,
+)
+def preview_node_decommission(
+    node_id: uuid.UUID, service: DecommissionService, _admin: Admin
+) -> NodeDecommissionPreview:
+    return service.preview(node_id)
+
+
+@router.post(
+    "/api/v1/admin/hardware/nodes/{node_id}/decommission",
+    response_model=NodeDecommissionResult,
+)
+def execute_node_decommission(
+    node_id: uuid.UUID,
+    request: NodeDecommissionExecute,
+    service: DecommissionService,
+    _admin: Admin,
+) -> NodeDecommissionResult:
+    return service.execute(node_id, request)
+
+
+@router.delete("/api/v1/admin/hardware/nodes/{node_id}/credential", response_model=ActionResponse)
+def revoke_credential(node_id: uuid.UUID, service: Service, _admin: Admin) -> ActionResponse:
+    service.revoke_credential(node_id)
+    return ActionResponse()
+
+
+@router.post(
+    "/api/v1/admin/hardware/nodes/{node_id}/credential/rotate",
+    response_model=NodeCredentialCreated,
+)
+def rotate_credential(
+    node_id: uuid.UUID, service: Service, _admin: Admin, response: Response
+) -> NodeCredentialCreated:
+    response.headers["Cache-Control"] = "no-store"
+    return service.rotate_credential(node_id)
+
+
+@router.post(
+    "/api/v1/agent/enroll",
+    response_model=EnrollmentResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def enroll(
+    request: EnrollmentRequest, http_request: Request, service: Service, response: Response
+) -> EnrollmentResponse:
+    enrollment_limiter.check(http_request.client.host if http_request.client else "unknown")
+    response.headers["Cache-Control"] = "no-store"
+    return service.enroll(request)
+
+
+def authenticated_node(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> tuple[NodeCredential, ComputeNode]:
+    evidence = getattr(request.state, "node_authentication", None)
+    if isinstance(evidence, NodeAuthenticationEvidence):
+        return service.reuse_authentication(evidence, authorization)
+    return service.authenticate(authorization)
+
+
+NodeIdentity = tuple[NodeCredential, ComputeNode]
+AgentIdentity = Annotated[NodeIdentity, Depends(authenticated_node)]
+
+
+def get_agent_acquisition_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> AcquisitionService:
+    token = settings.hf_token.get_secret_value() if settings.hf_token else None
+    return AcquisitionService(
+        session,
+        settings,
+        OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds),
+        actor_type="node_agent",
+        actor_id="authenticated-node",
+    )
+
+
+AgentAcquisition = Annotated[AcquisitionService, Depends(get_agent_acquisition_service)]
+
+
+def get_agent_runtime_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> RuntimeService:
+    return RuntimeService(
+        session,
+        settings,
+        actor_type="runtime_worker",
+        actor_id="authenticated-node",
+    )
+
+
+AgentRuntime = Annotated[RuntimeService, Depends(get_agent_runtime_service)]
+
+
+@router.post("/api/v1/agent/heartbeat", response_model=ObservationAck)
+def heartbeat(
+    request: HeartbeatRequest, service: Service, identity: AgentIdentity
+) -> ObservationAck:
+    _credential, node = identity
+    return service.heartbeat(node, request)
+
+
+@router.put("/api/v1/agent/inventory", response_model=ObservationAck)
+def publish_inventory(
+    request: InventoryReport, service: Service, identity: AgentIdentity
+) -> ObservationAck:
+    _credential, node = identity
+    return service.publish_inventory(node, request)
+
+
+@router.put("/api/v1/agent/telemetry", response_model=ObservationAck)
+def publish_telemetry(
+    request: TelemetryReport, service: Service, identity: AgentIdentity
+) -> ObservationAck:
+    _credential, node = identity
+    return service.publish_telemetry(node, request)
+
+
+@router.get(
+    "/api/v1/agent/artifact-jobs/next",
+    response_model=AgentArtifactJobLease | None,
+)
+def claim_artifact_job(
+    acquisition: AgentAcquisition, identity: AgentIdentity
+) -> AgentArtifactJobLease | None:
+    _credential, node = identity
+    return acquisition.claim_next(node)
+
+
+@router.post(
+    "/api/v1/agent/artifact-jobs/{job_id}/progress",
+    response_model=AgentJobControl,
+)
+def artifact_job_progress(
+    job_id: uuid.UUID,
+    request: AgentJobProgress,
+    acquisition: AgentAcquisition,
+    identity: AgentIdentity,
+) -> AgentJobControl:
+    _credential, node = identity
+    return acquisition.progress(job_id, node, request)
+
+
+@router.post(
+    "/api/v1/agent/artifact-jobs/{job_id}/complete",
+    response_model=ArtifactJobResponse,
+)
+def artifact_job_complete(
+    job_id: uuid.UUID,
+    request: AgentJobComplete,
+    acquisition: AgentAcquisition,
+    identity: AgentIdentity,
+) -> ArtifactJobResponse:
+    _credential, node = identity
+    return acquisition.complete(job_id, node, request)
+
+
+@router.post(
+    "/api/v1/agent/artifact-jobs/{job_id}/fail",
+    response_model=ArtifactJobResponse,
+)
+def artifact_job_fail(
+    job_id: uuid.UUID,
+    request: AgentJobFailure,
+    acquisition: AgentAcquisition,
+    identity: AgentIdentity,
+) -> ArtifactJobResponse:
+    _credential, node = identity
+    return acquisition.fail(job_id, node, request)
+
+
+@router.get(
+    "/api/v1/agent/runtime-probes/next",
+    response_model=AgentRuntimeProbeLease | None,
+)
+def claim_runtime_probe(
+    runtime: AgentRuntime, identity: AgentIdentity
+) -> AgentRuntimeProbeLease | None:
+    _credential, node = identity
+    return runtime.claim_next(node)
+
+
+@router.post(
+    "/api/v1/agent/runtime-probes/{probe_id}/progress",
+    response_model=AgentRuntimeProbeControl,
+)
+def runtime_probe_progress(
+    probe_id: uuid.UUID,
+    request: AgentRuntimeProbeProgress,
+    runtime: AgentRuntime,
+    identity: AgentIdentity,
+) -> AgentRuntimeProbeControl:
+    _credential, node = identity
+    return runtime.progress(probe_id, node, request)
+
+
+@router.post(
+    "/api/v1/agent/runtime-probes/{probe_id}/complete",
+    response_model=RuntimeProbeResponse,
+)
+def runtime_probe_complete(
+    probe_id: uuid.UUID,
+    request: AgentRuntimeProbeComplete,
+    runtime: AgentRuntime,
+    identity: AgentIdentity,
+) -> RuntimeProbeResponse:
+    _credential, node = identity
+    return runtime.complete(probe_id, node, request)
+
+
+@router.post(
+    "/api/v1/agent/runtime-probes/{probe_id}/fail",
+    response_model=RuntimeProbeResponse,
+)
+def runtime_probe_fail(
+    probe_id: uuid.UUID,
+    request: AgentRuntimeProbeFailure,
+    runtime: AgentRuntime,
+    identity: AgentIdentity,
+) -> RuntimeProbeResponse:
+    _credential, node = identity
+    return runtime.fail(probe_id, node, request)

backend/src/modelforge_api/api/routes/catalog.py

@@ -0,0 +1,273 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.schemas import (
+    CapabilityContractResponse,
+    CapabilityEstateResponse,
+    InstallationDependencyResponse,
+    ModelInstallationRationaleResponse,
+    ProjectBindingResponse,
+    ProjectResponse,
+)
+from modelforge_api.persistence.models import (
+    ArtifactSet,
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    CapabilityEvaluationRun,
+    CapabilityResourceEnvelope,
+    Model,
+    ModelRevision,
+    Project,
+    ProjectBinding,
+    ProjectFitEvidence,
+    ResidencyAllocation,
+    RuntimeProfile,
+    ServiceClient,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
+
+router = APIRouter(
+    prefix="/api/v1", tags=["registry"], dependencies=[Depends(require_viewer)]
+)
+
+
+@router.get("/capabilities", response_model=list[CapabilityContractResponse])
+def list_capabilities(
+    registry: ManifestRegistry = Depends(get_manifest_registry),
+    session: Session = Depends(get_session),
+) -> list[CapabilityContractResponse]:
+    responses = []
+    for contract in registry.capabilities():
+        stable = session.execute(
+            select(CapabilityDeployment)
+            .join(
+                CapabilityContract,
+                CapabilityContract.id == CapabilityDeployment.capability_contract_id,
+            )
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
+            .where(
+                Capability.key == contract.capability,
+                CapabilityContract.version == contract.version,
+                CapabilityDeployment.status == "stable",
+                CapabilityDeployment.production.is_(True),
+            )
+        ).scalar_one_or_none()
+        responses.append(
+            CapabilityContractResponse(
+                key=contract.capability,
+                version=contract.version,
+                description=contract.description,
+                contract=contract,
+                stable_deployment=(
+                    {
+                        "id": str(stable.id),
+                        "status": stable.status,
+                        "health": stable.health_status,
+                        "production": stable.production,
+                    }
+                    if stable
+                    else None
+                ),
+            )
+        )
+    return responses
+
+
+@router.get("/capability-estate", response_model=list[CapabilityEstateResponse])
+def capability_estate(
+    registry: ManifestRegistry = Depends(get_manifest_registry),
+    session: Session = Depends(get_session),
+) -> list[CapabilityEstateResponse]:
+    responses: list[CapabilityEstateResponse] = []
+    for manifest in registry.capabilities():
+        contract = session.scalar(
+            select(CapabilityContract)
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
+            .where(Capability.key == manifest.capability, CapabilityContract.version == manifest.version)
+        )
+        deployment = None
+        model = None
+        revision = None
+        profile = None
+        envelope = None
+        latest_evaluation = None
+        if contract:
+            deployment = session.scalar(
+                select(CapabilityDeployment)
+                .where(CapabilityDeployment.capability_contract_id == contract.id)
+                .order_by(
+                    CapabilityDeployment.production.desc(),
+                    CapabilityDeployment.created_at.desc(),
+                )
+                .limit(1)
+            )
+        if deployment:
+            artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
+            revision = session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
+            model = session.get(Model, revision.model_id) if revision else None
+            profile = session.get(RuntimeProfile, deployment.runtime_profile_id)
+            envelope = session.scalar(
+                select(CapabilityResourceEnvelope).where(
+                    CapabilityResourceEnvelope.capability_deployment_id == deployment.id
+                )
+            )
+            latest_evaluation = session.scalar(
+                select(CapabilityEvaluationRun)
+                .where(CapabilityEvaluationRun.capability_deployment_id == deployment.id)
+                .order_by(CapabilityEvaluationRun.created_at.desc())
+                .limit(1)
+            )
+        operational_state = (
+            deployment.status
+            if deployment
+            else ("blocked" if manifest.estate.stability == "blocked" else "not_deployed")
+        )
+        responses.append(
+            CapabilityEstateResponse(
+                capability=manifest.capability,
+                version=manifest.version,
+                category=manifest.estate.category,
+                purpose=manifest.estate.purpose,
+                declared_stability=manifest.estate.stability,
+                operational_state=operational_state,
+                current_deployment_id=deployment.id if deployment else None,
+                model=model.display_name if model else None,
+                revision=revision.resolved_commit_sha if revision else None,
+                runtime=profile.runtime_type if profile else None,
+                node=str(deployment.compute_node_id) if deployment else None,
+                resource_class=manifest.estate.resource_class,
+                measured_required_vram_bytes=envelope.required_vram_bytes if envelope else None,
+                consumers=manifest.estate.consumers,
+                privacy_class=manifest.privacy.classification,
+                evaluation_type=manifest.estate.evaluation_type,
+                evaluation_state=latest_evaluation.status if latest_evaluation else "not_evaluated",
+            )
+        )
+    return responses
+
+
+@router.get(
+    "/models/installation-rationale",
+    response_model=list[ModelInstallationRationaleResponse],
+)
+def model_installation_rationale(
+    session: Session = Depends(get_session),
+) -> list[ModelInstallationRationaleResponse]:
+    responses: list[ModelInstallationRationaleResponse] = []
+    for model in session.scalars(select(Model).order_by(Model.display_name)).all():
+        revisions = session.scalars(
+            select(ModelRevision).where(ModelRevision.model_id == model.id)
+        ).all()
+        revision_ids = [item.id for item in revisions]
+        sets = (
+            session.scalars(select(ArtifactSet).where(ArtifactSet.revision_id.in_(revision_ids))).all()
+            if revision_ids
+            else []
+        )
+        installed_sets = [item for item in sets if item.availability == "local"]
+        dependencies: list[InstallationDependencyResponse] = []
+        for artifact_set in installed_sets:
+            deployments = session.scalars(
+                select(CapabilityDeployment).where(
+                    CapabilityDeployment.artifact_set_id == artifact_set.id
+                )
+            ).all()
+            for deployment in deployments:
+                contract = session.get(CapabilityContract, deployment.capability_contract_id)
+                capability = session.get(Capability, contract.capability_id) if contract else None
+                if not contract or not capability:
+                    continue
+                projects = session.scalars(
+                    select(Project)
+                    .join(ProjectBinding, ProjectBinding.project_id == Project.id)
+                    .where(
+                        ProjectBinding.capability_contract_id == contract.id,
+                        ProjectBinding.deprecated_at.is_(None),
+                    )
+                    .distinct()
+                ).all()
+                active_projects = session.scalars(
+                    select(Project)
+                    .join(ProjectBinding, ProjectBinding.project_id == Project.id)
+                    .join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
+                    .where(
+                        ProjectBinding.capability_contract_id == contract.id,
+                        ServiceClient.status == "active",
+                    )
+                    .distinct()
+                ).all()
+                project_fit_ids = session.scalars(
+                    select(ProjectFitEvidence.id)
+                    .join(
+                        ProjectBinding,
+                        ProjectBinding.id == ProjectFitEvidence.project_binding_id,
+                    )
+                    .where(ProjectBinding.capability_contract_id == contract.id)
+                ).all()
+                evaluations = session.scalars(
+                    select(CapabilityEvaluationRun).where(
+                        CapabilityEvaluationRun.capability_deployment_id == deployment.id
+                    )
+                ).all()
+                residency = session.scalar(
+                    select(ResidencyAllocation).where(
+                        ResidencyAllocation.capability_deployment_id == deployment.id
+                    )
+                )
+                dependencies.append(
+                    InstallationDependencyResponse(
+                        capability=capability.key,
+                        version=contract.version,
+                        deployment_id=deployment.id,
+                        channel=deployment.channel,
+                        production=deployment.production,
+                        project_consumers=[item.key for item in projects],
+                        active_project_consumers=[item.key for item in active_projects],
+                        project_fit_evidence_ids=list(project_fit_ids),
+                        evaluation_run_ids=[item.id for item in evaluations],
+                        last_used_at=residency.last_used_at if residency else None,
+                    )
+                )
+        blockers = []
+        if dependencies:
+            blockers.append("capability_deployment_dependency")
+        if any(item.production for item in dependencies):
+            blockers.append("production_dependency")
+        responses.append(
+            ModelInstallationRationaleResponse(
+                model_id=model.id,
+                display_name=model.display_name,
+                upstream_source=model.upstream_source,
+                installed=bool(installed_sets),
+                installed_bytes=sum(item.total_size_bytes for item in installed_sets),
+                dependencies=dependencies,
+                can_delete=bool(installed_sets) and not blockers,
+                deletion_blockers=blockers,
+            )
+        )
+    return responses
+
+
+@router.get("/projects", response_model=list[ProjectResponse])
+def list_projects(
+    registry: ManifestRegistry = Depends(get_manifest_registry),
+) -> list[ProjectResponse]:
+    return [
+        ProjectResponse(
+            **project.project.model_dump(),
+            notes=project.notes,
+            bindings=[
+                ProjectBindingResponse(
+                    capability=capability,
+                    contract_version=binding.contract_version,
+                    binding=binding,
+                )
+                for capability, binding in project.bindings.items()
+            ],
+        )
+        for project in registry.projects()
+    ]

backend/src/modelforge_api/api/routes/evaluation.py

@@ -0,0 +1,387 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.capability_evaluation import (
+    CapabilityAdvisorResponse,
+    CapabilityEvaluationRunCreate,
+    CapabilityEvaluationRunResponse,
+    CapabilityEvaluationSuiteCreate,
+    CapabilityEvaluationSuiteResponse,
+)
+from modelforge_api.domain.evaluation import (
+    AdvisorPolicyResponse,
+    AdvisorPolicyUpdate,
+    AdvisorRecommendationCreate,
+    AdvisorRecommendationDismiss,
+    AdvisorRecommendationResponse,
+    DiscoveryCandidateAssessmentCreate,
+    DiscoveryCandidateAssessmentResponse,
+    EmbeddingMigrationCreate,
+    EmbeddingMigrationResponse,
+    EvaluationCaseDefinitionResponse,
+    EvaluationCaseResultResponse,
+    EvaluationComparisonCreate,
+    EvaluationComparisonResponse,
+    EvaluationRunComplete,
+    EvaluationRunCreate,
+    EvaluationRunResponse,
+    EvaluationSuiteCreate,
+    EvaluationSuiteResponse,
+    MigrationUpdate,
+    ModelComparisonCreate,
+    ModelComparisonResponse,
+    RerankingCaseResultResponse,
+    RerankingRunComplete,
+    RerankingRunCreate,
+    RerankingRunResponse,
+    RetrievalCandidatePoolCreate,
+    RetrievalCandidatePoolResponse,
+    RetrievalPipelineIdentityCreate,
+    RetrievalPipelineIdentityResponse,
+)
+from modelforge_api.services.capability_evaluation import CapabilityEvaluationService
+from modelforge_api.services.evaluation import EvaluationService
+
+router = APIRouter(
+    prefix="/api/v1",
+    tags=["project-evaluation"],
+    dependencies=[Depends(require_viewer)],
+)
+
+
+def service(session: Annotated[Session, Depends(get_session)]) -> EvaluationService:
+    return EvaluationService(session)
+
+
+Service = Annotated[EvaluationService, Depends(service)]
+
+
+def capability_service(
+    session: Annotated[Session, Depends(get_session)],
+) -> CapabilityEvaluationService:
+    return CapabilityEvaluationService(session)
+
+
+CapabilityService = Annotated[CapabilityEvaluationService, Depends(capability_service)]
+
+
+@router.get("/capability-advisor", response_model=list[CapabilityAdvisorResponse])
+def capability_advisor(svc: CapabilityService) -> list[CapabilityAdvisorResponse]:
+    return svc.advisor()
+
+
+@router.get(
+    "/capability-evaluation-suites",
+    response_model=list[CapabilityEvaluationSuiteResponse],
+)
+def capability_suites(svc: CapabilityService) -> list[CapabilityEvaluationSuiteResponse]:
+    return svc.suites()
+
+
+@router.post(
+    "/capability-evaluation-suites",
+    response_model=CapabilityEvaluationSuiteResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_capability_suite(
+    request: CapabilityEvaluationSuiteCreate, svc: CapabilityService, _admin: Admin
+) -> CapabilityEvaluationSuiteResponse:
+    return svc.create_suite(request)
+
+
+@router.get(
+    "/capability-evaluation-runs",
+    response_model=list[CapabilityEvaluationRunResponse],
+)
+def capability_runs(svc: CapabilityService) -> list[CapabilityEvaluationRunResponse]:
+    return svc.runs()
+
+
+@router.post(
+    "/capability-evaluation-runs",
+    response_model=CapabilityEvaluationRunResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_capability_run(
+    request: CapabilityEvaluationRunCreate, svc: CapabilityService, _admin: Admin
+) -> CapabilityEvaluationRunResponse:
+    return svc.create_run(request)
+
+
+@router.get("/evaluation-suites", response_model=list[EvaluationSuiteResponse])
+def suites(svc: Service) -> list[EvaluationSuiteResponse]:
+    return svc.suites()
+
+
+@router.post(
+    "/evaluation-suites",
+    response_model=EvaluationSuiteResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_suite(
+    request: EvaluationSuiteCreate, svc: Service, _admin: Admin
+) -> EvaluationSuiteResponse:
+    return svc.create_suite(request)
+
+
+@router.get(
+    "/evaluation-suites/{suite_id}/revisions/{revision_id}/cases",
+    response_model=list[EvaluationCaseDefinitionResponse],
+)
+def suite_cases(
+    suite_id: uuid.UUID, revision_id: uuid.UUID, svc: Service
+) -> list[EvaluationCaseDefinitionResponse]:
+    return svc.suite_cases(suite_id, revision_id)
+
+
+@router.get("/evaluation-runs", response_model=list[EvaluationRunResponse])
+def runs(svc: Service) -> list[EvaluationRunResponse]:
+    return svc.runs()
+
+
+@router.post(
+    "/evaluation-runs", response_model=EvaluationRunResponse, status_code=status.HTTP_201_CREATED
+)
+def create_run(request: EvaluationRunCreate, svc: Service, _admin: Admin) -> EvaluationRunResponse:
+    return svc.create_run(request)
+
+
+@router.get("/evaluation-runs/{run_id}", response_model=EvaluationRunResponse)
+def run(run_id: uuid.UUID, svc: Service) -> EvaluationRunResponse:
+    return svc.run(run_id)
+
+
+@router.post("/evaluation-runs/{run_id}/complete", response_model=EvaluationRunResponse)
+def complete_run(
+    run_id: uuid.UUID, request: EvaluationRunComplete, svc: Service, _admin: Admin
+) -> EvaluationRunResponse:
+    return svc.complete_run(run_id, request)
+
+
+@router.get("/evaluation-runs/{run_id}/cases", response_model=list[EvaluationCaseResultResponse])
+def case_results(run_id: uuid.UUID, svc: Service) -> list[EvaluationCaseResultResponse]:
+    return svc.case_results(run_id)
+
+
+@router.post(
+    "/retrieval-candidate-pools",
+    response_model=RetrievalCandidatePoolResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_candidate_pool(
+    request: RetrievalCandidatePoolCreate, svc: Service, _admin: Admin
+) -> RetrievalCandidatePoolResponse:
+    return svc.create_candidate_pool(request)
+
+
+@router.get("/retrieval-candidate-pools", response_model=list[RetrievalCandidatePoolResponse])
+def candidate_pools(svc: Service) -> list[RetrievalCandidatePoolResponse]:
+    return svc.candidate_pools()
+
+
+@router.post(
+    "/retrieval-pipeline-identities",
+    response_model=RetrievalPipelineIdentityResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_pipeline_identity(
+    request: RetrievalPipelineIdentityCreate, svc: Service, _admin: Admin
+) -> RetrievalPipelineIdentityResponse:
+    return svc.create_pipeline_identity(request)
+
+
+@router.get(
+    "/retrieval-pipeline-identities", response_model=list[RetrievalPipelineIdentityResponse]
+)
+def pipeline_identities(svc: Service) -> list[RetrievalPipelineIdentityResponse]:
+    return svc.pipeline_identities()
+
+
+@router.post(
+    "/reranking-runs",
+    response_model=RerankingRunResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_reranking_run(
+    request: RerankingRunCreate, svc: Service, _admin: Admin
+) -> RerankingRunResponse:
+    return svc.create_reranking_run(request)
+
+
+@router.get("/reranking-runs", response_model=list[RerankingRunResponse])
+def reranking_runs(svc: Service) -> list[RerankingRunResponse]:
+    return svc.reranking_runs()
+
+
+@router.post("/reranking-runs/{run_id}/complete", response_model=RerankingRunResponse)
+def complete_reranking_run(
+    run_id: uuid.UUID, request: RerankingRunComplete, svc: Service, _admin: Admin
+) -> RerankingRunResponse:
+    return svc.complete_reranking_run(run_id, request)
+
+
+@router.get("/reranking-runs/{run_id}/cases", response_model=list[RerankingCaseResultResponse])
+def reranking_case_results(run_id: uuid.UUID, svc: Service) -> list[RerankingCaseResultResponse]:
+    return svc.reranking_case_results(run_id)
+
+
+@router.post(
+    "/discovery-assessments",
+    response_model=DiscoveryCandidateAssessmentResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_discovery_assessment(
+    request: DiscoveryCandidateAssessmentCreate, svc: Service, _admin: Admin
+) -> DiscoveryCandidateAssessmentResponse:
+    return svc.create_discovery_assessment(request)
+
+
+@router.get("/discovery-assessments", response_model=list[DiscoveryCandidateAssessmentResponse])
+def discovery_assessments(svc: Service) -> list[DiscoveryCandidateAssessmentResponse]:
+    return svc.discovery_assessments()
+
+
+@router.post(
+    "/evaluation-comparisons",
+    response_model=EvaluationComparisonResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def compare(
+    request: EvaluationComparisonCreate, svc: Service, _admin: Admin
+) -> EvaluationComparisonResponse:
+    return svc.compare(request)
+
+
+@router.get("/evaluation-comparisons", response_model=list[EvaluationComparisonResponse])
+def comparisons(svc: Service) -> list[EvaluationComparisonResponse]:
+    return svc.comparisons()
+
+
+@router.get("/evaluation-comparisons/{comparison_id}", response_model=EvaluationComparisonResponse)
+def comparison(comparison_id: uuid.UUID, svc: Service) -> EvaluationComparisonResponse:
+    return svc.comparison(comparison_id)
+
+
+@router.post(
+    "/model-comparisons",
+    response_model=ModelComparisonResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_model_comparison(
+    request: ModelComparisonCreate, svc: Service, _admin: Admin
+) -> ModelComparisonResponse:
+    return svc.create_model_comparison(request)
+
+
+@router.get("/model-comparisons", response_model=list[ModelComparisonResponse])
+def model_comparisons(svc: Service) -> list[ModelComparisonResponse]:
+    return svc.model_comparisons()
+
+
+@router.get("/model-comparisons/{comparison_id}", response_model=ModelComparisonResponse)
+def model_comparison(comparison_id: uuid.UUID, svc: Service) -> ModelComparisonResponse:
+    return svc.model_comparison(comparison_id)
+
+
+@router.post(
+    "/model-comparisons/{comparison_id}/recommendations",
+    response_model=AdvisorRecommendationResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_recommendation(
+    comparison_id: uuid.UUID,
+    request: AdvisorRecommendationCreate,
+    svc: Service,
+    _admin: Admin,
+) -> AdvisorRecommendationResponse:
+    return svc.recommend(comparison_id, request)
+
+
+@router.get("/recommendations", response_model=list[AdvisorRecommendationResponse])
+def recommendations(svc: Service) -> list[AdvisorRecommendationResponse]:
+    return svc.recommendations()
+
+
+@router.get("/recommendations/{recommendation_id}", response_model=AdvisorRecommendationResponse)
+def recommendation(recommendation_id: uuid.UUID, svc: Service) -> AdvisorRecommendationResponse:
+    return svc.recommendation(recommendation_id)
+
+
+@router.post(
+    "/recommendations/{recommendation_id}/dismiss",
+    response_model=AdvisorRecommendationResponse,
+)
+def dismiss_recommendation(
+    recommendation_id: uuid.UUID,
+    request: AdvisorRecommendationDismiss,
+    svc: Service,
+    _admin: Admin,
+) -> AdvisorRecommendationResponse:
+    return svc.dismiss_recommendation(recommendation_id, request)
+
+
+@router.get("/admin/advisor-policies/current", response_model=AdvisorPolicyResponse)
+def advisor_policy(svc: Service, _admin: Admin) -> AdvisorPolicyResponse:
+    return svc.advisor_policy()
+
+
+@router.patch("/admin/advisor-policies/current", response_model=AdvisorPolicyResponse)
+def update_advisor_policy(
+    request: AdvisorPolicyUpdate, svc: Service, _admin: Admin
+) -> AdvisorPolicyResponse:
+    return svc.update_advisor_policy(request)
+
+
+@router.get("/embedding-migrations", response_model=list[EmbeddingMigrationResponse])
+def migrations(svc: Service) -> list[EmbeddingMigrationResponse]:
+    return svc.migrations()
+
+
+@router.post(
+    "/projects/{project_id}/embedding-migrations",
+    response_model=EmbeddingMigrationResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_migration(
+    project_id: uuid.UUID, request: EmbeddingMigrationCreate, svc: Service, _admin: Admin
+) -> EmbeddingMigrationResponse:
+    return svc.create_migration(project_id, request)
+
+
+@router.get("/embedding-migrations/{migration_id}", response_model=EmbeddingMigrationResponse)
+def migration(migration_id: uuid.UUID, svc: Service) -> EmbeddingMigrationResponse:
+    return svc.migration(migration_id)
+
+
+@router.post(
+    "/embedding-migrations/{migration_id}/state", response_model=EmbeddingMigrationResponse
+)
+def update_migration(
+    migration_id: uuid.UUID, request: MigrationUpdate, svc: Service, _admin: Admin
+) -> EmbeddingMigrationResponse:
+    return svc.update_migration(migration_id, request)
+
+
+@router.post(
+    "/embedding-migrations/{migration_id}/start", response_model=EmbeddingMigrationResponse
+)
+def start_migration(
+    migration_id: uuid.UUID, svc: Service, _admin: Admin
+) -> EmbeddingMigrationResponse:
+    return svc.start_migration(migration_id)
+
+
+@router.post(
+    "/embedding-migrations/{migration_id}/cancel", response_model=EmbeddingMigrationResponse
+)
+def cancel_migration(
+    migration_id: uuid.UUID, svc: Service, _admin: Admin
+) -> EmbeddingMigrationResponse:
+    return svc.cancel_migration(migration_id)

backend/src/modelforge_api/api/routes/hardware.py

@@ -0,0 +1,82 @@
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_operator, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.hardware import AcceleratorState, HardwareState, NodeState
+from modelforge_api.services.hardware_factory import build_hardware_service
+from modelforge_api.services.hardware_inventory import HardwareInventoryService, HardwareRefreshBusy
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+    prefix="/api/v1/hardware", tags=["hardware"], dependencies=[Depends(require_viewer)]
+)
+
+
+def get_hardware_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> HardwareInventoryService:
+    return build_hardware_service(session, settings)
+
+
+Service = Annotated[HardwareInventoryService, Depends(get_hardware_service)]
+
+
+@router.get("", response_model=HardwareState)
+def hardware_overview(service: Service) -> HardwareState:
+    return service.state()
+
+
+@router.get("/nodes", response_model=list[NodeState])
+def list_nodes(service: Service) -> list[NodeState]:
+    return service.state().nodes
+
+
+@router.get("/nodes/{node_id}", response_model=NodeState)
+def get_node(node_id: uuid.UUID, service: Service) -> NodeState:
+    node = next((item for item in service.state().nodes if item.id == str(node_id)), None)
+    if node is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="compute node not found")
+    return node
+
+
+@router.get("/accelerators", response_model=list[AcceleratorState])
+def list_accelerators(service: Service) -> list[AcceleratorState]:
+    return [accelerator for node in service.state().nodes for accelerator in node.accelerators]
+
+
+@router.get("/accelerators/{accelerator_id}", response_model=AcceleratorState)
+def get_accelerator(accelerator_id: uuid.UUID, service: Service) -> AcceleratorState:
+    accelerator = next(
+        (
+            item
+            for node in service.state().nodes
+            for item in node.accelerators
+            if item.id == str(accelerator_id)
+        ),
+        None,
+    )
+    if accelerator is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="accelerator not found")
+    return accelerator
+
+
+@router.post(
+    "/refresh",
+    response_model=HardwareState,
+    dependencies=[Depends(require_operator)],
+)
+def refresh_hardware(service: Service) -> HardwareState:
+    try:
+        return service.refresh()
+    except HardwareRefreshBusy as exc:
+        raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail=f"hardware inventory failed: {type(exc).__name__}",
+        ) from exc

backend/src/modelforge_api/api/routes/health.py

@@ -0,0 +1,109 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api import __version__
+from modelforge_api.api.authorization import require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.enums import HealthStatus
+from modelforge_api.domain.release import (
+    CURRENT_AGENT_PROTOCOL_VERSION,
+    MINIMUM_POSTGRES_MAJOR,
+    MINIMUM_UPGRADE_SOURCE,
+    PRODUCT_NAME,
+    RELEASE_CHANNEL,
+    SUPPORTED_AGENT_PROTOCOL_VERSIONS,
+    SUPPORTED_SCHEMA_REVISIONS,
+    TARGET_SCHEMA_REVISION,
+    build_identity,
+)
+from modelforge_api.domain.schemas import (
+    HealthResponse,
+    ReadinessResponse,
+    ReleaseCompatibility,
+    ReleaseInfo,
+    SystemMetadata,
+)
+from modelforge_api.persistence.models import CapabilityDeployment
+from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(prefix="/api/v1", tags=["system"])
+
+
+@router.get("/health/live", response_model=HealthResponse)
+def liveness() -> HealthResponse:
+    return HealthResponse(version=__version__)
+
+
+@router.get("/health/ready", response_model=ReadinessResponse)
+def readiness(registry: ManifestRegistry = Depends(get_manifest_registry)) -> ReadinessResponse:
+    registry.capabilities()
+    registry.projects()
+    registry.candidates()
+    registry.benchmarks()
+    registry.policies()
+    return ReadinessResponse(
+        status=HealthStatus.HEALTHY,
+        checks={"manifests": HealthStatus.HEALTHY},
+        version=__version__,
+    )
+
+
+@router.get(
+    "/system",
+    response_model=SystemMetadata,
+    dependencies=[Depends(require_viewer)],
+)
+def system_metadata(
+    settings: Settings = Depends(get_settings),
+    session: Session = Depends(get_session),
+) -> SystemMetadata:
+    production_count = int(
+        session.scalar(
+            select(func.count(CapabilityDeployment.id)).where(
+                CapabilityDeployment.status == "stable",
+                CapabilityDeployment.production.is_(True),
+            )
+        )
+        or 0
+    )
+    return SystemMetadata(
+        name=PRODUCT_NAME,
+        version=__version__,
+        environment=settings.env,
+        release_channel=RELEASE_CHANNEL,
+        production_inference_available=production_count > 0,
+    )
+
+
+@router.get("/version", response_model=ReleaseInfo)
+def release_info(settings: Settings = Depends(get_settings)) -> ReleaseInfo:
+    """Build identity and compatibility for the running process.
+
+    Unauthenticated on purpose: an operator diagnosing a deployment needs to know which build is
+    answering before they have credentials for it, and everything here is already implied by the
+    image they are running. Nothing configuration-derived or sensitive is exposed.
+    """
+
+    identity = build_identity(
+        source_commit=settings.build_commit,
+        built_at=settings.build_timestamp,
+        image_digest=settings.build_image_digest,
+    )
+    return ReleaseInfo(
+        name=PRODUCT_NAME,
+        version=identity.version,
+        release_channel=identity.channel,
+        source_commit=identity.source_commit,
+        built_at=identity.built_at,
+        image_digest=identity.image_digest,
+        compatibility=ReleaseCompatibility(
+            schema_revision=TARGET_SCHEMA_REVISION,
+            supported_schema_revisions=list(SUPPORTED_SCHEMA_REVISIONS),
+            agent_protocol_version=CURRENT_AGENT_PROTOCOL_VERSION,
+            supported_agent_protocol_versions=list(SUPPORTED_AGENT_PROTOCOL_VERSIONS),
+            minimum_upgrade_source=MINIMUM_UPGRADE_SOURCE,
+            minimum_postgres_major=MINIMUM_POSTGRES_MAJOR,
+        ),
+    )

backend/src/modelforge_api/api/routes/lifecycle.py

@@ -0,0 +1,248 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.lifecycle_contracts import (
+    ApprovalDecision,
+    ApprovalPolicyCreate,
+    ApprovalPolicyResponse,
+    ApprovalRequestCreate,
+    ApprovalRequestResponse,
+    CanaryObservation,
+    CleanupExecutionCreate,
+    CleanupPlanCreate,
+    CleanupPlanResponse,
+    LifecycleEventResponse,
+    LifecycleOperationResponse,
+    LifecycleSubjectCreate,
+    LifecycleSubjectResponse,
+    PlanExecutionCreate,
+    PromotionPlanCreate,
+    PromotionPlanResponse,
+    RetentionPolicyCreate,
+    RetentionPolicyResponse,
+)
+from modelforge_api.services.lifecycle import LifecycleService
+
+router = APIRouter(
+    prefix="/api/v1/admin/lifecycle",
+    tags=["lifecycle"],
+    dependencies=[Depends(require_viewer)],
+)
+
+
+def get_lifecycle_service(
+    session: Annotated[Session, Depends(get_session)],
+) -> LifecycleService:
+    return LifecycleService(session)
+
+
+Service = Annotated[LifecycleService, Depends(get_lifecycle_service)]
+
+
+@router.get("/approval-policies", response_model=list[ApprovalPolicyResponse])
+def approval_policies(service: Service, _admin: Admin) -> list[ApprovalPolicyResponse]:
+    return service.policies()
+
+
+@router.post(
+    "/approval-policies",
+    response_model=ApprovalPolicyResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_approval_policy(
+    request: ApprovalPolicyCreate, service: Service, _admin: Admin
+) -> ApprovalPolicyResponse:
+    return service.create_policy(request)
+
+
+@router.get("/subjects", response_model=list[LifecycleSubjectResponse])
+def subjects(service: Service, _admin: Admin) -> list[LifecycleSubjectResponse]:
+    return service.subjects()
+
+
+@router.post(
+    "/subjects",
+    response_model=LifecycleSubjectResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_subject(
+    request: LifecycleSubjectCreate, service: Service, _admin: Admin
+) -> LifecycleSubjectResponse:
+    return service.create_subject(request)
+
+
+@router.get("/approval-requests", response_model=list[ApprovalRequestResponse])
+def approval_requests(service: Service, _admin: Admin) -> list[ApprovalRequestResponse]:
+    return service.approval_requests()
+
+
+@router.post(
+    "/approval-requests",
+    response_model=ApprovalRequestResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def request_approval(
+    request: ApprovalRequestCreate, service: Service, _admin: Admin
+) -> ApprovalRequestResponse:
+    return service.request_approval(request)
+
+
+@router.get("/approval-requests/{approval_id}", response_model=ApprovalRequestResponse)
+def approval_request(
+    approval_id: uuid.UUID, service: Service, _admin: Admin
+) -> ApprovalRequestResponse:
+    return service.approval(approval_id)
+
+
+@router.post("/approval-requests/{approval_id}/approve", response_model=ApprovalRequestResponse)
+def approve_request(
+    approval_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
+) -> ApprovalRequestResponse:
+    return service.decide_approval(approval_id, request, "approve")
+
+
+@router.post("/approval-requests/{approval_id}/reject", response_model=ApprovalRequestResponse)
+def reject_request(
+    approval_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
+) -> ApprovalRequestResponse:
+    return service.decide_approval(approval_id, request, "reject")
+
+
+@router.post("/approval-requests/{approval_id}/revoke", response_model=ApprovalRequestResponse)
+def revoke_request(
+    approval_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
+) -> ApprovalRequestResponse:
+    return service.decide_approval(approval_id, request, "revoke")
+
+
+@router.get("/promotion-plans", response_model=list[PromotionPlanResponse])
+def promotion_plans(service: Service, _admin: Admin) -> list[PromotionPlanResponse]:
+    return service.plans()
+
+
+@router.post(
+    "/promotion-plans",
+    response_model=PromotionPlanResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_promotion_plan(
+    request: PromotionPlanCreate, service: Service, _admin: Admin
+) -> PromotionPlanResponse:
+    return service.create_plan(request)
+
+
+@router.get("/promotion-plans/{plan_id}", response_model=PromotionPlanResponse)
+def promotion_plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> PromotionPlanResponse:
+    return service.plan(plan_id)
+
+
+@router.post("/promotion-plans/{plan_id}/approve", response_model=PromotionPlanResponse)
+def approve_promotion_plan(
+    plan_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
+) -> PromotionPlanResponse:
+    return service.approve_plan(plan_id, request)
+
+
+@router.post("/promotion-plans/{plan_id}/execute", response_model=LifecycleOperationResponse)
+def execute_promotion_plan(
+    plan_id: uuid.UUID, request: PlanExecutionCreate, service: Service, _admin: Admin
+) -> LifecycleOperationResponse:
+    return service.execute_plan(plan_id, request)
+
+
+@router.get("/operations", response_model=list[LifecycleOperationResponse])
+def operations(service: Service, _admin: Admin) -> list[LifecycleOperationResponse]:
+    return service.operations()
+
+
+@router.get("/operations/{operation_id}", response_model=LifecycleOperationResponse)
+def operation(
+    operation_id: uuid.UUID, service: Service, _admin: Admin
+) -> LifecycleOperationResponse:
+    return service.operation(operation_id)
+
+
+@router.post("/operations/{operation_id}/canary", response_model=LifecycleOperationResponse)
+def observe_canary(
+    operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin
+) -> LifecycleOperationResponse:
+    return service.observe_canary(operation_id, request)
+
+
+@router.post("/operations/{operation_id}/commit", response_model=LifecycleOperationResponse)
+def commit_operation(
+    operation_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
+) -> LifecycleOperationResponse:
+    return service.commit_operation(operation_id, request)
+
+
+@router.post("/operations/{operation_id}/rollback", response_model=LifecycleOperationResponse)
+def rollback_operation(
+    operation_id: uuid.UUID, request: ApprovalDecision, service: Service, _admin: Admin
+) -> LifecycleOperationResponse:
+    return service.rollback_operation(operation_id, request)
+
+
+@router.post("/reconcile", response_model=dict[str, int])
+def reconcile(service: Service, _admin: Admin) -> dict[str, int]:
+    return {"reconciled_operations": service.reconcile_incomplete()}
+
+
+@router.get("/retention-policies", response_model=list[RetentionPolicyResponse])
+def retention_policies(service: Service, _admin: Admin) -> list[RetentionPolicyResponse]:
+    return service.retention_policies()
+
+
+@router.post(
+    "/retention-policies",
+    response_model=RetentionPolicyResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_retention_policy(
+    request: RetentionPolicyCreate, service: Service, _admin: Admin
+) -> RetentionPolicyResponse:
+    return service.create_retention_policy(request)
+
+
+@router.get("/cleanup-plans", response_model=list[CleanupPlanResponse])
+def cleanup_plans(service: Service, _admin: Admin) -> list[CleanupPlanResponse]:
+    return service.cleanup_plans()
+
+
+@router.post(
+    "/cleanup-plans",
+    response_model=CleanupPlanResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_cleanup_plan(
+    request: CleanupPlanCreate, service: Service, _admin: Admin
+) -> CleanupPlanResponse:
+    return service.create_cleanup_plan(request)
+
+
+@router.get("/cleanup-plans/{plan_id}", response_model=CleanupPlanResponse)
+def cleanup_plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> CleanupPlanResponse:
+    return service.cleanup_plan(plan_id)
+
+
+@router.post("/cleanup-plans/{plan_id}/execute", response_model=CleanupPlanResponse)
+def execute_cleanup_plan(
+    plan_id: uuid.UUID, request: CleanupExecutionCreate, service: Service, _admin: Admin
+) -> CleanupPlanResponse:
+    return service.execute_cleanup(plan_id, request)
+
+
+@router.get("/events", response_model=list[LifecycleEventResponse])
+def lifecycle_events(
+    service: Service,
+    _admin: Admin,
+    limit: Annotated[int, Query(ge=1, le=1000)] = 200,
+) -> list[LifecycleEventResponse]:
+    return service.events(limit)

backend/src/modelforge_api/api/routes/migrations.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.migration_contracts import (
+    BatchReport,
+    CutoverOperationResponse,
+    CutoverPrepare,
+    CutoverReport,
+    MigrationBatchResponse,
+    MigrationEventResponse,
+    MigrationPlanCreate,
+    MigrationPlanResponse,
+    MigrationValidationPolicyCreate,
+    MigrationValidationPolicyResponse,
+    PreflightReport,
+    ReconciliationReport,
+    RollbackReport,
+    ShadowReport,
+    StateAction,
+    ValidationReport,
+    ValidationSnapshotResponse,
+)
+from modelforge_api.services.migration_engine import MigrationEngineService
+
+router = APIRouter(
+    prefix="/api/v1/admin/migrations",
+    tags=["migrations"],
+    dependencies=[Depends(require_viewer)],
+)
+
+
+def get_service(session: Annotated[Session, Depends(get_session)]) -> MigrationEngineService:
+    return MigrationEngineService(session)
+
+
+Service = Annotated[MigrationEngineService, Depends(get_service)]
+
+
+@router.get("/validation-policies", response_model=list[MigrationValidationPolicyResponse])
+def validation_policies(service: Service, _admin: Admin) -> list[MigrationValidationPolicyResponse]:
+    return service.validation_policies()
+
+
+@router.post(
+    "/validation-policies",
+    response_model=MigrationValidationPolicyResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_validation_policy(
+    request: MigrationValidationPolicyCreate, service: Service, _admin: Admin
+) -> MigrationValidationPolicyResponse:
+    return service.create_validation_policy(request)
+
+
+@router.get("/plans", response_model=list[MigrationPlanResponse])
+def plans(service: Service, _admin: Admin) -> list[MigrationPlanResponse]:
+    return service.plans()
+
+
+@router.post("/plans", response_model=MigrationPlanResponse, status_code=status.HTTP_201_CREATED)
+def create_plan(
+    request: MigrationPlanCreate, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.create_plan(request)
+
+
+@router.get("/plans/{plan_id}", response_model=MigrationPlanResponse)
+def plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> MigrationPlanResponse:
+    return service.plan(plan_id)
+
+
+@router.post("/plans/{plan_id}/preflight", response_model=MigrationPlanResponse)
+def preflight(
+    plan_id: uuid.UUID, request: PreflightReport, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.preflight(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/backfill/start", response_model=MigrationPlanResponse)
+def start_backfill(
+    plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.start_backfill(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/backfill/pause", response_model=MigrationPlanResponse)
+def pause_backfill(
+    plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.pause_backfill(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/batches", response_model=MigrationBatchResponse)
+def record_batch(
+    plan_id: uuid.UUID, request: BatchReport, service: Service, _admin: Admin
+) -> MigrationBatchResponse:
+    return service.record_batch(plan_id, request)
+
+
+@router.get("/plans/{plan_id}/batches", response_model=list[MigrationBatchResponse])
+def batches(plan_id: uuid.UUID, service: Service, _admin: Admin) -> list[MigrationBatchResponse]:
+    return service.batches(plan_id)
+
+
+@router.post("/plans/{plan_id}/validation", response_model=ValidationSnapshotResponse)
+def validate_target(
+    plan_id: uuid.UUID, request: ValidationReport, service: Service, _admin: Admin
+) -> ValidationSnapshotResponse:
+    return service.validate(plan_id, request)
+
+
+@router.get("/plans/{plan_id}/validation", response_model=list[ValidationSnapshotResponse])
+def validation_snapshots(
+    plan_id: uuid.UUID, service: Service, _admin: Admin
+) -> list[ValidationSnapshotResponse]:
+    return service.validation_snapshots(plan_id)
+
+
+@router.post("/plans/{plan_id}/shadow/start", response_model=MigrationPlanResponse)
+def start_shadow(
+    plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.start_shadow(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/shadow/complete", response_model=MigrationPlanResponse)
+def complete_shadow(
+    plan_id: uuid.UUID, request: ShadowReport, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.complete_shadow(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/cutover/prepare", response_model=CutoverOperationResponse)
+def prepare_cutover(
+    plan_id: uuid.UUID, request: CutoverPrepare, service: Service, _admin: Admin
+) -> CutoverOperationResponse:
+    return service.prepare_cutover(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/cutover/report", response_model=CutoverOperationResponse)
+def report_cutover(
+    plan_id: uuid.UUID, request: CutoverReport, service: Service, _admin: Admin
+) -> CutoverOperationResponse:
+    return service.report_cutover(plan_id, request)
+
+
+@router.post("/plans/{plan_id}/rollback", response_model=CutoverOperationResponse)
+def rollback(
+    plan_id: uuid.UUID, request: RollbackReport, service: Service, _admin: Admin
+) -> CutoverOperationResponse:
+    return service.rollback(plan_id, request)
+
+
+@router.post("/reconcile", response_model=CutoverOperationResponse)
+def reconcile(
+    request: ReconciliationReport, service: Service, _admin: Admin
+) -> CutoverOperationResponse:
+    return service.reconcile(request)
+
+
+@router.post("/plans/{plan_id}/cancel", response_model=MigrationPlanResponse)
+def cancel(
+    plan_id: uuid.UUID, request: StateAction, service: Service, _admin: Admin
+) -> MigrationPlanResponse:
+    return service.cancel(plan_id, request)
+
+
+@router.get("/cutovers", response_model=list[CutoverOperationResponse])
+def cutovers(
+    service: Service,
+    _admin: Admin,
+    plan_id: Annotated[uuid.UUID | None, Query()] = None,
+) -> list[CutoverOperationResponse]:
+    return service.operations(plan_id)
+
+
+@router.get("/events", response_model=list[MigrationEventResponse])
+def events(
+    service: Service,
+    _admin: Admin,
+    plan_id: Annotated[uuid.UUID | None, Query()] = None,
+    limit: Annotated[int, Query(ge=1, le=1000)] = 200,
+) -> list[MigrationEventResponse]:
+    return service.events(plan_id, limit)

backend/src/modelforge_api/api/routes/observability.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Query, Response, status
+from sqlalchemy import select
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.observability import (
+    AlertAction,
+    AlertHistoryResponse,
+    AlertResponse,
+    AlertRuleCreate,
+    AlertRuleResponse,
+    CapacitySnapshotResponse,
+    IncidentResponse,
+    MaintenanceWindowCreate,
+    MaintenanceWindowResponse,
+    OperationsOverview,
+    SLIDefinitionResponse,
+    SLOEvaluationResponse,
+    SLOPolicyCreate,
+    SLOPolicyResponse,
+    TrendResponse,
+    metrics,
+)
+from modelforge_api.persistence.models import (
+    AlertHistoryEvent,
+    IncidentTimelineEvent,
+    OperationalIncident,
+)
+from modelforge_api.services.observability import ObservabilityService
+
+router = APIRouter(tags=["operations"], dependencies=[Depends(require_viewer)])
+
+
+def get_service(session: Annotated[Session, Depends(get_session)]) -> ObservabilityService:
+    return ObservabilityService(session)
+
+
+Service = Annotated[ObservabilityService, Depends(get_service)]
+
+
+@router.get("/metrics", response_class=Response)
+def prometheus_metrics(service: Service, _admin: Admin) -> Response:
+    """Admin-isolated Prometheus exposition; process metrics survive DB failure."""
+    try:
+        content = service.prometheus()
+        metrics.gauge("modelforge_observability_degraded", {}, 0)
+    except SQLAlchemyError:
+        service.session.rollback()
+        metrics.gauge("modelforge_observability_degraded", {}, 1)
+        content = metrics.render()
+    return Response(content=content, media_type="text/plain; version=0.0.4; charset=utf-8")
+
+
+@router.get("/api/v1/admin/operations/overview", response_model=OperationsOverview)
+def overview(service: Service, _admin: Admin) -> OperationsOverview:
+    return service.overview()
+
+
+@router.get(
+    "/api/v1/admin/operations/slis", response_model=list[SLIDefinitionResponse]
+)
+def sli_definitions(service: Service, _admin: Admin) -> list[SLIDefinitionResponse]:
+    return service.definitions()
+
+
+@router.get(
+    "/api/v1/admin/operations/slo-policies", response_model=list[SLOPolicyResponse]
+)
+def slo_policies(service: Service, _admin: Admin) -> list[SLOPolicyResponse]:
+    return service.policies()
+
+
+@router.post(
+    "/api/v1/admin/operations/slo-policies",
+    response_model=SLOPolicyResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_slo_policy(
+    request: SLOPolicyCreate, service: Service, _admin: Admin
+) -> SLOPolicyResponse:
+    return service.create_policy(request)
+
+
+@router.get(
+    "/api/v1/admin/operations/slo-evaluations", response_model=list[SLOEvaluationResponse]
+)
+def slo_evaluations(
+    service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
+) -> list[SLOEvaluationResponse]:
+    return service.evaluations(limit)
+
+
+@router.post(
+    "/api/v1/admin/operations/slo-evaluations/run",
+    response_model=list[SLOEvaluationResponse],
+)
+def evaluate_slos(service: Service, _admin: Admin) -> list[SLOEvaluationResponse]:
+    return service.evaluate_slos()
+
+
+@router.get(
+    "/api/v1/admin/operations/alert-rules", response_model=list[AlertRuleResponse]
+)
+def alert_rules(service: Service, _admin: Admin) -> list[AlertRuleResponse]:
+    return service.rules()
+
+
+@router.post(
+    "/api/v1/admin/operations/alert-rules",
+    response_model=AlertRuleResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_alert_rule(
+    request: AlertRuleCreate, service: Service, _admin: Admin
+) -> AlertRuleResponse:
+    return service.create_rule(request)
+
+
+@router.get("/api/v1/admin/operations/alerts", response_model=list[AlertResponse])
+def alerts(
+    service: Service, _admin: Admin, limit: int = Query(default=200, ge=1, le=1000)
+) -> list[AlertResponse]:
+    return service.alerts(limit)
+
+
+@router.post("/api/v1/admin/operations/alerts/evaluate", response_model=list[AlertResponse])
+def evaluate_alerts(service: Service, _admin: Admin) -> list[AlertResponse]:
+    return service.evaluate_alerts()
+
+
+@router.post(
+    "/api/v1/admin/operations/alerts/{alert_id}/acknowledge", response_model=AlertResponse
+)
+def acknowledge_alert(
+    alert_id: uuid.UUID, request: AlertAction, service: Service, _admin: Admin
+) -> AlertResponse:
+    return service.acknowledge(alert_id, request)
+
+
+@router.get(
+    "/api/v1/admin/operations/alerts/{alert_id}/history",
+    response_model=list[AlertHistoryResponse],
+)
+def alert_history(
+    alert_id: uuid.UUID, service: Service, _admin: Admin
+) -> list[AlertHistoryResponse]:
+    rows = service.session.scalars(
+        select(AlertHistoryEvent)
+        .where(AlertHistoryEvent.alert_id == alert_id)
+        .order_by(AlertHistoryEvent.occurred_at)
+    )
+    return [AlertHistoryResponse.model_validate(item) for item in rows]
+
+
+@router.get(
+    "/api/v1/admin/operations/maintenance-windows",
+    response_model=list[MaintenanceWindowResponse],
+)
+def maintenance_windows(service: Service, _admin: Admin) -> list[MaintenanceWindowResponse]:
+    return service.maintenance_windows()
+
+
+@router.post(
+    "/api/v1/admin/operations/maintenance-windows",
+    response_model=MaintenanceWindowResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_maintenance_window(
+    request: MaintenanceWindowCreate, service: Service, _admin: Admin
+) -> MaintenanceWindowResponse:
+    return service.create_maintenance_window(request)
+
+
+@router.get("/api/v1/admin/operations/capacity", response_model=list[CapacitySnapshotResponse])
+def capacity(
+    service: Service, _admin: Admin, limit: int = Query(default=500, ge=1, le=5000)
+) -> list[CapacitySnapshotResponse]:
+    return service.capacity(limit)
+
+
+@router.post(
+    "/api/v1/admin/operations/capacity/collect", response_model=list[CapacitySnapshotResponse]
+)
+def collect_capacity(service: Service, _admin: Admin) -> list[CapacitySnapshotResponse]:
+    return service.collect_capacity()
+
+
+@router.get("/api/v1/admin/operations/capacity/{node_id}/trend", response_model=TrendResponse)
+def capacity_trend(
+    node_id: uuid.UUID,
+    service: Service,
+    _admin: Admin,
+    hours: int = Query(default=24, ge=1, le=2160),
+) -> TrendResponse:
+    return service.capacity_trend(node_id, hours)
+
+
+@router.post("/api/v1/admin/operations/retention/run", response_model=dict[str, int])
+def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
+    return service.apply_retention()
+
+
+@router.get("/api/v1/admin/operations/incidents", response_model=list[IncidentResponse])
+def incidents(service: Service, _admin: Admin) -> list[IncidentResponse]:
+    rows = service.session.scalars(
+        select(OperationalIncident).order_by(OperationalIncident.last_seen_at.desc())
+    )
+    return [IncidentResponse.model_validate(item) for item in rows]
+
+
+@router.get("/api/v1/admin/operations/incidents/{incident_id}/timeline")
+def incident_timeline(
+    incident_id: uuid.UUID, service: Service, _admin: Admin
+) -> list[dict[str, Any]]:
+    rows = service.session.scalars(
+        select(IncidentTimelineEvent)
+        .where(IncidentTimelineEvent.incident_id == incident_id)
+        .order_by(IncidentTimelineEvent.occurred_at)
+    )
+    return [
+        {
+            "id": str(item.id),
+            "alert_id": str(item.alert_id) if item.alert_id else None,
+            "event_type": item.event_type,
+            "relation": item.relation,
+            "summary": item.summary,
+            "occurred_at": item.occurred_at,
+        }
+        for item in rows
+    ]

backend/src/modelforge_api/api/routes/recovery.py

@@ -0,0 +1,215 @@
+"""Operator-only recovery API.
+
+Backup creation, verification and restore *planning* are safe control-plane operations. Restore
+*execution* is deliberately restricted: it can only run against an isolated destination that is
+not this control plane's own database, and replacing a production database stays an operator
+runbook/CLI action rather than a remote call.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, Query, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.recovery import (
+    ArtifactRecoveryCreate,
+    ArtifactRecoveryResponse,
+    BackupCapacityEstimate,
+    BackupSetCreate,
+    BackupSetResponse,
+    RecoveryAssetResponse,
+    RecoveryDashboard,
+    RecoveryPolicyCreate,
+    RecoveryPolicyResponse,
+    RestoreAdvanceRequest,
+    RestoreOperationEventResponse,
+    RestoreOperationResponse,
+    RestorePlanCreate,
+    RestorePlanResponse,
+)
+from modelforge_api.services.recovery import RecoveryService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+    prefix="/api/v1/admin/recovery",
+    tags=["recovery"],
+    dependencies=[Depends(require_viewer)],
+)
+
+
+def get_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> RecoveryService:
+    return RecoveryService(session, settings)
+
+
+Service = Annotated[RecoveryService, Depends(get_service)]
+
+
+@router.get("/dashboard", response_model=RecoveryDashboard)
+def dashboard(service: Service, _admin: Admin) -> RecoveryDashboard:
+    return service.dashboard()
+
+
+@router.get("/policies", response_model=list[RecoveryPolicyResponse])
+def policies(service: Service, _admin: Admin) -> list[RecoveryPolicyResponse]:
+    return service.policies()
+
+
+@router.post(
+    "/policies",
+    response_model=RecoveryPolicyResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_policy(
+    request: RecoveryPolicyCreate, service: Service, _admin: Admin
+) -> RecoveryPolicyResponse:
+    return service.create_policy(request)
+
+
+@router.get("/assets", response_model=list[RecoveryAssetResponse])
+def assets(service: Service, _admin: Admin) -> list[RecoveryAssetResponse]:
+    return service.assets()
+
+
+@router.get("/capacity", response_model=BackupCapacityEstimate)
+def capacity(service: Service, _admin: Admin) -> BackupCapacityEstimate:
+    return service.estimate_capacity()
+
+
+@router.get("/backups", response_model=list[BackupSetResponse])
+def backups(
+    service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
+) -> list[BackupSetResponse]:
+    return service.backups(limit)
+
+
+@router.post(
+    "/backups", response_model=BackupSetResponse, status_code=status.HTTP_201_CREATED
+)
+def create_backup(
+    request: BackupSetCreate, service: Service, _admin: Admin
+) -> BackupSetResponse:
+    return service.create_backup(request)
+
+
+@router.get("/backups/{backup_set_id}", response_model=BackupSetResponse)
+def backup(backup_set_id: uuid.UUID, service: Service, _admin: Admin) -> BackupSetResponse:
+    return service.backup(backup_set_id)
+
+
+@router.post("/backups/{backup_set_id}/verify", response_model=BackupSetResponse)
+def verify_backup(
+    backup_set_id: uuid.UUID, service: Service, _admin: Admin
+) -> BackupSetResponse:
+    return service.verify_backup(backup_set_id)
+
+
+@router.post("/retention/run", response_model=dict[str, int])
+def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
+    return service.apply_retention()
+
+
+@router.get("/restore-plans", response_model=list[RestorePlanResponse])
+def restore_plans(
+    service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
+) -> list[RestorePlanResponse]:
+    return service.restore_plans(limit)
+
+
+@router.post(
+    "/restore-plans", response_model=RestorePlanResponse, status_code=status.HTTP_201_CREATED
+)
+def create_restore_plan(
+    request: RestorePlanCreate, service: Service, _admin: Admin
+) -> RestorePlanResponse:
+    return service.create_restore_plan(request)
+
+
+@router.get("/restore-plans/{plan_id}", response_model=RestorePlanResponse)
+def restore_plan(plan_id: uuid.UUID, service: Service, _admin: Admin) -> RestorePlanResponse:
+    return service.restore_plan(plan_id)
+
+
+@router.post("/restore-plans/{plan_id}/preflight", response_model=RestorePlanResponse)
+def preflight(plan_id: uuid.UUID, service: Service, _admin: Admin) -> RestorePlanResponse:
+    return service.preflight(plan_id)
+
+
+@router.post(
+    "/restore-plans/{plan_id}/start",
+    response_model=RestoreOperationResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def start_restore(
+    plan_id: uuid.UUID,
+    request: RestoreAdvanceRequest,
+    service: Service,
+    _admin: Admin,
+) -> RestoreOperationResponse:
+    return service.start_restore(plan_id, request)
+
+
+@router.get("/restore-operations", response_model=list[RestoreOperationResponse])
+def restore_operations(
+    service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
+) -> list[RestoreOperationResponse]:
+    return service.restore_operations(limit)
+
+
+@router.get("/restore-operations/{operation_id}", response_model=RestoreOperationResponse)
+def restore_operation(
+    operation_id: uuid.UUID, service: Service, _admin: Admin
+) -> RestoreOperationResponse:
+    return service.restore_operation(operation_id)
+
+
+@router.post("/restore-operations/{operation_id}/advance", response_model=RestoreOperationResponse)
+def advance_restore(
+    operation_id: uuid.UUID,
+    request: RestoreAdvanceRequest,
+    service: Service,
+    _admin: Admin,
+) -> RestoreOperationResponse:
+    return service.advance_restore(operation_id, request)
+
+
+@router.get(
+    "/restore-operations/{operation_id}/events",
+    response_model=list[RestoreOperationEventResponse],
+)
+def restore_events(
+    operation_id: uuid.UUID, service: Service, _admin: Admin
+) -> list[RestoreOperationEventResponse]:
+    return service.restore_events(operation_id)
+
+
+@router.get("/artifact-recoveries", response_model=list[ArtifactRecoveryResponse])
+def artifact_recoveries(
+    service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
+) -> list[ArtifactRecoveryResponse]:
+    return service.artifact_recoveries(limit)
+
+
+@router.post(
+    "/artifact-recoveries",
+    response_model=ArtifactRecoveryResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def plan_artifact_recovery(
+    request: ArtifactRecoveryCreate, service: Service, _admin: Admin
+) -> ArtifactRecoveryResponse:
+    return service.plan_artifact_recovery(request)
+
+
+@router.get("/fingerprint", response_model=dict[str, Any])
+def fingerprint(service: Service, _admin: Admin) -> dict[str, Any]:
+    """Bounded semantic fingerprint of the live control plane; contains no secret material."""
+
+    return service.fingerprint()

backend/src/modelforge_api/api/routes/registry.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import math
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query, Response, status
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_admin, require_operator, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.registry import (
+    ArtifactCreate,
+    ArtifactResponse,
+    CapacityDecision,
+    DerivedArtifactCreate,
+    DerivedArtifactResponse,
+    ModelCreate,
+    ModelResponse,
+    ModelUpdate,
+    Page,
+    RevisionCreate,
+    RevisionResponse,
+    StorageRootCreate,
+    StorageRootObservation,
+    StorageRootResponse,
+    StorageRootUpdate,
+    VerifyResponse,
+)
+from modelforge_api.services.registry import RegistryService
+
+router = APIRouter(
+    prefix="/api/v1", tags=["model-registry"], dependencies=[Depends(require_viewer)]
+)
+
+
+def get_registry_service(session: Annotated[Session, Depends(get_session)]) -> RegistryService:
+    return RegistryService(session)
+
+
+Service = Annotated[RegistryService, Depends(get_registry_service)]
+
+
+def page_values[T: BaseModel](items: list[T], page: int, page_size: int) -> Page[T]:
+    total = len(items)
+    start = (page - 1) * page_size
+    return Page(
+        items=items[start : start + page_size],
+        page=page,
+        page_size=page_size,
+        total=total,
+        pages=math.ceil(total / page_size) if total else 0,
+    )
+
+
+@router.get("/models", response_model=Page[ModelResponse])
+def list_models(
+    service: Service,
+    page: Annotated[int, Query(ge=1)] = 1,
+    page_size: Annotated[int, Query(ge=1, le=100)] = 20,
+    search: str | None = None,
+    lifecycle: str | None = None,
+    source_type: str | None = None,
+) -> Page[ModelResponse]:
+    return service.list_models(
+        page=page,
+        page_size=page_size,
+        search=search,
+        lifecycle=lifecycle,
+        source_type=source_type,
+    )
+
+
+@router.post(
+    "/models",
+    response_model=ModelResponse,
+    status_code=status.HTTP_201_CREATED,
+    dependencies=[Depends(require_operator)],
+)
+def create_model(request: ModelCreate, service: Service) -> ModelResponse:
+    return service.create_model(request)
+
+
+@router.get("/models/{model_id}", response_model=ModelResponse)
+def get_model(model_id: uuid.UUID, service: Service) -> ModelResponse:
+    return service.get_model(model_id)
+
+
+@router.patch(
+    "/models/{model_id}",
+    response_model=ModelResponse,
+    dependencies=[Depends(require_operator)],
+)
+def update_model(model_id: uuid.UUID, request: ModelUpdate, service: Service) -> ModelResponse:
+    return service.update_model(model_id, request)
+
+
+@router.post(
+    "/models/{model_id}/deprecate",
+    response_model=ModelResponse,
+    dependencies=[Depends(require_admin)],
+)
+def deprecate_model(model_id: uuid.UUID, service: Service) -> ModelResponse:
+    return service.deprecate_model(model_id)
+
+
+@router.post(
+    "/models/{model_id}/archive",
+    response_model=ModelResponse,
+    dependencies=[Depends(require_admin)],
+)
+def archive_model(model_id: uuid.UUID, service: Service) -> ModelResponse:
+    return service.archive_model(model_id)
+
+
+@router.delete(
+    "/models/{model_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    dependencies=[Depends(require_admin)],
+)
+def delete_model(model_id: uuid.UUID, service: Service) -> Response:
+    service.delete("model", model_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.get("/models/{model_id}/revisions", response_model=Page[RevisionResponse])
+def list_revisions(
+    model_id: uuid.UUID,
+    service: Service,
+    page: Annotated[int, Query(ge=1)] = 1,
+    page_size: Annotated[int, Query(ge=1, le=100)] = 20,
+) -> Page[RevisionResponse]:
+    return page_values(service.revisions(model_id), page, page_size)
+
+
+@router.post(
+    "/models/{model_id}/revisions",
+    response_model=RevisionResponse,
+    status_code=status.HTTP_201_CREATED,
+    dependencies=[Depends(require_operator)],
+)
+def create_revision(
+    model_id: uuid.UUID, request: RevisionCreate, service: Service
+) -> RevisionResponse:
+    return service.create_revision(model_id, request)
+
+
+@router.delete(
+    "/revisions/{revision_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    dependencies=[Depends(require_admin)],
+)
+def delete_revision(revision_id: uuid.UUID, service: Service) -> Response:
+    service.delete("model_revision", revision_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.get("/revisions/{revision_id}/artifacts", response_model=Page[ArtifactResponse])
+def list_artifacts(
+    revision_id: uuid.UUID,
+    service: Service,
+    page: Annotated[int, Query(ge=1)] = 1,
+    page_size: Annotated[int, Query(ge=1, le=100)] = 20,
+) -> Page[ArtifactResponse]:
+    return page_values(service.artifacts(revision_id), page, page_size)
+
+
+@router.post(
+    "/revisions/{revision_id}/artifacts",
+    response_model=ArtifactResponse,
+    status_code=status.HTTP_201_CREATED,
+    dependencies=[Depends(require_operator)],
+)
+def create_artifact(
+    revision_id: uuid.UUID, request: ArtifactCreate, service: Service
+) -> ArtifactResponse:
+    return service.create_artifact(revision_id, request)
+
+
+@router.delete(
+    "/artifacts/{artifact_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    dependencies=[Depends(require_admin)],
+)
+def delete_artifact(artifact_id: uuid.UUID, service: Service) -> Response:
+    service.delete("model_artifact", artifact_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.post(
+    "/artifacts/{artifact_id}/verify",
+    response_model=VerifyResponse,
+    dependencies=[Depends(require_operator)],
+)
+def verify_artifact(
+    artifact_id: uuid.UUID, location_id: uuid.UUID, service: Service
+) -> VerifyResponse:
+    return service.verify_artifact(artifact_id, location_id)
+
+
+@router.get(
+    "/revisions/{revision_id}/derived-artifacts", response_model=Page[DerivedArtifactResponse]
+)
+def list_derived_artifacts(
+    revision_id: uuid.UUID,
+    service: Service,
+    page: Annotated[int, Query(ge=1)] = 1,
+    page_size: Annotated[int, Query(ge=1, le=100)] = 20,
+) -> Page[DerivedArtifactResponse]:
+    return page_values(service.derived(revision_id), page, page_size)
+
+
+@router.post(
+    "/derived-artifacts",
+    response_model=DerivedArtifactResponse,
+    status_code=status.HTTP_201_CREATED,
+    dependencies=[Depends(require_operator)],
+)
+def create_derived_artifact(
+    request: DerivedArtifactCreate, service: Service
+) -> DerivedArtifactResponse:
+    return service.create_derived(request)
+
+
+@router.delete(
+    "/derived-artifacts/{artifact_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    dependencies=[Depends(require_admin)],
+)
+def delete_derived_artifact(artifact_id: uuid.UUID, service: Service) -> Response:
+    service.delete("derived_artifact", artifact_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.get("/storage-roots", response_model=list[StorageRootResponse])
+def list_storage_roots(
+    service: Service, node_id: uuid.UUID | None = None
+) -> list[StorageRootResponse]:
+    return service.storage_roots(node_id)
+
+
+@router.post(
+    "/storage-roots",
+    response_model=StorageRootResponse,
+    status_code=status.HTTP_201_CREATED,
+    dependencies=[Depends(require_admin)],
+)
+def create_storage_root(request: StorageRootCreate, service: Service) -> StorageRootResponse:
+    return service.create_storage_root(request)
+
+
+@router.patch(
+    "/storage-roots/{root_id}",
+    response_model=StorageRootResponse,
+    dependencies=[Depends(require_admin)],
+)
+def update_storage_root(
+    root_id: uuid.UUID, request: StorageRootUpdate, service: Service
+) -> StorageRootResponse:
+    return service.update_storage_root(root_id, request)
+
+
+@router.post(
+    "/storage-roots/{root_id}/observations",
+    response_model=StorageRootResponse,
+    dependencies=[Depends(require_operator)],
+)
+def observe_storage_root(
+    root_id: uuid.UUID, request: StorageRootObservation, service: Service
+) -> StorageRootResponse:
+    return service.observe_storage_root(root_id, request)
+
+
+@router.get(
+    "/storage-roots/{root_id}/capacity",
+    response_model=CapacityDecision,
+    dependencies=[Depends(require_operator)],
+)
+def check_capacity(root_id: uuid.UUID, requested_bytes: int, service: Service) -> CapacityDecision:
+    return service.check_capacity(root_id, requested_bytes)
+
+
+@router.delete(
+    "/storage-roots/{root_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    dependencies=[Depends(require_admin)],
+)
+def delete_storage_root(root_id: uuid.UUID, service: Service) -> Response:
+    service.delete("storage_root", root_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

backend/src/modelforge_api/api/routes/runtime.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import require_admin, require_operator, require_viewer
+from modelforge_api.db import get_session
+from modelforge_api.domain.runtime import (
+    CompatibilityAssessmentCreate,
+    CompatibilityAssessmentResponse,
+    DeploymentCandidateResponse,
+    ExecutionApprovalCreate,
+    ExecutionApprovalResponse,
+    RuntimeEnvironmentCreate,
+    RuntimeEnvironmentResponse,
+    RuntimeProbeCreate,
+    RuntimeProbeResponse,
+    RuntimeProfileCreate,
+    RuntimeProfileResponse,
+)
+from modelforge_api.services.runtime import RuntimeService
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(
+    prefix="/api/v1", tags=["runtime-plane"], dependencies=[Depends(require_viewer)]
+)
+
+
+def get_runtime_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+) -> RuntimeService:
+    return RuntimeService(session, settings)
+
+
+Service = Annotated[RuntimeService, Depends(get_runtime_service)]
+
+
+@router.post(
+    "/runtime-environments",
+    response_model=RuntimeEnvironmentResponse,
+    status_code=201,
+    dependencies=[Depends(require_operator)],
+)
+def create_environment(
+    request: RuntimeEnvironmentCreate, service: Service
+) -> RuntimeEnvironmentResponse:
+    return service.create_environment(request)
+
+
+@router.get("/runtime-environments", response_model=list[RuntimeEnvironmentResponse])
+def environments(service: Service) -> list[RuntimeEnvironmentResponse]:
+    return service.environments()
+
+
+@router.get("/runtime-environments/{environment_id}", response_model=RuntimeEnvironmentResponse)
+def environment(environment_id: uuid.UUID, service: Service) -> RuntimeEnvironmentResponse:
+    return service.environment(environment_id)
+
+
+@router.post(
+    "/runtime-profiles",
+    response_model=RuntimeProfileResponse,
+    status_code=201,
+    dependencies=[Depends(require_operator)],
+)
+def create_profile(request: RuntimeProfileCreate, service: Service) -> RuntimeProfileResponse:
+    return service.create_profile(request)
+
+
+@router.get("/runtime-profiles", response_model=list[RuntimeProfileResponse])
+def profiles(service: Service) -> list[RuntimeProfileResponse]:
+    return service.profiles()
+
+
+@router.get("/runtime-profiles/{profile_id}", response_model=RuntimeProfileResponse)
+def profile(profile_id: uuid.UUID, service: Service) -> RuntimeProfileResponse:
+    return service.profile(profile_id)
+
+
+@router.post(
+    "/artifact-sets/{artifact_set_id}/compatibility-assessments",
+    response_model=CompatibilityAssessmentResponse,
+    status_code=201,
+    dependencies=[Depends(require_operator)],
+)
+def assess(
+    artifact_set_id: uuid.UUID,
+    request: CompatibilityAssessmentCreate,
+    service: Service,
+) -> CompatibilityAssessmentResponse:
+    profile = service.repo.profile(request.runtime_profile_id)
+    if not profile or profile.artifact_set_id != artifact_set_id:
+        from modelforge_api.services.registry import RegistryConflict
+
+        raise RegistryConflict("runtime profile belongs to another artifact set")
+    return service.assess(request)
+
+
+@router.get("/compatibility-assessments", response_model=list[CompatibilityAssessmentResponse])
+def assessments(
+    service: Service,
+    artifact_set_id: Annotated[uuid.UUID | None, Query()] = None,
+    model_id: Annotated[uuid.UUID | None, Query()] = None,
+    compute_node_id: Annotated[uuid.UUID | None, Query()] = None,
+    runtime_profile_id: Annotated[uuid.UUID | None, Query()] = None,
+    status: Annotated[str | None, Query()] = None,
+) -> list[CompatibilityAssessmentResponse]:
+    return service.assessments(
+        artifact_set_id=artifact_set_id,
+        model_id=model_id,
+        compute_node_id=compute_node_id,
+        runtime_profile_id=runtime_profile_id,
+        status=status,
+    )
+
+
+@router.get(
+    "/compatibility-assessments/{assessment_id}",
+    response_model=CompatibilityAssessmentResponse,
+)
+def assessment(assessment_id: uuid.UUID, service: Service) -> CompatibilityAssessmentResponse:
+    return service.assessment(assessment_id)
+
+
+@router.post(
+    "/artifact-sets/{artifact_set_id}/execution-approvals",
+    response_model=ExecutionApprovalResponse,
+    status_code=201,
+    dependencies=[Depends(require_admin)],
+)
+def approve_execution(
+    artifact_set_id: uuid.UUID,
+    request: ExecutionApprovalCreate,
+    service: Service,
+) -> ExecutionApprovalResponse:
+    return service.approve(artifact_set_id, request)
+
+
+@router.get("/execution-approvals", response_model=list[ExecutionApprovalResponse])
+def approvals(
+    service: Service,
+    artifact_set_id: Annotated[uuid.UUID | None, Query()] = None,
+) -> list[ExecutionApprovalResponse]:
+    return service.approvals(artifact_set_id)
+
+
+@router.post(
+    "/runtime-probes",
+    response_model=RuntimeProbeResponse,
+    status_code=201,
+    dependencies=[Depends(require_operator)],
+)
+def create_probe(request: RuntimeProbeCreate, service: Service) -> RuntimeProbeResponse:
+    return service.create_probe(request)
+
+
+@router.get("/runtime-probes", response_model=list[RuntimeProbeResponse])
+def probes(service: Service) -> list[RuntimeProbeResponse]:
+    return service.probes()
+
+
+@router.get("/runtime-probes/{probe_id}", response_model=RuntimeProbeResponse)
+def probe(probe_id: uuid.UUID, service: Service) -> RuntimeProbeResponse:
+    return service.probe(probe_id)
+
+
+@router.post(
+    "/runtime-probes/{probe_id}/cancel",
+    response_model=RuntimeProbeResponse,
+    dependencies=[Depends(require_operator)],
+)
+def cancel_probe(probe_id: uuid.UUID, service: Service) -> RuntimeProbeResponse:
+    return service.cancel(probe_id)
+
+
+@router.get("/deployment-candidates", response_model=list[DeploymentCandidateResponse])
+def deployment_candidates(service: Service) -> list[DeploymentCandidateResponse]:
+    return service.candidates()
+
+
+@router.get("/deployment-candidates/{candidate_id}", response_model=DeploymentCandidateResponse)
+def deployment_candidate(candidate_id: uuid.UUID, service: Service) -> DeploymentCandidateResponse:
+    return service.candidate(candidate_id)

backend/src/modelforge_api/api/routes/serving.py

@@ -0,0 +1,582 @@
+from __future__ import annotations
+
+import uuid
+from functools import lru_cache
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Header, Query, Request, Response, status
+from sqlalchemy.orm import Session
+
+from modelforge_api.api.authorization import Admin, require_viewer
+from modelforge_api.api.routes.agent import AgentIdentity
+from modelforge_api.db import get_session
+from modelforge_api.domain.serving import (
+    AgentServingJobComplete,
+    AgentServingJobFailure,
+    AgentServingJobLease,
+    AgentServingStateAck,
+    AgentServingStateReport,
+    CapabilityDeploymentResponse,
+    CapabilityExperimentCreate,
+    CapabilityExperimentResponse,
+    CapabilityPromotionCreate,
+    CoResidencyEvidenceCreate,
+    CoResidencyEvidenceResponse,
+    EmbeddingInvokeRequest,
+    EmbeddingInvokeResponse,
+    GatewayRequestResponse,
+    OCRInvokeRequest,
+    OCRInvokeResponse,
+    OpenAIEmbeddingItem,
+    OpenAIEmbeddingRequest,
+    OpenAIEmbeddingResponse,
+    OpenAIUsage,
+    PlacementPlanRequest,
+    PlacementPlanResponse,
+    ProductionApprovalCreate,
+    ProductionApprovalResponse,
+    ProjectFitEvidenceCreate,
+    ProjectFitEvidenceResponse,
+    ProjectIntegrationResponse,
+    RerankingInvokeRequest,
+    RerankingInvokeResponse,
+    ResidencyPolicyUpdate,
+    SchedulerBudgetResponse,
+    SchedulerMetricsResponse,
+    SchedulerPolicyResponse,
+    SchedulerPolicyUpdate,
+    ServiceClientCreate,
+    ServiceClientCreated,
+    ServiceClientResponse,
+    SpeechTranscriptionInvokeRequest,
+    SpeechTranscriptionInvokeResponse,
+    StableEmbeddingInvokeResponse,
+    VisionEmbeddingInvokeRequest,
+    VisionEmbeddingInvokeResponse,
+)
+from modelforge_api.persistence.models import ServiceClient
+from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
+from modelforge_api.services.serving import (
+    CapabilityAuthenticationEvidence,
+    ServingError,
+    ServingService,
+)
+from modelforge_api.services.transient_payloads import RedisPayloadStore
+from modelforge_api.settings import Settings, get_settings
+
+router = APIRouter(tags=["capability-serving"])
+
+
+@lru_cache
+def payload_store() -> RedisPayloadStore:
+    return RedisPayloadStore(get_settings().redis_url)
+
+
+def get_serving_service(
+    session: Annotated[Session, Depends(get_session)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    manifests: Annotated[ManifestRegistry, Depends(get_manifest_registry)],
+) -> ServingService:
+    return ServingService(session, settings, manifests, payload_store())
+
+
+Service = Annotated[ServingService, Depends(get_serving_service)]
+
+
+def _authenticate_capability_client(
+    request: Request,
+    service: ServingService,
+    authorization: str | None,
+    capability: str,
+) -> ServiceClient:
+    evidence = getattr(request.state, "capability_authentication", None)
+    if isinstance(evidence, CapabilityAuthenticationEvidence):
+        return service.reuse_authentication(evidence, authorization, capability)
+    return service.authenticate(authorization, capability)
+
+
+def authenticate_rag_embedding_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    return _authenticate_capability_client(request, service, authorization, "rag.embedding@1")
+
+
+def authenticate_rag_reranking_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    return _authenticate_capability_client(request, service, authorization, "rag.reranking@1")
+
+
+def authenticate_document_ocr_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    return _authenticate_capability_client(request, service, authorization, "document.ocr@1")
+
+
+def authenticate_vision_embedding_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    return _authenticate_capability_client(request, service, authorization, "vision.embedding@1")
+
+
+def authenticate_speech_transcription_client(
+    request: Request,
+    service: Service,
+    authorization: Annotated[str | None, Header(alias="Authorization")] = None,
+) -> ServiceClient:
+    return _authenticate_capability_client(
+        request, service, authorization, "speech.transcription@1"
+    )
+
+
+RagEmbeddingClient = Annotated[ServiceClient, Depends(authenticate_rag_embedding_client)]
+RagRerankingClient = Annotated[ServiceClient, Depends(authenticate_rag_reranking_client)]
+DocumentOcrClient = Annotated[ServiceClient, Depends(authenticate_document_ocr_client)]
+VisionEmbeddingClient = Annotated[ServiceClient, Depends(authenticate_vision_embedding_client)]
+SpeechTranscriptionClient = Annotated[
+    ServiceClient, Depends(authenticate_speech_transcription_client)
+]
+
+
+@router.post(
+    "/api/v1/admin/deployment-candidates/{candidate_id}/production-approvals",
+    response_model=ProductionApprovalResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def approve_production(
+    candidate_id: uuid.UUID,
+    request: ProductionApprovalCreate,
+    service: Service,
+    _admin: Admin,
+) -> ProductionApprovalResponse:
+    return service.approve_production(candidate_id, request)
+
+
+@router.get(
+    "/api/v1/admin/production-approvals",
+    response_model=list[ProductionApprovalResponse],
+)
+def production_approvals(service: Service, _admin: Admin) -> list[ProductionApprovalResponse]:
+    return service.approvals()
+
+
+@router.post(
+    "/api/v1/admin/deployment-candidates/{candidate_id}/promote",
+    response_model=CapabilityDeploymentResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def promote_candidate(
+    candidate_id: uuid.UUID,
+    request: CapabilityPromotionCreate,
+    service: Service,
+    _admin: Admin,
+) -> CapabilityDeploymentResponse:
+    return service.promote(candidate_id, request)
+
+
+@router.get(
+    "/api/v1/capability-deployments",
+    response_model=list[CapabilityDeploymentResponse],
+    dependencies=[Depends(require_viewer)],
+)
+def capability_deployments(service: Service) -> list[CapabilityDeploymentResponse]:
+    return service.deployments()
+
+
+@router.post(
+    "/api/v1/admin/deployment-candidates/{candidate_id}/experiments",
+    response_model=CapabilityExperimentResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_capability_experiment(
+    candidate_id: uuid.UUID,
+    request: CapabilityExperimentCreate,
+    service: Service,
+    _admin: Admin,
+) -> CapabilityExperimentResponse:
+    return service.create_experiment(candidate_id, request)
+
+
+@router.get(
+    "/api/v1/capability-experiments",
+    response_model=list[CapabilityExperimentResponse],
+    dependencies=[Depends(require_viewer)],
+)
+def capability_experiments(service: Service) -> list[CapabilityExperimentResponse]:
+    return service.experiments()
+
+
+@router.post(
+    "/api/v1/admin/capability-experiments/{experiment_id}/deactivate",
+    response_model=CapabilityExperimentResponse,
+)
+def deactivate_capability_experiment(
+    experiment_id: uuid.UUID, service: Service, _admin: Admin
+) -> CapabilityExperimentResponse:
+    return service.deactivate_experiment(experiment_id)
+
+
+@router.post(
+    "/api/v1/admin/capability-deployments/{deployment_id}/unload",
+    response_model=CapabilityDeploymentResponse,
+)
+def unload_deployment(
+    deployment_id: uuid.UUID, service: Service, _admin: Admin
+) -> CapabilityDeploymentResponse:
+    return service.request_unload(deployment_id)
+
+
+@router.post(
+    "/api/v1/admin/capability-deployments/{deployment_id}/drain",
+    response_model=CapabilityDeploymentResponse,
+)
+def drain_deployment(
+    deployment_id: uuid.UUID, service: Service, _admin: Admin
+) -> CapabilityDeploymentResponse:
+    return service.request_unload(deployment_id, drain=True)
+
+
+@router.post(
+    "/api/v1/admin/service-clients",
+    response_model=ServiceClientCreated,
+    status_code=status.HTTP_201_CREATED,
+)
+def create_service_client(
+    request: ServiceClientCreate,
+    service: Service,
+    _admin: Admin,
+    response: Response,
+) -> ServiceClientCreated:
+    response.headers["Cache-Control"] = "no-store"
+    return service.create_client(request)
+
+
+@router.get("/api/v1/admin/service-clients", response_model=list[ServiceClientResponse])
+def service_clients(service: Service, _admin: Admin) -> list[ServiceClientResponse]:
+    return service.clients()
+
+
+@router.delete(
+    "/api/v1/admin/service-clients/{client_id}/credential",
+    response_model=ServiceClientResponse,
+)
+def revoke_service_credential(
+    client_id: uuid.UUID, service: Service, _admin: Admin
+) -> ServiceClientResponse:
+    return service.revoke_client_credential(client_id)
+
+
+@router.post(
+    "/api/v1/admin/service-clients/{client_id}/credential/rotate",
+    response_model=ServiceClientCreated,
+)
+def rotate_service_credential(
+    client_id: uuid.UUID, service: Service, _admin: Admin, response: Response
+) -> ServiceClientCreated:
+    response.headers["Cache-Control"] = "no-store"
+    return service.rotate_client_credential(client_id)
+
+
+@router.get(
+    "/api/v1/project-integrations",
+    response_model=list[ProjectIntegrationResponse],
+    dependencies=[Depends(require_viewer)],
+)
+def project_integrations(service: Service) -> list[ProjectIntegrationResponse]:
+    return service.project_integrations()
+
+
+@router.post(
+    "/api/v1/admin/project-fit-evidence",
+    response_model=ProjectFitEvidenceResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def record_project_fit_evidence(
+    request: ProjectFitEvidenceCreate, service: Service, _admin: Admin
+) -> ProjectFitEvidenceResponse:
+    return service.record_project_fit(request)
+
+
+@router.get(
+    "/api/v1/scheduler",
+    response_model=list[SchedulerBudgetResponse],
+    dependencies=[Depends(require_viewer)],
+)
+def scheduler_overview(service: Service) -> list[SchedulerBudgetResponse]:
+    return service.scheduler_overview()
+
+
+@router.get(
+    "/api/v1/scheduler/co-residency",
+    response_model=list[CoResidencyEvidenceResponse],
+    dependencies=[Depends(require_viewer)],
+)
+def co_residency_matrix(service: Service) -> list[CoResidencyEvidenceResponse]:
+    return service.co_residency_matrix()
+
+
+@router.post(
+    "/api/v1/admin/scheduler/co-residency-evidence",
+    response_model=CoResidencyEvidenceResponse,
+    status_code=status.HTTP_201_CREATED,
+)
+def record_co_residency_evidence(
+    request: CoResidencyEvidenceCreate, service: Service, _admin: Admin
+) -> CoResidencyEvidenceResponse:
+    return service.record_co_residency_evidence(request)
+
+
+@router.get("/api/v1/admin/scheduler/policy", response_model=SchedulerPolicyResponse)
+def scheduler_policy(service: Service, _admin: Admin) -> SchedulerPolicyResponse:
+    return service.scheduler_policy()
+
+
+@router.get("/api/v1/admin/scheduler/metrics", response_model=SchedulerMetricsResponse)
+def scheduler_metrics(service: Service, _admin: Admin) -> SchedulerMetricsResponse:
+    return service.scheduler_metrics()
+
+
+@router.put("/api/v1/admin/scheduler/policy", response_model=SchedulerPolicyResponse)
+def update_scheduler_policy(
+    request: SchedulerPolicyUpdate, service: Service, _admin: Admin
+) -> SchedulerPolicyResponse:
+    return service.update_scheduler_policy(request)
+
+
+@router.post(
+    "/api/v1/admin/scheduler/placements/{deployment_id}/dry-run",
+    response_model=PlacementPlanResponse,
+)
+def dry_run_placement(
+    deployment_id: uuid.UUID,
+    request: PlacementPlanRequest,
+    service: Service,
+    _admin: Admin,
+) -> PlacementPlanResponse:
+    return service.dry_run_placement(deployment_id, request)
+
+
+@router.get(
+    "/api/v1/admin/scheduler/placements",
+    response_model=list[PlacementPlanResponse],
+)
+def placement_history(
+    service: Service,
+    _admin: Admin,
+    limit: Annotated[int, Query(ge=1, le=500)] = 100,
+) -> list[PlacementPlanResponse]:
+    return service.placement_history(limit)
+
+
+@router.put(
+    "/api/v1/admin/capability-deployments/{deployment_id}/residency-policy",
+    response_model=CapabilityDeploymentResponse,
+)
+def update_residency_policy(
+    deployment_id: uuid.UUID,
+    request: ResidencyPolicyUpdate,
+    service: Service,
+    _admin: Admin,
+) -> CapabilityDeploymentResponse:
+    return service.update_residency_policy(deployment_id, request)
+
+
+@router.get("/api/v1/gateway/requests", response_model=list[GatewayRequestResponse])
+def gateway_requests(
+    service: Service,
+    _admin: Admin,
+    limit: Annotated[int, Query(ge=1, le=500)] = 100,
+) -> list[GatewayRequestResponse]:
+    return service.request_history(limit)
+
+
+@router.get("/api/v1/admin/latency-traces", response_model=list[GatewayRequestResponse])
+def latency_traces(
+    service: Service,
+    _admin: Admin,
+    limit: Annotated[int, Query(ge=1, le=500)] = 100,
+) -> list[GatewayRequestResponse]:
+    """Return bounded span summaries; request content and input digests are excluded."""
+    return service.request_history(limit)
+
+
+@router.post(
+    "/api/v1/capabilities/rag.embedding@1/invoke",
+    response_model=StableEmbeddingInvokeResponse,
+)
+def invoke_embedding(
+    request: EmbeddingInvokeRequest,
+    service: Service,
+    client: RagEmbeddingClient,
+) -> StableEmbeddingInvokeResponse:
+    return StableEmbeddingInvokeResponse.model_validate(service.invoke(request, client))
+
+
+@router.post(
+    "/api/v1/capabilities/rag.reranking@1/invoke",
+    response_model=RerankingInvokeResponse,
+)
+def invoke_reranking(
+    request: RerankingInvokeRequest,
+    service: Service,
+    client: RagRerankingClient,
+) -> RerankingInvokeResponse:
+    return service.invoke_reranking(request, client)
+
+
+@router.post(
+    "/api/v1/capabilities/document.ocr@1/invoke",
+    response_model=OCRInvokeResponse,
+)
+def invoke_ocr(
+    request: OCRInvokeRequest,
+    service: Service,
+    client: DocumentOcrClient,
+) -> OCRInvokeResponse:
+    result, request_id, execution = service.invoke_modality(
+        "document.ocr", request.model_dump(mode="json"), client
+    )
+    return OCRInvokeResponse(request_id=request_id, execution=execution, **result)
+
+
+@router.post(
+    "/api/v1/capabilities/vision.embedding@1/invoke",
+    response_model=VisionEmbeddingInvokeResponse,
+)
+def invoke_vision_embedding(
+    request: VisionEmbeddingInvokeRequest,
+    service: Service,
+    client: VisionEmbeddingClient,
+) -> VisionEmbeddingInvokeResponse:
+    result, request_id, execution = service.invoke_modality(
+        "vision.embedding", request.model_dump(mode="json"), client
+    )
+    return VisionEmbeddingInvokeResponse(
+        request_id=request_id,
+        execution=execution,
+        dimension=int(result["dimension"]),
+        embedding_space_id=result["embedding_space_id"],
+        data=result["vectors"],
+    )
+
+
+@router.post(
+    "/api/v1/capabilities/speech.transcription@1/invoke",
+    response_model=SpeechTranscriptionInvokeResponse,
+)
+def invoke_speech_transcription(
+    request: SpeechTranscriptionInvokeRequest,
+    service: Service,
+    client: SpeechTranscriptionClient,
+) -> SpeechTranscriptionInvokeResponse:
+    result, request_id, execution = service.invoke_modality(
+        "speech.transcription", request.model_dump(mode="json"), client
+    )
+    return SpeechTranscriptionInvokeResponse(
+        request_id=request_id,
+        execution=execution,
+        **result,
+    )
+
+
+@router.post(
+    "/api/v1/capability-experiments/{route_key}/invoke",
+    response_model=EmbeddingInvokeResponse,
+)
+def invoke_embedding_experiment(
+    route_key: str,
+    request: EmbeddingInvokeRequest,
+    service: Service,
+    client: RagEmbeddingClient,
+) -> EmbeddingInvokeResponse:
+    route = service.repo.experiment_route(route_key)
+    deployment = (
+        service.repo.deployment(route.capability_deployment_id)
+        if route and route.status == "active"
+        else None
+    )
+    if not deployment:
+        raise ServingError(404, "EXPERIMENT_NOT_FOUND", "capability experiment is unavailable")
+    return service.invoke(
+        request,
+        client,
+        deployment=deployment,
+        experiment_route=route_key,
+    )
+
+
+@router.post("/v1/embeddings", response_model=OpenAIEmbeddingResponse)
+def openai_embeddings(
+    request: OpenAIEmbeddingRequest,
+    service: Service,
+    client: RagEmbeddingClient,
+) -> OpenAIEmbeddingResponse:
+    native = service.invoke(EmbeddingInvokeRequest(input=request.input), client)
+    return OpenAIEmbeddingResponse(
+        data=[
+            OpenAIEmbeddingItem(index=index, embedding=embedding)
+            for index, embedding in enumerate(native.data)
+        ],
+        usage=OpenAIUsage(
+            prompt_tokens=native.usage.input_tokens,
+            total_tokens=native.usage.input_tokens,
+        ),
+    )
+
+
+@router.get(
+    "/api/v1/agent/serving-jobs/next",
+    response_model=AgentServingJobLease | None,
+)
+def claim_serving_job(
+    service: Service,
+    identity: AgentIdentity,
+    wait_seconds: Annotated[float, Query(ge=0.0, le=5.0)] = 0.0,
+) -> AgentServingJobLease | None:
+    _credential, node = identity
+    return service.claim_next(node, wait_seconds=wait_seconds)
+
+
+@router.post("/api/v1/agent/serving-jobs/{job_id}/complete")
+def complete_serving_job(
+    job_id: uuid.UUID,
+    request: AgentServingJobComplete,
+    service: Service,
+    identity: AgentIdentity,
+) -> dict[str, str]:
+    _credential, node = identity
+    service.complete_job(job_id, node, request)
+    return {"status": "accepted"}
+
+
+@router.post("/api/v1/agent/serving-jobs/{job_id}/fail")
+def fail_serving_job(
+    job_id: uuid.UUID,
+    request: AgentServingJobFailure,
+    service: Service,
+    identity: AgentIdentity,
+) -> dict[str, str]:
+    _credential, node = identity
+    service.fail_job(job_id, node, request)
+    return {"status": "accepted"}
+
+
+@router.post(
+    "/api/v1/agent/serving-state",
+    response_model=AgentServingStateAck,
+)
+def report_serving_state(
+    request: AgentServingStateReport,
+    service: Service,
+    identity: AgentIdentity,
+) -> AgentServingStateAck:
+    _credential, node = identity
+    return service.report_state(node, request)

backend/src/modelforge_api/cli/__init__.py

@@ -0,0 +1 @@
+"""ModelForge operator command-line surfaces."""

backend/src/modelforge_api/cli/dr.py

@@ -0,0 +1,334 @@
+"""Operator disaster-recovery CLI.
+
+Restoring over a live database is an operator action with a runbook behind it, not a remote API
+call, so the destructive half of recovery lives here. Every subcommand takes typed arguments and
+runs fixed operations: there is no pass-through shell, and no argument reaches a shell.
+
+    python -m modelforge_api.cli.dr create-backup --backup-id m15-rehearsal-a --reason "..."
+    python -m modelforge_api.cli.dr verify-backup --backup-id m15-rehearsal-a
+    python -m modelforge_api.cli.dr plan-restore --backup-id m15-rehearsal-a --target-url ...
+    python -m modelforge_api.cli.dr validate-restore --plan-id <uuid>
+    python -m modelforge_api.cli.dr run-restore --plan-id <uuid> --reason "..."
+    python -m modelforge_api.cli.dr readiness
+    python -m modelforge_api.cli.dr bundle --backup-id m15-rehearsal-a
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import uuid
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.db import build_engine
+from modelforge_api.domain.recovery import (
+    POINT_IN_TIME_SUPPORT,
+    BackupSetCreate,
+    RestoreAdvanceRequest,
+    RestoreMode,
+    RestorePlanCreate,
+    redact_database_url,
+)
+from modelforge_api.persistence.models import BackupSet
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.settings import get_settings
+
+
+def _service(session: Session) -> RecoveryService:
+    return RecoveryService(session, get_settings(), "operator", "dr-cli")
+
+
+def _emit(payload: Any) -> None:
+    print(json.dumps(payload, indent=2, default=str))
+
+
+def _backup_by_id(session: Session, backup_id: str) -> BackupSet:
+    record = session.scalar(select(BackupSet).where(BackupSet.backup_id == backup_id))
+    if record is None:
+        raise RecoveryError(404, "backup_not_found", f"no backup set named {backup_id}")
+    return record
+
+
+def _cmd_create_backup(session: Session, args: argparse.Namespace) -> int:
+    service = _service(session)
+    service.ensure_defaults()
+    response = service.create_backup(
+        BackupSetCreate(
+            backup_id=args.backup_id,
+            reason=args.reason,
+            milestone=args.milestone,
+            legal_hold=args.legal_hold,
+            created_by=args.actor,
+        )
+    )
+    _emit(json.loads(response.model_dump_json()))
+    return 0 if response.state.value in {"CREATED", "VERIFIED"} else 1
+
+
+def _cmd_verify_backup(session: Session, args: argparse.Namespace) -> int:
+    service = _service(session)
+    record = _backup_by_id(session, args.backup_id)
+    response = service.verify_backup(record.id)
+    _emit(
+        {
+            "backup_id": response.backup_id,
+            "state": response.state.value,
+            "restore_eligible": response.restore_eligible,
+            "manifest_sha256": response.manifest_sha256,
+            "failure_code": response.failure_code,
+            "failure_reason": response.failure_reason,
+            "verification": response.verification_details.get("verification", {}),
+        }
+    )
+    return 0 if response.restore_eligible else 1
+
+
+def _cmd_list_backups(session: Session, args: argparse.Namespace) -> int:
+    service = _service(session)
+    _emit(
+        [
+            {
+                "backup_id": item.backup_id,
+                "state": item.state.value,
+                "restore_eligible": item.restore_eligible,
+                "schema_revision": item.schema_revision,
+                "payload_bytes": item.payload_bytes,
+                "encrypted": item.encrypted,
+                "verified_at": item.verified_at,
+                "expires_at": item.expires_at,
+            }
+            for item in service.backups(args.limit)
+        ]
+    )
+    return 0
+
+
+def _cmd_plan_restore(session: Session, args: argparse.Namespace) -> int:
+    service = _service(session)
+    record = _backup_by_id(session, args.backup_id)
+    response = service.create_restore_plan(
+        RestorePlanCreate(
+            backup_set_id=record.id,
+            mode=RestoreMode(args.mode),
+            target_environment=args.target_environment,
+            target_label=args.target_label,
+            database_destination=args.target_url,
+            artifact_strategy=args.artifact_strategy,
+            secret_strategy=args.secret_strategy,
+            node_strategy=args.node_strategy,
+            reason=args.reason,
+            created_by=args.actor,
+        )
+    )
+    _emit(json.loads(response.model_dump_json()))
+    return 0
+
+
+def _cmd_validate_restore(session: Session, args: argparse.Namespace) -> int:
+    service = _service(session)
+    response = service.preflight(uuid.UUID(args.plan_id))
+    _emit(
+        {
+            "plan_id": str(response.id),
+            "backup_id": response.backup_id,
+            "state": response.state.value,
+            "destination": response.database_destination_redacted,
+            "preflight": response.preflight,
+        }
+    )
+    return 0 if response.preflight.get("status") == "PASS" else 1
+
+
+def _cmd_run_restore(session: Session, args: argparse.Namespace) -> int:
+    service = _service(session)
+    plan_id = uuid.UUID(args.plan_id)
+    request = RestoreAdvanceRequest(actor=args.actor, reason=args.reason)
+    operation = service.start_restore(plan_id, request)
+    operation = service.advance_restore(operation.id, request)
+    _emit(
+        {
+            "operation_id": str(operation.id),
+            "backup_id": operation.backup_id,
+            "state": operation.state.value,
+            "attempt": operation.attempt,
+            "phase_durations": operation.phase_durations,
+            "rto_seconds": operation.rto_seconds,
+            "rpo_seconds": operation.rpo_seconds,
+            "validation": operation.validation_result,
+            "fingerprint_diff": operation.fingerprint_diff,
+            "failure_code": operation.failure_code,
+            "failure_reason": operation.failure_reason,
+        }
+    )
+    return 0 if operation.state.value == "READY" else 1
+
+
+def _cmd_readiness(session: Session, _args: argparse.Namespace) -> int:
+    service = _service(session)
+    dashboard = service.dashboard()
+    _emit(json.loads(dashboard.model_dump_json()))
+    return 0 if not dashboard.unprotected_assets and not dashboard.stale_backup else 1
+
+
+def _cmd_bundle(session: Session, args: argparse.Namespace) -> int:
+    """Print the bounded DR bundle manifest: what an operator needs to rebuild from nothing."""
+
+    service = _service(session)
+    record = _backup_by_id(session, args.backup_id)
+    backup = service.backup(record.id)
+    settings = get_settings()
+    _emit(
+        {
+            "modelforge": {
+                "version": backup.modelforge_version,
+                "commit": backup.modelforge_commit,
+                "reference": backup.environment_fingerprint.get("source_reference"),
+                "repository": record.source_repository,
+            },
+            "database_backup": {
+                "backup_id": backup.backup_id,
+                "state": backup.state.value,
+                "restore_eligible": backup.restore_eligible,
+                "destination_root": backup.destination_root,
+                "manifest": backup.manifest_relative_path,
+                "manifest_sha256": backup.manifest_sha256,
+                "payload_bytes": backup.payload_bytes,
+                "schema_revision": backup.schema_revision,
+                "database_identity": backup.database_identity,
+                "point_in_time_support": POINT_IN_TIME_SUPPORT,
+            },
+            "encryption": {
+                "encrypted": backup.encrypted,
+                "algorithm": backup.encryption_algorithm,
+                "key_id": backup.encryption_key_id,
+                "key_requirement": (
+                    "MODELFORGE_BACKUP_ENCRYPTION_KEY must be supplied by the operator; it is "
+                    "never written into a backup"
+                ),
+            },
+            "host_configuration": {
+                "backup_root": str(settings.backup_root),
+                "artifact_root": settings.artifact_root,
+                "quarantine_root": settings.quarantine_root,
+                "config_root": str(settings.config_root),
+                "database_url": redact_database_url(settings.database_url),
+            },
+            "artifact_recovery_plan": [
+                {
+                    "object": entry.object_name,
+                    "type": entry.logical_asset_type,
+                    "sha256": entry.sha256,
+                    "size_bytes": entry.size_bytes,
+                }
+                for entry in backup.entries
+            ],
+            "node_enrollment": (
+                "Issue a fresh enrollment token through /api/v1/admin/node-enrollments and "
+                "revoke any credential believed lost; hardware identity is persisted so a "
+                "recovered node keeps its node id."
+            ),
+            "external_dependencies": [
+                item.asset_key
+                for item in service.readiness()
+                if item.readiness.value == "EXTERNAL_DEPENDENCY"
+            ],
+            "runbooks": [
+                "docs/operations/RUNBOOK_FULL_DR.md",
+                "docs/operations/RUNBOOK_DATABASE_RESTORE.md",
+                "docs/operations/RUNBOOK_CONTROL_PLANE_LOSS.md",
+                "docs/operations/RUNBOOK_NODE_LOSS.md",
+                "docs/operations/RUNBOOK_ARTIFACT_LOSS.md",
+                "docs/operations/RUNBOOK_BACKUP_FAILURE.md",
+            ],
+        }
+    )
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="modelforge-dr", description="ModelForge disaster-recovery operator commands"
+    )
+    parser.add_argument("--database-url", default=None, help="override the control-plane database")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    create = subparsers.add_parser("create-backup", help="create a new backup set")
+    create.add_argument("--backup-id", required=True)
+    create.add_argument("--reason", required=True)
+    create.add_argument("--milestone", default=None)
+    create.add_argument("--legal-hold", action="store_true")
+    create.add_argument("--actor", default="operator")
+    create.set_defaults(handler=_cmd_create_backup)
+
+    verify = subparsers.add_parser("verify-backup", help="verify a backup set end to end")
+    verify.add_argument("--backup-id", required=True)
+    verify.set_defaults(handler=_cmd_verify_backup)
+
+    listing = subparsers.add_parser("list-backups", help="list journaled backup sets")
+    listing.add_argument("--limit", type=int, default=25)
+    listing.set_defaults(handler=_cmd_list_backups)
+
+    plan = subparsers.add_parser("plan-restore", help="create a restore plan")
+    plan.add_argument("--backup-id", required=True)
+    plan.add_argument("--target-url", required=True)
+    plan.add_argument("--target-label", required=True)
+    plan.add_argument(
+        "--mode", choices=[item.value for item in RestoreMode], default=RestoreMode.VALIDATION.value
+    )
+    plan.add_argument(
+        "--target-environment", choices=["ISOLATED", "STAGING", "PRODUCTION"], default="ISOLATED"
+    )
+    plan.add_argument(
+        "--artifact-strategy",
+        choices=["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"],
+        default="MANIFEST_ONLY",
+    )
+    plan.add_argument(
+        "--secret-strategy", choices=["ROTATE", "RESTORE_HASHES", "MANUAL"], default="RESTORE_HASHES"
+    )
+    plan.add_argument(
+        "--node-strategy", choices=["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"], default="NONE"
+    )
+    plan.add_argument("--reason", required=True)
+    plan.add_argument("--actor", default="operator")
+    plan.set_defaults(handler=_cmd_plan_restore)
+
+    validate = subparsers.add_parser("validate-restore", help="run the restore preflight")
+    validate.add_argument("--plan-id", required=True)
+    validate.set_defaults(handler=_cmd_validate_restore)
+
+    run = subparsers.add_parser("run-restore", help="execute a restore plan to completion")
+    run.add_argument("--plan-id", required=True)
+    run.add_argument("--reason", required=True)
+    run.add_argument("--actor", default="operator")
+    run.set_defaults(handler=_cmd_run_restore)
+
+    readiness = subparsers.add_parser("readiness", help="print recovery readiness")
+    readiness.set_defaults(handler=_cmd_readiness)
+
+    bundle = subparsers.add_parser("bundle", help="print the operator DR bundle manifest")
+    bundle.add_argument("--backup-id", required=True)
+    bundle.set_defaults(handler=_cmd_bundle)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    engine = build_engine(args.database_url)
+    try:
+        with Session(engine) as session:
+            handler = args.handler
+            return int(handler(session, args))
+    except RecoveryError as error:
+        _emit({"error": {"code": error.code, "message": error.message, "details": error.details}})
+        return 2
+    finally:
+        engine.dispose()
+
+
+if __name__ == "__main__":  # pragma: no cover - operator entry point
+    sys.exit(main())

backend/src/modelforge_api/db.py

@@ -0,0 +1,38 @@
+from collections.abc import Generator
+from typing import Any, cast
+
+from sqlalchemy import CursorResult, Engine, create_engine
+from sqlalchemy.engine import Result
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.runtime_engine import register_application_engine
+from modelforge_api.settings import get_settings
+
+
+def rows_affected(result: Result[Any]) -> int:
+    """How many rows a DML statement actually changed.
+
+    `Session.execute` is typed as returning `Result`, which carries no `rowcount`; a DML statement
+    always returns a `CursorResult`, which does. SQLAlchemy 2.0.52 narrowed that return type, so
+    reading `.rowcount` directly stopped type-checking — while continuing to work at runtime.
+
+    The cast is where that knowledge lives, once, rather than at ten call sites. It matters more
+    than it looks: every single-use claim in the platform is a conditional UPDATE whose row count
+    decides the winner, and that is what makes enrolment tokens and lifecycle claims atomic instead
+    of merely usually-correct.
+    """
+
+    return cast("CursorResult[Any]", result).rowcount
+
+
+def build_engine(database_url: str | None = None) -> Engine:
+    url = database_url or get_settings().database_url
+    return register_application_engine(create_engine(url, pool_pre_ping=True))
+
+
+engine = build_engine()
+
+
+def get_session() -> Generator[Session, None, None]:
+    with Session(engine) as session:
+        yield session

backend/src/modelforge_api/domain/acquisition.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class AcquisitionModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class DiscoverySearchRequest(AcquisitionModel):
+    query: str = Field(min_length=1, max_length=255)
+    limit: int = Field(default=20, ge=1, le=100)
+    sort: Literal["downloads", "likes", "last_modified"] = "downloads"
+    pipeline_tag: str | None = Field(default=None, max_length=128)
+
+
+class DiscoveryCandidate(AcquisitionModel):
+    repository_id: str
+    resolved_commit_sha: str | None = None
+    access_state: str
+    pipeline_tag: str | None = None
+    library_name: str | None = None
+    tags: list[str] = Field(default_factory=list)
+    downloads: int | None = None
+    likes: int | None = None
+    last_modified: datetime | None = None
+    matched_model_id: uuid.UUID | None = None
+    upstream_facts: dict[str, Any] = Field(default_factory=dict)
+    local_interpretation: dict[str, Any] = Field(default_factory=dict)
+
+
+class UpstreamRefreshRequest(AcquisitionModel):
+    revision: str = Field(default="main", min_length=1, max_length=255)
+
+
+class UpstreamFileResponse(AcquisitionModel):
+    id: uuid.UUID
+    path: str
+    size_bytes: int | None
+    blob_id: str | None
+    upstream_sha256: str | None
+    file_format: str
+    role: str
+    risk_flags: list[str]
+    metadata_snapshot: dict[str, Any]
+
+
+class UpstreamSnapshotResponse(AcquisitionModel):
+    id: uuid.UUID
+    model_id: uuid.UUID | None
+    provider: str
+    repository_id: str
+    requested_revision: str
+    resolved_commit_sha: str
+    access_state: str
+    metadata_snapshot: dict[str, Any]
+    card_metadata: dict[str, Any]
+    security_metadata: dict[str, Any]
+    source_updated_at: datetime | None
+    observed_at: datetime
+    stale_after: datetime
+    stale: bool
+    files: list[UpstreamFileResponse]
+
+
+class ArtifactSetResponse(AcquisitionModel):
+    id: uuid.UUID
+    revision_id: uuid.UUID
+    snapshot_id: uuid.UUID
+    variant_key: str
+    label: str
+    selection_reason: str
+    selected_paths: list[str]
+    total_size_bytes: int
+    file_count: int
+    availability: str
+    status: str
+    completeness: str
+    security_status: str
+    license_status: str
+    immutable_at: datetime
+    created_at: datetime
+    updated_at: datetime
+
+
+class DownloadPlanCreate(AcquisitionModel):
+    artifact_set_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    storage_root_id: uuid.UUID
+    expires_in_seconds: int = Field(default=3600, ge=300, le=86400)
+
+
+class DownloadPlanFileResponse(AcquisitionModel):
+    ordinal: int
+    path: str
+    size_bytes: int
+    upstream_sha256: str | None
+    file_format: str
+    role: str
+    risk_flags: list[str]
+
+
+class DownloadPlanResponse(AcquisitionModel):
+    id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    storage_root_id: uuid.UUID
+    repository_id: str
+    resolved_commit_sha: str
+    total_size_bytes: int
+    file_count: int
+    status: str
+    idempotency_key: str
+    preflight: dict[str, Any]
+    immutable_payload: dict[str, Any]
+    planned_at: datetime
+    expires_at: datetime
+    immutable_at: datetime
+    created_at: datetime
+    updated_at: datetime
+    stale: bool
+    files: list[DownloadPlanFileResponse]
+
+
+class ArtifactJobResponse(AcquisitionModel):
+    id: uuid.UUID
+    plan_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    storage_root_id: uuid.UUID
+    status: str
+    attempt_count: int
+    progress_bytes: int
+    total_bytes: int
+    current_file: str | None
+    cancel_requested: bool
+    quarantine_relative_path: str | None
+    promoted_relative_path: str | None
+    error_code: str | None
+    error_message: str | None
+    result: dict[str, Any]
+    started_at: datetime | None
+    completed_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+
+class AgentArtifactJobFile(AcquisitionModel):
+    ordinal: int
+    path: str
+    size_bytes: int
+    upstream_sha256: str | None
+    file_format: str
+    role: str
+    risk_flags: list[str]
+
+
+class AgentArtifactJobLease(AcquisitionModel):
+    job_id: uuid.UUID
+    lease_token: str
+    lease_expires_at: datetime
+    repository_id: str
+    resolved_commit_sha: str
+    storage_root_id: uuid.UUID
+    target_root: str
+    total_size_bytes: int
+    reserve_bytes: int
+    reserve_percent: int
+    files: list[AgentArtifactJobFile]
+
+
+class AgentJobProgress(AcquisitionModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    status: Literal["claimed", "downloading", "verifying", "promoting"]
+    progress_bytes: int = Field(ge=0)
+    current_file: str | None = Field(default=None, max_length=2048)
+    quarantine_relative_path: str | None = Field(default=None, max_length=2048)
+
+
+class AgentJobControl(AcquisitionModel):
+    accepted: bool
+    cancel_requested: bool
+    lease_expires_at: datetime
+
+
+class CompletedFile(AcquisitionModel):
+    path: str
+    relative_path: str
+    size_bytes: int = Field(ge=0)
+    sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+    inspections: list[dict[str, Any]] = Field(default_factory=list)
+
+    @field_validator("path", "relative_path")
+    @classmethod
+    def safe_path(cls, value: str) -> str:
+        normalized = value.replace("\\", "/")
+        if normalized.startswith("/") or ".." in normalized.split("/"):
+            raise ValueError("path must be relative and confined")
+        return normalized
+
+
+class AgentJobComplete(AcquisitionModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    promoted_relative_path: str
+    capacity_observation: dict[str, Any]
+    files: list[CompletedFile] = Field(min_length=1)
+
+
+class AgentJobFailure(AcquisitionModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    error_code: str = Field(min_length=1, max_length=64)
+    error_message: str = Field(min_length=1, max_length=2000)
+    retryable: bool = False
+    details: dict[str, Any] = Field(default_factory=dict)

backend/src/modelforge_api/domain/agent_protocol.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from modelforge_api.domain.enums import Availability
+from modelforge_api.domain.hardware import (
+    AcceleratorInventory,
+    AcceleratorTelemetry,
+    HostInventory,
+    ObservedValue,
+    StorageObservation,
+)
+
+AGENT_PROTOCOL_VERSION = 1
+AGENT_PROTOCOL_CAPABILITIES = [
+    "hardware.inventory",
+    "hardware.telemetry",
+    "artifact.acquire.v1",
+    "runtime.probe.v1",
+    "runtime.health.v1",
+    "runtime.unload.v1",
+    "deployment.load.v1",
+    "deployment.invoke.v1",
+    "deployment.health.v1",
+    "deployment.drain.v1",
+    "deployment.unload.v1",
+]
+
+
+class AgentProtocolModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class AgentMetadata(AgentProtocolModel):
+    agent_version: str
+    protocol_version: int
+    supported_capabilities: list[str] = Field(default_factory=list)
+    started_at: datetime
+
+
+class EnrollmentRequest(AgentProtocolModel):
+    enrollment_token: str = Field(min_length=32, max_length=512)
+    identity_key: str = Field(min_length=1, max_length=128)
+    identity_source: str = Field(min_length=1, max_length=32)
+    hostname: str = Field(min_length=1, max_length=255)
+    display_name: str = Field(min_length=1, max_length=255)
+    metadata: AgentMetadata
+
+
+class EnrollmentResponse(AgentProtocolModel):
+    node_id: str
+    credential_id: str
+    node_credential: str
+    protocol_version: int = AGENT_PROTOCOL_VERSION
+
+
+class NodeCredentialCreated(AgentProtocolModel):
+    node_id: str
+    credential_id: str
+    node_credential: str
+
+
+class HeartbeatRequest(AgentProtocolModel):
+    identity_key: str
+    metadata: AgentMetadata
+    observed_at: datetime
+    last_error: str | None = Field(default=None, max_length=1000)
+
+
+class InventoryNvidiaPayload(AgentProtocolModel):
+    availability: Availability
+    reason: str | None = None
+    inventory: list[AcceleratorInventory] = Field(default_factory=list)
+
+
+class InventoryReport(AgentProtocolModel):
+    identity_key: str
+    protocol_version: int
+    sequence: int = Field(ge=1, le=9_223_372_036_854_775_807)
+    observed_at: datetime
+    host: HostInventory
+    nvidia: InventoryNvidiaPayload
+
+
+class TelemetryReport(AgentProtocolModel):
+    identity_key: str
+    protocol_version: int
+    sequence: int = Field(ge=1, le=9_223_372_036_854_775_807)
+    observed_at: datetime
+    available_ram_bytes: ObservedValue[int]
+    storage: list[StorageObservation] = Field(default_factory=list)
+    accelerators: list[AcceleratorTelemetry] = Field(default_factory=list)
+
+
+class ObservationAck(AgentProtocolModel):
+    accepted: bool
+    reason: str | None = None
+    received_at: datetime
+
+
+class EnrollmentTokenCreate(AgentProtocolModel):
+    expires_in_seconds: int = Field(default=900, ge=60, le=86400)
+    display_name: str | None = Field(default=None, max_length=255)
+    role: str | None = Field(default=None, max_length=64)
+    labels: dict[str, str | bool] = Field(default_factory=dict)
+    production_eligible: bool = False
+    lab_eligible: bool = True
+    benchmark_eligible: bool = False
+
+
+class EnrollmentTokenCreated(AgentProtocolModel):
+    id: str
+    enrollment_token: str
+    expires_at: datetime
+    setup_environment: dict[str, str]
+
+
+class EnrollmentTokenSummary(AgentProtocolModel):
+    id: str
+    created_at: datetime
+    expires_at: datetime
+    used_at: datetime | None
+    revoked_at: datetime | None
+
+
+class NodeManagementUpdate(AgentProtocolModel):
+    display_name: str | None = Field(default=None, min_length=1, max_length=255)
+    role: str | None = Field(default=None, max_length=64)
+    labels: dict[str, str | bool] | None = None
+    enabled: bool | None = None
+    production_eligible: bool | None = None
+    lab_eligible: bool | None = None
+    benchmark_eligible: bool | None = None

backend/src/modelforge_api/domain/audit.py

@@ -0,0 +1,140 @@
+"""Stable audit-chain format identifiers shared by persistence and services."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+
+AUDIT_CHAIN_SINGLETON_ID = 1
+AUDIT_HASH_FORMAT_V1 = "v1"
+AUDIT_HASH_FORMAT_V2 = "v2"
+AUDIT_CURRENT_HASH_FORMAT = AUDIT_HASH_FORMAT_V2
+AUDIT_LEGACY_PREFIX_DOMAIN = b"modelforge:audit:legacy-prefix:v1\n"
+AUDIT_EMPTY_LEGACY_PREFIX_SEAL = hashlib.sha256(AUDIT_LEGACY_PREFIX_DOMAIN).hexdigest()
+
+
+def normalise_audit_timestamp(value: datetime | str) -> str:
+    """Return the stable UTC/microsecond representation used by v2 audit hashes."""
+
+    moment: datetime
+    if isinstance(value, datetime):
+        moment = value
+    elif isinstance(value, str):
+        candidate = value.strip()
+        if candidate.endswith("Z"):
+            candidate = candidate[:-1] + "+00:00"
+        try:
+            moment = datetime.fromisoformat(candidate)
+        except ValueError as error:
+            raise ValueError("audit occurred_at is not an ISO-8601 timestamp") from error
+    else:
+        raise TypeError("audit occurred_at must be a datetime or ISO-8601 string")
+    if moment.tzinfo is None:
+        moment = moment.replace(tzinfo=UTC)
+    return moment.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def normalise_audit_event_id(value: Any) -> str:
+    try:
+        return str(uuid.UUID(str(value)))
+    except (AttributeError, TypeError, ValueError) as error:
+        raise ValueError("audit event id is not a UUID") from error
+
+
+def canonical_audit_payload_and_hash(
+    *,
+    correlation_id: str,
+    actor_type: str,
+    actor_id: str,
+    action: str,
+    resource_type: str,
+    resource_id: str | None,
+    outcome: str,
+    details: dict[str, Any],
+    previous_event_hash: str | None,
+    hash_format: str = AUDIT_HASH_FORMAT_V1,
+    event_id: Any | None = None,
+    occurred_at: datetime | str | None = None,
+) -> tuple[dict[str, Any], str]:
+    """Return detached stored fields and their versioned SHA-256 identity."""
+
+    stored_payload, _encoded_hash, event_hash = canonical_audit_payload_text_and_hash(
+        correlation_id=correlation_id,
+        actor_type=actor_type,
+        actor_id=actor_id,
+        action=action,
+        resource_type=resource_type,
+        resource_id=resource_id,
+        outcome=outcome,
+        details=details,
+        previous_event_hash=previous_event_hash,
+        hash_format=hash_format,
+        event_id=event_id,
+        occurred_at=occurred_at,
+    )
+    return stored_payload, event_hash
+
+
+def canonical_audit_payload_text_and_hash(
+    *,
+    correlation_id: str,
+    actor_type: str,
+    actor_id: str,
+    action: str,
+    resource_type: str,
+    resource_id: str | None,
+    outcome: str,
+    details: dict[str, Any],
+    previous_event_hash: str | None,
+    hash_format: str = AUDIT_HASH_FORMAT_V1,
+    event_id: Any | None = None,
+    occurred_at: datetime | str | None = None,
+) -> tuple[dict[str, Any], str, str]:
+    """Return stored fields, the exact hashed text, and its SHA-256 digest.
+
+    The exact text is persisted for v2 events. That makes PostgreSQL's SECURITY DEFINER writer
+    authoritative for its own canonical encoding without asking Python and PostgreSQL to reproduce
+    each other's JSON lexical representation. Strict verification hashes the stored bytes and then
+    independently checks that the decoded object is exactly the event's semantic payload.
+    """
+
+    if not isinstance(details, dict):
+        raise TypeError("audit details must be an object")
+    stored_payload: dict[str, Any] = {
+        "correlation_id": correlation_id,
+        "actor_type": actor_type,
+        "actor_id": actor_id,
+        "action": action,
+        "resource_type": resource_type,
+        "resource_id": resource_id,
+        "outcome": outcome,
+        "details": details,
+        "previous_event_hash": previous_event_hash,
+    }
+    encoded_stored = json.dumps(stored_payload, sort_keys=True, separators=(",", ":"))
+    canonical_stored = json.loads(encoded_stored)
+    if not isinstance(canonical_stored, dict):  # pragma: no cover - constructed above
+        raise TypeError("canonical audit payload must be an object")
+
+    if hash_format == AUDIT_HASH_FORMAT_V1:
+        hash_payload = canonical_stored
+    elif hash_format == AUDIT_HASH_FORMAT_V2:
+        if event_id is None or occurred_at is None:
+            raise ValueError("v2 audit hashes require an event id and occurred_at")
+        hash_payload = {
+            "hash_format": AUDIT_HASH_FORMAT_V2,
+            "id": normalise_audit_event_id(event_id),
+            "occurred_at": normalise_audit_timestamp(occurred_at),
+            **canonical_stored,
+        }
+    else:
+        raise ValueError(f"unsupported audit hash format {hash_format!r}")
+    encoded_hash = json.dumps(hash_payload, sort_keys=True, separators=(",", ":"))
+    return (
+        canonical_stored,
+        encoded_hash,
+        hashlib.sha256(encoded_hash.encode("utf-8")).hexdigest(),
+    )

backend/src/modelforge_api/domain/capability_evaluation.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import math
+import re
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+EvaluationType = Literal[
+    "retrieval", "ocr", "visual-retrieval", "asr", "tts", "generation", "llm"
+]
+CapabilityRecommendationState = Literal[
+    "KEEP_CURRENT",
+    "LAB_READY",
+    "PROMOTION_ELIGIBLE",
+    "REQUIRES_MORE_EVIDENCE",
+    "BLOCKED",
+]
+MetricDirection = Literal["higher_is_better", "lower_is_better", "informational"]
+_KEY = re.compile(r"^[a-z0-9][a-z0-9_-]{0,127}$")
+
+ALLOWED_METRICS: dict[str, frozenset[str]] = {
+    "retrieval": frozenset({"recall_at_5", "recall_at_10", "mrr", "ndcg_at_10"}),
+    "ocr": frozenset({"cer", "wer", "text_accuracy", "field_accuracy", "layout_accuracy", "latency_ms", "peak_vram_bytes"}),
+    "visual-retrieval": frozenset({"recall_at_1", "recall_at_5", "recall_at_10", "mrr", "latency_ms", "peak_vram_bytes"}),
+    "asr": frozenset({"wer", "real_time_factor", "latency_ms", "peak_vram_bytes"}),
+    "tts": frozenset({"latency_ms", "real_time_factor", "peak_vram_bytes"}),
+    "generation": frozenset({"latency_ms", "peak_vram_bytes"}),
+    "llm": frozenset({"task_success", "structured_output_validity", "latency_ms", "peak_vram_bytes"}),
+}
+
+
+class EvaluationModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class MetricDefinition(EvaluationModel):
+    name: str
+    direction: MetricDirection
+    unit: str = Field(min_length=1, max_length=64)
+    minimum: float | None = None
+    maximum: float | None = None
+
+    @model_validator(mode="after")
+    def valid_range(self) -> MetricDefinition:
+        if self.minimum is not None and self.maximum is not None and self.minimum > self.maximum:
+            raise ValueError("metric minimum cannot exceed maximum")
+        return self
+
+
+class CapabilityEvaluationCase(EvaluationModel):
+    key: str
+    fixture_ref: str = Field(min_length=1, max_length=512)
+    fixture_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+    ground_truth: dict[str, Any]
+    labels: list[str] = Field(default_factory=list)
+    critical: bool = False
+
+    @field_validator("key")
+    @classmethod
+    def valid_key(cls, value: str) -> str:
+        if not _KEY.fullmatch(value):
+            raise ValueError("case key must be a safe identifier")
+        return value
+
+
+class CapabilityEvaluationSuiteCreate(EvaluationModel):
+    capability: str = Field(pattern=r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
+    contract_version: int = Field(default=1, ge=1)
+    key: str
+    evaluation_type: EvaluationType
+    revision: str
+    dataset_revision: str
+    metrics: list[MetricDefinition] = Field(min_length=1)
+    cases: list[CapabilityEvaluationCase] = Field(min_length=1, max_length=500)
+    thresholds: dict[str, float] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def validate_definition(self) -> CapabilityEvaluationSuiteCreate:
+        for key in (self.key, self.revision, self.dataset_revision):
+            if not _KEY.fullmatch(key):
+                raise ValueError("suite identifiers must be safe")
+        metric_names = [item.name for item in self.metrics]
+        if len(metric_names) != len(set(metric_names)):
+            raise ValueError("metric names must be unique")
+        unsupported = set(metric_names) - ALLOWED_METRICS[self.evaluation_type]
+        if unsupported:
+            raise ValueError(f"metrics are invalid for {self.evaluation_type}: {sorted(unsupported)}")
+        if set(self.thresholds) - set(metric_names):
+            raise ValueError("thresholds must refer to declared metrics")
+        if len({item.key for item in self.cases}) != len(self.cases):
+            raise ValueError("case keys must be unique")
+        return self
+
+
+class CapabilityEvaluationSuiteResponse(EvaluationModel):
+    id: uuid.UUID
+    capability: str
+    contract_version: int
+    key: str
+    evaluation_type: EvaluationType
+    revision: str
+    dataset_revision: str
+    definition_digest: str
+    metrics: list[MetricDefinition]
+    cases: list[CapabilityEvaluationCase]
+    thresholds: dict[str, float]
+    created_at: datetime
+
+
+class CapabilityCaseResult(EvaluationModel):
+    case_key: str
+    metrics: dict[str, float]
+    output_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+    status: Literal["passed", "failed", "error"]
+    error_code: str | None = Field(default=None, max_length=64)
+
+
+class CapabilityEvaluationRunCreate(EvaluationModel):
+    suite_id: uuid.UUID
+    capability_deployment_id: uuid.UUID
+    status: Literal["completed", "failed"]
+    metrics: dict[str, float]
+    cases: list[CapabilityCaseResult] = Field(min_length=1, max_length=500)
+    resource_metrics: dict[str, float]
+    environment_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
+    evidence: dict[str, Any] = Field(default_factory=dict)
+    started_at: datetime
+    completed_at: datetime
+
+    @field_validator("metrics", "resource_metrics")
+    @classmethod
+    def finite_values(cls, value: dict[str, float]) -> dict[str, float]:
+        if any(not math.isfinite(item) for item in value.values()):
+            raise ValueError("metric values must be finite")
+        return value
+
+    @model_validator(mode="after")
+    def chronological(self) -> CapabilityEvaluationRunCreate:
+        if self.completed_at < self.started_at:
+            raise ValueError("evaluation completion precedes its start")
+        return self
+
+
+class CapabilityEvaluationRunResponse(EvaluationModel):
+    id: uuid.UUID
+    suite_id: uuid.UUID
+    capability_deployment_id: uuid.UUID
+    evaluation_type: EvaluationType
+    status: str
+    metrics: dict[str, float]
+    cases: list[CapabilityCaseResult]
+    resource_metrics: dict[str, float]
+    environment_fingerprint: str
+    evidence_digest: str
+    evidence: dict[str, Any]
+    started_at: datetime
+    completed_at: datetime
+    created_at: datetime
+
+
+class CapabilityAdvisorResponse(EvaluationModel):
+    capability: str
+    contract_version: int
+    state: CapabilityRecommendationState
+    deployment_id: uuid.UUID | None = None
+    evaluation_run_id: uuid.UUID | None = None
+    reasons: list[str]
+    automatic_promotion: Literal[False] = False

backend/src/modelforge_api/domain/configuration_reference.py

@@ -0,0 +1,282 @@
+"""Documentation for every setting, held next to the settings themselves.
+
+`.env.example` and `docs/CONFIGURATION.md` are generated from this table joined with the typed
+defaults, and a test fails when a setting exists without an entry here. Hand-maintained
+configuration documentation drifts silently — the operator finds out when a production deployment
+does something the manual said it would not.
+
+`required_in_production` means startup validation refuses to run production without it, not merely
+that it is recommended.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class Sensitivity(StrEnum):
+    PUBLIC = "public"
+    #: Reveals deployment topology. Not a credential, but not for a public issue tracker either.
+    INTERNAL = "internal"
+    #: A credential or key. Never logged, never packaged, never echoed in an error.
+    SECRET = "secret"  # noqa: S105 - a classification label, not a credential
+
+
+@dataclass(frozen=True, slots=True)
+class SettingDoc:
+    description: str
+    sensitivity: Sensitivity = Sensitivity.PUBLIC
+    required_in_production: bool = False
+    #: False when the value is re-read per request or per poll rather than only at startup.
+    restart_required: bool = True
+
+
+def _p(description: str, **kwargs: object) -> SettingDoc:
+    return SettingDoc(description, **kwargs)  # type: ignore[arg-type]
+
+
+def _secret(description: str, *, required: bool = False) -> SettingDoc:
+    return SettingDoc(description, Sensitivity.SECRET, required_in_production=required)
+
+
+def _internal(description: str, *, required: bool = False) -> SettingDoc:
+    return SettingDoc(description, Sensitivity.INTERNAL, required_in_production=required)
+
+
+SETTING_DOCS: dict[str, SettingDoc] = {
+    # ---------------------------------------------------------------- process
+    "env": _p(
+        "Deployment profile. 'production' turns on every fail-closed startup rule; "
+        "'development' and 'test' report the same problems without refusing to start."
+    ),
+    "api_host": _p("Interface the API binds inside its container. Leave at 0.0.0.0."),
+    "api_port": _p("Port the API listens on inside its container."),
+    "control_plane_max_payload_bytes": _p(
+        "Pre-parser request-body limit for public and operator control-plane routes."
+    ),
+    "service_name": _p("Name this process reports in logs and audit events."),
+    "log_level": _p("Structured log level: DEBUG, INFO, WARNING or ERROR.", restart_required=True),
+    # ---------------------------------------------------------------- dependencies
+    "database_url": _secret(
+        "SQLAlchemy URL for the API's non-owner modelforge_runtime role. It may read the audit "
+        "trail and execute the canonical append function, but cannot mutate audit tables directly.",
+        required=True,
+    ),
+    "migration_database_url": _secret(
+        "SQLAlchemy URL for the non-superuser modelforge schema-owner role. Set only in the "
+        "one-shot migration process; production API startup refuses when this secret is present."
+    ),
+    "redis_url": _internal("Redis URL for transient request payloads and queues.", required=True),
+    "cors_origins": _p(
+        "Comma-separated exact origins allowed to call the API from a browser. "
+        "A wildcard is refused in production because requests are credentialed."
+    ),
+    # ---------------------------------------------------------------- credentials
+    "operator_api_key": _secret(
+        "Operator API key guarding every admin route. Generate at least 32 random characters; "
+        "ModelForge never mints one for you.",
+        required=True,
+    ),
+    "hf_token": _secret(
+        "Optional Hugging Face token, used only for acquiring gated repositories. "
+        "It is never passed to a runtime and never leaves the control plane."
+    ),
+    "backup_encryption_key": _secret(
+        "Base64 AES-256 key for backup encryption. Without it no backup can be produced, and "
+        "without the same key no backup can be restored — store it outside this deployment.",
+        required=True,
+    ),
+    "backup_encryption_key_id": _p(
+        "Identifier recorded in each backup manifest so a restore can name the key it needs."
+    ),
+    # ---------------------------------------------------------------- storage
+    "hf_home": _p("Hugging Face cache root inside the container."),
+    "artifact_root": _p("Verified model artifact root. Must exist and be writable."),
+    "quarantine_root": _p("Where acquired artifacts are held until their checks pass."),
+    "runtime_artifact_root": _p("Artifact root as a runtime worker sees it on a compute node."),
+    "config_root": _p("Directory holding the capability, project and policy manifests."),
+    "backup_root": _p("Backup destination. Must exist and be writable, or backups fail closed."),
+    "backup_restore_root": _p("Working directory a restore stages into before it commits."),
+    "alembic_directory": _p("Override for the migration directory. Leave empty in a container."),
+    # ---------------------------------------------------------------- acquisition
+    "hf_timeout_seconds": _p("Per-request timeout for Hugging Face metadata calls."),
+    "hf_snapshot_ttl_seconds": _p("How long a resolved upstream snapshot stays cached."),
+    "allow_remote_code": _p(
+        "Whether model repositories may execute their own Python. Always false in production; "
+        "startup refuses any other value there."
+    ),
+    # ---------------------------------------------------------------- hardware and nodes
+    "enable_gpu_telemetry": _p("Collect GPU telemetry on this host."),
+    "hardware_refresh_on_startup": _p("Run a hardware inventory pass when the process starts."),
+    "hardware_poll_interval_seconds": _p("Interval between hardware inventory passes."),
+    "node_identity": _internal("Explicit node identity. Leave empty to use the persisted file."),
+    "node_identity_mode": _p("'persisted' keeps a node's identity across restarts; 'auto' derives it."),
+    "node_identity_file": _p("Where a persisted node identity is stored."),
+    "node_stale_after_seconds": _p("Silence after which a node is considered stale."),
+    "node_offline_after_seconds": _p(
+        "Silence after which a node is considered offline. Must exceed the stale threshold."
+    ),
+    "liveness_poll_interval_seconds": _p("How often node liveness is re-evaluated."),
+    "agent_max_clock_skew_seconds": _p("Clock skew tolerated on an agent report before refusal."),
+    "node_agent_max_payload_bytes": _p(
+        "Pre-parser request-body limit for enrollment and authenticated Node Agent reports."
+    ),
+    "node_liveness_monitor_enabled": _p("Run the node liveness monitor in this process."),
+    "agent_protocol_version": _p("Agent protocol version this control plane speaks."),
+    # ---------------------------------------------------------------- gateway
+    "gateway_max_batch_size": _p("Maximum inputs accepted in a single capability invocation."),
+    "gateway_max_input_characters": _p("Maximum characters per input item."),
+    "gateway_max_payload_bytes": _p("Maximum accepted request body size."),
+    "gateway_request_timeout_seconds": _p(
+        "Total time a capability invocation may take. Must exceed the queue timeout."
+    ),
+    "gateway_queue_timeout_seconds": _p("How long a request may wait for capacity before rejection."),
+    "serving_job_lease_seconds": _p("Lease held by a serving job before it is reclaimed."),
+    "serving_payload_ttl_seconds": _p("How long a request payload survives in Redis."),
+    # ---------------------------------------------------------------- scheduler
+    "scheduler_safety_reserve_bytes": _p("VRAM never offered to a placement, as an absolute floor."),
+    "scheduler_safety_reserve_percentage": _p(
+        "VRAM never offered to a placement, as a fraction. Half a device leaves nothing schedulable."
+    ),
+    "scheduler_runtime_margin_bytes": _p("Headroom reserved for runtime overhead per node."),
+    "scheduler_deployment_margin_bytes": _p("Absolute headroom added to each deployment estimate."),
+    "scheduler_deployment_margin_percentage": _p("Proportional headroom added to each estimate."),
+    "scheduler_global_queue_limit": _p("Queued requests accepted before capacity rejection begins."),
+    "scheduler_telemetry_stale_seconds": _p(
+        "Telemetry age past which admission is blocked rather than extrapolated."
+    ),
+    "scheduler_pressure_stable_seconds": _p("How long pressure must hold before the state changes."),
+    "scheduler_eviction_cooldown_seconds": _p("Minimum interval between evictions on a node."),
+    "scheduler_placement_history_limit": _p("Placement decisions retained for inspection."),
+    # ---------------------------------------------------------------- background work
+    "registry_seed_on_startup": _p("Seed the candidate and project registries from manifests."),
+    "serving_reconciliation_enabled": _p("Reconcile abandoned serving work in this process."),
+    "serving_reconciliation_interval_seconds": _p("Interval between serving reconciliation passes."),
+    "lifecycle_reconciliation_enabled": _p("Roll back incomplete lifecycle operations at startup."),
+    "migration_reconciliation_enabled": _p(
+        "Report interrupted migration cutovers at startup. They are never auto-resolved: external "
+        "alias truth cannot be inferred after a crash."
+    ),
+    "observability_monitor_enabled": _p("Run SLO and alert evaluation in this process."),
+    "observability_poll_interval_seconds": _p("Interval between observability evaluation passes."),
+    "recovery_reconciliation_enabled": _p("Reconcile interrupted backups and restores at startup."),
+    # ---------------------------------------------------------------- recovery
+    "backup_pg_dump_path": _p("pg_dump executable. Must match the server major version."),
+    "backup_pg_restore_path": _p("pg_restore executable."),
+    "backup_psql_path": _p("psql executable."),
+    "backup_command_timeout_seconds": _p("Timeout for a dump or restore command."),
+    "backup_stale_after_seconds": _p("Age past which the newest verified backup raises BACKUP_STALE."),
+    "backup_minimum_free_bytes": _p("Free space below which a backup refuses to start."),
+    "backup_capacity_headroom_ratio": _p("Required free space as a multiple of the estimated size."),
+    "restore_allow_production_target": _p(
+        "Whether a restore may overwrite the live database. Keep false outside a rehearsal."
+    ),
+    # ---------------------------------------------------------------- build identity
+    "build_commit": _p("Source commit stamped into the image at build time. Never set by hand."),
+    "build_timestamp": _p("Build time stamped into the image. Never set by hand."),
+    "build_image_digest": _p("Image digest recorded at deployment. Never set by hand."),
+}
+
+
+# --------------------------------------------------------------------------- deployment variables
+#
+# Variables the deployment reads rather than the control-plane process: Compose interpolation, the
+# Node Agent and the Runtime Worker. They are not `Settings` fields, but an operator still has to
+# set them, so leaving them out of the reference would recreate exactly the undocumented-variable
+# problem this table exists to prevent.
+
+DEPLOYMENT_DOCS: dict[str, SettingDoc] = {
+    # ---------------------------------------------------------------- network bindings
+    "MODELFORGE_POSTGRES_BIND": _p(
+        "Host address the control-plane database is published on. Defaults to 127.0.0.1; "
+        "publishing it more widely exposes provenance, credential hashes and the audit trail."
+    ),
+    "MODELFORGE_REDIS_BIND": _p("Host address Redis is published on. Defaults to 127.0.0.1."),
+    "MODELFORGE_API_BIND": _p(
+        "Host address the API is published on. Defaults to 0.0.0.0 deliberately: the console and "
+        "compute nodes need it, and every admin route is operator-authenticated."
+    ),
+    "MODELFORGE_WEB_BIND": _p("Host address the console is published on. Defaults to 127.0.0.1."),
+    "MODELFORGE_POSTGRES_PORT": _p("Host port the database is published on. Defaults to 5432."),
+    "MODELFORGE_REDIS_PORT": _p("Host port Redis is published on. Defaults to 6379."),
+    "MODELFORGE_API_PUBLISHED_PORT": _p("Host port the API is published on. Defaults to 8000."),
+    "MODELFORGE_WEB_PORT": _p("Host port the console is published on. Defaults to 3000."),
+    "MODELFORGE_DR_POSTGRES_BIND": _p("Host address for the DR rehearsal database. Loopback only."),
+    "MODELFORGE_DR_API_BIND": _p("Host address for the DR rehearsal API. Loopback only."),
+    "VITE_API_BASE_URL": _p(
+        "API base URL compiled into the console. Vite inlines it at build time, so changing it "
+        "requires rebuilding the console image, not restarting it."
+    ),
+    # ---------------------------------------------------------------- production identity
+    "MODELFORGE_POSTGRES_DB": _p("Production database name. Required by the production overlay."),
+    "MODELFORGE_POSTGRES_ADMIN_USER": _internal(
+        "Bootstrap/admin role used only by PostgreSQL provisioning; defaults to postgres."
+    ),
+    "MODELFORGE_POSTGRES_ADMIN_PASSWORD": _secret(
+        "Bootstrap/admin password; never passed to the migration or API container.", required=True
+    ),
+    "MODELFORGE_MIGRATION_DB_PASSWORD": _secret(
+        "Raw password supplied to provisioning for the non-superuser modelforge owner role.",
+        required=True,
+    ),
+    "MODELFORGE_RUNTIME_DB_PASSWORD": _secret(
+        "Raw password supplied to provisioning for the non-owner modelforge_runtime role.",
+        required=True,
+    ),
+    "MODELFORGE_RUNTIME_DATABASE_URL": _secret(
+        "Non-owner runtime-role URL passed only to the API container.", required=True
+    ),
+    "MODELFORGE_VERSION": _p(
+        "Exact version tag applied to built images and required when the production overlay is "
+        "not given explicit API and web image references."
+    ),
+    "MODELFORGE_COMMIT": _p("Source commit stamped into images at build time."),
+    "MODELFORGE_BUILT_AT": _p("Build timestamp stamped into images."),
+    "MODELFORGE_API_IMAGE": _p(
+        "Exact tag or digest the production overlay runs for the API; never use latest."
+    ),
+    "MODELFORGE_WEB_IMAGE": _p(
+        "Exact tag or digest the production overlay runs for the console; never use latest."
+    ),
+    "MODELFORGE_NODE_AGENT_IMAGE": _p(
+        "Exact release tag or digest for the standalone Node Agent. The local-build fallback is "
+        "named local and never resolves to latest."
+    ),
+    "MODELFORGE_API_IMAGE_DIGEST": _p("Digest recorded as the running API build identity."),
+    # ---------------------------------------------------------------- volumes
+    "MODELFORGE_BACKUP_VOLUME": _p("Volume or bind path backing the backup root."),
+    "MODELFORGE_RESTORE_VOLUME": _p("Volume or bind path backing the restore staging root."),
+    "MODELFORGE_AGENT_STATE_VOLUME": _p("Volume or bind path holding the agent's persisted identity."),
+    "MODELFORGE_AGENT_HF_CACHE_VOLUME": _p("Volume or bind path for the agent's Hugging Face cache."),
+    "MODELFORGE_AGENT_ARTIFACT_VOLUME": _p("Volume or bind path for verified artifacts on a node."),
+    "MODELFORGE_AGENT_QUARANTINE_VOLUME": _p("Volume or bind path for the node's quarantine area."),
+    # ---------------------------------------------------------------- node agent
+    "MODELFORGE_AGENT_CONTROL_PLANE_URL": _p(
+        "URL the agent reports to. Outbound only; the control plane never dials a node."
+    ),
+    "MODELFORGE_AGENT_ENROLLMENT_TOKEN": _secret(
+        "Single-use enrolment token. Consumed atomically: a storm against one token produces "
+        "exactly one identity."
+    ),
+    "MODELFORGE_AGENT_HOSTNAME": _p("Hostname the agent enrols under."),
+    "MODELFORGE_AGENT_ACCELERATOR_MODE": _p(
+        "Accelerator contract: nvidia fails closed unless NVML inventory and telemetry are valid; "
+        "cpu permits a legitimate CPU-only node; auto requires NVIDIA when injected devices are "
+        "observed. Canonical GPU Compose deployments set nvidia explicitly."
+    ),
+    "MODELFORGE_AGENT_TLS_VERIFY": _p(
+        "Whether the agent verifies the control plane's certificate. True wherever TLS is real."
+    ),
+    "MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS": _internal(
+        "Host address mapped for a private-CA deployment."
+    ),
+    "MODELFORGE_AGENT_CA_CERT_PATH": _p("Path to the private CA certificate the agent trusts."),
+    # ---------------------------------------------------------------- runtime worker
+    "MODELFORGE_RUNTIME_WORKER_ARTIFACT_ROOT": _p("Artifact root as the runtime worker sees it."),
+    "MODELFORGE_RUNTIME_WORKER_POLL_INTERVAL_SECONDS": _p("Worker poll interval, in seconds."),
+    # ---------------------------------------------------------------- provenance passthrough
+    "MODELFORGE_SOURCE_COMMIT": _p("Source commit reported by the deployment."),
+    "MODELFORGE_SOURCE_REFERENCE": _p("Git reference reported by the deployment."),
+    "MODELFORGE_SOURCE_REPOSITORY": _p("Repository URL reported by the deployment."),
+}

backend/src/modelforge_api/domain/contracts.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import UTC, datetime
+from typing import Any, Literal, Protocol
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from .enums import (
+    DeploymentChannel,
+    FailureCode,
+    FailureOwner,
+    HealthStatus,
+    MigrationStatus,
+    ResidencyPolicy,
+    UpgradeClass,
+    VerificationStatus,
+    WorkloadPriority,
+)
+
+CAPABILITY_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$")
+SHA256 = re.compile(r"^[a-f0-9]{64}$")
+COMMIT_SHA = re.compile(r"^[a-f0-9]{40,64}$")
+
+
+class StrictModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class SchemaDocument(StrictModel):
+    schema_: dict[str, Any] = Field(alias="schema")
+
+
+class ModalityContract(StrictModel):
+    input: list[str] = Field(min_length=1)
+    output: list[str] = Field(min_length=1)
+
+
+class VectorContract(StrictModel):
+    dimensionality: int | None = Field(default=None, gt=0)
+    normalized: bool | None = None
+    cross_deployment_compatible: bool = False
+
+
+class SLOContract(StrictModel):
+    latency_p95_ms: int | None = Field(default=None, gt=0)
+    availability_percent: float | None = Field(default=None, ge=0, le=100)
+    max_concurrency: int | None = Field(default=None, gt=0)
+
+
+class PrivacyContract(StrictModel):
+    classification: Literal["public", "internal", "confidential", "restricted"] = "internal"
+    allow_persistence: bool = False
+    allow_logging_payloads: bool = False
+    allow_network_egress: bool = False
+
+
+class ResourceRequirement(StrictModel):
+    accelerator_required: bool = False
+    minimum_vram_mb: int | None = Field(default=None, ge=0)
+    cpu_fallback_allowed: bool = False
+
+
+CapabilityCategory = Literal[
+    "TEXT", "RAG", "DOCUMENT", "VISION", "AUDIO", "GENERATION", "ASSISTANTS"
+]
+CapabilityStability = Literal["stable", "experimental", "blocked", "planned"]
+EvaluationType = Literal[
+    "retrieval", "ocr", "visual-retrieval", "asr", "tts", "generation", "llm"
+]
+ResourceClass = Literal["LIGHT", "MEDIUM", "HEAVY", "EXCLUSIVE_GPU"]
+
+
+class PayloadLimits(StrictModel):
+    max_bytes: int = Field(ge=1, le=67_108_864)
+    max_batch_count: int = Field(default=1, ge=1, le=64)
+    max_width: int | None = Field(default=None, ge=1, le=16_384)
+    max_height: int | None = Field(default=None, ge=1, le=16_384)
+    max_pages: int | None = Field(default=None, ge=1, le=128)
+    max_duration_seconds: float | None = Field(default=None, gt=0, le=3600)
+
+
+class CapabilityEstateMetadata(StrictModel):
+    category: CapabilityCategory
+    purpose: str = Field(min_length=1, max_length=1000)
+    stability: CapabilityStability
+    resource_class: ResourceClass
+    evaluation_type: EvaluationType
+    consumers: list[str] = Field(default_factory=list)
+    payload_limits: PayloadLimits
+
+
+class FallbackContract(StrictModel):
+    allowed: bool = False
+    mode: Literal["none", "compatible_deployment", "degrade", "hard_fail"] = "hard_fail"
+    capability: str | None = None
+
+    @model_validator(mode="after")
+    def validate_fallback(self) -> FallbackContract:
+        if not self.allowed and self.mode not in {"none", "hard_fail"}:
+            raise ValueError("disabled fallback must use none or hard_fail")
+        return self
+
+
+class CapabilityContractManifest(StrictModel):
+    capability: str
+    version: int = Field(ge=1)
+    description: str = Field(min_length=1)
+    input_schema: dict[str, Any]
+    output_schema: dict[str, Any]
+    modalities: ModalityContract
+    languages: list[str] = Field(default_factory=list)
+    streaming: bool = False
+    structured_output: bool = False
+    vector: VectorContract | None = None
+    slo: SLOContract = Field(default_factory=SLOContract)
+    quality_metrics: list[str] = Field(min_length=1)
+    upgrade_class: UpgradeClass
+    fallback: FallbackContract = Field(default_factory=FallbackContract)
+    privacy: PrivacyContract = Field(default_factory=PrivacyContract)
+    resources: ResourceRequirement = Field(default_factory=ResourceRequirement)
+    production_priority: WorkloadPriority
+    default_residency: ResidencyPolicy
+    estate: CapabilityEstateMetadata
+
+    @field_validator("capability")
+    @classmethod
+    def validate_key(cls, value: str) -> str:
+        if not CAPABILITY_KEY.fullmatch(value):
+            raise ValueError("capability must be a dotted, lowercase logical key")
+        return value
+
+    @model_validator(mode="after")
+    def vector_safety(self) -> CapabilityContractManifest:
+        vector_output = self.vector is not None or "embedding" in self.capability
+        if vector_output and not self.vector:
+            raise ValueError("vector-producing capabilities require a vector contract")
+        if (
+            vector_output
+            and self.vector is not None
+            and not self.vector.cross_deployment_compatible
+            and self.upgrade_class is not UpgradeClass.REQUIRES_REINDEX
+        ):
+            raise ValueError("incompatible vector spaces require requires_reindex")
+        return self
+
+
+class ProjectDefinition(StrictModel):
+    id: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
+    name: str = Field(min_length=1)
+    description: str = Field(min_length=1)
+
+
+class ProjectBindingManifest(StrictModel):
+    contract_version: int = Field(ge=1)
+    channel: DeploymentChannel
+    priority: WorkloadPriority
+    optional: bool = False
+    fallback: FallbackContract = Field(default_factory=FallbackContract)
+    migration_support: Literal["none", "reindex", "schema"] = "none"
+    slo: SLOContract = Field(default_factory=SLOContract)
+    benchmark_suites: list[str] = Field(default_factory=list)
+
+
+class ProjectManifest(StrictModel):
+    project: ProjectDefinition
+    bindings: dict[str, ProjectBindingManifest]
+    notes: list[str] = Field(default_factory=list)
+
+    @field_validator("bindings")
+    @classmethod
+    def validate_bindings(
+        cls, value: dict[str, ProjectBindingManifest]
+    ) -> dict[str, ProjectBindingManifest]:
+        for key in value:
+            if not CAPABILITY_KEY.fullmatch(key):
+                raise ValueError(f"invalid capability binding key: {key}")
+        return value
+
+
+class CandidateManifest(StrictModel):
+    id: str
+    display_name: str
+    source: str
+    intended_capabilities: list[str] = Field(min_length=1)
+    proposed_role: str
+    preferred_runtime: str | None = None
+    verification_status: VerificationStatus = VerificationStatus.UNVERIFIED
+    deployment_status: Literal["not_deployed"] = "not_deployed"
+
+
+class CandidateRegistryManifest(StrictModel):
+    candidates: list[CandidateManifest]
+
+
+class BenchmarkSuiteDefinition(StrictModel):
+    id: str
+    capability: str
+    version: int = Field(ge=1)
+    purpose: str
+    dataset_revision: str
+    runnable: bool = False
+
+
+class BenchmarkSuiteManifest(StrictModel):
+    suite: BenchmarkSuiteDefinition
+    metrics: dict[str, list[str]]
+    performance: dict[str, list[str]]
+    release_policy: dict[str, Any]
+
+    @model_validator(mode="after")
+    def require_pinned_dataset_for_runnable_suite(self) -> BenchmarkSuiteManifest:
+        if self.suite.runnable and self.suite.dataset_revision.startswith("pending"):
+            raise ValueError("runnable benchmark suites require a pinned dataset revision")
+        return self
+
+
+class SecurityDefaults(StrictModel):
+    trust_remote_code: Literal[False] = False
+    prefer_safetensors: Literal[True] = True
+    require_exact_revision_for_approval: Literal[True] = True
+    require_artifact_digest: Literal[True] = True
+    inference_network_egress: Literal[False] = False
+    expose_runtime_workers: Literal[False] = False
+
+
+class LifecycleDefaults(StrictModel):
+    automatic_production_promotion: Literal[False] = False
+    require_local_benchmark_for_stable: Literal[True] = True
+    retain_previous_stable_as_rollback_target: Literal[True] = True
+    destructive_cleanup_requires_explicit_approval: Literal[True] = True
+
+
+class SchedulerDefaults(StrictModel):
+    priority_order: list[WorkloadPriority]
+    reserve_vram_mb: int = Field(ge=0)
+    default_warm_ttl_seconds: int = Field(ge=0)
+    benchmark_may_disrupt_production: Literal[False] = False
+
+
+class UpgradeDefaults(StrictModel):
+    default_behavioral_change_class: Literal[UpgradeClass.BEHAVIORAL]
+    vector_producer_default_change_class: Literal[UpgradeClass.REQUIRES_REINDEX]
+    schema_change_requires_contract_version_bump: Literal[True] = True
+
+
+class PolicyDefaults(StrictModel):
+    security: SecurityDefaults
+    lifecycle: LifecycleDefaults
+    scheduler: SchedulerDefaults
+    upgrades: UpgradeDefaults
+
+
+class ArtifactProvenance(StrictModel):
+    upstream_repository: str
+    resolved_commit_sha: str
+    filename: str
+    artifact_type: str
+    sha256: str
+    source_artifact_sha256: str | None = None
+    derivation_tool: str | None = None
+    derivation_tool_version: str | None = None
+    derivation_arguments: dict[str, Any] = Field(default_factory=dict)
+    imported_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    @field_validator("resolved_commit_sha")
+    @classmethod
+    def validate_commit(cls, value: str) -> str:
+        value = value.lower()
+        if not COMMIT_SHA.fullmatch(value):
+            raise ValueError("resolved commit must be a 40-64 character hexadecimal digest")
+        return value
+
+    @field_validator("sha256", "source_artifact_sha256")
+    @classmethod
+    def validate_digest(cls, value: str | None) -> str | None:
+        if value is not None and not SHA256.fullmatch(value.lower()):
+            raise ValueError("artifact digest must be lowercase SHA-256")
+        return value.lower() if value else None
+
+    @model_validator(mode="after")
+    def validate_lineage(self) -> ArtifactProvenance:
+        derived = self.source_artifact_sha256 is not None
+        if derived and (not self.derivation_tool or not self.derivation_tool_version):
+            raise ValueError("derived artifacts require derivation tool and version")
+        return self
+
+
+class RuntimeProfileContract(StrictModel):
+    runtime_type: Literal["vllm", "transformers", "diffusers", "llama_cpp", "custom"]
+    runtime_version: str
+    runtime_image_digest: str | None = Field(default=None, pattern=r"^sha256:[a-f0-9]{64}$")
+    artifact_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+    quantization: str | None = None
+    context_length: int | None = Field(default=None, gt=0)
+    max_concurrency: int = Field(default=1, gt=0)
+    launch_arguments: dict[str, Any] = Field(default_factory=dict)
+    environment_constraints: dict[str, Any] = Field(default_factory=dict)
+    trust_remote_code: bool = False
+    network_egress: bool = False
+
+    @property
+    def fingerprint(self) -> str:
+        payload = self.model_dump(mode="json", exclude_none=True)
+        return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+
+
+class ResourceEnvelopeContract(StrictModel):
+    deployment_id: str
+    accelerator_kind: str
+    context_length: int | None = Field(default=None, gt=0)
+    concurrency: int = Field(ge=1)
+    batch_size: int = Field(ge=1)
+    idle_vram_mb: int = Field(ge=0)
+    peak_vram_mb: int = Field(ge=0)
+    safety_margin_mb: int = Field(default=1024, ge=0)
+
+    @model_validator(mode="after")
+    def peak_not_below_idle(self) -> ResourceEnvelopeContract:
+        if self.peak_vram_mb < self.idle_vram_mb:
+            raise ValueError("peak VRAM cannot be below idle VRAM")
+        return self
+
+
+class LeaseRequest(StrictModel):
+    deployment_id: str
+    priority: WorkloadPriority
+    residency: ResidencyPolicy
+    envelope: ResourceEnvelopeContract
+    warm_ttl_seconds: int = Field(default=900, ge=0)
+    exclusive: bool = False
+    allow_cpu_fallback: bool = False
+
+
+class BenchmarkEnvironmentFingerprint(StrictModel):
+    runtime_type: str
+    runtime_version: str
+    runtime_image_digest: str | None = None
+    launch_arguments: dict[str, Any]
+    cuda_version: str | None
+    driver_version: str | None
+    accelerator_name: str
+    accelerator_uuid: str | None
+    context_length: int | None
+    concurrency: int = Field(ge=1)
+    seed: int | None
+    operating_system: str
+    python_version: str | None = None
+
+    @property
+    def digest(self) -> str:
+        raw = json.dumps(self.model_dump(mode="json"), sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(raw.encode()).hexdigest()
+
+    def comparable_with(self, other: BenchmarkEnvironmentFingerprint) -> bool:
+        relevant = (
+            "runtime_type",
+            "runtime_version",
+            "runtime_image_digest",
+            "launch_arguments",
+            "cuda_version",
+            "driver_version",
+            "accelerator_name",
+            "context_length",
+            "concurrency",
+        )
+        return all(getattr(self, field) == getattr(other, field) for field in relevant)
+
+
+class BenchmarkRunContract(StrictModel):
+    deployment_id: str
+    model_revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
+    artifact_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+    suite_key: str
+    suite_revision: str
+    dataset_revision: str
+    environment: BenchmarkEnvironmentFingerprint
+    started_at: datetime
+    completed_at: datetime | None = None
+
+
+class MigrationContract(StrictModel):
+    project_id: str
+    capability: str
+    source_deployment_id: str
+    target_deployment_id: str
+    upgrade_class: UpgradeClass
+    current_index_ref: str
+    shadow_index_ref: str
+    status: MigrationStatus = MigrationStatus.PLANNED
+    rollback_retain_until: datetime
+
+    @model_validator(mode="after")
+    def require_reindex(self) -> MigrationContract:
+        if (
+            "embedding" in self.capability
+            and self.upgrade_class is not UpgradeClass.REQUIRES_REINDEX
+        ):
+            raise ValueError("embedding migrations must be classified requires_reindex")
+        if self.current_index_ref == self.shadow_index_ref:
+            raise ValueError("shadow index must be distinct from current production index")
+        return self
+
+
+class LayerHealth(StrictModel):
+    process: HealthStatus
+    runtime: HealthStatus
+    model: HealthStatus
+    capability: HealthStatus
+    project: HealthStatus
+    reasons: list[str] = Field(default_factory=list)
+
+
+class FailurePolicy(StrictModel):
+    code: FailureCode
+    owner: FailureOwner
+    retryable: bool
+    fallback_allowed: bool
+    hard_failure: bool
+
+
+class RuntimeAdapter(Protocol):
+    """Stable control-plane boundary; concrete adapters arrive in M5."""
+
+    runtime_type: str
+
+    def validate_profile(self, profile: RuntimeProfileContract) -> None: ...
+    def probe(self, profile: RuntimeProfileContract) -> LayerHealth: ...
+    def load(self, profile: RuntimeProfileContract, lease_id: str) -> str: ...
+    def unload(self, deployment_id: str) -> None: ...

backend/src/modelforge_api/domain/enums.py

@@ -0,0 +1,200 @@
+from enum import StrEnum
+
+
+class ModelLifecycle(StrEnum):
+    DISCOVERED = "discovered"
+    CANDIDATE = "candidate"
+    DOWNLOADING = "downloading"
+    QUARANTINED = "quarantined"
+    VERIFIED = "verified"
+    TESTING = "testing"
+    APPROVED = "approved"
+    STANDBY = "standby"
+    ACTIVE = "active"
+    DEPRECATED = "deprecated"
+    ARCHIVED = "archived"
+    REJECTED = "rejected"
+    INCOMPATIBLE = "incompatible"
+    SECURITY_BLOCKED = "security_blocked"
+    LICENSE_BLOCKED = "license_blocked"
+
+
+class UpgradeClass(StrEnum):
+    TRANSPARENT = "transparent"
+    BEHAVIORAL = "behavioral"
+    REQUIRES_REINDEX = "requires_reindex"
+    SCHEMA_BREAKING = "schema_breaking"
+
+
+class DeploymentChannel(StrEnum):
+    STABLE = "stable"
+    CANDIDATE = "candidate"
+    EXPERIMENTAL = "experimental"
+    ARCHIVE = "archive"
+
+
+class ResidencyPolicy(StrEnum):
+    ALWAYS_WARM = "always_warm"
+    KEEP_WARM = "keep_warm"
+    LOAD_ON_DEMAND = "load_on_demand"
+    EXCLUSIVE = "exclusive"
+    LAB_ONLY = "lab_only"
+
+
+class WorkloadPriority(StrEnum):
+    PRODUCTION = "production"
+    INTERACTIVE = "interactive"
+    BACKGROUND = "background"
+    BENCHMARK = "benchmark"
+    MAINTENANCE = "maintenance"
+
+
+class VerificationStatus(StrEnum):
+    UNKNOWN = "unknown"
+    UNVERIFIED = "unverified"
+    QUARANTINED = "quarantined"
+    VERIFIED = "verified"
+    BLOCKED = "blocked"
+
+
+class LicenseStatus(StrEnum):
+    UNKNOWN = "unknown"
+    REVIEW_REQUIRED = "review_required"
+    APPROVED = "approved"
+    BLOCKED = "blocked"
+
+
+class ArtifactStatus(StrEnum):
+    REMOTE = "remote"
+    LOCAL = "local"
+    VERIFYING = "verifying"
+    VERIFIED = "verified"
+    MISSING = "missing"
+    CORRUPT = "corrupt"
+    QUARANTINED = "quarantined"
+    ARCHIVED = "archived"
+    UNREACHABLE = "unreachable"
+
+
+class StorageRootStatus(StrEnum):
+    UNKNOWN = "unknown"
+    READY = "ready"
+    READ_ONLY = "read_only"
+    CAPACITY_BLOCKED = "capacity_blocked"
+    UNAVAILABLE = "unavailable"
+    DEPRECATED = "deprecated"
+
+
+class DeploymentStatus(StrEnum):
+    DRAFT = "draft"
+    VALIDATING = "validating"
+    READY = "ready"
+    ACTIVE = "active"
+    UNHEALTHY = "unhealthy"
+    RETIRED = "retired"
+
+
+class MigrationStatus(StrEnum):
+    PLANNED = "planned"
+    BACKFILLING = "backfilling"
+    VALIDATING = "validating"
+    SHADOWING = "shadowing"
+    READY = "ready"
+    PROMOTED = "promoted"
+    ROLLED_BACK = "rolled_back"
+    FAILED = "failed"
+
+
+class HealthStatus(StrEnum):
+    HEALTHY = "healthy"
+    DEGRADED = "degraded"
+    UNHEALTHY = "unhealthy"
+    UNAVAILABLE = "unavailable"
+    UNKNOWN = "unknown"
+
+
+class FailureCode(StrEnum):
+    MODEL_LOAD_FAILED = "MODEL_LOAD_FAILED"
+    GPU_OOM = "GPU_OOM"
+    RUNTIME_CRASH = "RUNTIME_CRASH"
+    TIMEOUT = "TIMEOUT"
+    INVALID_OUTPUT = "INVALID_OUTPUT"
+    HEALTHCHECK_FAILED = "HEALTHCHECK_FAILED"
+    ARTIFACT_CORRUPT = "ARTIFACT_CORRUPT"
+    DRIVER_ERROR = "DRIVER_ERROR"
+    CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE"
+    RUNTIME_INCOMPATIBLE = "RUNTIME_INCOMPATIBLE"
+    ARTIFACT_INCOMPLETE = "ARTIFACT_INCOMPLETE"
+    EXECUTION_NOT_APPROVED = "EXECUTION_NOT_APPROVED"
+    RUNTIME_DEPENDENCY_MISSING = "RUNTIME_DEPENDENCY_MISSING"
+    OFFLINE_LOAD_VIOLATION = "OFFLINE_LOAD_VIOLATION"
+    UNLOAD_FAILED = "UNLOAD_FAILED"
+    GPU_MEMORY_NOT_RECLAIMED = "GPU_MEMORY_NOT_RECLAIMED"
+    NO_ELIGIBLE_NODE = "NO_ELIGIBLE_NODE"
+    INSUFFICIENT_SCHEDULABLE_VRAM = "INSUFFICIENT_SCHEDULABLE_VRAM"
+    QUEUE_FULL = "QUEUE_FULL"
+    LEASE_TIMEOUT = "LEASE_TIMEOUT"
+    RESIDENCY_LOAD_FAILED = "RESIDENCY_LOAD_FAILED"
+    RESIDENCY_UNLOAD_FAILED = "RESIDENCY_UNLOAD_FAILED"
+    AUTHORIZATION_DENIED = "AUTHORIZATION_DENIED"
+    RATE_LIMITED = "RATE_LIMITED"
+    EXTERNAL_GPU_PRESSURE = "EXTERNAL_GPU_PRESSURE"
+    STALE_RESOURCE_ENVELOPE = "STALE_RESOURCE_ENVELOPE"
+
+
+class FailureOwner(StrEnum):
+    RUNTIME_ADAPTER = "runtime_adapter"
+    SCHEDULER = "scheduler"
+    GATEWAY = "gateway"
+    CONTROL_PLANE = "control_plane"
+    OPERATOR = "operator"
+
+
+class Availability(StrEnum):
+    KNOWN = "known"
+    UNKNOWN = "unknown"
+    UNSUPPORTED = "unsupported"
+    UNAVAILABLE = "unavailable"
+    TEMPORARILY_FAILED = "temporarily_failed"
+
+
+class HardwareStatus(StrEnum):
+    UNKNOWN = "unknown"
+    ACTIVE = "active"
+    MISSING = "missing"
+    UNAVAILABLE = "unavailable"
+    DEGRADED = "degraded"
+    PENDING = "pending"
+    DECOMMISSIONED = "decommissioned"
+
+
+class InventorySource(StrEnum):
+    HOST = "host"
+    NVIDIA_NVML = "nvidia_nvml"
+
+
+class InventoryRunStatus(StrEnum):
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    DEGRADED = "degraded"
+    FAILED = "failed"
+
+
+class NodeLiveness(StrEnum):
+    ONLINE = "online"
+    STALE = "stale"
+    OFFLINE = "offline"
+    DISABLED = "disabled"
+    DECOMMISSIONED = "decommissioned"
+
+
+class AgentHealth(StrEnum):
+    HEALTHY = "healthy"
+    INCOMPATIBLE = "incompatible"
+    REVOKED = "revoked"
+    UNKNOWN = "unknown"
+
+
+class ObservationSource(StrEnum):
+    LOCAL_CONTROL_PLANE = "local_control_plane"
+    REMOTE_AGENT = "remote_agent"

backend/src/modelforge_api/domain/evaluation.py

@@ -0,0 +1,593 @@
+from __future__ import annotations
+
+import math
+import re
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+MetricName = Literal["recall_at_5", "recall_at_10", "mrr", "ndcg_at_10"]
+TargetKind = Literal["current", "shadow"]
+_KEY = re.compile(r"^[a-z0-9][a-z0-9_-]{0,127}$")
+
+
+class EvaluationModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class EvaluationCaseCreate(EvaluationModel):
+    case_key: str
+    query: str = Field(min_length=1, max_length=4000)
+    relevant_chunk_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
+    relevant_document_ids: list[uuid.UUID] = Field(default_factory=list, max_length=100)
+    relevance_grades: dict[str, int] = Field(default_factory=dict)
+    label_provenance: dict[str, Any]
+    critical: bool = False
+    review_status: Literal["reviewed", "approved"]
+
+    @field_validator("case_key")
+    @classmethod
+    def valid_key(cls, value: str) -> str:
+        if not _KEY.fullmatch(value):
+            raise ValueError("case_key must be a safe identifier")
+        return value
+
+    @field_validator("relevant_chunk_ids")
+    @classmethod
+    def unique_relevance(cls, value: list[uuid.UUID]) -> list[uuid.UUID]:
+        if len(set(value)) != len(value):
+            raise ValueError("relevant chunk ids must be unique")
+        return value
+
+
+class EvaluationRevisionCreate(EvaluationModel):
+    revision: str
+    dataset_revision: str
+    metrics: list[MetricName] = ["recall_at_5", "recall_at_10", "mrr", "ndcg_at_10"]
+    top_k: int = Field(default=10, ge=10, le=100)
+    retrieval_settings: dict[str, Any]
+    thresholds: dict[str, float] = Field(default_factory=dict)
+    cases: list[EvaluationCaseCreate] = Field(min_length=1, max_length=500)
+
+    @model_validator(mode="after")
+    def validate_revision(self) -> EvaluationRevisionCreate:
+        if not _KEY.fullmatch(self.revision) or not _KEY.fullmatch(self.dataset_revision):
+            raise ValueError("revision identifiers must be safe")
+        if len(set(self.metrics)) != len(self.metrics):
+            raise ValueError("metrics must be unique")
+        if len({case.case_key for case in self.cases}) != len(self.cases):
+            raise ValueError("case keys must be unique")
+        return self
+
+
+class EvaluationSuiteCreate(EvaluationModel):
+    project_id: uuid.UUID
+    key: str
+    name: str = Field(min_length=1, max_length=255)
+    description: str = Field(min_length=1, max_length=4000)
+    revision: EvaluationRevisionCreate
+
+    @field_validator("key")
+    @classmethod
+    def valid_key(cls, value: str) -> str:
+        if not _KEY.fullmatch(value):
+            raise ValueError("suite key must be a safe identifier")
+        return value
+
+
+class EvaluationSuiteResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    key: str
+    name: str
+    description: str
+    latest_revision_id: uuid.UUID
+    latest_revision: str
+    case_count: int
+    critical_case_count: int
+    created_at: datetime
+
+
+class EvaluationCaseDefinitionResponse(EvaluationModel):
+    id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    case_key: str
+    query: str
+    relevant_chunk_ids: list[str]
+    relevant_document_ids: list[str]
+    relevance_grades: dict[str, int]
+    label_provenance: dict[str, Any]
+    critical: bool
+    review_status: str
+    created_at: datetime
+
+
+class EvaluationRunCreate(EvaluationModel):
+    project_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    target_kind: TargetKind
+    target_index_ref: str = Field(min_length=1, max_length=512)
+    embedding_space_ref: str = Field(min_length=1, max_length=255)
+    capability_deployment_id: uuid.UUID | None = None
+    corpus_revision: str = Field(min_length=1, max_length=128)
+    retrieval_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    environment_fingerprint: dict[str, Any]
+
+
+class RankedResult(EvaluationModel):
+    chunk_id: uuid.UUID
+    document_id: uuid.UUID | None = None
+    score: float
+
+    @field_validator("score")
+    @classmethod
+    def finite_score(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("score must be finite")
+        return value
+
+
+class CandidatePoolEntry(EvaluationModel):
+    id: str = Field(min_length=1, max_length=255)
+    document_id: str | None = Field(default=None, max_length=255)
+    score: float
+    content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+
+    @field_validator("score")
+    @classmethod
+    def finite_score(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("score must be finite")
+        return value
+
+
+class RetrievalCandidatePoolCreate(EvaluationModel):
+    project_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    evaluation_case_id: uuid.UUID
+    source_embedding_space: str = Field(min_length=1, max_length=255)
+    source_index_ref: str = Field(min_length=1, max_length=512)
+    corpus_revision: str = Field(min_length=1, max_length=128)
+    retrieval_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    ordered_candidates: list[CandidatePoolEntry] = Field(min_length=1, max_length=40)
+
+    @model_validator(mode="after")
+    def unique_ordered_ids(self) -> RetrievalCandidatePoolCreate:
+        ids = [item.id for item in self.ordered_candidates]
+        if len(set(ids)) != len(ids):
+            raise ValueError("candidate pool ids must be unique")
+        return self
+
+
+class RetrievalCandidatePoolResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    evaluation_case_id: uuid.UUID
+    source_embedding_space: str
+    source_index_ref: str
+    corpus_revision: str
+    retrieval_config_digest: str
+    candidate_count: int
+    ordered_candidates: list[dict[str, Any]]
+    fingerprint: str
+    created_at: datetime
+    immutable_at: datetime
+
+
+class RetrievalPipelineIdentityCreate(EvaluationModel):
+    project_id: uuid.UUID
+    embedding_space_ref: str = Field(min_length=1, max_length=255)
+    sparse_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    fusion_config_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    reranker_deployment_id: uuid.UUID | None = None
+    reranker_config_digest: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$")
+    candidate_k: Literal[40] = 40
+    output_k: int = Field(default=10, ge=1, le=10)
+    configuration: dict[str, Any] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def reranker_identity_is_complete(self) -> RetrievalPipelineIdentityCreate:
+        if (self.reranker_deployment_id is None) != (self.reranker_config_digest is None):
+            raise ValueError("reranker deployment and config digest must be supplied together")
+        return self
+
+
+class RetrievalPipelineIdentityResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    embedding_space_ref: str
+    sparse_config_digest: str
+    fusion_config_digest: str
+    reranker_deployment_id: uuid.UUID | None
+    reranker_config_digest: str | None
+    candidate_k: int
+    output_k: int
+    identity_digest: str
+    configuration: dict[str, Any]
+    migration_class: str
+    created_at: datetime
+    immutable_at: datetime
+
+
+class RerankingRunCreate(EvaluationModel):
+    project_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    pipeline_identity_id: uuid.UUID
+    control_pipeline_identity_id: uuid.UUID
+    candidate_pool_ids: list[uuid.UUID] = Field(min_length=1, max_length=500)
+    corpus_revision: str = Field(min_length=1, max_length=128)
+    environment_fingerprint: dict[str, Any]
+
+
+class RerankingCaseResultCreate(EvaluationModel):
+    case_id: uuid.UUID
+    candidate_pool_id: uuid.UUID
+    ranked_results: list[RankedResult] = Field(min_length=1, max_length=10)
+    retrieval_latency_ms: float = Field(ge=0, le=3_600_000)
+    rerank_latency_ms: float = Field(ge=0, le=3_600_000)
+    total_latency_ms: float = Field(ge=0, le=3_600_000)
+    error_code: str | None = Field(default=None, max_length=64)
+
+
+class RerankingRunComplete(EvaluationModel):
+    results: list[RerankingCaseResultCreate] = Field(min_length=1, max_length=500)
+
+
+class RerankingRunResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    pipeline_identity_id: uuid.UUID
+    control_pipeline_identity_id: uuid.UUID
+    reranker_deployment_id: uuid.UUID | None
+    status: str
+    corpus_revision: str
+    candidate_pool_set_fingerprint: str
+    environment_fingerprint: dict[str, Any]
+    environment_digest: str
+    expected_cases: int
+    completed_cases: int
+    error_count: int
+    aggregate_metrics: dict[str, float]
+    latency_metrics: dict[str, float]
+    started_at: datetime | None
+    completed_at: datetime | None
+    created_at: datetime
+
+
+class RerankingCaseResultResponse(EvaluationModel):
+    id: uuid.UUID
+    run_id: uuid.UUID
+    case_id: uuid.UUID
+    case_key: str
+    critical: bool
+    candidate_pool_id: uuid.UUID
+    ranked_results: list[dict[str, Any]]
+    relevant_results: list[str]
+    first_relevant_rank: int | None
+    metrics: dict[str, float]
+    retrieval_latency_ms: float
+    rerank_latency_ms: float
+    total_latency_ms: float
+    error_code: str | None
+
+
+class DiscoveryCandidateAssessmentCreate(EvaluationModel):
+    model_id: uuid.UUID | None = None
+    upstream_snapshot_id: uuid.UUID
+    candidate_key: str
+    repository_id: str = Field(min_length=3, max_length=255)
+    resolved_commit_sha: str = Field(pattern=r"^[0-9a-f]{40,64}$")
+    artifact_evidence: dict[str, Any]
+    security_state: dict[str, Any]
+    license_state: dict[str, Any]
+    gpu_fit: dict[str, Any]
+    status: Literal[
+        "shortlisted",
+        "preflight_passed",
+        "evaluated",
+        "discovery_security_blocked",
+        "discovery_gpu_fit_blocked",
+        "license_blocked",
+        "not_selected",
+    ]
+    rationale: str = Field(min_length=10, max_length=4000)
+
+    @field_validator("candidate_key")
+    @classmethod
+    def valid_candidate_key(cls, value: str) -> str:
+        if not _KEY.fullmatch(value):
+            raise ValueError("candidate_key must be a safe identifier")
+        return value
+
+
+class DiscoveryCandidateAssessmentResponse(DiscoveryCandidateAssessmentCreate):
+    id: uuid.UUID
+    evidence_fingerprint: str
+    created_at: datetime
+    immutable_at: datetime
+
+
+class EvaluationCaseResultCreate(EvaluationModel):
+    case_id: uuid.UUID
+    ranked_results: list[RankedResult] = Field(max_length=100)
+    latency_ms: float = Field(ge=0, le=3_600_000)
+    error_code: str | None = Field(default=None, max_length=64)
+
+
+class EvaluationRunComplete(EvaluationModel):
+    results: list[EvaluationCaseResultCreate] = Field(min_length=1, max_length=500)
+
+
+class EvaluationRunResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    target_kind: str
+    target_index_ref: str
+    embedding_space_ref: str
+    capability_deployment_id: uuid.UUID | None
+    status: str
+    corpus_revision: str
+    retrieval_config_digest: str
+    environment_fingerprint: dict[str, Any]
+    environment_digest: str
+    expected_cases: int
+    completed_cases: int
+    error_count: int
+    aggregate_metrics: dict[str, float]
+    started_at: datetime | None
+    completed_at: datetime | None
+    created_at: datetime
+
+
+class EvaluationCaseResultResponse(EvaluationModel):
+    id: uuid.UUID
+    run_id: uuid.UUID
+    case_id: uuid.UUID
+    case_key: str
+    critical: bool
+    ranked_results: list[dict[str, Any]]
+    relevant_results: list[str]
+    first_relevant_rank: int | None
+    metrics: dict[str, float]
+    latency_ms: float
+    error_code: str | None
+
+
+class EvaluationComparisonCreate(EvaluationModel):
+    baseline_run_id: uuid.UUID
+    candidate_run_id: uuid.UUID
+    require_no_critical_regressions: bool = True
+    maximum_error_rate: float = Field(default=0.0, ge=0, le=1)
+    minimum_recall_at_10_delta: float = Field(default=0.0, ge=-1, le=1)
+
+
+class EvaluationComparisonResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    baseline_run_id: uuid.UUID
+    candidate_run_id: uuid.UUID
+    comparability: str
+    comparability_evidence: dict[str, Any]
+    metric_deltas: dict[str, float]
+    improved_cases: int
+    unchanged_cases: int
+    regressed_cases: int
+    critical_regressions: int
+    case_comparisons: list[dict[str, Any]]
+    promotion_eligibility: str
+    eligibility_evidence: dict[str, Any]
+    created_at: datetime
+
+
+class ModelComparisonCandidateCreate(EvaluationModel):
+    candidate_key: str
+    label: str = Field(min_length=1, max_length=255)
+    status: Literal["evaluated", "blocked", "unknown"]
+    evaluation_run_id: uuid.UUID | None = None
+    candidate_deployment_id: uuid.UUID | None = None
+    embedding_space: str | None = Field(default=None, max_length=255)
+    artifact_size_bytes: int | None = Field(default=None, ge=0)
+    latency_ms: dict[str, float] = Field(default_factory=dict)
+    resource_evidence: dict[str, Any] = Field(default_factory=dict)
+    migration_impact: dict[str, Any] = Field(default_factory=dict)
+    security_state: dict[str, Any] = Field(default_factory=dict)
+    provenance: dict[str, Any] = Field(default_factory=dict)
+    blockers: list[str] = Field(default_factory=list, max_length=32)
+    candidate_kind: Literal["embedding_deployment", "retrieval_pipeline"] = "embedding_deployment"
+    pipeline_identity_id: uuid.UUID | None = None
+
+    @field_validator("candidate_key")
+    @classmethod
+    def valid_candidate_key(cls, value: str) -> str:
+        if not _KEY.fullmatch(value):
+            raise ValueError("candidate_key must be a safe identifier")
+        return value
+
+    @model_validator(mode="after")
+    def evaluated_run_required(self) -> ModelComparisonCandidateCreate:
+        if (
+            self.status == "evaluated"
+            and self.candidate_kind == "embedding_deployment"
+            and self.evaluation_run_id is None
+        ):
+            raise ValueError("evaluated embedding candidate requires evaluation_run_id")
+        if self.candidate_kind == "retrieval_pipeline" and self.evaluation_run_id is not None:
+            raise ValueError("retrieval pipeline evidence must come from a fixed-pool run")
+        if self.status != "evaluated" and self.evaluation_run_id is not None:
+            raise ValueError("blocked or unknown candidate cannot claim an evaluation run")
+        if self.candidate_kind == "retrieval_pipeline" and self.pipeline_identity_id is None:
+            raise ValueError("retrieval pipeline candidate requires pipeline_identity_id")
+        return self
+
+
+class ModelComparisonCreate(EvaluationModel):
+    project_id: uuid.UUID
+    capability_contract_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    current_run_id: uuid.UUID
+    title: str = Field(min_length=1, max_length=255)
+    candidates: list[ModelComparisonCandidateCreate] = Field(min_length=1, max_length=20)
+
+    @model_validator(mode="after")
+    def unique_candidates(self) -> ModelComparisonCreate:
+        if len({item.candidate_key for item in self.candidates}) != len(self.candidates):
+            raise ValueError("candidate keys must be unique")
+        return self
+
+
+class ModelComparisonResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    capability_contract_id: uuid.UUID
+    suite_revision_id: uuid.UUID
+    current_run_id: uuid.UUID
+    title: str
+    candidates: list[dict[str, Any]]
+    comparability: str
+    evidence_fingerprint: str
+    created_at: datetime
+
+
+class AdvisorPolicyUpdate(EvaluationModel):
+    maximum_latency_p95_regression_ratio: float = Field(ge=0, le=10)
+    minimum_metric_deltas: dict[MetricName, float]
+    rationale: str = Field(min_length=20, max_length=4000)
+
+
+class AdvisorPolicyResponse(EvaluationModel):
+    id: uuid.UUID
+    key: str
+    required_evidence_level: str
+    critical_regression_hard_block: bool
+    maximum_latency_p95_regression_ratio: float
+    minimum_metric_deltas: dict[str, float]
+    require_verified_supply_chain: bool
+    require_runtime_fit: bool
+    rationale: str
+    version: int
+    created_at: datetime
+    updated_at: datetime
+
+
+class AdvisorRecommendationCreate(EvaluationModel):
+    candidate_key: str
+
+    @field_validator("candidate_key")
+    @classmethod
+    def valid_candidate_key(cls, value: str) -> str:
+        if not _KEY.fullmatch(value):
+            raise ValueError("candidate_key must be a safe identifier")
+        return value
+
+
+class AdvisorRecommendationDismiss(EvaluationModel):
+    dismissed_by: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=10, max_length=2000)
+
+
+class AdvisorRecommendationResponse(EvaluationModel):
+    id: uuid.UUID
+    comparison_id: uuid.UUID
+    project_id: uuid.UUID
+    capability_contract_id: uuid.UUID
+    policy_id: uuid.UUID
+    candidate_key: str
+    current_deployment_id: uuid.UUID | None
+    candidate_deployment_id: uuid.UUID | None
+    current_embedding_space: str
+    candidate_embedding_space: str | None
+    target_kind: Literal["embedding_deployment", "retrieval_pipeline"]
+    current_pipeline_identity_id: uuid.UUID | None
+    candidate_pipeline_identity_id: uuid.UUID | None
+    verdict: Literal[
+        "KEEP_CURRENT",
+        "KEEP_CURRENT_EMBEDDING_ADD_RERANKER_CANDIDATE",
+        "PROMOTION_ELIGIBLE",
+        "PROMOTION_NOT_RECOMMENDED",
+        "REQUIRES_MORE_EVIDENCE",
+    ]
+    confidence: str
+    evidence_level: str
+    quality_deltas: dict[str, float]
+    latency_deltas: dict[str, Any]
+    resource_deltas: dict[str, Any]
+    migration_impact: dict[str, Any]
+    security_state: dict[str, Any]
+    key_improvements: list[str]
+    blockers: list[str]
+    policy_snapshot: dict[str, Any]
+    evidence_fingerprint: str
+    status: str
+    generated_at: datetime
+    dismissed_at: datetime | None
+    dismissed_by: str | None
+    dismissal_reason: str | None
+
+
+class EmbeddingMigrationCreate(EvaluationModel):
+    source_embedding_space: str = Field(min_length=1, max_length=255)
+    target_embedding_space_id: uuid.UUID
+    source_index_ref: str = Field(min_length=1, max_length=512)
+    target_index_ref: str = Field(min_length=1, max_length=512)
+    corpus_revision: str = Field(min_length=1, max_length=128)
+    total_chunks: int = Field(ge=1, le=10_000_000)
+    batch_size: int = Field(default=32, ge=1, le=256)
+    concurrency: int = Field(default=1, ge=1, le=8)
+
+
+class MigrationUpdate(EvaluationModel):
+    status: Literal[
+        "preflight",
+        "backfilling",
+        "validating",
+        "ready_for_evaluation",
+        "evaluated",
+        "promotion_eligible",
+        "not_eligible",
+        "failed",
+        "cancelled",
+    ]
+    completed_chunks: int = Field(ge=0)
+    failed_chunks: int = Field(ge=0)
+    retried_chunks: int = Field(ge=0)
+    preflight_evidence: dict[str, Any] = Field(default_factory=dict)
+    progress_evidence: dict[str, Any] = Field(default_factory=dict)
+    validation_evidence: dict[str, Any] = Field(default_factory=dict)
+    operational_metrics: dict[str, Any] = Field(default_factory=dict)
+    failure_code: str | None = Field(default=None, max_length=64)
+    failure_message: str | None = Field(default=None, max_length=2000)
+
+
+class EmbeddingMigrationResponse(EvaluationModel):
+    id: uuid.UUID
+    project_id: uuid.UUID
+    source_embedding_space: str
+    target_embedding_space_id: uuid.UUID
+    source_index_ref: str
+    target_index_ref: str
+    corpus_revision: str
+    status: str
+    total_chunks: int
+    completed_chunks: int
+    failed_chunks: int
+    retried_chunks: int
+    batch_size: int
+    concurrency: int
+    priority: str
+    preflight_evidence: dict[str, Any]
+    progress_evidence: dict[str, Any]
+    validation_evidence: dict[str, Any]
+    operational_metrics: dict[str, Any]
+    evaluation_eligibility: bool
+    cancel_requested: bool
+    failure_code: str | None
+    failure_message: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime

backend/src/modelforge_api/domain/hardware.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import UTC, datetime
+from typing import Protocol
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from modelforge_api.domain.enums import (
+    AgentHealth,
+    Availability,
+    HardwareStatus,
+    InventorySource,
+    NodeLiveness,
+    ObservationSource,
+)
+
+
+class HardwareModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class ObservedValue[T](HardwareModel):
+    value: T | None = None
+    availability: Availability
+    reason: str | None = None
+
+    @model_validator(mode="after")
+    def value_matches_availability(self) -> ObservedValue[T]:
+        if self.availability is Availability.KNOWN and self.value is None:
+            raise ValueError("known observations require a value")
+        if self.availability is not Availability.KNOWN and self.value is not None:
+            raise ValueError("non-known observations cannot carry a value")
+        return self
+
+    @classmethod
+    def known(cls, value: T) -> ObservedValue[T]:
+        return cls(value=value, availability=Availability.KNOWN)
+
+    @classmethod
+    def absent(cls, availability: Availability, reason: str | None = None) -> ObservedValue[T]:
+        return cls(availability=availability, reason=reason)
+
+
+class StorageObservation(HardwareModel):
+    purpose: str
+    path: str
+    total_bytes: ObservedValue[int]
+    used_bytes: ObservedValue[int]
+    free_bytes: ObservedValue[int]
+
+
+class HostInventory(HardwareModel):
+    identity_key: str
+    identity_source: str
+    hostname: str
+    display_name: str
+    os_name: str
+    os_version: ObservedValue[str]
+    architecture: str
+    kernel_version: ObservedValue[str]
+    cpu_model: ObservedValue[str]
+    logical_cpu_count: ObservedValue[int]
+    physical_core_count: ObservedValue[int]
+    total_ram_bytes: ObservedValue[int]
+    available_ram_bytes: ObservedValue[int]
+    agent_version: str
+    inventory_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    storage: list[StorageObservation] = Field(default_factory=list)
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+
+class AcceleratorInventory(HardwareModel):
+    device_index: int
+    device_uuid: str
+    pci_bus_id: ObservedValue[str]
+    name: str
+    vendor: str = "NVIDIA"
+    architecture: ObservedValue[str]
+    compute_capability_major: ObservedValue[int]
+    compute_capability_minor: ObservedValue[int]
+    total_vram_bytes: ObservedValue[int]
+    driver_version: ObservedValue[str]
+    cuda_driver_version: ObservedValue[str]
+    mig_mode_current: ObservedValue[bool]
+    inventory_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    source: InventorySource = InventorySource.NVIDIA_NVML
+
+
+class AcceleratorTelemetry(HardwareModel):
+    device_uuid: str
+    observed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    used_vram_bytes: ObservedValue[int]
+    free_vram_bytes: ObservedValue[int]
+    gpu_utilization_percent: ObservedValue[int]
+    memory_utilization_percent: ObservedValue[int]
+    temperature_c: ObservedValue[int]
+    power_draw_w: ObservedValue[float]
+    power_limit_w: ObservedValue[float]
+    graphics_clock_mhz: ObservedValue[int]
+    memory_clock_mhz: ObservedValue[int]
+    fan_speed_percent: ObservedValue[int]
+    performance_state: ObservedValue[str]
+
+
+class NvidiaCollection(HardwareModel):
+    availability: Availability
+    reason: str | None = None
+    inventory: list[AcceleratorInventory] = Field(default_factory=list)
+    telemetry: list[AcceleratorTelemetry] = Field(default_factory=list)
+    observed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+
+class HardwareSnapshot(HardwareModel):
+    host: HostInventory
+    nvidia: NvidiaCollection
+
+    @property
+    def fingerprint(self) -> str:
+        facts = {
+            "node_identity": self.host.identity_key,
+            "os": self.host.os_name,
+            "os_version": self.host.os_version.model_dump(mode="json"),
+            "architecture": self.host.architecture,
+            "kernel": self.host.kernel_version.model_dump(mode="json"),
+            "accelerators": [
+                {
+                    "uuid": item.device_uuid,
+                    "name": item.name,
+                    "pci": item.pci_bus_id.model_dump(mode="json"),
+                    "vram": item.total_vram_bytes.model_dump(mode="json"),
+                    "compute_major": item.compute_capability_major.model_dump(mode="json"),
+                    "compute_minor": item.compute_capability_minor.model_dump(mode="json"),
+                    "driver": item.driver_version.model_dump(mode="json"),
+                    "cuda_driver": item.cuda_driver_version.model_dump(mode="json"),
+                }
+                for item in sorted(self.nvidia.inventory, key=lambda value: value.device_uuid)
+            ],
+        }
+        canonical = json.dumps(facts, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+class HostCollector(Protocol):
+    def collect(self) -> HostInventory: ...
+
+
+class AcceleratorCollector(Protocol):
+    def collect(self) -> NvidiaCollection: ...
+
+
+class HardwareOverview(HardwareModel):
+    status: HardwareStatus
+    inventory_state: HardwareStatus
+    node_count: int
+    accelerator_count: int
+    last_inventory_at: datetime | None
+    reason: str | None = None
+
+
+class StorageState(HardwareModel):
+    id: str
+    purpose: str
+    path: str
+    total_bytes: ObservedValue[int]
+    used_bytes: ObservedValue[int]
+    free_bytes: ObservedValue[int]
+    observed_at: datetime
+
+
+class AcceleratorState(HardwareModel):
+    id: str
+    node_id: str
+    status: HardwareStatus
+    status_reason: str | None
+    device_index: int
+    device_uuid: str
+    pci_bus_id: ObservedValue[str]
+    name: str
+    vendor: str
+    architecture: ObservedValue[str]
+    compute_capability_major: ObservedValue[int]
+    compute_capability_minor: ObservedValue[int]
+    total_vram_bytes: ObservedValue[int]
+    driver_version: ObservedValue[str]
+    cuda_driver_version: ObservedValue[str]
+    mig_mode_current: ObservedValue[bool]
+    first_seen_at: datetime
+    last_seen_at: datetime | None
+    inventory_at: datetime | None
+    telemetry: AcceleratorTelemetry | None
+
+
+class NodeState(HardwareModel):
+    id: str
+    identity_key: str
+    identity_source: str
+    hostname: str
+    display_name: str
+    status: HardwareStatus
+    status_reason: str | None
+    os_name: str | None
+    os_version: ObservedValue[str]
+    architecture: str | None
+    kernel_version: ObservedValue[str]
+    cpu_model: ObservedValue[str]
+    logical_cpu_count: ObservedValue[int]
+    physical_core_count: ObservedValue[int]
+    total_ram_bytes: ObservedValue[int]
+    available_ram_bytes: ObservedValue[int]
+    agent_version: str | None
+    first_seen_at: datetime
+    last_seen_at: datetime | None
+    inventory_at: datetime | None
+    hardware_fingerprint: str | None
+    enabled: bool = True
+    liveness: NodeLiveness = NodeLiveness.OFFLINE
+    agent_health: AgentHealth = AgentHealth.UNKNOWN
+    observation_source: ObservationSource = ObservationSource.LOCAL_CONTROL_PLANE
+    protocol_version: int | None = None
+    supported_capabilities: list[str] = Field(default_factory=list)
+    agent_started_at: datetime | None = None
+    last_heartbeat_at: datetime | None = None
+    last_inventory_received_at: datetime | None = None
+    last_telemetry_received_at: datetime | None = None
+    inventory_age_seconds: int | None = None
+    telemetry_age_seconds: int | None = None
+    last_connection_error: str | None = None
+    role: str | None = None
+    labels: dict[str, str | bool] = Field(default_factory=dict)
+    production_eligible: bool = False
+    lab_eligible: bool = True
+    benchmark_eligible: bool = False
+    generation: int = 1
+    decommissioned_at: datetime | None = None
+    decommission_reason: str | None = None
+    decommissioned_by: str | None = None
+    environment: str | None = None
+    storage: list[StorageState]
+    accelerators: list[AcceleratorState]
+
+
+class HardwareState(HardwareModel):
+    overview: HardwareOverview
+    nodes: list[NodeState]

backend/src/modelforge_api/domain/lifecycle.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from .enums import MigrationStatus, ModelLifecycle, UpgradeClass, VerificationStatus
+
+
+class InvalidTransition(ValueError):
+    pass
+
+
+MODEL_TRANSITIONS: dict[ModelLifecycle, frozenset[ModelLifecycle]] = {
+    ModelLifecycle.DISCOVERED: frozenset(
+        {ModelLifecycle.CANDIDATE, ModelLifecycle.REJECTED, ModelLifecycle.DEPRECATED}
+    ),
+    ModelLifecycle.CANDIDATE: frozenset(
+        {ModelLifecycle.DOWNLOADING, ModelLifecycle.REJECTED, ModelLifecycle.DEPRECATED}
+    ),
+    ModelLifecycle.DOWNLOADING: frozenset({ModelLifecycle.QUARANTINED, ModelLifecycle.REJECTED}),
+    ModelLifecycle.QUARANTINED: frozenset(
+        {ModelLifecycle.VERIFIED, ModelLifecycle.SECURITY_BLOCKED, ModelLifecycle.LICENSE_BLOCKED}
+    ),
+    ModelLifecycle.VERIFIED: frozenset({ModelLifecycle.TESTING, ModelLifecycle.INCOMPATIBLE}),
+    ModelLifecycle.TESTING: frozenset(
+        {ModelLifecycle.APPROVED, ModelLifecycle.INCOMPATIBLE, ModelLifecycle.REJECTED}
+    ),
+    ModelLifecycle.APPROVED: frozenset(
+        {ModelLifecycle.STANDBY, ModelLifecycle.ACTIVE, ModelLifecycle.DEPRECATED}
+    ),
+    ModelLifecycle.STANDBY: frozenset({ModelLifecycle.ACTIVE, ModelLifecycle.DEPRECATED}),
+    ModelLifecycle.ACTIVE: frozenset({ModelLifecycle.STANDBY, ModelLifecycle.DEPRECATED}),
+    ModelLifecycle.DEPRECATED: frozenset({ModelLifecycle.ARCHIVED, ModelLifecycle.ACTIVE}),
+    ModelLifecycle.ARCHIVED: frozenset(),
+    ModelLifecycle.REJECTED: frozenset(),
+    ModelLifecycle.INCOMPATIBLE: frozenset({ModelLifecycle.CANDIDATE}),
+    ModelLifecycle.SECURITY_BLOCKED: frozenset(
+        {ModelLifecycle.QUARANTINED, ModelLifecycle.REJECTED}
+    ),
+    ModelLifecycle.LICENSE_BLOCKED: frozenset(
+        {ModelLifecycle.QUARANTINED, ModelLifecycle.REJECTED}
+    ),
+}
+
+
+def assert_model_transition(current: ModelLifecycle, target: ModelLifecycle) -> None:
+    if target not in MODEL_TRANSITIONS[current]:
+        raise InvalidTransition(f"model lifecycle transition {current} -> {target} is not allowed")
+
+
+def assert_promotion_allowed(
+    *,
+    upgrade_class: UpgradeClass,
+    verification_status: VerificationStatus,
+    local_benchmark_ids: list[str],
+    project_benchmark_ids: list[str],
+    operator_approval_id: str | None,
+    rollback_deployment_id: str | None,
+    migration_status: MigrationStatus | None = None,
+) -> None:
+    blockers: list[str] = []
+    if verification_status is not VerificationStatus.VERIFIED:
+        blockers.append("artifact is not verified")
+    if not local_benchmark_ids:
+        blockers.append("local benchmark evidence is missing")
+    if not project_benchmark_ids:
+        blockers.append("project benchmark evidence is missing")
+    if not operator_approval_id:
+        blockers.append("operator approval is missing")
+    if not rollback_deployment_id:
+        blockers.append("rollback target is missing")
+    if (
+        upgrade_class is UpgradeClass.REQUIRES_REINDEX
+        and migration_status is not MigrationStatus.READY
+    ):
+        blockers.append("ready migration is required for reindexing change")
+    if blockers:
+        raise InvalidTransition("promotion blocked: " + "; ".join(blockers))

backend/src/modelforge_api/domain/lifecycle_contracts.py

@@ -0,0 +1,412 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class LifecycleModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class LifecycleEnvironment(StrEnum):
+    LAB = "LAB"
+    PRODUCTION = "PRODUCTION"
+
+
+class DeploymentLifecycleState(StrEnum):
+    CANDIDATE = "CANDIDATE"
+    LAB_READY = "LAB_READY"
+    PROMOTION_ELIGIBLE = "PROMOTION_ELIGIBLE"
+    CANARY = "CANARY"
+    LAB_STABLE = "LAB_STABLE"
+    STABLE = "STABLE"
+    DRAINING = "DRAINING"
+    DEPRECATED = "DEPRECATED"
+    ARCHIVED = "ARCHIVED"
+    MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
+
+
+class ApprovalStatus(StrEnum):
+    PENDING = "PENDING"
+    APPROVED = "APPROVED"
+    REJECTED = "REJECTED"
+    BLOCKED = "BLOCKED"
+    EXPIRED = "EXPIRED"
+    STALE = "STALE"
+    REVOKED = "REVOKED"
+
+
+class PromotionPlanStatus(StrEnum):
+    DRAFT = "DRAFT"
+    APPROVED = "APPROVED"
+    EXECUTING = "EXECUTING"
+    COMMITTED = "COMMITTED"
+    ROLLED_BACK = "ROLLED_BACK"
+    FAILED = "FAILED"
+
+
+class OperationStage(StrEnum):
+    PLANNED = "PLANNED"
+    APPROVED = "APPROVED"
+    PREPARING = "PREPARING"
+    ACTIVATING = "ACTIVATING"
+    VERIFYING = "VERIFYING"
+    COMMITTED = "COMMITTED"
+    ROLLING_BACK = "ROLLING_BACK"
+    ROLLED_BACK = "ROLLED_BACK"
+    FAILED = "FAILED"
+
+
+class CanaryStatus(StrEnum):
+    PLANNED = "PLANNED"
+    RUNNING = "RUNNING"
+    READY_FOR_PROMOTION_REVIEW = "READY_FOR_PROMOTION_REVIEW"
+    ABORTED = "ABORTED"
+
+
+class RetentionState(StrEnum):
+    ACTIVE = "ACTIVE"
+    ROLLBACK_RETAINED = "ROLLBACK_RETAINED"
+    DEPRECATED = "DEPRECATED"
+    ARCHIVABLE = "ARCHIVABLE"
+    ARCHIVED = "ARCHIVED"
+
+
+class CleanupStatus(StrEnum):
+    READY = "READY"
+    BLOCKED = "BLOCKED"
+    STALE = "STALE"
+    EXECUTED = "EXECUTED"
+
+
+ALLOWED_DEPLOYMENT_TRANSITIONS: dict[
+    DeploymentLifecycleState, frozenset[DeploymentLifecycleState]
+] = {
+    DeploymentLifecycleState.CANDIDATE: frozenset(
+        {DeploymentLifecycleState.LAB_READY, DeploymentLifecycleState.DEPRECATED}
+    ),
+    DeploymentLifecycleState.LAB_READY: frozenset(
+        {
+            DeploymentLifecycleState.PROMOTION_ELIGIBLE,
+            DeploymentLifecycleState.CANARY,
+            DeploymentLifecycleState.DEPRECATED,
+        }
+    ),
+    DeploymentLifecycleState.PROMOTION_ELIGIBLE: frozenset(
+        {DeploymentLifecycleState.CANARY, DeploymentLifecycleState.DEPRECATED}
+    ),
+    DeploymentLifecycleState.CANARY: frozenset(
+        {
+            DeploymentLifecycleState.LAB_READY,
+            DeploymentLifecycleState.LAB_STABLE,
+            DeploymentLifecycleState.STABLE,
+        }
+    ),
+    DeploymentLifecycleState.LAB_STABLE: frozenset(
+        {
+            DeploymentLifecycleState.CANARY,
+            DeploymentLifecycleState.LAB_READY,
+            DeploymentLifecycleState.DEPRECATED,
+        }
+    ),
+    DeploymentLifecycleState.STABLE: frozenset(
+        {DeploymentLifecycleState.CANARY, DeploymentLifecycleState.DRAINING}
+    ),
+    DeploymentLifecycleState.DRAINING: frozenset(
+        {DeploymentLifecycleState.STABLE, DeploymentLifecycleState.DEPRECATED}
+    ),
+    DeploymentLifecycleState.DEPRECATED: frozenset({DeploymentLifecycleState.ARCHIVED}),
+    DeploymentLifecycleState.ARCHIVED: frozenset(),
+    DeploymentLifecycleState.MANUAL_INTERVENTION_REQUIRED: frozenset(),
+}
+
+
+def assert_deployment_transition(
+    current: DeploymentLifecycleState,
+    target: DeploymentLifecycleState,
+) -> None:
+    if target not in ALLOWED_DEPLOYMENT_TRANSITIONS[current]:
+        raise ValueError(f"lifecycle transition {current} -> {target} is not allowed")
+
+
+class LifecycleEvidenceBundle(LifecycleModel):
+    artifact_set_id: uuid.UUID | None = None
+    model_revision_id: uuid.UUID | None = None
+    runtime_profile_id: uuid.UUID | None = None
+    runtime_probe_id: uuid.UUID | None = None
+    production_execution_approval_id: uuid.UUID | None = None
+    runtime_image_digest: str | None = Field(default=None, max_length=128)
+    artifact_digests: list[str] = Field(default_factory=list, max_length=1000)
+    capability_deployment_id: uuid.UUID | None = None
+    project_binding_id: uuid.UUID | None = None
+    project_fit_evidence_id: uuid.UUID | None = None
+    evaluation_run_ids: list[uuid.UUID] = Field(default_factory=list, max_length=1000)
+    resource_envelope_id: uuid.UUID | None = None
+    embedding_space_id: uuid.UUID | None = None
+    retrieval_pipeline_id: uuid.UUID | None = None
+    migration_id: uuid.UUID | None = None
+    rollback_target_ref: str | None = Field(default=None, max_length=255)
+    integrity: Literal["UNKNOWN", "VERIFIED", "CORRUPT"] = "UNKNOWN"
+    security: Literal["UNREVIEWED", "APPROVED", "BLOCKED"] = "UNREVIEWED"
+    license: Literal["UNKNOWN", "APPROVED", "BLOCKED"] = "UNKNOWN"
+    runtime: Literal["UNPROBED", "PROVEN", "INCOMPATIBLE"] = "UNPROBED"
+    evaluation: Literal["NOT_EVALUATED", "EVALUATED", "REGRESSED", "PROMOTION_ELIGIBLE"] = (
+        "NOT_EVALUATED"
+    )
+    project_fit: Literal[
+        "UNKNOWN",
+        "REQUIRES_MORE_EVIDENCE",
+        "ELIGIBLE",
+        "BLOCKED",
+        "DEFERRED_EXTERNAL_VALIDATION",
+        "KEEP_LAB",
+    ] = "UNKNOWN"
+    engineering_integration: Literal["UNKNOWN", "PASS", "INCOMPLETE", "BLOCKED"] = "UNKNOWN"
+    platform_readiness: Literal["UNKNOWN", "LAB_READY", "PROMOTION_ELIGIBLE", "STABLE"] = (
+        "UNKNOWN"
+    )
+    production_validation: Literal[
+        "NOT_REQUIRED", "REQUIRED", "DEFERRED_EXTERNAL_VALIDATION", "SATISFIED"
+    ] = "REQUIRED"
+    scheduler_readiness: Literal["UNKNOWN", "READY", "BLOCKED"] = "UNKNOWN"
+    critical_regressions: int = Field(default=0, ge=0)
+    evidence_revisions: dict[str, str] = Field(default_factory=dict)
+
+
+class ApprovalPolicyCreate(LifecycleModel):
+    key: str = Field(pattern=r"^[a-z][a-z0-9-]+$", max_length=128)
+    scope: Literal["LAB_PROMOTION", "CAPABILITY_PRODUCTION", "PROJECT_PRODUCTION"]
+    requirements: dict[str, Any]
+    created_by: str = Field(min_length=1, max_length=255)
+
+
+class ApprovalPolicyResponse(LifecycleModel):
+    id: uuid.UUID
+    key: str
+    revision: int
+    scope: str
+    requirements: dict[str, Any]
+    fingerprint: str
+    active: bool
+    created_by: str
+    created_at: datetime
+
+
+class LifecycleSubjectCreate(LifecycleModel):
+    target_type: Literal[
+        "CAPABILITY_DEPLOYMENT", "PROJECT_BINDING", "ARTIFACT_SET", "LAB_REHEARSAL"
+    ]
+    target_ref: str = Field(min_length=1, max_length=255)
+    environment: LifecycleEnvironment
+    state: DeploymentLifecycleState
+
+
+class LifecycleSubjectResponse(LifecycleModel):
+    id: uuid.UUID
+    target_type: str
+    target_ref: str
+    environment: str
+    state: str
+    version: int
+    superseded_by_ref: str | None
+    created_at: datetime
+    updated_at: datetime
+
+
+class ApprovalRequestCreate(LifecycleModel):
+    policy_key: str = Field(pattern=r"^[a-z][a-z0-9-]+$", max_length=128)
+    subject_id: uuid.UUID | None = None
+    target_type: Literal[
+        "CAPABILITY_DEPLOYMENT", "PROJECT_BINDING", "CAPABILITY", "ARTIFACT_SET", "LAB_REHEARSAL"
+    ]
+    target_ref: str = Field(min_length=1, max_length=255)
+    environment: LifecycleEnvironment
+    requested_transition: DeploymentLifecycleState
+    evidence: LifecycleEvidenceBundle
+    requested_by: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=8, max_length=4000)
+    expires_at: datetime | None = None
+
+
+class ApprovalDecision(LifecycleModel):
+    actor: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=8, max_length=4000)
+
+
+class ApprovalRequestResponse(LifecycleModel):
+    id: uuid.UUID
+    policy_revision_id: uuid.UUID
+    policy_key: str
+    policy_revision: int
+    version: int
+    subject_id: uuid.UUID | None
+    target_type: str
+    target_ref: str
+    environment: str
+    requested_transition: str
+    evidence_snapshot: dict[str, Any]
+    evidence_fingerprint: str
+    status: str
+    blockers: list[str]
+    warnings: list[str]
+    requested_by: str
+    approved_by: str | None
+    reason: str
+    expires_at: datetime | None
+    stale_at: datetime | None
+    decided_at: datetime | None
+    created_at: datetime
+
+
+class PromotionPlanCreate(LifecycleModel):
+    approval_request_id: uuid.UUID
+    subject_id: uuid.UUID
+    desired_state: DeploymentLifecycleState
+    migration_class: Literal["transparent", "behavioral", "requires_reindex", "schema_breaking"]
+    candidate_deployment_id: uuid.UUID | None = None
+    rollback_target_ref: str = Field(min_length=1, max_length=255)
+    project_consumers: list[str] = Field(default_factory=list, max_length=1000)
+    affected_identities: dict[str, Any] = Field(default_factory=dict)
+    migration_id: uuid.UUID | None = None
+    canary_strategy: dict[str, Any] = Field(default_factory=dict)
+    drain_strategy: dict[str, Any] = Field(default_factory=dict)
+    health_gates: dict[str, Any] = Field(default_factory=dict)
+    automatic_abort_conditions: list[str] = Field(default_factory=list)
+    created_by: str = Field(min_length=1, max_length=255)
+
+
+class PromotionPlanResponse(LifecycleModel):
+    id: uuid.UUID
+    approval_request_id: uuid.UUID
+    subject_id: uuid.UUID
+    current_state: str
+    desired_state: str
+    migration_class: str
+    candidate_deployment_id: uuid.UUID | None
+    rollback_target_ref: str
+    project_consumers: list[str]
+    affected_identities: dict[str, Any]
+    impact_analysis: dict[str, Any]
+    migration_id: uuid.UUID | None
+    canary_strategy: dict[str, Any]
+    drain_strategy: dict[str, Any]
+    health_gates: dict[str, Any]
+    automatic_abort_conditions: list[str]
+    plan_fingerprint: str
+    status: str
+    version: int
+    created_by: str
+    approved_by: str | None
+    immutable_at: datetime | None
+    created_at: datetime
+
+
+class PlanExecutionCreate(LifecycleModel):
+    executor: str = Field(min_length=1, max_length=255)
+    idempotency_key: str = Field(min_length=8, max_length=128)
+    expected_subject_version: int = Field(ge=1)
+    rehearsal_pause_stage: Literal["PREPARING"] | None = None
+
+
+class CanaryObservation(LifecycleModel):
+    request_count: int = Field(ge=0)
+    error_count: int = Field(ge=0)
+    latency_p95_ms: float = Field(ge=0)
+    capability_health: Literal["HEALTHY", "UNHEALTHY"]
+    scheduler_ready: bool
+    critical_project_failures: int = Field(default=0, ge=0)
+    external_pressure: bool = False
+    worker_healthy: bool = True
+
+    @model_validator(mode="after")
+    def validate_counts(self) -> CanaryObservation:
+        if self.error_count > self.request_count:
+            raise ValueError("error_count cannot exceed request_count")
+        return self
+
+
+class LifecycleOperationResponse(LifecycleModel):
+    id: uuid.UUID
+    promotion_plan_id: uuid.UUID
+    version: int
+    stage: str
+    idempotency_key: str
+    expected_subject_version: int
+    requester: str
+    approver: str
+    executor: str
+    failure_code: str | None
+    failure_details: dict[str, Any]
+    rollback_duration_ms: float | None
+    started_at: datetime
+    finished_at: datetime | None
+    canary: dict[str, Any] | None = None
+
+
+class RetentionPolicyCreate(LifecycleModel):
+    key: str = Field(pattern=r"^[a-z][a-z0-9-]+$", max_length=128)
+    minimum_rollback_days: int = Field(ge=1, le=3650)
+    requirements: dict[str, Any] = Field(default_factory=dict)
+    created_by: str = Field(min_length=1, max_length=255)
+
+
+class RetentionPolicyResponse(LifecycleModel):
+    id: uuid.UUID
+    key: str
+    revision: int
+    minimum_rollback_days: int
+    requirements: dict[str, Any]
+    fingerprint: str
+    active: bool
+    created_by: str
+    created_at: datetime
+
+
+class CleanupPlanCreate(LifecycleModel):
+    target_type: Literal["ARTIFACT_SET", "ARTIFACT_LOCATION"]
+    target_ref: uuid.UUID
+    action: Literal["ARCHIVE_METADATA", "RECORD_LOCATION_REMOVAL"]
+    created_by: str = Field(min_length=1, max_length=255)
+
+
+class CleanupPlanResponse(LifecycleModel):
+    id: uuid.UUID
+    target_type: str
+    target_ref: str
+    action: str
+    dependencies: list[dict[str, Any]]
+    dependency_digest: str
+    reclaimable_bytes: int
+    retention_state: str
+    blockers: list[str]
+    status: str
+    created_by: str
+    created_at: datetime
+    executed_at: datetime | None
+
+
+class CleanupExecutionCreate(LifecycleModel):
+    executor: str = Field(min_length=1, max_length=255)
+    confirm: Literal[True]
+    physical_removal_confirmed: bool = False
+
+
+class LifecycleEventResponse(LifecycleModel):
+    id: uuid.UUID
+    event_type: str
+    object_type: str
+    object_ref: str
+    from_state: str | None
+    to_state: str | None
+    actor: str
+    actor_role: str
+    policy_revision_id: uuid.UUID | None
+    evidence_ids: list[str]
+    reason: str
+    change_id: str
+    details: dict[str, Any]
+    occurred_at: datetime

backend/src/modelforge_api/domain/migration_contracts.py

@@ -0,0 +1,543 @@
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+class MigrationModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class MigrationClass(StrEnum):
+    TRANSPARENT = "TRANSPARENT"
+    BEHAVIORAL = "BEHAVIORAL"
+    REQUIRES_REINDEX = "REQUIRES_REINDEX"
+    SCHEMA_BREAKING = "SCHEMA_BREAKING"
+
+
+class MigrationState(StrEnum):
+    PLANNED = "PLANNED"
+    PREFLIGHT = "PREFLIGHT"
+    READY = "READY"
+    BACKFILLING = "BACKFILLING"
+    BACKFILL_PAUSED = "BACKFILL_PAUSED"
+    BACKFILL_COMPLETE = "BACKFILL_COMPLETE"
+    VALIDATING = "VALIDATING"
+    VALIDATION_FAILED = "VALIDATION_FAILED"
+    READY_FOR_SHADOW = "READY_FOR_SHADOW"
+    SHADOWING = "SHADOWING"
+    READY_FOR_CUTOVER = "READY_FOR_CUTOVER"
+    CUTOVER_PREPARING = "CUTOVER_PREPARING"
+    CUTTING_OVER = "CUTTING_OVER"
+    VERIFYING_CUTOVER = "VERIFYING_CUTOVER"
+    CUTOVER_COMMITTED = "CUTOVER_COMMITTED"
+    ROLLING_BACK = "ROLLING_BACK"
+    ROLLED_BACK = "ROLLED_BACK"
+    FAILED = "FAILED"
+    CANCELLED = "CANCELLED"
+    MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
+
+
+class CutoverStage(StrEnum):
+    PREPARING = "PREPARING"
+    LOCKING = "LOCKING"
+    SWITCHING = "SWITCHING"
+    VERIFYING = "VERIFYING"
+    COMMITTING = "COMMITTING"
+    COMMITTED = "COMMITTED"
+    ROLLING_BACK = "ROLLING_BACK"
+    ROLLED_BACK = "ROLLED_BACK"
+    FAILED = "FAILED"
+
+
+class BatchState(StrEnum):
+    PLANNED = "PLANNED"
+    RUNNING = "RUNNING"
+    COMPLETED = "COMPLETED"
+    RETRYABLE_FAILED = "RETRYABLE_FAILED"
+    PERMANENTLY_FAILED = "PERMANENTLY_FAILED"
+
+
+class MigrationFailureCode(StrEnum):
+    SOURCE_CHANGED = "SOURCE_CHANGED"
+    STALE_SOURCE = "STALE_SOURCE"
+    TARGET_CONFLICT = "TARGET_CONFLICT"
+    CAPABILITY_UNAVAILABLE = "CAPABILITY_UNAVAILABLE"
+    BATCH_FAILED = "BATCH_FAILED"
+    VECTOR_INVALID = "VECTOR_INVALID"
+    WRITE_FAILED = "WRITE_FAILED"
+    CHECKPOINT_FAILED = "CHECKPOINT_FAILED"
+    CONTENT_HASH_MISMATCH = "CONTENT_HASH_MISMATCH"
+    TARGET_SCHEMA_MISMATCH = "TARGET_SCHEMA_MISMATCH"
+    VALIDATION_FAILED = "VALIDATION_FAILED"
+    HEALTH_CHECK_FAILED = "HEALTH_CHECK_FAILED"
+    ROLLBACK_FAILED = "ROLLBACK_FAILED"
+    STALE_APPROVAL = "STALE_APPROVAL"
+
+
+ALLOWED_TRANSITIONS: dict[MigrationState, frozenset[MigrationState]] = {
+    MigrationState.PLANNED: frozenset(
+        {MigrationState.PREFLIGHT, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.PREFLIGHT: frozenset(
+        {MigrationState.READY, MigrationState.FAILED, MigrationState.CANCELLED}
+    ),
+    MigrationState.READY: frozenset(
+        {MigrationState.BACKFILLING, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.BACKFILLING: frozenset(
+        {
+            MigrationState.BACKFILL_PAUSED,
+            MigrationState.BACKFILL_COMPLETE,
+            MigrationState.CANCELLED,
+            MigrationState.FAILED,
+        }
+    ),
+    MigrationState.BACKFILL_PAUSED: frozenset(
+        {MigrationState.BACKFILLING, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.BACKFILL_COMPLETE: frozenset(
+        {MigrationState.VALIDATING, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.VALIDATING: frozenset(
+        {
+            MigrationState.VALIDATION_FAILED,
+            MigrationState.READY_FOR_SHADOW,
+            MigrationState.FAILED,
+            MigrationState.CANCELLED,
+        }
+    ),
+    MigrationState.VALIDATION_FAILED: frozenset(
+        {MigrationState.VALIDATING, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.READY_FOR_SHADOW: frozenset(
+        {MigrationState.SHADOWING, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.SHADOWING: frozenset(
+        {MigrationState.READY_FOR_CUTOVER, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.READY_FOR_CUTOVER: frozenset(
+        {MigrationState.CUTOVER_PREPARING, MigrationState.CANCELLED, MigrationState.FAILED}
+    ),
+    MigrationState.CUTOVER_PREPARING: frozenset(
+        {MigrationState.CUTTING_OVER, MigrationState.FAILED, MigrationState.CANCELLED}
+    ),
+    MigrationState.CUTTING_OVER: frozenset(
+        {
+            MigrationState.VERIFYING_CUTOVER,
+            MigrationState.ROLLING_BACK,
+            MigrationState.MANUAL_INTERVENTION_REQUIRED,
+        }
+    ),
+    MigrationState.VERIFYING_CUTOVER: frozenset(
+        {
+            MigrationState.CUTOVER_COMMITTED,
+            MigrationState.ROLLING_BACK,
+            MigrationState.MANUAL_INTERVENTION_REQUIRED,
+        }
+    ),
+    MigrationState.CUTOVER_COMMITTED: frozenset({MigrationState.ROLLING_BACK}),
+    MigrationState.ROLLING_BACK: frozenset(
+        {MigrationState.ROLLED_BACK, MigrationState.MANUAL_INTERVENTION_REQUIRED}
+    ),
+    MigrationState.ROLLED_BACK: frozenset(),
+    MigrationState.FAILED: frozenset(),
+    MigrationState.CANCELLED: frozenset(),
+    MigrationState.MANUAL_INTERVENTION_REQUIRED: frozenset(),
+}
+
+
+def assert_migration_transition(before: str, after: str) -> None:
+    current = MigrationState(before)
+    target = MigrationState(after)
+    if target not in ALLOWED_TRANSITIONS[current]:
+        raise ValueError(f"invalid migration transition {current.value} -> {target.value}")
+
+
+class AdapterContract(MigrationModel):
+    key: str = Field(pattern=r"^[a-z0-9][a-z0-9._-]{1,127}$")
+    version: str = Field(min_length=1, max_length=64)
+    operations: frozenset[
+        Literal[
+            "preflight",
+            "create_shadow_target",
+            "enumerate_source_items",
+            "transform_batch",
+            "write_batch",
+            "validate_batch",
+            "finalize_backfill",
+            "validate_target",
+            "shadow_compare",
+            "cutover",
+            "rollback",
+            "inspect_external_state",
+        ]
+    ]
+    fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    schema_operations: frozenset[str] = Field(default_factory=frozenset, max_length=32)
+
+    @field_validator("schema_operations")
+    @classmethod
+    def validate_schema_operations(cls, value: frozenset[str]) -> frozenset[str]:
+        if any(not re.fullmatch(r"[a-z0-9][a-z0-9._-]{1,127}", item) for item in value):
+            raise ValueError("schema operations must be typed adapter keys")
+        return value
+
+
+class SchemaMigrationStep(MigrationModel):
+    operation: str = Field(min_length=1, max_length=64)
+    adapter_step: str = Field(default="manual-boundary", min_length=1, max_length=128)
+    preconditions: list[str] = Field(default_factory=list, max_length=32)
+    required_application_versions: dict[str, str] = Field(default_factory=dict)
+    compatibility_window: str = Field(default="unspecified", min_length=1, max_length=255)
+    rollback_feasible: bool = False
+    irreversible: bool = False
+
+
+class MigrationPlanCreate(MigrationModel):
+    project_id: uuid.UUID
+    project_binding_id: uuid.UUID
+    capability_contract_id: uuid.UUID
+    migration_class: MigrationClass
+    environment: Literal["LAB", "PRODUCTION"]
+    adapter: AdapterContract
+    source_identity: dict[str, Any]
+    target_identity: dict[str, Any]
+    source_data_target: str = Field(min_length=1, max_length=512)
+    target_shadow_target: str = Field(min_length=1, max_length=512)
+    source_space_ref: str = Field(min_length=1, max_length=255)
+    target_space_id: uuid.UUID
+    corpus_revision: str = Field(min_length=1, max_length=128)
+    migration_policy_revision: str = Field(min_length=1, max_length=128)
+    validation_policy_revision_id: uuid.UUID
+    lifecycle_approval_id: uuid.UUID
+    promotion_plan_id: uuid.UUID | None = None
+    rollback_target_ref: str = Field(min_length=1, max_length=512)
+    total_expected_items: int = Field(ge=1, le=10_000_000)
+    batch_size: int = Field(default=32, ge=1, le=256)
+    max_in_flight_batches: int = Field(default=1, ge=1, le=8)
+    concurrency: int = Field(default=1, ge=1, le=8)
+    priority: Literal["BACKGROUND"] = "BACKGROUND"
+    target_storage: dict[str, Any]
+    shadow_policy: dict[str, Any]
+    cutover_policy: dict[str, Any]
+    rollback_retention_days: int = Field(default=30, ge=30, le=3650)
+    environment_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    idempotency_key: str = Field(min_length=8, max_length=128)
+    created_by: str = Field(min_length=1, max_length=255)
+    irreversible: bool = False
+    schema_steps: list[SchemaMigrationStep] = Field(default_factory=list, max_length=32)
+
+    @model_validator(mode="after")
+    def identities_are_isolated(self) -> MigrationPlanCreate:
+        if self.migration_class is MigrationClass.REQUIRES_REINDEX:
+            if self.source_data_target == self.target_shadow_target:
+                raise ValueError("reindex target must be separate from source")
+            if self.source_space_ref == str(self.target_space_id):
+                raise ValueError("reindex requires a distinct target embedding space")
+        if self.migration_class is MigrationClass.SCHEMA_BREAKING and not self.schema_steps:
+            raise ValueError("schema-breaking plans require typed adapter steps")
+        return self
+
+
+class MigrationPlanResponse(MigrationPlanCreate):
+    id: uuid.UUID
+    state: MigrationState
+    version: int
+    generation: int
+    plan_fingerprint: str
+    approval_fingerprint: str
+    completed_items: int
+    failed_items: int
+    retryable_items: int
+    permanent_failed_items: int
+    last_cursor: str | None
+    cancel_requested: bool
+    failure_code: str | None
+    failure_details: dict[str, Any]
+    started_at: datetime | None
+    completed_at: datetime | None
+    immutable_at: datetime
+    created_at: datetime
+    updated_at: datetime
+
+
+class MigrationValidationPolicyCreate(MigrationModel):
+    key: str = Field(min_length=1, max_length=128)
+    revision: int = Field(ge=1)
+    required_completeness: float = Field(default=1.0, ge=0, le=1)
+    allowed_failures: int = Field(default=0, ge=0)
+    required_evaluation: bool = True
+    critical_regressions_allowed: int = Field(default=0, ge=0)
+    maximum_latency_regression_ratio: float | None = Field(default=None, ge=0)
+    require_project_fit: bool = True
+    require_external_validation: bool = True
+    require_security_approved: bool = True
+    allow_isolated_lab_cutover: bool = False
+    created_by: str = Field(min_length=1, max_length=255)
+
+
+class MigrationValidationPolicyResponse(MigrationValidationPolicyCreate):
+    id: uuid.UUID
+    fingerprint: str
+    active: bool
+    created_at: datetime
+    immutable_at: datetime
+
+
+class PreflightReport(MigrationModel):
+    expected_version: int = Field(ge=1)
+    adapter_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    source_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    source_exists: bool
+    source_healthy: bool
+    source_count: int = Field(ge=0)
+    target_conflict_free: bool
+    target_space_valid: bool
+    capability_healthy: bool
+    project_credential_valid: bool
+    storage_sufficient: bool
+    scheduler_capacity: bool
+    adapter_available: bool
+    rollback_source_retained: bool
+    evaluation_suite_available: bool
+    lifecycle_approval_current: bool
+    evidence: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def passed(self) -> bool:
+        return all(
+            (
+                self.source_exists,
+                self.source_healthy,
+                self.target_conflict_free,
+                self.target_space_valid,
+                self.capability_healthy,
+                self.project_credential_valid,
+                self.storage_sufficient,
+                self.scheduler_capacity,
+                self.adapter_available,
+                self.rollback_source_retained,
+                self.evaluation_suite_available,
+                self.lifecycle_approval_current,
+            )
+        )
+
+
+class BatchReport(MigrationModel):
+    expected_version: int = Field(ge=1)
+    generation: int = Field(ge=1)
+    batch_number: int = Field(ge=0)
+    cursor_start: str = Field(min_length=1, max_length=255)
+    cursor_end: str = Field(min_length=1, max_length=255)
+    item_count: int = Field(ge=1, le=256)
+    completed_items: int = Field(ge=0, le=256)
+    failed_items: int = Field(ge=0, le=256)
+    retryable_items: int = Field(ge=0, le=256)
+    permanent_failed_items: int = Field(ge=0, le=256)
+    item_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    result_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    output_shape_valid: bool
+    finite: bool
+    target_space_matches: bool
+    destination_committed: bool
+    content_hashes_match: bool
+    duration_ms: float = Field(ge=0)
+    retries: int = Field(default=0, ge=0, le=20)
+    error_code: MigrationFailureCode | None = None
+    bounded_errors: list[dict[str, Any]] = Field(default_factory=list, max_length=20)
+
+    @model_validator(mode="after")
+    def totals_are_consistent(self) -> BatchReport:
+        if self.completed_items + self.failed_items != self.item_count:
+            raise ValueError("batch outcome count must equal item count")
+        if self.retryable_items + self.permanent_failed_items != self.failed_items:
+            raise ValueError("failed item taxonomy is incomplete")
+        return self
+
+
+class StateAction(MigrationModel):
+    expected_version: int = Field(ge=1)
+    actor: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=1, max_length=2000)
+
+
+class ValidationReport(MigrationModel):
+    expected_version: int = Field(ge=1)
+    generation: int = Field(ge=1)
+    expected_count: int = Field(ge=0)
+    actual_count: int = Field(ge=0)
+    missing_count: int = Field(ge=0)
+    duplicate_count: int = Field(ge=0)
+    malformed_count: int = Field(ge=0)
+    non_finite_count: int = Field(ge=0)
+    wrong_dimension_count: int = Field(ge=0)
+    content_hash_mismatch_count: int = Field(ge=0)
+    wrong_space_count: int = Field(ge=0)
+    index_schema_matches: bool
+    distance_metric_matches: bool
+    payload_integrity: bool
+    target_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    evaluation_run_ids: list[uuid.UUID] = Field(default_factory=list, max_length=32)
+    comparable: bool
+    critical_regressions: int = Field(ge=0)
+    latency_regression_ratio: float | None = Field(default=None, ge=0)
+    project_fit_eligible: bool
+    external_validation_satisfied: bool
+    security_approved: bool
+    evidence: dict[str, Any] = Field(default_factory=dict)
+
+
+class ValidationSnapshotResponse(ValidationReport):
+    id: uuid.UUID
+    migration_plan_id: uuid.UUID
+    validation_policy_revision_id: uuid.UUID
+    snapshot_fingerprint: str
+    passed: bool
+    technical_cutover_eligible: bool
+    project_promotion_eligible: bool
+    blockers: list[str]
+    created_at: datetime
+    immutable_at: datetime
+
+
+class ShadowReport(MigrationModel):
+    expected_version: int = Field(ge=1)
+    generation: int = Field(ge=1)
+    request_count: int = Field(ge=1)
+    source_error_count: int = Field(ge=0)
+    target_error_count: int = Field(ge=0)
+    source_latency_p95_ms: float = Field(ge=0)
+    target_latency_p95_ms: float = Field(ge=0)
+    critical_regressions: int = Field(ge=0)
+    metrics: dict[str, float]
+    evidence_refs: list[str] = Field(default_factory=list, max_length=64)
+    result_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+
+
+class CutoverPrepare(MigrationModel):
+    expected_version: int = Field(ge=1)
+    idempotency_key: str = Field(min_length=8, max_length=128)
+    actor: str = Field(min_length=1, max_length=255)
+    expected_external_source: str = Field(min_length=1, max_length=512)
+    observed_external_source: str = Field(min_length=1, max_length=512)
+    external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    configuration_version: str = Field(min_length=1, max_length=128)
+
+
+class CutoverReport(MigrationModel):
+    expected_version: int = Field(ge=1)
+    operation_id: uuid.UUID
+    generation: int = Field(ge=1)
+    external_source_before: str = Field(min_length=1, max_length=512)
+    external_target_after: str = Field(min_length=1, max_length=512)
+    external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    switch_duration_ms: float = Field(ge=0)
+    target_reachable: bool
+    expected_identity: bool
+    capability_healthy: bool
+    project_read_path_healthy: bool
+    error_rate: float = Field(ge=0, le=1)
+    smoke_query_count: int = Field(ge=0, le=100)
+    smoke_error_count: int = Field(ge=0, le=100)
+    evidence: dict[str, Any] = Field(default_factory=dict)
+
+
+class RollbackReport(MigrationModel):
+    expected_version: int = Field(ge=1)
+    operation_id: uuid.UUID
+    generation: int = Field(ge=1)
+    restored_external_target: str = Field(min_length=1, max_length=512)
+    external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    elapsed_ms: float = Field(ge=0)
+    source_reachable: bool
+    exact_identity_restored: bool
+    capability_healthy: bool
+    project_read_path_healthy: bool
+    evidence: dict[str, Any] = Field(default_factory=dict)
+
+
+class ReconciliationReport(MigrationModel):
+    operation_id: uuid.UUID
+    generation: int = Field(ge=1)
+    observed_external_target: str = Field(min_length=1, max_length=512)
+    external_state_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$")
+    target_healthy: bool
+    source_healthy: bool
+    switch_duration_ms: float | None = Field(default=None, ge=0)
+    smoke_query_count: int = Field(default=0, ge=0, le=100)
+    smoke_error_count: int = Field(default=0, ge=0, le=100)
+    evidence: dict[str, Any] = Field(default_factory=dict)
+    actor: Literal["migration-reconciler"] = "migration-reconciler"
+
+    @model_validator(mode="after")
+    def smoke_totals_are_consistent(self) -> ReconciliationReport:
+        if self.smoke_error_count > self.smoke_query_count:
+            raise ValueError("smoke errors cannot exceed smoke queries")
+        return self
+
+
+class MigrationBatchResponse(MigrationModel):
+    id: uuid.UUID
+    migration_plan_id: uuid.UUID
+    batch_number: int
+    generation: int
+    cursor_start: str
+    cursor_end: str
+    item_count: int
+    completed_items: int
+    failed_items: int
+    retryable_items: int
+    permanent_failed_items: int
+    item_fingerprint: str
+    result_fingerprint: str | None
+    status: BatchState
+    attempts: int
+    duration_ms: float | None
+    error_code: str | None
+    bounded_errors: list[dict[str, Any]]
+    started_at: datetime | None
+    completed_at: datetime | None
+
+
+class CutoverOperationResponse(MigrationModel):
+    id: uuid.UUID
+    migration_plan_id: uuid.UUID
+    stage: CutoverStage
+    idempotency_key: str
+    generation: int
+    expected_plan_version: int
+    source_before: str
+    target_after: str
+    external_state_fingerprint: str
+    health_evidence: dict[str, Any]
+    failure_code: str | None
+    failure_details: dict[str, Any]
+    switch_duration_ms: float | None
+    rollback_duration_ms: float | None
+    started_at: datetime
+    finished_at: datetime | None
+
+
+class MigrationEventResponse(MigrationModel):
+    id: uuid.UUID
+    migration_plan_id: uuid.UUID
+    operation_id: uuid.UUID | None
+    event_type: str
+    before_state: str | None
+    after_state: str | None
+    actor: str
+    policy_revision: str
+    evidence_refs: list[str]
+    reason: str
+    source_identity: dict[str, Any]
+    target_identity: dict[str, Any]
+    generation: int
+    change_id: str
+    details: dict[str, Any]
+    occurred_at: datetime

backend/src/modelforge_api/domain/node_decommission.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class NodeDecommissionModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class DecommissionBlocker(NodeDecommissionModel):
+    code: str
+    message: str
+    record_type: str
+    count: int = Field(ge=1)
+    resource_ids: list[str] = Field(default_factory=list)
+
+
+class DecommissionRecordCount(NodeDecommissionModel):
+    record_type: str
+    count: int = Field(ge=0)
+    action: str
+
+
+class NodeDecommissionPreview(NodeDecommissionModel):
+    node_id: uuid.UUID
+    persisted_identity: str
+    hostname: str
+    display_name: str
+    current_state: dict[str, object]
+    node_generation: int = Field(ge=1)
+    dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    safe: bool
+    blockers: list[DecommissionBlocker]
+    cleanup: list[DecommissionRecordCount]
+    preserved: list[DecommissionRecordCount]
+    dependent_records: list[DecommissionRecordCount]
+
+
+class NodeDecommissionExecute(NodeDecommissionModel):
+    expected_generation: int = Field(ge=1)
+    preview_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
+    idempotency_key: str = Field(min_length=8, max_length=128)
+    operator: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=10, max_length=2000)
+    confirmation: str = Field(min_length=1, max_length=255)
+
+
+class NodeDecommissionResult(NodeDecommissionModel):
+    operation_id: uuid.UUID
+    node_id: uuid.UUID
+    persisted_identity: str
+    status: str
+    decommissioned_at: datetime
+    cleanup_summary: dict[str, int]
+    previous_state: dict[str, object]
+    credential_revocations: int = Field(ge=0)
+    idempotent_replay: bool = False

backend/src/modelforge_api/domain/observability.py

@@ -0,0 +1,399 @@
+from __future__ import annotations
+
+import re
+import threading
+import time
+import uuid
+from collections import defaultdict
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+
+class TelemetryType(StrEnum):
+    COUNTER = "COUNTER"
+    GAUGE = "GAUGE"
+    HISTOGRAM = "HISTOGRAM"
+    EVENT = "EVENT"
+    STATE = "STATE"
+
+
+class SLOState(StrEnum):
+    HEALTHY = "HEALTHY"
+    AT_RISK = "AT_RISK"
+    BREACHED = "BREACHED"
+    INSUFFICIENT_DATA = "INSUFFICIENT_DATA"
+    STALE = "STALE"
+    DISABLED = "DISABLED"
+
+
+class AlertState(StrEnum):
+    PENDING = "PENDING"
+    FIRING = "FIRING"
+    ACKNOWLEDGED = "ACKNOWLEDGED"
+    RESOLVED = "RESOLVED"
+    SUPPRESSED = "SUPPRESSED"
+
+
+class AlertSeverity(StrEnum):
+    INFO = "INFO"
+    WARNING = "WARNING"
+    CRITICAL = "CRITICAL"
+
+
+class OperationalModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class MetricDefinition(OperationalModel):
+    name: str = Field(pattern=r"^modelforge_[a-z][a-z0-9_]*$")
+    type: TelemetryType
+    help: str = Field(min_length=3, max_length=500)
+    labels: tuple[str, ...] = ()
+
+    @field_validator("labels")
+    @classmethod
+    def bounded_labels(cls, value: tuple[str, ...]) -> tuple[str, ...]:
+        forbidden = {
+            "request_id",
+            "artifact_sha",
+            "query",
+            "project_uuid",
+            "filename",
+            "error_text",
+        }
+        if len(value) > 6 or len(set(value)) != len(value) or forbidden.intersection(value):
+            raise ValueError("metric labels must be unique, bounded and non-sensitive")
+        if any(not re.fullmatch(r"[a-z][a-z0-9_]*", item) for item in value):
+            raise ValueError("metric label names must use snake_case")
+        return value
+
+
+class MetricRegistry:
+    """Process-local Prometheus registry; DB history is deliberately a separate concern."""
+
+    def __init__(self) -> None:
+        self._definitions: dict[str, MetricDefinition] = {}
+        self._values: dict[tuple[str, tuple[tuple[str, str], ...]], float] = defaultdict(float)
+        self._histograms: dict[tuple[str, tuple[tuple[str, str], ...]], list[float]] = defaultdict(
+            list
+        )
+        self._lock = threading.Lock()
+        self.started_at = time.time()
+
+    def define(self, definition: MetricDefinition) -> None:
+        current = self._definitions.get(definition.name)
+        if current and current != definition:
+            raise ValueError(f"metric {definition.name} is already defined differently")
+        self._definitions[definition.name] = definition
+
+    def _key(self, name: str, labels: dict[str, str]) -> tuple[str, tuple[tuple[str, str], ...]]:
+        definition = self._definitions[name]
+        if set(labels) != set(definition.labels):
+            raise ValueError(f"metric {name} requires labels {definition.labels}")
+        if any(len(value) > 128 or "\n" in value for value in labels.values()):
+            raise ValueError("metric label values must be bounded single-line values")
+        return name, tuple(sorted(labels.items()))
+
+    def increment(self, name: str, labels: dict[str, str], value: float = 1.0) -> None:
+        if self._definitions[name].type is not TelemetryType.COUNTER or value < 0:
+            raise ValueError("only counters accept non-negative increments")
+        with self._lock:
+            self._values[self._key(name, labels)] += value
+
+    def gauge(self, name: str, labels: dict[str, str], value: float) -> None:
+        if self._definitions[name].type is not TelemetryType.GAUGE:
+            raise ValueError("only gauges accept current values")
+        with self._lock:
+            self._values[self._key(name, labels)] = value
+
+    def observe(self, name: str, labels: dict[str, str], value: float) -> None:
+        if self._definitions[name].type is not TelemetryType.HISTOGRAM or value < 0:
+            raise ValueError("only histograms accept non-negative observations")
+        with self._lock:
+            samples = self._histograms[self._key(name, labels)]
+            samples.append(value)
+            if len(samples) > 10_000:
+                del samples[: len(samples) - 10_000]
+
+    def histogram(self, name: str, labels: dict[str, str]) -> list[float]:
+        with self._lock:
+            return list(self._histograms.get(self._key(name, labels), []))
+
+    def samples(self, name: str) -> list[tuple[dict[str, str], float]]:
+        with self._lock:
+            return [
+                (dict(labels), value)
+                for (metric, labels), value in self._values.items()
+                if metric == name
+            ]
+
+    @staticmethod
+    def _label_text(labels: tuple[tuple[str, str], ...]) -> str:
+        if not labels:
+            return ""
+        escaped = [f'{key}="{value.replace(chr(92), chr(92) * 2).replace(chr(34), chr(92) + chr(34))}"' for key, value in labels]
+        return "{" + ",".join(escaped) + "}"
+
+    def render(self) -> str:
+        buckets = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
+        lines: list[str] = []
+        with self._lock:
+            for name, definition in sorted(self._definitions.items()):
+                lines.extend((f"# HELP {name} {definition.help}", f"# TYPE {name} {definition.type.value.lower()}"))
+                for (metric, labels), value in sorted(self._values.items()):
+                    if metric == name:
+                        lines.append(f"{name}{self._label_text(labels)} {value}")
+                for (metric, labels), values in sorted(self._histograms.items()):
+                    if metric != name:
+                        continue
+                    label_text = self._label_text(labels)
+                    for upper_bound in buckets:
+                        bucket_labels = tuple(sorted((*labels, ("le", str(upper_bound)))))
+                        count = sum(value <= upper_bound for value in values)
+                        lines.append(f"{name}_bucket{self._label_text(bucket_labels)} {count}")
+                    infinite_labels = tuple(sorted((*labels, ("le", "+Inf"))))
+                    lines.append(
+                        f"{name}_bucket{self._label_text(infinite_labels)} {len(values)}"
+                    )
+                    lines.append(f"{name}_count{label_text} {len(values)}")
+                    lines.append(f"{name}_sum{label_text} {sum(values)}")
+        lines.append(f"modelforge_process_uptime_seconds {max(0.0, time.time() - self.started_at)}")
+        return "\n".join(lines) + "\n"
+
+
+metrics = MetricRegistry()
+for _definition in (
+    MetricDefinition(name="modelforge_api_requests_total", type=TelemetryType.COUNTER, help="Bounded API request outcomes", labels=("method", "route_class", "status_class")),
+    MetricDefinition(name="modelforge_api_request_duration_seconds", type=TelemetryType.HISTOGRAM, help="API request duration", labels=("method", "route_class")),
+    MetricDefinition(name="modelforge_observability_degraded", type=TelemetryType.GAUGE, help="Historical observability persistence is degraded", labels=()),
+):
+    metrics.define(_definition)
+
+
+class SLIDefinitionCreate(OperationalModel):
+    key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
+    name: str = Field(min_length=3, max_length=255)
+    service: str = Field(min_length=2, max_length=128)
+    capability: str | None = Field(default=None, max_length=128)
+    measurement: Literal["SUCCESS_RATIO", "LATENCY_P95", "FRESHNESS", "CORRECTNESS_RATIO"]
+    valid_population: dict[str, Any]
+    success_condition: dict[str, Any]
+    default_window_seconds: int = Field(ge=60, le=2_592_000)
+
+
+class SLOPolicyCreate(OperationalModel):
+    key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
+    sli_definition_id: uuid.UUID
+    objective: float = Field(gt=0, le=1)
+    threshold_ms: float | None = Field(default=None, gt=0)
+    rolling_window_seconds: int = Field(ge=60, le=2_592_000)
+    minimum_sample_count: int = Field(ge=1, le=1_000_000)
+    severity: AlertSeverity
+    environment: Literal["PRODUCTION", "LAB", "BACKGROUND"]
+    effective_from: datetime
+    rationale: str = Field(min_length=10, max_length=4000)
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+
+class AlertRuleCreate(OperationalModel):
+    key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
+    alert_type: str = Field(pattern=r"^[A-Z][A-Z0-9_]{2,63}$")
+    signal: str = Field(min_length=3, max_length=128)
+    slo_policy_id: uuid.UUID | None = None
+    condition: dict[str, Any]
+    pending_seconds: int = Field(ge=0, le=86_400)
+    severity: AlertSeverity
+    labels: dict[str, str] = Field(default_factory=dict)
+    cooldown_seconds: int = Field(ge=0, le=604_800)
+    recovery_condition: dict[str, Any]
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+    @field_validator("labels")
+    @classmethod
+    def validate_labels(cls, value: dict[str, str]) -> dict[str, str]:
+        if len(value) > 8 or any(len(key) > 64 or len(item) > 128 for key, item in value.items()):
+            raise ValueError("alert labels must be bounded")
+        return value
+
+
+class MaintenanceWindowCreate(OperationalModel):
+    name: str = Field(min_length=3, max_length=255)
+    starts_at: datetime
+    ends_at: datetime
+    matcher: dict[str, str]
+    reason: str = Field(min_length=5, max_length=2000)
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+    @model_validator(mode="after")
+    def ordered(self) -> MaintenanceWindowCreate:
+        if self.ends_at <= self.starts_at:
+            raise ValueError("maintenance window end must follow start")
+        return self
+
+
+class AlertAction(OperationalModel):
+    actor: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=3, max_length=2000)
+
+
+class SLIDefinitionResponse(SLIDefinitionCreate):
+    id: uuid.UUID
+    enabled: bool
+    created_at: datetime
+
+
+class SLOPolicyResponse(SLOPolicyCreate):
+    id: uuid.UUID
+    revision: int
+    active: bool
+    fingerprint: str
+    created_at: datetime
+
+
+class AlertRuleResponse(AlertRuleCreate):
+    id: uuid.UUID
+    revision: int
+    active: bool
+    fingerprint: str
+    created_at: datetime
+
+
+class MaintenanceWindowResponse(MaintenanceWindowCreate):
+    id: uuid.UUID
+    active: bool
+    created_at: datetime
+
+
+class AlertHistoryResponse(OperationalModel):
+    id: uuid.UUID
+    alert_id: uuid.UUID
+    from_state: str | None
+    to_state: str
+    actor: str
+    reason: str
+    evidence: dict[str, Any]
+    occurred_at: datetime
+
+
+class IncidentResponse(OperationalModel):
+    id: uuid.UUID
+    fingerprint: str
+    title: str
+    state: str
+    severity: str
+    root_subject_type: str
+    root_subject_ref: str
+    correlation: Literal["RELATED", "LIKELY_ROOT", "DOWNSTREAM", "UNKNOWN"]
+    first_seen_at: datetime
+    last_seen_at: datetime
+    resolved_at: datetime | None
+
+
+class SLOEvaluationResponse(OperationalModel):
+    id: uuid.UUID
+    policy_id: uuid.UUID
+    policy_key: str
+    policy_revision: int
+    sli_key: str
+    environment: str
+    objective: float
+    observed_value: float | None
+    threshold_ms: float | None
+    sample_count: int
+    good_count: int
+    bad_count: int
+    state: SLOState
+    window_start: datetime
+    window_end: datetime
+    observed_at: datetime
+    freshness_seconds: float | None
+    allowed_bad: float | None
+    consumed_bad: int | None
+    remaining_bad: float | None
+    short_burn_rate: float | None
+    long_burn_rate: float | None
+    evidence: dict[str, Any]
+
+
+class AlertResponse(OperationalModel):
+    id: uuid.UUID
+    rule_id: uuid.UUID
+    fingerprint: str
+    alert_type: str
+    severity: str
+    state: AlertState
+    source: str
+    subject_type: str
+    subject_ref: str
+    summary: str
+    details: dict[str, Any]
+    first_seen_at: datetime
+    last_seen_at: datetime
+    firing_at: datetime | None
+    acknowledged_at: datetime | None
+    acknowledged_by: str | None
+    resolved_at: datetime | None
+    suppressed_until: datetime | None
+    occurrence_count: int
+
+
+class CapacitySnapshotResponse(OperationalModel):
+    id: uuid.UUID
+    observed_at: datetime
+    received_at: datetime
+    node_id: uuid.UUID
+    node_name: str
+    accelerator_id: uuid.UUID | None
+    gpu_total_bytes: int | None
+    gpu_observed_bytes: int | None
+    gpu_external_bytes: int | None
+    gpu_managed_resident_bytes: int | None
+    gpu_leased_bytes: int | None
+    gpu_reserve_bytes: int | None
+    gpu_schedulable_bytes: int | None
+    pressure_state: str
+    system_ram_total_bytes: int | None
+    system_ram_available_bytes: int | None
+    storage_total_bytes: int | None
+    storage_free_bytes: int | None
+    availability: str
+    freshness_seconds: float
+
+
+class TrendResponse(OperationalModel):
+    subject: str
+    sample_count: int
+    period_start: datetime | None
+    period_end: datetime | None
+    status: Literal["AVAILABLE", "INSUFFICIENT_DATA", "STALE"]
+    metrics: dict[str, float | int | None]
+    forecast: dict[str, Any]
+
+
+class OperationsOverview(OperationalModel):
+    status: Literal["HEALTHY", "DEGRADED", "OBSERVABILITY_DEGRADED"]
+    observed_at: datetime
+    active_alerts: list[AlertResponse]
+    slo_evaluations: list[SLOEvaluationResponse]
+    capacity: list[CapacitySnapshotResponse]
+    capability_health: list[dict[str, Any]]
+    project_health: list[dict[str, Any]]
+    recent_failures: list[dict[str, Any]]
+    history_available: bool
+
+
+def route_class(path: str) -> str:
+    if path.startswith("/api/v1/capabilities/") or path == "/v1/embeddings":
+        return "gateway"
+    if path.startswith("/api/v1/agent/"):
+        return "agent"
+    if path.startswith("/api/v1/admin/"):
+        return "admin"
+    if path.startswith("/api/v1/health/"):
+        return "health"
+    if path == "/metrics":
+        return "metrics"
+    return "control_plane"

backend/src/modelforge_api/domain/recovery.py

@@ -0,0 +1,471 @@
+"""M15 backup, restore and disaster-recovery contracts.
+
+Recovery is a first-class control-plane concern: what ModelForge owns authoritatively,
+what it can rebuild from an exact upstream identity, what is deliberately discarded and
+what is never exportable are separate classifications with separate guarantees.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+BACKUP_ID_PATTERN = r"^[a-z0-9][a-z0-9-]{6,62}$"
+SHA256_PATTERN = r"^[0-9a-f]{64}$"
+
+# ModelForge stores verified logical snapshots. WAL archiving and continuous point-in-time
+# recovery are deliberately out of scope for v1; the value is exported so operators and the
+# recovery dashboard never have to infer it.
+POINT_IN_TIME_SUPPORT: Literal["SUPPORTED", "NOT_SUPPORTED"] = "NOT_SUPPORTED"
+
+
+class RecoveryAssetClass(StrEnum):
+    """Top-level classification that decides whether state must be copied at all."""
+
+    AUTHORITATIVE = "AUTHORITATIVE"
+    REBUILDABLE = "REBUILDABLE"
+    EPHEMERAL = "EPHEMERAL"
+    EXTERNAL = "EXTERNAL"
+    SECRET = "SECRET"  # noqa: S105 - asset classification, not a credential
+
+
+class SecretRecoveryClass(StrEnum):
+    RESTORABLE_SECRET = "RESTORABLE_SECRET"  # noqa: S105 - asset classification, not a credential
+    ROTATABLE_SECRET = "ROTATABLE_SECRET"  # noqa: S105 - asset classification, not a credential
+    NON_EXPORTABLE_SECRET = "NON_EXPORTABLE_SECRET"  # noqa: S105 - asset classification, not a credential
+
+
+class ArtifactRecoveryClass(StrEnum):
+    REHYDRATABLE = "REHYDRATABLE"
+    NON_REHYDRATABLE = "NON_REHYDRATABLE"
+    DERIVED = "DERIVED"
+    LOCAL_ONLY = "LOCAL_ONLY"
+
+
+class ConfigurationClass(StrEnum):
+    SOURCE_CONTROLLED = "SOURCE_CONTROLLED"
+    SECRET = "SECRET"  # noqa: S105 - asset classification, not a credential
+    GENERATED = "GENERATED"
+    HOST_LOCAL = "HOST_LOCAL"
+
+
+class BackupState(StrEnum):
+    PLANNED = "PLANNED"
+    CREATING = "CREATING"
+    CREATED = "CREATED"
+    VERIFYING = "VERIFYING"
+    VERIFIED = "VERIFIED"
+    FAILED = "FAILED"
+    EXPIRED = "EXPIRED"
+    DELETED = "DELETED"
+
+
+RESTORE_ELIGIBLE_STATES = frozenset({BackupState.VERIFIED})
+BACKUP_TERMINAL_STATES = frozenset({BackupState.FAILED, BackupState.EXPIRED, BackupState.DELETED})
+
+
+class BackupMethod(StrEnum):
+    POSTGRES_LOGICAL_CUSTOM = "POSTGRES_LOGICAL_CUSTOM"
+    MANIFEST_ONLY = "MANIFEST_ONLY"
+    FILE_COPY = "FILE_COPY"
+    SOURCE_CONTROL_REFERENCE = "SOURCE_CONTROL_REFERENCE"
+    NOT_BACKED_UP = "NOT_BACKED_UP"
+
+
+class RestoreMode(StrEnum):
+    """VALIDATION never touches an active target; the destructive modes are operator-gated."""
+
+    VALIDATION = "VALIDATION"
+    REPLACEMENT = "REPLACEMENT"
+    DISASTER_RECOVERY = "DISASTER_RECOVERY"
+
+
+class RestorePlanState(StrEnum):
+    DRAFT = "DRAFT"
+    PREFLIGHT_PASSED = "PREFLIGHT_PASSED"
+    PREFLIGHT_FAILED = "PREFLIGHT_FAILED"
+    CONSUMED = "CONSUMED"
+
+
+class RestoreState(StrEnum):
+    PLANNED = "PLANNED"
+    PREFLIGHT = "PREFLIGHT"
+    RESTORING_DATABASE = "RESTORING_DATABASE"
+    RESTORING_CONFIGURATION = "RESTORING_CONFIGURATION"
+    REHYDRATING_ARTIFACTS = "REHYDRATING_ARTIFACTS"
+    RECONCILING = "RECONCILING"
+    VALIDATING = "VALIDATING"
+    READY = "READY"
+    FAILED = "FAILED"
+    MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
+
+
+RESTORE_TERMINAL_STATES = frozenset(
+    {RestoreState.READY, RestoreState.FAILED, RestoreState.MANUAL_INTERVENTION_REQUIRED}
+)
+
+RESTORE_PHASE_ORDER: tuple[RestoreState, ...] = (
+    RestoreState.PREFLIGHT,
+    RestoreState.RESTORING_DATABASE,
+    RestoreState.RESTORING_CONFIGURATION,
+    RestoreState.REHYDRATING_ARTIFACTS,
+    RestoreState.RECONCILING,
+    RestoreState.VALIDATING,
+    RestoreState.READY,
+)
+
+
+class ArtifactRecoveryState(StrEnum):
+    PLANNED = "PLANNED"
+    REHYDRATING = "REHYDRATING"
+    VERIFYING = "VERIFYING"
+    RECOVERED = "RECOVERED"
+    BLOCKED = "BLOCKED"
+    FAILED = "FAILED"
+
+
+class RecoveryReadiness(StrEnum):
+    PROTECTED = "PROTECTED"
+    REHYDRATABLE = "REHYDRATABLE"
+    ROTATION_REQUIRED = "ROTATION_REQUIRED"
+    EXTERNAL_DEPENDENCY = "EXTERNAL_DEPENDENCY"
+    UNPROTECTED = "UNPROTECTED"
+
+
+class RecoveryFailureCode(StrEnum):
+    HASH_MISMATCH = "HASH_MISMATCH"
+    MANIFEST_HASH_MISMATCH = "MANIFEST_HASH_MISMATCH"
+    MANIFEST_INCOMPLETE = "MANIFEST_INCOMPLETE"
+    PAYLOAD_MISSING = "PAYLOAD_MISSING"
+    BACKUP_NOT_RESTORE_ELIGIBLE = "BACKUP_NOT_RESTORE_ELIGIBLE"
+    ENCRYPTION_KEY_UNAVAILABLE = "ENCRYPTION_KEY_UNAVAILABLE"
+    DECRYPTION_FAILED = "DECRYPTION_FAILED"
+    SCHEMA_TOO_NEW = "SCHEMA_TOO_NEW"
+    SCHEMA_UNKNOWN = "SCHEMA_UNKNOWN"
+    POSTGRES_VERSION_INCOMPATIBLE = "POSTGRES_VERSION_INCOMPATIBLE"
+    DESTINATION_NOT_ISOLATED = "DESTINATION_NOT_ISOLATED"
+    DESTINATION_NOT_EMPTY = "DESTINATION_NOT_EMPTY"
+    INSUFFICIENT_CAPACITY = "INSUFFICIENT_CAPACITY"
+    DESTINATION_UNAVAILABLE = "DESTINATION_UNAVAILABLE"
+    BACKUP_TOOL_UNAVAILABLE = "BACKUP_TOOL_UNAVAILABLE"
+    BACKUP_TOOL_FAILED = "BACKUP_TOOL_FAILED"
+    CONCURRENT_OPERATION = "CONCURRENT_OPERATION"
+    PATH_NOT_ALLOWED = "PATH_NOT_ALLOWED"
+    ARCHIVE_UNSAFE = "ARCHIVE_UNSAFE"
+    ARTIFACT_REHYDRATION_BLOCKED = "ARTIFACT_REHYDRATION_BLOCKED"
+    ARTIFACT_HASH_MISMATCH = "ARTIFACT_HASH_MISMATCH"
+    ARTIFACT_NOT_REHYDRATABLE = "ARTIFACT_NOT_REHYDRATABLE"
+    AUDIT_CHAIN_CORRUPT = "AUDIT_CHAIN_CORRUPT"
+    RECOVERY_RECONCILIATION_REQUIRED = "RECOVERY_RECONCILIATION_REQUIRED"
+    SOURCE_CONTROL_REVISION_UNAVAILABLE = "SOURCE_CONTROL_REVISION_UNAVAILABLE"
+
+
+class RecoveryModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class RecoveryPolicyCreate(RecoveryModel):
+    """A versioned recovery contract per asset class; never scattered inline rules."""
+
+    key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
+    name: str = Field(min_length=3, max_length=255)
+    asset_class: RecoveryAssetClass
+    backup_method: BackupMethod
+    retention_days: int = Field(ge=1, le=3650)
+    minimum_verified_backups: int = Field(ge=1, le=100)
+    rpo_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
+    rto_target_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
+    restore_verification: Literal["FULL_RESTORE", "HASH_ONLY", "MANIFEST_ONLY", "NOT_APPLICABLE"]
+    encryption_required: bool
+    external_dependency: bool = False
+    rehydration_allowed: bool = False
+    secret_class: SecretRecoveryClass | None = None
+    rationale: str = Field(min_length=10, max_length=4000)
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+    @model_validator(mode="after")
+    def coherent(self) -> RecoveryPolicyCreate:
+        if self.asset_class is RecoveryAssetClass.AUTHORITATIVE:
+            if self.backup_method in {BackupMethod.NOT_BACKED_UP, BackupMethod.MANIFEST_ONLY}:
+                raise ValueError("authoritative state requires a payload-bearing backup method")
+            if self.rpo_seconds is None:
+                raise ValueError("authoritative state requires an explicit RPO target")
+        if self.asset_class is RecoveryAssetClass.EPHEMERAL and self.backup_method not in {
+            BackupMethod.NOT_BACKED_UP,
+            BackupMethod.MANIFEST_ONLY,
+        }:
+            raise ValueError("ephemeral state must not claim a payload backup")
+        if self.asset_class is RecoveryAssetClass.EXTERNAL and not self.external_dependency:
+            raise ValueError("external state must be marked as an external dependency")
+        if self.asset_class is RecoveryAssetClass.SECRET and self.secret_class is None:
+            raise ValueError("secret state requires an explicit secret recovery class")
+        if self.rehydration_allowed and self.asset_class is not RecoveryAssetClass.REBUILDABLE:
+            raise ValueError("only rebuildable state may be rehydrated instead of copied")
+        return self
+
+
+class RecoveryPolicyResponse(RecoveryPolicyCreate):
+    id: uuid.UUID
+    revision: int
+    active: bool
+    fingerprint: str
+    created_at: datetime
+
+
+class RecoveryAssetResponse(RecoveryModel):
+    id: uuid.UUID
+    key: str
+    name: str
+    asset_class: RecoveryAssetClass
+    owner: str
+    location: str
+    backup_method: BackupMethod
+    restore_method: str
+    rebuild_method: str | None
+    rpo_seconds: int | None
+    readiness: RecoveryReadiness
+    dependencies: list[str]
+    notes: str
+    policy_key: str
+    updated_at: datetime
+
+
+class BackupSetCreate(RecoveryModel):
+    backup_id: str = Field(pattern=BACKUP_ID_PATTERN)
+    reason: str = Field(min_length=5, max_length=2000)
+    milestone: str | None = Field(default=None, max_length=64)
+    legal_hold: bool = False
+    include_artifact_manifest: bool = True
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+
+class BackupManifestEntryResponse(RecoveryModel):
+    id: uuid.UUID
+    logical_asset_type: str
+    object_name: str
+    relative_path: str
+    size_bytes: int
+    sha256: str
+    source_generation: str
+    schema_version: str | None
+    dependency_refs: dict[str, Any]
+
+
+class BackupSetResponse(RecoveryModel):
+    id: uuid.UUID
+    backup_id: str
+    state: BackupState
+    policy_key: str
+    policy_revision: int
+    modelforge_version: str
+    modelforge_commit: str | None
+    schema_revision: str | None
+    environment_fingerprint: dict[str, Any]
+    database_identity: dict[str, Any]
+    destination_root: str
+    manifest_relative_path: str | None
+    manifest_sha256: str | None
+    included_asset_classes: list[str]
+    excluded_asset_classes: list[str]
+    payload_bytes: int
+    encrypted: bool
+    encryption_algorithm: str | None
+    encryption_key_id: str | None
+    verification_details: dict[str, Any]
+    verified_at: datetime | None
+    failure_code: str | None
+    failure_reason: str | None
+    milestone: str | None
+    legal_hold: bool
+    restore_eligible: bool
+    reason: str
+    created_by: str
+    started_at: datetime | None
+    completed_at: datetime | None
+    expires_at: datetime | None
+    created_at: datetime
+    entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
+
+
+class RestorePlanCreate(RecoveryModel):
+    backup_set_id: uuid.UUID
+    mode: RestoreMode
+    target_environment: Literal["ISOLATED", "STAGING", "PRODUCTION"]
+    target_label: str = Field(min_length=3, max_length=128)
+    database_destination: str = Field(min_length=8, max_length=1024)
+    artifact_strategy: Literal["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"]
+    secret_strategy: Literal["ROTATE", "RESTORE_HASHES", "MANUAL"]
+    node_strategy: Literal["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"]
+    expected_modelforge_version: str | None = Field(default=None, max_length=64)
+    validation_requirements: dict[str, Any] = Field(default_factory=dict)
+    reason: str = Field(min_length=10, max_length=4000)
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+    @field_validator("database_destination")
+    @classmethod
+    def supported_destination(cls, value: str) -> str:
+        if not value.startswith("postgresql+psycopg://"):
+            raise ValueError("restore destinations must be PostgreSQL SQLAlchemy URLs")
+        return value
+
+    @model_validator(mode="after")
+    def guarded(self) -> RestorePlanCreate:
+        if self.mode is RestoreMode.VALIDATION and self.target_environment == "PRODUCTION":
+            raise ValueError("validation restores must never target a production environment")
+        if self.target_environment == "PRODUCTION" and self.mode is not RestoreMode.DISASTER_RECOVERY:
+            raise ValueError("production targets require an explicit disaster-recovery restore")
+        return self
+
+
+class RestorePlanResponse(RecoveryModel):
+    id: uuid.UUID
+    backup_set_id: uuid.UUID
+    backup_id: str
+    mode: RestoreMode
+    state: RestorePlanState
+    target_environment: str
+    target_label: str
+    database_destination_redacted: str
+    artifact_strategy: str
+    secret_strategy: str
+    node_strategy: str
+    expected_modelforge_version: str | None
+    preflight: dict[str, Any]
+    validation_requirements: dict[str, Any]
+    fingerprint: str
+    reason: str
+    created_by: str
+    created_at: datetime
+
+
+class RestoreOperationResponse(RecoveryModel):
+    id: uuid.UUID
+    plan_id: uuid.UUID
+    backup_set_id: uuid.UUID
+    backup_id: str
+    state: RestoreState
+    mode: RestoreMode
+    attempt: int
+    idempotency_key: str
+    preflight_result: dict[str, Any]
+    phase_durations: dict[str, float]
+    source_fingerprint: dict[str, Any]
+    restored_fingerprint: dict[str, Any]
+    fingerprint_diff: dict[str, Any]
+    validation_result: dict[str, Any]
+    rpo_seconds: float | None
+    rto_seconds: float | None
+    failure_code: str | None
+    failure_reason: str | None
+    started_at: datetime
+    updated_at: datetime
+    ready_at: datetime | None
+
+
+class RestoreOperationEventResponse(RecoveryModel):
+    id: uuid.UUID
+    restore_operation_id: uuid.UUID
+    from_state: str | None
+    to_state: str
+    phase: str
+    actor: str
+    reason: str
+    evidence: dict[str, Any]
+    occurred_at: datetime
+
+
+class RestoreAdvanceRequest(RecoveryModel):
+    actor: str = Field(default="operator", min_length=1, max_length=255)
+    reason: str = Field(min_length=5, max_length=2000)
+    stop_after: RestoreState | None = None
+    simulate_interruption_after: RestoreState | None = None
+
+
+class ArtifactRecoveryCreate(RecoveryModel):
+    artifact_set_id: uuid.UUID
+    target_storage_root_id: uuid.UUID
+    reason: str = Field(min_length=10, max_length=2000)
+    restore_operation_id: uuid.UUID | None = None
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+
+class ArtifactRecoveryResponse(RecoveryModel):
+    id: uuid.UUID
+    restore_operation_id: uuid.UUID | None
+    artifact_set_id: uuid.UUID
+    model_revision_id: uuid.UUID
+    recovery_class: ArtifactRecoveryClass
+    state: ArtifactRecoveryState
+    upstream_repository: str | None
+    upstream_commit_sha: str | None
+    target_storage_root_id: uuid.UUID
+    expected_files: list[dict[str, Any]]
+    verified_files: list[dict[str, Any]]
+    bytes_total: int
+    bytes_recovered: int
+    download_plan_id: uuid.UUID | None
+    artifact_job_id: uuid.UUID | None
+    lineage: dict[str, Any]
+    duration_seconds: float | None
+    failure_code: str | None
+    failure_reason: str | None
+    started_at: datetime
+    completed_at: datetime | None
+
+
+class RecoveryReadinessEntry(RecoveryModel):
+    asset_key: str
+    asset_name: str
+    asset_class: RecoveryAssetClass
+    readiness: RecoveryReadiness
+    policy_key: str
+    rpo_seconds: int | None
+    detail: str
+
+
+class RecoveryDashboard(RecoveryModel):
+    """Every field is measured; unknown values stay null instead of becoming a fiction."""
+
+    observed_at: datetime
+    point_in_time_support: Literal["SUPPORTED", "NOT_SUPPORTED"]
+    latest_verified_backup_id: str | None
+    latest_verified_backup_at: datetime | None
+    latest_verified_backup_age_seconds: float | None
+    latest_verified_schema_revision: str | None
+    backup_states: dict[str, int]
+    verified_backup_count: int
+    stale_backup: bool
+    backup_staleness_threshold_seconds: int
+    last_restore_rehearsal_at: datetime | None
+    last_restore_rehearsal_state: str | None
+    observed_restore_seconds: float | None
+    observed_rpo_seconds: float | None
+    protected_asset_count: int
+    unprotected_assets: list[str]
+    readiness: list[RecoveryReadinessEntry]
+    coverage_ratio: float
+    estimated_protected_bytes: int
+    estimated_rehydratable_bytes: int
+    destination_capacity_bytes: int | None
+    destination_free_bytes: int | None
+
+
+class BackupCapacityEstimate(RecoveryModel):
+    bytes_to_copy: int
+    bytes_manifest_only: int
+    estimated_protected_bytes: int
+    available_bytes: int | None
+    capacity_bytes: int | None
+    sufficient: bool
+    detail: str
+
+
+def redact_database_url(url: str) -> str:
+    """Never journal or return a restore DSN that still carries its password."""
+
+    return re.sub(r"://([^:/@]+):[^@]*@", r"://\1:***@", url)

backend/src/modelforge_api/domain/registry.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from modelforge_api.domain.enums import ArtifactStatus, ModelLifecycle, StorageRootStatus
+
+
+class RegistryModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class Page[Item](RegistryModel):
+    items: list[Item]
+    page: int
+    page_size: int
+    total: int
+    pages: int
+
+
+class ModelCreate(RegistryModel):
+    key: str = Field(min_length=1, max_length=128, pattern=r"^[a-z0-9][a-z0-9._-]*$")
+    display_name: str = Field(min_length=1, max_length=255)
+    description: str | None = None
+    source_type: Literal["huggingface", "local", "custom"] = "huggingface"
+    upstream_provider: str = Field(min_length=1, max_length=64)
+    upstream_source: str = Field(min_length=1, max_length=255)
+    upstream_metadata: dict[str, Any] = Field(default_factory=dict)
+    local_metadata: dict[str, Any] = Field(default_factory=dict)
+    interpretation_metadata: dict[str, Any] = Field(default_factory=dict)
+    family: str | None = None
+    modalities: list[str] = Field(default_factory=list)
+    parameter_metadata: dict[str, Any] = Field(default_factory=dict)
+    license_metadata: dict[str, Any] = Field(default_factory=lambda: {"status": "unknown"})
+    lifecycle: ModelLifecycle = ModelLifecycle.CANDIDATE
+
+
+class ModelUpdate(RegistryModel):
+    display_name: str | None = Field(default=None, min_length=1, max_length=255)
+    description: str | None = None
+    local_metadata: dict[str, Any] | None = None
+    interpretation_metadata: dict[str, Any] | None = None
+    lifecycle: ModelLifecycle | None = None
+
+
+class ModelResponse(ModelCreate):
+    id: uuid.UUID
+    created_at: datetime
+    updated_at: datetime
+    deprecated_at: datetime | None = None
+    revision_count: int = 0
+    artifact_count: int = 0
+    verification_status: str = "unverified"
+    deployment_status: str = "not_deployed"
+
+
+class RevisionCreate(RegistryModel):
+    upstream_revision: str = Field(min_length=1, max_length=255)
+    resolved_commit_sha: str = Field(pattern=r"^[0-9a-f]{40,64}$")
+    metadata_snapshot: dict[str, Any] = Field(default_factory=dict)
+
+
+class RevisionResponse(RevisionCreate):
+    id: uuid.UUID
+    model_id: uuid.UUID
+    discovered_at: datetime
+    approved_at: datetime | None = None
+    immutable_at: datetime
+    deprecated_at: datetime | None = None
+    archived_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class ArtifactLocationCreate(RegistryModel):
+    storage_root_id: uuid.UUID
+    relative_path: str = Field(min_length=1)
+    status: ArtifactStatus = ArtifactStatus.LOCAL
+
+    @field_validator("relative_path")
+    @classmethod
+    def safe_relative_path(cls, value: str) -> str:
+        normalized = value.replace("\\", "/")
+        if normalized.startswith("/") or ".." in normalized.split("/"):
+            raise ValueError("relative_path must remain within the storage root")
+        return normalized
+
+
+class ArtifactLocationResponse(ArtifactLocationCreate):
+    id: uuid.UUID
+    artifact_id: uuid.UUID | None = None
+    derived_artifact_id: uuid.UUID | None = None
+    size_bytes: int | None = None
+    observed_sha256: str | None = None
+    last_checked_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class ArtifactCreate(RegistryModel):
+    filename: str = Field(min_length=1, max_length=512)
+    artifact_type: str = Field(min_length=1, max_length=64)
+    serialization_format: str = Field(min_length=1, max_length=64)
+    sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+    size_bytes: int = Field(ge=0)
+    status: ArtifactStatus = ArtifactStatus.REMOTE
+    security_status: str = "unverified"
+    license_status: str = "unknown"
+    locations: list[ArtifactLocationCreate] = Field(default_factory=list)
+
+
+class ArtifactResponse(RegistryModel):
+    id: uuid.UUID
+    revision_id: uuid.UUID
+    filename: str
+    artifact_type: str
+    serialization_format: str
+    sha256: str
+    size_bytes: int
+    status: ArtifactStatus
+    security_status: str
+    license_status: str
+    quarantined: bool
+    verification_details: dict[str, Any]
+    verified_at: datetime | None = None
+    immutable_at: datetime | None = None
+    deprecated_at: datetime | None = None
+    archived_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+    locations: list[ArtifactLocationResponse] = Field(default_factory=list)
+
+
+class DerivedArtifactCreate(RegistryModel):
+    revision_id: uuid.UUID
+    source_artifact_ids: list[uuid.UUID] = Field(min_length=1)
+    filename: str = Field(min_length=1, max_length=512)
+    artifact_type: str = Field(min_length=1, max_length=64)
+    sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
+    size_bytes: int = Field(ge=0)
+    transformation_type: str = Field(min_length=1, max_length=64)
+    tool: str = Field(min_length=1, max_length=128)
+    tool_version: str = Field(min_length=1, max_length=128)
+    configuration: dict[str, Any] = Field(default_factory=dict)
+    environment_snapshot: dict[str, Any] = Field(default_factory=dict)
+    status: ArtifactStatus = ArtifactStatus.REMOTE
+    locations: list[ArtifactLocationCreate] = Field(default_factory=list)
+
+
+class DerivedSourceResponse(RegistryModel):
+    artifact_id: uuid.UUID
+    sha256: str
+    ordinal: int
+
+
+class DerivedArtifactResponse(RegistryModel):
+    id: uuid.UUID
+    revision_id: uuid.UUID
+    filename: str
+    artifact_type: str
+    sha256: str
+    size_bytes: int
+    transformation_type: str
+    tool: str
+    tool_version: str
+    configuration: dict[str, Any]
+    environment_snapshot: dict[str, Any]
+    status: ArtifactStatus
+    immutable_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+    sources: list[DerivedSourceResponse]
+    locations: list[ArtifactLocationResponse]
+
+
+class StorageRootCreate(RegistryModel):
+    compute_node_id: uuid.UUID
+    name: str = Field(min_length=1, max_length=128)
+    purpose: str = Field(default="model_artifacts", min_length=1, max_length=64)
+    path: str = Field(min_length=1)
+    agent_path: str | None = Field(default=None, min_length=1)
+    reserve_bytes: int = Field(default=0, ge=0)
+    reserve_percent: int = Field(default=10, ge=0, le=100)
+
+
+class StorageRootObservation(RegistryModel):
+    writable: bool
+    capacity_bytes: int | None = Field(default=None, ge=0)
+    free_bytes: int | None = Field(default=None, ge=0)
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class StorageRootUpdate(RegistryModel):
+    agent_path: str = Field(min_length=1)
+
+
+class StorageRootResponse(StorageRootCreate):
+    id: uuid.UUID
+    status: StorageRootStatus
+    writable: bool
+    capacity_bytes: int | None = None
+    free_bytes: int | None = None
+    capacity_observed_at: datetime | None = None
+    validation_details: dict[str, Any]
+    deprecated_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class CapacityDecision(RegistryModel):
+    allowed: bool
+    status: StorageRootStatus
+    requested_bytes: int
+    usable_bytes: int | None
+    reason: str
+
+
+class VerifyResponse(RegistryModel):
+    artifact_id: uuid.UUID
+    status: ArtifactStatus
+    expected_sha256: str
+    observed_sha256: str | None
+    observed_size_bytes: int | None
+    checked_at: datetime
+    location_id: uuid.UUID
+
+
+class DependencyReference(RegistryModel):
+    resource_type: str
+    resource_id: uuid.UUID
+    relation: str
+
+
+class DependencyConflict(RegistryModel):
+    message: str
+    dependencies: list[DependencyReference]

backend/src/modelforge_api/domain/release.py

@@ -0,0 +1,223 @@
+"""The v1 release contract.
+
+One authoritative product version, and the compatibility ranges that version promises. Before M17
+the version existed four times — backend, Node Agent, Runtime Worker and the console each declared
+``0.1.0`` independently — which is three opportunities for a release to describe itself wrongly.
+
+The version lives in the repository's ``VERSION`` file. Every packaged manifest is checked against
+it rather than trusted to agree, because a manifest that has drifted looks exactly like one that has
+not.
+
+Compatibility is stated, not implied. An application refusing to start against a schema it does not
+support is a far better outcome than one that starts and writes rows the next version cannot read.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import StrEnum
+from pathlib import Path
+
+# --------------------------------------------------------------------------- version
+
+#: Schema revisions this application version can run against, oldest first. The application refuses
+#: to serve against anything outside this list rather than guessing.
+SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260830_0024",)
+
+#: The revision a clean installation and a completed upgrade must both arrive at.
+TARGET_SCHEMA_REVISION = SUPPORTED_SCHEMA_REVISIONS[-1]
+
+#: Schema revisions from which the upgrade tool may migrate to the target. These are deliberately
+#: separate from runtime compatibility: the node-decommission code reads columns that do not exist
+#: at 0021, so serving on 0021 would be unsafe even though migrating from it is supported.
+SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS: tuple[str, ...] = (
+    "20260827_0021",
+    "20260828_0022",
+    "20260830_0023",
+    TARGET_SCHEMA_REVISION,
+)
+
+#: The oldest release an in-place upgrade to this version is supported from.
+MINIMUM_UPGRADE_SOURCE = "v1.0.0"
+
+#: Agent protocol versions this control plane accepts.
+SUPPORTED_AGENT_PROTOCOL_VERSIONS: tuple[int, ...] = (1,)
+
+#: The protocol version this control plane itself speaks.
+CURRENT_AGENT_PROTOCOL_VERSION = SUPPORTED_AGENT_PROTOCOL_VERSIONS[-1]
+
+#: Release channels. v1 ships one; the enum exists so adding another is a typed change.
+RELEASE_CHANNEL = "stable"
+
+#: Minimum PostgreSQL major version. The schema uses partial unique indexes and generated columns
+#: that older majors either lack or plan differently.
+MINIMUM_POSTGRES_MAJOR = 16
+
+
+def _read_version_file() -> str:
+    """Read the repository's VERSION file, falling back to the packaged distribution metadata."""
+
+    here = Path(__file__).resolve()
+    for parent in here.parents:
+        candidate = parent / "VERSION"
+        if candidate.is_file():
+            return candidate.read_text("utf-8").strip()
+    # An installed wheel has no VERSION file beside it; the distribution metadata is authoritative
+    # there, and the packaging test proves the two agree at build time.
+    from importlib.metadata import PackageNotFoundError, version
+
+    try:
+        return version("modelforge-api")
+    except PackageNotFoundError:  # pragma: no cover - only in a broken install
+        return "0.0.0"
+
+
+PRODUCT_VERSION = _read_version_file()
+PRODUCT_NAME = "ITWorx ModelForge"
+
+
+# --------------------------------------------------------------------------- semver
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticVersion:
+    major: int
+    minor: int
+    patch: int
+    prerelease: str | None = None
+
+    @classmethod
+    def parse(cls, value: str) -> SemanticVersion:
+        core, _, prerelease = value.partition("-")
+        parts = core.split(".")
+        if len(parts) != 3 or not all(part.isdigit() for part in parts):
+            raise ValueError(f"{value!r} is not a MAJOR.MINOR.PATCH version")
+        major, minor, patch = (int(part) for part in parts)
+        return cls(major, minor, patch, prerelease or None)
+
+    def __str__(self) -> str:
+        core = f"{self.major}.{self.minor}.{self.patch}"
+        return f"{core}-{self.prerelease}" if self.prerelease else core
+
+    @property
+    def is_prerelease(self) -> bool:
+        return self.prerelease is not None
+
+
+# --------------------------------------------------------------------------- compatibility
+
+
+class Compatibility(StrEnum):
+    """The four answers a compatibility question can have. There is no fifth, and no silent pass."""
+
+    COMPATIBLE = "COMPATIBLE"
+    TOO_OLD = "TOO_OLD"
+    TOO_NEW = "TOO_NEW"
+    UNKNOWN = "UNKNOWN"
+
+
+def schema_compatibility(revision: str | None) -> Compatibility:
+    """Is this application version able to run against that schema revision?"""
+
+    if revision is None:
+        return Compatibility.UNKNOWN
+    if revision in SUPPORTED_SCHEMA_REVISIONS:
+        return Compatibility.COMPATIBLE
+    # Revisions are date-ordered identifiers, so a straight comparison against the oldest and newest
+    # supported revision tells old from new without a migration graph walk.
+    if revision < SUPPORTED_SCHEMA_REVISIONS[0]:
+        return Compatibility.TOO_OLD
+    if revision > SUPPORTED_SCHEMA_REVISIONS[-1]:
+        return Compatibility.TOO_NEW
+    return Compatibility.UNKNOWN
+
+
+def agent_protocol_compatibility(protocol_version: int | None) -> Compatibility:
+    """Can an agent speaking that protocol version talk to this control plane?"""
+
+    if protocol_version is None:
+        return Compatibility.UNKNOWN
+    if protocol_version in SUPPORTED_AGENT_PROTOCOL_VERSIONS:
+        return Compatibility.COMPATIBLE
+    if protocol_version < SUPPORTED_AGENT_PROTOCOL_VERSIONS[0]:
+        return Compatibility.TOO_OLD
+    return Compatibility.TOO_NEW
+
+
+def upgrade_required(compatibility: Compatibility) -> str | None:
+    """What the operator has to do, in one sentence, or None when nothing is required."""
+
+    match compatibility:
+        case Compatibility.COMPATIBLE:
+            return None
+        case Compatibility.TOO_OLD:
+            return "upgrade the component to a release that speaks the current protocol"
+        case Compatibility.TOO_NEW:
+            return "upgrade the control plane, which is older than the component reporting to it"
+        case Compatibility.UNKNOWN:
+            return "the version could not be determined and is refused rather than assumed"
+
+
+# --------------------------------------------------------------------------- build identity
+
+
+@dataclass(frozen=True, slots=True)
+class BuildIdentity:
+    """What a running binary can say about where it came from.
+
+    Everything here is either compiled in at build time or read from the environment the image was
+    built with. Nothing is inferred at runtime, because a build identity that a running process can
+    talk itself into is worth nothing during an incident.
+    """
+
+    version: str
+    source_commit: str | None
+    built_at: str | None
+    image_digest: str | None
+    channel: str
+    schema_revision: str
+    agent_protocol_version: int
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "version": self.version,
+            "source_commit": self.source_commit,
+            "built_at": self.built_at,
+            "image_digest": self.image_digest,
+            "channel": self.channel,
+            "schema_revision": self.schema_revision,
+            "agent_protocol_version": self.agent_protocol_version,
+        }
+
+
+def _clean(value: str | None) -> str | None:
+    """Treat an unsubstituted build argument as absent rather than reporting it as a fact."""
+
+    if value is None:
+        return None
+    stripped = value.strip()
+    if not stripped or stripped.lower() in {"unknown", "none", "null"}:
+        return None
+    return stripped
+
+
+def build_identity(
+    *,
+    source_commit: str | None = None,
+    built_at: str | None = None,
+    image_digest: str | None = None,
+) -> BuildIdentity:
+    return BuildIdentity(
+        version=PRODUCT_VERSION,
+        source_commit=_clean(source_commit),
+        built_at=_clean(built_at),
+        image_digest=_clean(image_digest),
+        channel=RELEASE_CHANNEL,
+        schema_revision=TARGET_SCHEMA_REVISION,
+        agent_protocol_version=CURRENT_AGENT_PROTOCOL_VERSION,
+    )
+
+
+def utc_now_iso() -> str:
+    return datetime.now(UTC).isoformat()

backend/src/modelforge_api/domain/runtime.py

@@ -0,0 +1,249 @@
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+RuntimeAdapterName = Literal[
+    "sentence_transformers",
+    "qwen3_reranker",
+    "transformers_trocr",
+    "transformers_siglip2",
+    "transformers_whisper",
+    "transformers",
+    "vllm",
+    "llama_cpp",
+    "diffusers",
+    "custom",
+]
+CompatibilityStatus = Literal["compatible", "incompatible", "unknown", "requires_probe", "blocked"]
+ProbeStatus = Literal[
+    "queued",
+    "preparing",
+    "loading",
+    "healthchecking",
+    "ready",
+    "unloading",
+    "completed",
+    "failed",
+    "cancelled",
+]
+
+
+class RuntimeModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class RuntimeEnvironmentCreate(RuntimeModel):
+    name: str = Field(min_length=1, max_length=255)
+    adapter: RuntimeAdapterName
+    runtime_version: str = Field(min_length=1, max_length=128)
+    image_repository: str = Field(min_length=1, max_length=255)
+    image_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
+    python_version: str = Field(min_length=1, max_length=64)
+    cuda_runtime_version: str | None = Field(default=None, max_length=64)
+    package_versions: dict[str, str] = Field(default_factory=dict)
+    supported_model_types: list[str] = Field(default_factory=list)
+    supported_formats: list[str] = Field(default_factory=list)
+    supported_modalities: list[str] = Field(default_factory=list)
+    network_policy: Literal["offline_control_plane_only"] = "offline_control_plane_only"
+
+
+class RuntimeEnvironmentResponse(RuntimeEnvironmentCreate):
+    id: uuid.UUID
+    fingerprint: str
+    immutable_at: datetime
+    created_at: datetime
+
+
+class RuntimeProfileCreate(RuntimeModel):
+    name: str = Field(min_length=1, max_length=255)
+    runtime_environment_id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    dtype: Literal["float32", "float16", "bfloat16"] = "bfloat16"
+    quantization: str | None = Field(default=None, max_length=64)
+    modality: Literal[
+        "embedding", "reranking", "text_generation", "vision", "document", "audio", "diffusion"
+    ]
+    max_sequence_length: int = Field(default=128, ge=1, le=131072)
+    batch_size: int = Field(default=1, ge=1, le=128)
+    concurrency: int = Field(default=1, ge=1, le=128)
+    device_policy: Literal["cuda_required", "cuda_preferred", "cpu_only"] = "cuda_required"
+    gpu_memory_policy: dict[str, Any] = Field(default_factory=dict)
+    launch_parameters: dict[str, Any] = Field(default_factory=dict)
+    environment_variables: dict[str, str] = Field(default_factory=dict)
+    trust_remote_code: Literal[False] = False
+    network_egress: Literal[False] = False
+
+    @field_validator("environment_variables")
+    @classmethod
+    def reject_secrets(cls, value: dict[str, str]) -> dict[str, str]:
+        forbidden = {"TOKEN", "SECRET", "PASSWORD", "KEY", "CREDENTIAL"}
+        if any(
+            forbidden.intersection(filter(None, re.split(r"[^A-Z0-9]+", key.upper())))
+            for key in value
+        ):
+            raise ValueError("runtime profile environment cannot contain secrets")
+        return value
+
+    @model_validator(mode="after")
+    def fixed_probe_shape(self) -> RuntimeProfileCreate:
+        if self.modality == "embedding" and self.batch_size != 1:
+            raise ValueError("M4 embedding probes require batch_size=1")
+        return self
+
+
+class RuntimeProfileResponse(RuntimeProfileCreate):
+    id: uuid.UUID
+    adapter: str
+    runtime_version: str
+    image_digest: str
+    version: int
+    fingerprint: str
+    health_contract: dict[str, Any]
+    immutable_at: datetime
+    created_at: datetime
+
+
+class CompatibilityAssessmentCreate(RuntimeModel):
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+
+
+class CompatibilityAssessmentResponse(RuntimeModel):
+    id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    adapter: str
+    runtime_version: str
+    status: CompatibilityStatus
+    static_result: dict[str, Any]
+    evidence: dict[str, Any]
+    blockers: list[str]
+    warnings: list[str]
+    required_approvals: list[str]
+    hardware_facts: dict[str, Any]
+    artifact_facts: dict[str, Any]
+    environment_fingerprint: str
+    stale: bool
+    stale_reason: str | None
+    created_at: datetime
+
+
+class ExecutionApprovalCreate(RuntimeModel):
+    scope: Literal["lab_execution"] = "lab_execution"
+    reason: str = Field(min_length=8, max_length=2000)
+    approved_by: str = Field(min_length=1, max_length=255)
+    expires_at: datetime | None = None
+
+
+class ExecutionApprovalResponse(RuntimeModel):
+    id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    scope: str
+    status: str
+    evidence_fingerprint: str
+    reason: str
+    approved_by: str
+    approved_at: datetime
+    expires_at: datetime | None
+    revoked_at: datetime | None
+    stale: bool
+
+
+class RuntimeProbeCreate(RuntimeModel):
+    compatibility_assessment_id: uuid.UUID
+    execution_approval_id: uuid.UUID
+    input_text: Literal["ModelForge runtime compatibility probe"] = (
+        "ModelForge runtime compatibility probe"
+    )
+
+
+class RuntimeProbeResponse(RuntimeModel):
+    id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    compatibility_assessment_id: uuid.UUID
+    execution_approval_id: uuid.UUID
+    status: ProbeStatus
+    phase: str | None
+    attempt_count: int
+    cancel_requested: bool
+    load_result: dict[str, Any]
+    health_result: dict[str, Any]
+    inference_result: dict[str, Any]
+    unload_result: dict[str, Any]
+    measured_resources: dict[str, Any]
+    runtime_facts: dict[str, Any]
+    environment_fingerprint: str
+    failure_code: str | None
+    failure_message: str | None
+    logs_reference: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+
+class DeploymentCandidateResponse(RuntimeModel):
+    id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    compatibility_assessment_id: uuid.UUID
+    runtime_probe_id: uuid.UUID
+    channel: Literal["lab"]
+    status: Literal["lab_ready"]
+    production: Literal[False]
+    health_contract: dict[str, Any]
+    measured_resources: dict[str, Any]
+    created_at: datetime
+
+
+class AgentRuntimeProbeLease(RuntimeModel):
+    probe_id: uuid.UUID
+    lease_token: str
+    lease_expires_at: datetime
+    artifact_set_id: uuid.UUID
+    revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
+    artifact_root: str
+    artifact_relative_path: str
+    expected_manifest: dict[str, Any]
+    runtime_profile: dict[str, Any]
+    runtime_environment: dict[str, Any]
+    probe_input: Literal["ModelForge runtime compatibility probe"]
+
+
+class AgentRuntimeProbeProgress(RuntimeModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    status: Literal["preparing", "loading", "healthchecking", "ready", "unloading"]
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class AgentRuntimeProbeControl(RuntimeModel):
+    accepted: bool
+    cancel_requested: bool
+    lease_expires_at: datetime
+
+
+class AgentRuntimeProbeComplete(RuntimeModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    load_result: dict[str, Any]
+    health_result: dict[str, Any]
+    inference_result: dict[str, Any]
+    unload_result: dict[str, Any]
+    measured_resources: dict[str, Any]
+    runtime_facts: dict[str, Any]
+    environment_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
+
+
+class AgentRuntimeProbeFailure(RuntimeModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    failure_code: str = Field(min_length=1, max_length=64)
+    failure_message: str = Field(min_length=1, max_length=2000)
+    details: dict[str, Any] = Field(default_factory=dict)

backend/src/modelforge_api/domain/schemas.py

@@ -0,0 +1,172 @@
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .contracts import (
+    CapabilityCategory,
+    CapabilityContractManifest,
+    CapabilityStability,
+    EvaluationType,
+    ProjectBindingManifest,
+    ResourceClass,
+)
+from .enums import (
+    HealthStatus,
+    ModelLifecycle,
+    VerificationStatus,
+)
+
+
+class HealthResponse(BaseModel):
+    status: str = "ok"
+    service: str = "modelforge-api"
+    version: str
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+
+class ReadinessResponse(BaseModel):
+    status: HealthStatus
+    checks: dict[str, HealthStatus]
+    version: str
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+
+class SystemMetadata(BaseModel):
+    name: str
+    version: str
+    environment: str
+    api_version: str = "v1"
+    release_channel: str = "stable"
+    production_inference_available: bool = False
+
+
+class ReleaseCompatibility(BaseModel):
+    """What this build will and will not talk to, stated rather than implied."""
+
+    schema_revision: str
+    supported_schema_revisions: list[str]
+    agent_protocol_version: int
+    supported_agent_protocol_versions: list[int]
+    minimum_upgrade_source: str
+    minimum_postgres_major: int
+
+
+class ReleaseInfo(BaseModel):
+    """Build identity for the running process.
+
+    Deliberately free of anything sensitive: no paths, no configuration values, no credentials —
+    only what an operator needs to answer "which build is this, and what does it support?".
+    """
+
+    name: str
+    version: str
+    release_channel: str
+    source_commit: str | None = None
+    built_at: str | None = None
+    image_digest: str | None = None
+    compatibility: ReleaseCompatibility
+
+
+class ErrorDetail(BaseModel):
+    code: str
+    message: str
+    correlation_id: str
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class ErrorEnvelope(BaseModel):
+    error: ErrorDetail
+
+
+class GPUInfo(BaseModel):
+    index: int
+    name: str
+    uuid: str | None = None
+    memory_total_mb: int | None = None
+    memory_used_mb: int | None = None
+    utilization_gpu_percent: int | None = None
+    utilization_memory_percent: int | None = None
+    temperature_c: int | None = None
+    power_draw_w: float | None = None
+    telemetry_available: bool = True
+    error: str | None = None
+
+
+class CapabilityContractResponse(BaseModel):
+    key: str
+    version: int
+    description: str
+    contract: CapabilityContractManifest
+    stable_deployment: dict[str, Any] | None = None
+
+
+class CapabilityEstateResponse(BaseModel):
+    capability: str
+    version: int
+    category: CapabilityCategory
+    purpose: str
+    declared_stability: CapabilityStability
+    operational_state: str
+    current_deployment_id: uuid.UUID | None = None
+    model: str | None = None
+    revision: str | None = None
+    runtime: str | None = None
+    node: str | None = None
+    resource_class: ResourceClass
+    measured_required_vram_bytes: int | None = None
+    consumers: list[str]
+    privacy_class: str
+    evaluation_type: EvaluationType
+    evaluation_state: str
+
+
+class InstallationDependencyResponse(BaseModel):
+    capability: str
+    version: int
+    deployment_id: uuid.UUID
+    channel: str
+    production: bool
+    project_consumers: list[str]
+    active_project_consumers: list[str]
+    project_fit_evidence_ids: list[uuid.UUID]
+    evaluation_run_ids: list[uuid.UUID]
+    last_used_at: datetime | None
+
+
+class ModelInstallationRationaleResponse(BaseModel):
+    model_id: uuid.UUID
+    display_name: str
+    upstream_source: str
+    installed: bool
+    installed_bytes: int
+    dependencies: list[InstallationDependencyResponse]
+    can_delete: bool
+    deletion_blockers: list[str]
+
+
+class CandidateSummary(BaseModel):
+    id: str
+    display_name: str
+    source: str
+    intended_capabilities: list[str]
+    proposed_role: str
+    preferred_runtime: str | None = None
+    lifecycle: ModelLifecycle = ModelLifecycle.CANDIDATE
+    verification_status: VerificationStatus = VerificationStatus.UNVERIFIED
+    deployment_status: str = "not_deployed"
+
+
+class ProjectBindingResponse(BaseModel):
+    capability: str
+    contract_version: int
+    binding: ProjectBindingManifest
+
+
+class ProjectResponse(BaseModel):
+    id: str
+    name: str
+    description: str
+    bindings: list[ProjectBindingResponse]
+    notes: list[str] = Field(default_factory=list)

backend/src/modelforge_api/domain/serving.py

@@ -0,0 +1,764 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import math
+import re
+import uuid
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+CAPABILITY_VERSION_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+@[1-9][0-9]*$")
+ServingOperation = Literal["load", "invoke", "health", "drain", "unload"]
+
+
+class ServingModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class SupplyChainReview(ServingModel):
+    exact_revision_reviewed: Literal[True]
+    artifact_hashes_reviewed: Literal[True]
+    safetensors_only: Literal[True]
+    remote_code_required: Literal[False]
+    pickle_present: Literal[False]
+    scanner_evidence_reviewed: Literal[True]
+    provenance_complete: Literal[True]
+    runtime_offline_reviewed: Literal[True]
+    dependency_provenance_reviewed: Literal[True]
+    license_reviewed: Literal[True]
+    license_identifier: str = Field(min_length=1, max_length=255)
+
+
+class ProductionApprovalCreate(ServingModel):
+    capability: Literal["rag.embedding"] = "rag.embedding"
+    contract_version: Literal[1] = 1
+    approved_by: str = Field(min_length=1, max_length=255)
+    reason: str = Field(min_length=16, max_length=4000)
+    supply_chain_review: SupplyChainReview
+    deployment_config: dict[str, Any] = Field(default_factory=dict)
+
+
+class ProductionApprovalResponse(ServingModel):
+    id: uuid.UUID
+    deployment_candidate_id: uuid.UUID
+    capability_contract_id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    deployment_config: dict[str, Any]
+    supply_chain_evidence: dict[str, Any]
+    evidence_fingerprint: str
+    status: str
+    approved_by: str
+    reason: str
+    approved_at: datetime
+    revoked_at: datetime | None
+    stale: bool = False
+
+
+class CapabilityPromotionCreate(ServingModel):
+    production_approval_id: uuid.UUID
+    residency_policy: Literal["always_warm", "keep_warm", "load_on_demand"] = "keep_warm"
+    keep_warm_seconds: int = Field(default=900, ge=5, le=86400)
+    max_concurrency: Literal[1] = 1
+    max_queue_depth: int = Field(default=16, ge=1, le=1024)
+    routing_weight: int = Field(default=100, ge=0, le=100)
+    rollback_policy: dict[str, Any] = Field(
+        default_factory=lambda: {"mode": "drain_to_unavailable", "previous_deployment_id": None}
+    )
+
+
+class EmbeddingSpaceResponse(ServingModel):
+    id: uuid.UUID
+    identity_digest: str
+    dimension: int
+    normalized: bool
+    migration_class: Literal["requires_reindex"]
+    identity_facts: dict[str, Any]
+    created_at: datetime
+
+
+class ResourceEnvelopeResponse(ServingModel):
+    id: uuid.UUID
+    runtime_probe_id: uuid.UUID
+    accelerator_kind: str
+    accelerator_uuid: str
+    environment_fingerprint: str
+    concurrency: int
+    batch_size: int
+    max_sequence_length: int
+    baseline_vram_bytes: int
+    resident_vram_bytes: int
+    peak_vram_bytes: int
+    required_vram_bytes: int
+    cold_load_time_ms: float
+    inference_latency_ms: float
+    stale: bool
+    stale_reason: str | None
+
+
+class ResidencyResponse(ServingModel):
+    id: uuid.UUID
+    state: str
+    worker_instance_id: str | None
+    load_count: int
+    active_requests: int
+    measured_resident_vram_bytes: int
+    external_baseline_vram_bytes: int
+    health: dict[str, Any]
+    resident_since: datetime | None
+    last_used_at: datetime | None
+    failure_code: str | None
+    failure_message: str | None
+    generation: int = 1
+    transition_reason: str | None = None
+
+
+class CapabilityDeploymentResponse(ServingModel):
+    id: uuid.UUID
+    capability: str
+    contract_version: int
+    deployment_candidate_id: uuid.UUID
+    production_approval_id: uuid.UUID | None
+    execution_approval_id: uuid.UUID | None
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    compute_node_id: uuid.UUID
+    accelerator_id: uuid.UUID
+    embedding_space: EmbeddingSpaceResponse | None
+    resource_envelope: ResourceEnvelopeResponse
+    residency: ResidencyResponse
+    channel: str
+    status: str
+    production: bool
+    health_status: str
+    routing_weight: int
+    fallback_policy: dict[str, Any]
+    residency_policy: str
+    keep_warm_seconds: int
+    max_concurrency: int
+    max_queue_depth: int
+    config_fingerprint: str
+    provenance: dict[str, Any]
+    rollback_policy: dict[str, Any]
+    promoted_at: datetime | None
+    created_at: datetime
+
+
+class ServiceClientCreate(ServingModel):
+    name: str = Field(min_length=1, max_length=255, pattern=r"^[a-z][a-z0-9-]*$")
+    allowed_capabilities: list[str] = Field(min_length=1, max_length=32)
+    requests_per_minute: int = Field(default=60, ge=1, le=10000)
+    max_concurrent_requests: int = Field(default=1, ge=1, le=64)
+    workload_priority: Literal["production", "interactive", "background", "benchmark"] = (
+        "production"
+    )
+    project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]*$")
+    integration_environment: Literal["production", "shadow", "evaluation", "lab"] | None = None
+    purpose: str | None = Field(default=None, min_length=3, max_length=500)
+    credential_expires_at: datetime | None = None
+
+    @field_validator("allowed_capabilities")
+    @classmethod
+    def validate_capability_scopes(cls, value: list[str]) -> list[str]:
+        if len(set(value)) != len(value) or any(
+            not CAPABILITY_VERSION_KEY.fullmatch(v) for v in value
+        ):
+            raise ValueError("capability scopes must be unique versioned capability keys")
+        return value
+
+    @model_validator(mode="after")
+    def validate_project_scope(self) -> ServiceClientCreate:
+        project_fields = (self.project_key, self.integration_environment, self.purpose)
+        if any(value is not None for value in project_fields) and not all(
+            value is not None for value in project_fields
+        ):
+            raise ValueError("project_key, integration_environment and purpose are required together")
+        if self.project_key is not None and len(self.allowed_capabilities) != 1:
+            raise ValueError("a project service identity must bind exactly one capability")
+        return self
+
+
+class ServiceClientResponse(ServingModel):
+    id: uuid.UUID
+    name: str
+    status: str
+    allowed_capabilities: list[str]
+    requests_per_minute: int
+    max_concurrent_requests: int
+    workload_priority: str
+    project_key: str | None
+    project_binding_id: uuid.UUID | None
+    integration_environment: str | None
+    purpose: str | None
+    credential_prefix: str | None
+    credential_created_at: datetime | None
+    credential_expires_at: datetime | None
+    credential_revoked_at: datetime | None
+    last_used_at: datetime | None
+    created_at: datetime
+
+
+class ServiceClientCreated(ServiceClientResponse):
+    credential: str
+
+
+class ProjectFitEvidenceCreate(ServingModel):
+    project_key: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
+    capability: str = Field(pattern=r"^[a-z][a-z0-9.]*@[1-9][0-9]*$")
+    environment: Literal["production", "shadow", "evaluation", "lab"]
+    recommendation: Literal[
+        "PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"
+    ]
+    evidence_class: Literal[
+        "OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"
+    ]
+    engineering_integration: Literal["PASS", "INCOMPLETE", "BLOCKED"]
+    production_validation: Literal[
+        "DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"
+    ]
+    production_action: Literal["NONE"] = "NONE"
+    case_count: int = Field(ge=0, le=1_000_000)
+    metric_values: dict[str, float | int | str | bool | None]
+    critical_errors: int = Field(ge=0)
+    blockers: list[str] = Field(default_factory=list, max_length=100)
+    evidence: dict[str, Any] = Field(default_factory=dict)
+    capability_deployment_id: uuid.UUID | None = None
+
+
+class ProjectFitEvidenceResponse(ServingModel):
+    id: uuid.UUID
+    project_key: str
+    capability: str
+    environment: str
+    recommendation: str
+    evidence_class: str
+    engineering_integration: str
+    production_validation: str
+    production_action: str
+    case_count: int
+    metric_values: dict[str, Any]
+    critical_errors: int
+    blockers: list[str]
+    evidence_digest: str
+    evidence: dict[str, Any]
+    capability_deployment_id: uuid.UUID | None
+    created_at: datetime
+
+
+class ProjectIntegrationUsage(ServingModel):
+    request_volume: int
+    successful_requests: int
+    error_count: int
+    last_used_at: datetime | None
+    latency_p50_ms: float | None
+    latency_p95_ms: float | None
+
+
+class ProjectIntegrationResponse(ServingModel):
+    project_key: str
+    project_name: str
+    capability: str
+    environment: str
+    state: str
+    purpose: str
+    client_id: uuid.UUID
+    client_name: str
+    deployment_id: uuid.UUID | None
+    resource_impact_bytes: int | None
+    usage: ProjectIntegrationUsage
+    project_fit: ProjectFitEvidenceResponse | None
+
+
+class EmbeddingInvokeRequest(ServingModel):
+    input: str | list[str]
+    input_type: Literal["raw", "query", "document"] = "raw"
+
+    @model_validator(mode="after")
+    def validate_input(self) -> EmbeddingInvokeRequest:
+        values = [self.input] if isinstance(self.input, str) else self.input
+        if not values or any(not item.strip() for item in values):
+            raise ValueError("input must contain at least one non-empty string")
+        return self
+
+    def inputs(self) -> list[str]:
+        return [self.input] if isinstance(self.input, str) else self.input
+
+
+class CapabilityExperimentCreate(ServingModel):
+    route_key: str = Field(min_length=3, max_length=128, pattern=r"^[a-z][a-z0-9-]+$")
+    execution_approval_id: uuid.UUID
+    capability: Literal[
+        "rag.embedding",
+        "rag.reranking",
+        "document.ocr",
+        "vision.embedding",
+        "speech.transcription",
+    ] = "rag.embedding"
+    purpose: str = Field(min_length=16, max_length=2000)
+    expected_dimension: int = Field(default=1024, ge=32, le=8192)
+    residency_policy: Literal["keep_warm", "load_on_demand"] = "keep_warm"
+    keep_warm_seconds: int = Field(default=300, ge=5, le=86400)
+    max_queue_depth: int = Field(default=16, ge=1, le=1024)
+
+
+class CapabilityExperimentResponse(ServingModel):
+    id: uuid.UUID
+    route_key: str
+    capability: str
+    contract_version: int
+    capability_deployment_id: uuid.UUID
+    status: str
+    purpose: str
+    evidence: dict[str, Any]
+    deployment: CapabilityDeploymentResponse
+    created_at: datetime
+
+
+class GatewayTiming(ServingModel):
+    validation_ms: float = 0.0
+    resolution_ms: float = 0.0
+    scheduling_ms: float = 0.0
+    payload_ms: float = 0.0
+    dispatch_ms: float = 0.0
+    worker_preprocess_ms: float = 0.0
+    worker_serialize_ms: float = 0.0
+    worker_total_ms: float = 0.0
+    completion_transport_ms: float = 0.0
+    result_ms: float = 0.0
+    gateway_ms: float
+    # Backwards-compatible alias for the scheduler/lease wait used by M5/M6.
+    queue_ms: float
+    load_ms: float
+    inference_ms: float
+    total_ms: float
+
+
+class GatewayExecution(ServingModel):
+    cold: bool
+    node: str
+    residency: str
+    load_count: int
+    timings: GatewayTiming
+
+
+class GatewayUsage(ServingModel):
+    input_count: int
+    input_tokens: int
+
+
+class EmbeddingInvokeResponse(ServingModel):
+    capability: Literal["rag.embedding@1"] = "rag.embedding@1"
+    dimension: int = Field(ge=1, le=65_536)
+    normalized: Literal[True] = True
+    embedding_space_id: uuid.UUID
+    data: list[list[float]]
+    request_id: uuid.UUID
+    execution: GatewayExecution
+    usage: GatewayUsage
+
+
+class StableEmbeddingInvokeResponse(EmbeddingInvokeResponse):
+    """Public stable contract; lab experiment routes may use another dimension."""
+
+    dimension: Literal[1024] = 1024
+
+
+class RerankDocument(ServingModel):
+    id: str = Field(min_length=1, max_length=255)
+    text: str = Field(min_length=1, max_length=16_384)
+
+
+class RerankingInvokeRequest(ServingModel):
+    query: str = Field(min_length=1, max_length=4_000)
+    documents: list[RerankDocument] = Field(min_length=1, max_length=40)
+    top_n: int = Field(default=10, ge=1, le=10)
+
+    @model_validator(mode="after")
+    def validate_reranking_input(self) -> RerankingInvokeRequest:
+        if not self.query.strip() or any(not document.text.strip() for document in self.documents):
+            raise ValueError("query and document text must be non-empty")
+        if len({document.id for document in self.documents}) != len(self.documents):
+            raise ValueError("document ids must be unique")
+        if self.top_n > len(self.documents):
+            raise ValueError("top_n cannot exceed the document count")
+        if len(self.query) + sum(len(document.text) for document in self.documents) > 262_144:
+            raise ValueError("rerank request is too large")
+        return self
+
+
+class RerankingResult(ServingModel):
+    id: str
+    score: float
+    rank: int = Field(ge=1, le=10)
+
+    @field_validator("score")
+    @classmethod
+    def finite_score(cls, value: float) -> float:
+        if not math.isfinite(value):
+            raise ValueError("reranking score must be finite")
+        return value
+
+
+class RerankingInvokeResponse(ServingModel):
+    capability: Literal["rag.reranking@1"] = "rag.reranking@1"
+    results: list[RerankingResult]
+    request_id: uuid.UUID
+    execution: GatewayExecution
+    usage: GatewayUsage
+
+
+def _bounded_base64(value: str, maximum_bytes: int) -> str:
+    try:
+        decoded = base64.b64decode(value, validate=True)
+    except (binascii.Error, ValueError) as exc:
+        raise ValueError("content must be canonical base64") from exc
+    if not decoded or len(decoded) > maximum_bytes:
+        raise ValueError(f"decoded content must be between 1 and {maximum_bytes} bytes")
+    return value
+
+
+class OCRInvokeRequest(ServingModel):
+    content_base64: str
+    media_type: Literal["image/png", "image/jpeg"]
+    language_hint: Literal["nl", "en", "auto"] = "auto"
+
+    @field_validator("content_base64")
+    @classmethod
+    def bounded_content(cls, value: str) -> str:
+        return _bounded_base64(value, 8 * 1024 * 1024)
+
+
+class OCRBlock(ServingModel):
+    text: str
+    order: int = Field(ge=0)
+    bbox: list[int] | None = Field(default=None, min_length=4, max_length=4)
+    confidence: float | None = Field(default=None, ge=0, le=1)
+
+
+class OCRPage(ServingModel):
+    page: Literal[1] = 1
+    width: int = Field(ge=1, le=4096)
+    height: int = Field(ge=1, le=4096)
+    blocks: list[OCRBlock]
+
+
+class OCRInvokeResponse(ServingModel):
+    capability: Literal["document.ocr@1"] = "document.ocr@1"
+    text: str
+    pages: list[OCRPage] = Field(min_length=1, max_length=1)
+    confidence: float | None = Field(default=None, ge=0, le=1)
+    request_id: uuid.UUID
+    execution: GatewayExecution
+
+
+class VisionEmbeddingItem(ServingModel):
+    image_base64: str | None = None
+    media_type: Literal["image/png", "image/jpeg"] | None = None
+    text: str | None = Field(default=None, min_length=1, max_length=4096)
+
+    @model_validator(mode="after")
+    def exactly_one_modality(self) -> VisionEmbeddingItem:
+        image = self.image_base64 is not None
+        text = self.text is not None
+        if image == text or image != (self.media_type is not None):
+            raise ValueError("each item must contain exactly one typed image or text")
+        if self.image_base64 is not None:
+            _bounded_base64(self.image_base64, 8 * 1024 * 1024)
+        return self
+
+
+class VisionEmbeddingInvokeRequest(ServingModel):
+    items: list[VisionEmbeddingItem] = Field(min_length=1, max_length=4)
+
+    @model_validator(mode="after")
+    def bounded_total(self) -> VisionEmbeddingInvokeRequest:
+        encoded = sum(len(item.image_base64 or "") for item in self.items)
+        if encoded > 11_184_812:
+            raise ValueError("combined image payload exceeds 8 MiB")
+        return self
+
+
+class VisionEmbeddingInvokeResponse(ServingModel):
+    capability: Literal["vision.embedding@1"] = "vision.embedding@1"
+    dimension: int = Field(ge=32, le=8192)
+    normalized: Literal[True] = True
+    embedding_space_id: uuid.UUID
+    data: list[list[float]] = Field(min_length=1, max_length=4)
+    request_id: uuid.UUID
+    execution: GatewayExecution
+
+
+class SpeechTranscriptionInvokeRequest(ServingModel):
+    audio_base64: str
+    media_type: Literal["audio/wav"] = "audio/wav"
+    language: Literal["nl", "en", "fr", "auto"] = "auto"
+
+    @field_validator("audio_base64")
+    @classmethod
+    def bounded_audio(cls, value: str) -> str:
+        return _bounded_base64(value, 16 * 1024 * 1024)
+
+
+class TranscriptionSegment(ServingModel):
+    start_seconds: float = Field(ge=0)
+    end_seconds: float = Field(ge=0)
+    text: str
+
+
+class SpeechTranscriptionInvokeResponse(ServingModel):
+    capability: Literal["speech.transcription@1"] = "speech.transcription@1"
+    text: str
+    language: str
+    duration_seconds: float = Field(gt=0, le=120)
+    segments: list[TranscriptionSegment] = Field(default_factory=list)
+    request_id: uuid.UUID
+    execution: GatewayExecution
+
+
+class OpenAIEmbeddingRequest(ServingModel):
+    model: Literal["rag.embedding", "rag.embedding@1"]
+    input: str | list[str]
+
+
+class OpenAIEmbeddingItem(ServingModel):
+    object: Literal["embedding"] = "embedding"
+    index: int
+    embedding: list[float]
+
+
+class OpenAIUsage(ServingModel):
+    prompt_tokens: int
+    total_tokens: int
+
+
+class OpenAIEmbeddingResponse(ServingModel):
+    object: Literal["list"] = "list"
+    data: list[OpenAIEmbeddingItem]
+    model: Literal["rag.embedding"] = "rag.embedding"
+    usage: OpenAIUsage
+
+
+class SchedulerBudgetResponse(ServingModel):
+    compute_node_id: uuid.UUID
+    node_name: str
+    accelerator_id: uuid.UUID
+    accelerator_uuid: str
+    accelerator_name: str
+    total_vram_bytes: int
+    observed_used_vram_bytes: int
+    external_vram_bytes: int
+    resident_vram_bytes: int
+    leased_vram_bytes: int
+    safety_reserve_bytes: int
+    schedulable_free_vram_bytes: int
+    pressure: bool
+    pressure_state: Literal["NORMAL", "ELEVATED", "HIGH", "CRITICAL"] = "NORMAL"
+    attribution_confidence: Literal["KNOWN", "ESTIMATED", "UNKNOWN"] = "UNKNOWN"
+    invariant_delta_bytes: int = 0
+    policy_revision: str = "m10-v1"
+    observed_at: datetime | None
+
+
+class PlacementPlanRequest(ServingModel):
+    priority: Literal["production", "interactive", "background", "lab"] = "interactive"
+    deadline_remaining_ms: float | None = Field(default=None, gt=0, le=900_000)
+
+
+class PlacementEvictionResponse(ServingModel):
+    deployment_id: uuid.UUID
+    capability: str
+    expected_reclaimed_bytes: int
+    reason: str
+
+
+class PlacementPlanResponse(ServingModel):
+    id: uuid.UUID | None = None
+    deployment_id: uuid.UUID
+    capability: str
+    node_id: uuid.UUID
+    accelerator_id: uuid.UUID
+    policy_revision: str
+    verdict: Literal[
+        "ADMIT",
+        "ADMIT_AFTER_EVICTION",
+        "QUEUE",
+        "REJECT_CAPACITY",
+        "REJECT_HEALTH",
+        "REJECT_POLICY",
+    ]
+    reason_codes: list[str]
+    required_vram_bytes: int
+    headroom_before_bytes: int
+    headroom_after_bytes: int
+    expected_cold_load_ms: float
+    evictions: list[PlacementEvictionResponse] = Field(default_factory=list)
+    decision_fingerprint: str
+    dry_run: bool
+    created_at: datetime
+
+
+class SchedulerPolicyResponse(ServingModel):
+    revision: str
+    active: bool
+    configuration: dict[str, Any]
+    created_at: datetime
+
+
+class SchedulerMetricsResponse(ServingModel):
+    counters: dict[str, int]
+    gauges: dict[str, int | str]
+    queue_seconds: dict[str, float]
+
+
+class SchedulerPolicyUpdate(ServingModel):
+    lab_paused: bool
+
+
+class CoResidencyEvidenceResponse(ServingModel):
+    left_deployment_id: uuid.UUID
+    left_capability: str
+    right_deployment_id: uuid.UUID
+    right_capability: str
+    left_alone_bytes: int
+    right_alone_bytes: int
+    expected_combined_bytes: int
+    measured_combined_bytes: int | None
+    status: Literal["PROVEN_SAFE", "EXPECTED_SAFE", "NOT_SAFE", "UNKNOWN"]
+    evidence: dict[str, Any] = Field(default_factory=dict)
+    measured_at: datetime | None
+
+
+class CoResidencyEvidenceCreate(ServingModel):
+    left_deployment_id: uuid.UUID
+    right_deployment_id: uuid.UUID
+
+    @model_validator(mode="after")
+    def distinct_deployments(self) -> CoResidencyEvidenceCreate:
+        if self.left_deployment_id == self.right_deployment_id:
+            raise ValueError("co-residency evidence requires two distinct deployments")
+        return self
+
+
+class ResidencyPolicyUpdate(ServingModel):
+    residency_policy: Literal["always_warm", "keep_warm", "load_on_demand", "lab_only"]
+    keep_warm_seconds: int = Field(default=900, ge=5, le=86400)
+
+
+class GatewayRequestResponse(ServingModel):
+    request_id: uuid.UUID
+    capability: str
+    client: str | None
+    status: str
+    cold: bool | None
+    queue_time_ms: float | None
+    load_time_ms: float | None
+    inference_time_ms: float | None
+    total_latency_ms: float | None
+    failure_code: str | None
+    latency_breakdown: dict[str, float] = Field(default_factory=dict)
+    created_at: datetime
+
+
+class AgentServingJobLease(ServingModel):
+    job_id: uuid.UUID
+    lease_token: str = Field(min_length=32, max_length=512)
+    lease_expires_at: datetime
+    operation: ServingOperation
+    deployment_id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
+    artifact_root: str
+    artifact_relative_path: str
+    expected_manifest: dict[str, Any]
+    runtime_profile: dict[str, Any]
+    runtime_environment: dict[str, Any]
+    capability: Literal[
+        "rag.embedding",
+        "rag.reranking",
+        "document.ocr",
+        "vision.embedding",
+        "speech.transcription",
+    ] = "rag.embedding"
+    embedding_space_id: uuid.UUID | None = None
+    expected_dimension: int | None = Field(default=None, ge=32, le=8192)
+    normalize: Literal[True] | None = True
+    input: list[str] | None = None
+    input_type: Literal["raw", "query", "document"] = "raw"
+    rerank_query: str | None = None
+    rerank_documents: list[RerankDocument] | None = None
+    top_n: int | None = Field(default=None, ge=1, le=10)
+    modality_payload: dict[str, Any] | None = None
+    request_id: uuid.UUID | None = None
+
+    @model_validator(mode="after")
+    def validate_capability_payload(self) -> AgentServingJobLease:
+        if self.operation != "invoke":
+            return self
+        if self.capability == "rag.embedding":
+            if self.embedding_space_id is None or self.expected_dimension is None:
+                raise ValueError("embedding invoke requires embedding identity")
+            if self.input is None or self.rerank_query is not None:
+                raise ValueError("embedding invoke payload is invalid")
+        elif self.capability == "rag.reranking" and (
+            self.rerank_query is None
+            or self.rerank_documents is None
+            or self.top_n is None
+            or self.input is not None
+        ):
+            raise ValueError("reranking invoke payload is invalid")
+        elif self.capability not in {"rag.embedding", "rag.reranking"} and (
+            self.modality_payload is None or self.input is not None or self.rerank_query is not None
+        ):
+            raise ValueError("modality invoke requires a typed modality payload")
+        return self
+
+
+class AgentServingJobComplete(ServingModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    worker_instance_id: str = Field(min_length=1, max_length=255)
+    state: str
+    result: dict[str, Any] = Field(default_factory=dict)
+    metrics: dict[str, Any] = Field(default_factory=dict)
+    health: dict[str, Any] = Field(default_factory=dict)
+    runtime_facts: dict[str, Any] = Field(default_factory=dict)
+
+
+class AgentServingJobFailure(ServingModel):
+    lease_token: str = Field(min_length=32, max_length=512)
+    worker_instance_id: str = Field(min_length=1, max_length=255)
+    failure_code: str = Field(min_length=1, max_length=64)
+    failure_message: str = Field(min_length=1, max_length=2000)
+    state: str = "failed"
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class AgentResidentState(ServingModel):
+    deployment_id: uuid.UUID
+    artifact_set_id: uuid.UUID
+    runtime_profile_id: uuid.UUID
+    runtime_environment_fingerprint: str = Field(pattern=r"^[a-f0-9]{64}$")
+    revision_sha: str = Field(pattern=r"^[a-f0-9]{40,64}$")
+    state: Literal["loading", "warm", "busy", "draining", "unloading", "failed"]
+    load_count: int = Field(ge=1)
+    resident_vram_bytes: int = Field(ge=0)
+    health: dict[str, Any] = Field(default_factory=dict)
+
+
+class AgentServingStateReport(ServingModel):
+    worker_instance_id: str = Field(min_length=1, max_length=255)
+    deployment_id: uuid.UUID | None
+    state: Literal["cold", "loading", "warm", "busy", "draining", "unloading", "failed"]
+    load_count: int = Field(ge=0)
+    health: dict[str, Any] = Field(default_factory=dict)
+    observed_at: datetime
+    generation: int = Field(default=1, ge=1)
+    residencies: list[AgentResidentState] = Field(default_factory=list, max_length=32)
+
+
+class AgentServingStateAck(ServingModel):
+    accepted: bool = True
+    desired_operation: Literal["none", "unload"] = "none"

backend/src/modelforge_api/hardware/__init__.py

@@ -0,0 +1,3 @@
+from .collectors import NvidiaNvmlCollector, SystemHostCollector, build_nvml_collector
+
+__all__ = ["NvidiaNvmlCollector", "SystemHostCollector", "build_nvml_collector"]

backend/src/modelforge_api/hardware/collectors.py

@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+import platform
+import socket
+import sys
+import uuid
+from collections.abc import Callable
+from contextlib import suppress
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, TypeVar
+
+import psutil  # type: ignore[import-untyped]
+
+from modelforge_api import __version__
+from modelforge_api.domain.enums import Availability
+from modelforge_api.domain.hardware import (
+    AcceleratorInventory,
+    AcceleratorTelemetry,
+    HostInventory,
+    NvidiaCollection,
+    ObservedValue,
+    StorageObservation,
+)
+
+T = TypeVar("T")
+
+
+class NodeIdentityProvider:
+    """Stable node identity: explicit override, OS machine ID, then persisted UUID."""
+
+    def __init__(
+        self,
+        identity_file: Path,
+        explicit_identity: str | None = None,
+        force_persisted: bool = False,
+    ) -> None:
+        self.identity_file = identity_file
+        self.explicit_identity = explicit_identity
+        self.force_persisted = force_persisted
+
+    def resolve(self) -> tuple[str, str]:
+        if self.explicit_identity:
+            return self.explicit_identity, "configured"
+        if not self.force_persisted:
+            system_id = self._system_machine_id()
+            if system_id:
+                return str(uuid.uuid5(uuid.NAMESPACE_OID, system_id)), "os_machine_id"
+        try:
+            if self.identity_file.exists():
+                return self.identity_file.read_text(encoding="utf-8").strip(), "persisted_uuid"
+            value = str(uuid.uuid4())
+            self.identity_file.parent.mkdir(parents=True, exist_ok=True)
+            self.identity_file.write_text(value, encoding="utf-8")
+            return value, "persisted_uuid"
+        except OSError as exc:
+            raise RuntimeError("unable to establish stable node identity") from exc
+
+    @staticmethod
+    def _system_machine_id() -> str | None:
+        # `sys.platform`, not `os.name`: a type checker narrows on the former and not on the latter,
+        # so under `os.name` this block was analysed on Linux too — where winreg has no attributes.
+        # The images run on Linux, so the platform the release actually ships on was the one the
+        # type check could not see. They mean the same thing at runtime.
+        if sys.platform == "win32":
+            try:
+                import winreg
+
+                with winreg.OpenKey(
+                    winreg.HKEY_LOCAL_MACHINE,
+                    r"SOFTWARE\Microsoft\Cryptography",
+                ) as key:
+                    return str(winreg.QueryValueEx(key, "MachineGuid")[0])
+            except (OSError, ImportError):
+                return None
+        for path in (Path("/etc/machine-id"), Path("/var/lib/dbus/machine-id")):
+            try:
+                value = path.read_text(encoding="utf-8").strip()
+                if value:
+                    return value
+            except OSError:
+                continue
+        return None
+
+
+class SystemHostCollector:
+    def __init__(
+        self,
+        identity_provider: NodeIdentityProvider,
+        storage_paths: dict[str, Path],
+    ) -> None:
+        self.identity_provider = identity_provider
+        self.storage_paths = storage_paths
+
+    @staticmethod
+    def _known_or_unknown(value: T | None, reason: str) -> ObservedValue[T]:
+        return (
+            ObservedValue.known(value)
+            if value is not None
+            else ObservedValue.absent(Availability.UNKNOWN, reason)
+        )
+
+    def collect(self) -> HostInventory:
+        identity_key, identity_source = self.identity_provider.resolve()
+        memory = psutil.virtual_memory()
+        cpu_model = platform.processor().strip() or None
+        physical = psutil.cpu_count(logical=False)
+        logical = psutil.cpu_count(logical=True)
+        storage: list[StorageObservation] = []
+        for purpose, path in self.storage_paths.items():
+            try:
+                usage = psutil.disk_usage(str(path))
+                storage.append(
+                    StorageObservation(
+                        purpose=purpose,
+                        path=str(path.resolve()),
+                        total_bytes=ObservedValue.known(usage.total),
+                        used_bytes=ObservedValue.known(usage.used),
+                        free_bytes=ObservedValue.known(usage.free),
+                    )
+                )
+            except OSError as exc:
+                reason = f"{type(exc).__name__}: path unavailable"
+                unavailable = ObservedValue[int].absent(Availability.UNAVAILABLE, reason)
+                storage.append(
+                    StorageObservation(
+                        purpose=purpose,
+                        path=str(path),
+                        total_bytes=unavailable,
+                        used_bytes=unavailable,
+                        free_bytes=unavailable,
+                    )
+                )
+        release = platform.release()
+        environment = "wsl2" if "microsoft" in release.lower() else "native"
+        return HostInventory(
+            identity_key=identity_key,
+            identity_source=identity_source,
+            hostname=socket.gethostname(),
+            display_name=socket.gethostname(),
+            os_name=platform.system() or "unknown",
+            os_version=self._known_or_unknown(platform.version() or None, "OS version unavailable"),
+            architecture=platform.machine() or "unknown",
+            kernel_version=self._known_or_unknown(release or None, "kernel unavailable"),
+            cpu_model=self._known_or_unknown(cpu_model, "CPU model unavailable"),
+            logical_cpu_count=self._known_or_unknown(logical, "logical CPU count unavailable"),
+            physical_core_count=self._known_or_unknown(physical, "physical core count unavailable"),
+            total_ram_bytes=ObservedValue.known(memory.total),
+            available_ram_bytes=ObservedValue.known(memory.available),
+            agent_version=__version__,
+            storage=storage,
+            metadata={"environment": environment},
+        )
+
+
+class NvidiaNvmlCollector:
+    def __init__(self, api: Any) -> None:
+        self.api = api
+        self.nvml_error = getattr(api, "NVMLError", Exception)
+        self.not_supported = getattr(api, "NVMLError_NotSupported", ())
+
+    def _optional(
+        self, call: Callable[[], T], transform: Callable[[T], Any] | None = None
+    ) -> ObservedValue[Any]:
+        try:
+            value = call()
+            return ObservedValue.known(transform(value) if transform else value)
+        except self.not_supported:
+            return ObservedValue.absent(Availability.UNSUPPORTED, "NVML metric not supported")
+        except self.nvml_error as exc:
+            return ObservedValue.absent(Availability.TEMPORARILY_FAILED, type(exc).__name__)
+
+    @staticmethod
+    def _text(value: Any) -> str:
+        return value.decode("utf-8", errors="replace") if isinstance(value, bytes) else str(value)
+
+    @staticmethod
+    def _cuda_version(value: int) -> str:
+        return f"{value // 1000}.{(value % 1000) // 10}"
+
+    def _architecture(self, handle: Any) -> ObservedValue[str]:
+        def transform(value: Any) -> str:
+            for name in dir(self.api):
+                if name.startswith("NVML_DEVICE_ARCH_") and getattr(self.api, name) == value:
+                    return name.removeprefix("NVML_DEVICE_ARCH_").lower()
+            return f"nvml_arch_{value}"
+
+        function = getattr(self.api, "nvmlDeviceGetArchitecture", None)
+        if function is None:
+            return ObservedValue.absent(Availability.UNSUPPORTED, "architecture API unavailable")
+        return self._optional(lambda: function(handle), transform)
+
+    def collect(self) -> NvidiaCollection:
+        initialized = False
+        try:
+            self.api.nvmlInit()
+            initialized = True
+            count = self.api.nvmlDeviceGetCount()
+            driver = self._optional(self.api.nvmlSystemGetDriverVersion, self._text)
+            cuda_function = getattr(self.api, "nvmlSystemGetCudaDriverVersion_v2", None) or getattr(
+                self.api, "nvmlSystemGetCudaDriverVersion", None
+            )
+            cuda_driver = (
+                self._optional(cuda_function, self._cuda_version)
+                if cuda_function
+                else ObservedValue.absent(Availability.UNSUPPORTED, "CUDA driver API unavailable")
+            )
+            inventory: list[AcceleratorInventory] = []
+            telemetry: list[AcceleratorTelemetry] = []
+            failures: list[str] = []
+            observed_at = datetime.now(UTC)
+            for index in range(count):
+                try:
+                    handle = self.api.nvmlDeviceGetHandleByIndex(index)
+                    device_uuid = self._text(self.api.nvmlDeviceGetUUID(handle))
+                    name = self._text(self.api.nvmlDeviceGetName(handle))
+                    memory = self.api.nvmlDeviceGetMemoryInfo(handle)
+                    pci = self._optional(
+                        lambda: self.api.nvmlDeviceGetPciInfo(handle),
+                        lambda value: self._text(value.busId),
+                    )
+                    compute = self._optional(
+                        lambda: self.api.nvmlDeviceGetCudaComputeCapability(handle)
+                    )
+                    major = (
+                        ObservedValue.known(int(compute.value[0]))
+                        if compute.availability is Availability.KNOWN and compute.value is not None
+                        else ObservedValue.absent(compute.availability, compute.reason)
+                    )
+                    minor = (
+                        ObservedValue.known(int(compute.value[1]))
+                        if compute.availability is Availability.KNOWN and compute.value is not None
+                        else ObservedValue.absent(compute.availability, compute.reason)
+                    )
+                    mig_function = getattr(self.api, "nvmlDeviceGetMigMode", None)
+                    mig = (
+                        self._optional(lambda: mig_function(handle), lambda value: bool(value[0]))
+                        if mig_function
+                        else ObservedValue.absent(Availability.UNSUPPORTED, "MIG API unavailable")
+                    )
+                    inventory.append(
+                        AcceleratorInventory(
+                            device_index=index,
+                            device_uuid=device_uuid,
+                            pci_bus_id=pci,
+                            name=name,
+                            architecture=self._architecture(handle),
+                            compute_capability_major=major,
+                            compute_capability_minor=minor,
+                            total_vram_bytes=ObservedValue.known(int(memory.total)),
+                            driver_version=driver,
+                            cuda_driver_version=cuda_driver,
+                            mig_mode_current=mig,
+                            inventory_at=observed_at,
+                        )
+                    )
+                    utilization = self._optional(
+                        lambda: self.api.nvmlDeviceGetUtilizationRates(handle)
+                    )
+                    gpu_util = (
+                        ObservedValue.known(int(utilization.value.gpu))
+                        if utilization.availability is Availability.KNOWN
+                        and utilization.value is not None
+                        else ObservedValue.absent(utilization.availability, utilization.reason)
+                    )
+                    mem_util = (
+                        ObservedValue.known(int(utilization.value.memory))
+                        if utilization.availability is Availability.KNOWN
+                        and utilization.value is not None
+                        else ObservedValue.absent(utilization.availability, utilization.reason)
+                    )
+                    telemetry.append(
+                        AcceleratorTelemetry(
+                            device_uuid=device_uuid,
+                            observed_at=observed_at,
+                            used_vram_bytes=ObservedValue.known(int(memory.used)),
+                            free_vram_bytes=ObservedValue.known(int(memory.free)),
+                            gpu_utilization_percent=gpu_util,
+                            memory_utilization_percent=mem_util,
+                            temperature_c=self._optional(
+                                lambda: self.api.nvmlDeviceGetTemperature(
+                                    handle, self.api.NVML_TEMPERATURE_GPU
+                                ),
+                                int,
+                            ),
+                            power_draw_w=self._optional(
+                                lambda: self.api.nvmlDeviceGetPowerUsage(handle),
+                                lambda value: round(value / 1000, 3),
+                            ),
+                            power_limit_w=self._optional(
+                                lambda: self.api.nvmlDeviceGetEnforcedPowerLimit(handle),
+                                lambda value: round(value / 1000, 3),
+                            ),
+                            graphics_clock_mhz=self._optional(
+                                lambda: self.api.nvmlDeviceGetClockInfo(
+                                    handle, self.api.NVML_CLOCK_GRAPHICS
+                                ),
+                                int,
+                            ),
+                            memory_clock_mhz=self._optional(
+                                lambda: self.api.nvmlDeviceGetClockInfo(
+                                    handle, self.api.NVML_CLOCK_MEM
+                                ),
+                                int,
+                            ),
+                            fan_speed_percent=self._optional(
+                                lambda: self.api.nvmlDeviceGetFanSpeed(handle), int
+                            ),
+                            performance_state=self._optional(
+                                lambda: self.api.nvmlDeviceGetPerformanceState(handle),
+                                lambda value: f"P{value}",
+                            ),
+                        )
+                    )
+                except self.nvml_error as exc:
+                    failures.append(f"device {index}: {type(exc).__name__}")
+            availability = Availability.KNOWN if not failures else Availability.TEMPORARILY_FAILED
+            return NvidiaCollection(
+                availability=availability,
+                reason="; ".join(failures)
+                or ("no NVIDIA devices detected" if count == 0 else None),
+                inventory=inventory,
+                telemetry=telemetry,
+                observed_at=observed_at,
+            )
+        except self.nvml_error as exc:
+            return NvidiaCollection(
+                availability=Availability.UNAVAILABLE, reason=type(exc).__name__
+            )
+        finally:
+            if initialized:
+                with suppress(self.nvml_error):
+                    self.api.nvmlShutdown()
+
+
+def build_nvml_collector() -> NvidiaNvmlCollector:
+    import pynvml  # type: ignore[import-untyped]
+
+    return NvidiaNvmlCollector(pynvml)

backend/src/modelforge_api/main.py

@@ -0,0 +1,660 @@
+import asyncio
+import math
+import time
+import uuid
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
+from contextlib import asynccontextmanager, contextmanager, suppress
+from typing import Any, cast
+
+import structlog
+from fastapi import FastAPI, HTTPException, Request, Response
+from fastapi.concurrency import run_in_threadpool
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from modelforge_api import __version__
+from modelforge_api.api.authorization import (
+    AccessBoundary,
+    access_boundary_for_request,
+    authenticate_operator_token,
+    request_body_limit_bytes,
+    required_capability_for_request,
+)
+from modelforge_api.api.request_limits import RequestBodyLimitMiddleware
+from modelforge_api.api.routes import (
+    acquisition,
+    agent,
+    catalog,
+    evaluation,
+    hardware,
+    health,
+    lifecycle,
+    migrations,
+    observability,
+    recovery,
+    registry,
+    runtime,
+    serving,
+)
+from modelforge_api.db import engine, get_session
+from modelforge_api.domain.observability import metrics, route_class
+from modelforge_api.domain.release import build_identity
+from modelforge_api.services.evaluation import EvaluationError
+from modelforge_api.services.hardware_polling import HardwarePollingService
+from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.migration_engine import (
+    MigrationEngineError,
+    MigrationEngineService,
+)
+from modelforge_api.services.node_agent import (
+    AgentProtocolError,
+    NodeAgentService,
+    NodeAuthenticationEvidence,
+)
+from modelforge_api.services.node_decommission import NodeDecommissionError
+from modelforge_api.services.node_liveness import NodeLivenessPollingService
+from modelforge_api.services.observability import ObservabilityError, ObservabilityService
+from modelforge_api.services.observability_polling import ObservabilityPollingService
+from modelforge_api.services.project_registry import sync_project_registry
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.registry import RegistryError, seed_candidate_registry
+from modelforge_api.services.serving import (
+    CapabilityAuthenticationEvidence,
+    ServingError,
+    ServingService,
+)
+from modelforge_api.services.serving_reconciliation import ServingReconciliationService
+from modelforge_api.services.startup_validation import enforce_startup
+from modelforge_api.settings import Settings, get_settings
+
+settings = get_settings()
+structlog.configure(processors=[structlog.processors.JSONRenderer()])
+logger = structlog.get_logger()
+
+
+def interactive_api_enabled_for(environment: str) -> bool:
+    """Keep schema explorers off deployed/test surfaces unless development is explicit."""
+
+    return environment == "development"
+
+
+@asynccontextmanager
+async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
+    # Configuration and schema compatibility are checked before anything else runs. In production a
+    # problem here stops the process; elsewhere it is logged, because finding out at startup rather
+    # than at first use is the point either way.
+    identity = build_identity(
+        source_commit=settings.build_commit,
+        built_at=settings.build_timestamp,
+        image_digest=settings.build_image_digest,
+    )
+    logger.info("modelforge_starting", **identity.as_dict(), environment=settings.env)
+    validation = enforce_startup(settings, engine)
+    for problem in validation.problems:
+        logger.error(
+            "startup_configuration_problem",
+            code=str(problem.code),
+            setting=problem.setting,
+            detail=problem.message,
+        )
+    for problem in validation.warnings:
+        logger.warning(
+            "startup_configuration_warning",
+            code=str(problem.code),
+            setting=problem.setting,
+            detail=problem.message,
+        )
+
+    if settings.lifecycle_reconciliation_enabled:
+        with Session(engine) as session:
+            lifecycle_service = LifecycleService(session)
+            lifecycle_service.ensure_defaults()
+            reconciled = lifecycle_service.reconcile_incomplete()
+            if reconciled:
+                logger.warning("lifecycle_operations_reconciled", count=reconciled)
+    if settings.migration_reconciliation_enabled:
+        with Session(engine) as session:
+            migration_service = MigrationEngineService(session)
+            migration_service.ensure_defaults()
+            pending_migrations = migration_service.pending_reconciliation_count()
+            if pending_migrations:
+                # External state is deliberately never guessed during startup. An operator or
+                # adapter must report observed truth through the reconciliation endpoint.
+                logger.warning(
+                    "migration_cutovers_require_reconciliation", count=pending_migrations
+                )
+    if settings.recovery_reconciliation_enabled:
+        with Session(engine) as session:
+            recovery_service = RecoveryService(session, settings, "control_plane", "startup")
+            recovery_service.ensure_defaults()
+            interrupted_backups = recovery_service.reconcile_interrupted_backups()
+            interrupted_restores = recovery_service.reconcile_interrupted_restores()
+            if interrupted_backups or interrupted_restores:
+                # An interrupted backup is never restore eligible and an interrupted restore is
+                # never silently resumed; both surface for explicit operator review.
+                logger.warning(
+                    "recovery_operations_reconciled",
+                    interrupted_backups=interrupted_backups,
+                    interrupted_restores=interrupted_restores,
+                )
+    if settings.registry_seed_on_startup:
+        with Session(engine) as session:
+            manifests = ManifestRegistry(settings.config_root)
+            seed_candidate_registry(session, manifests)
+            sync_result = sync_project_registry(session, manifests)
+            if sync_result.unavailable_contracts:
+                logger.warning(
+                    "project_bindings_waiting_for_contracts",
+                    bindings=sync_result.unavailable_contracts,
+                )
+    poller = HardwarePollingService(settings)
+    hardware_task = (
+        asyncio.create_task(poller.run()) if settings.hardware_refresh_on_startup else None
+    )
+    liveness = NodeLivenessPollingService(settings)
+    liveness_task = (
+        asyncio.create_task(liveness.run()) if settings.node_liveness_monitor_enabled else None
+    )
+    serving_reconciliation = ServingReconciliationService(settings, engine)
+    serving_task = (
+        asyncio.create_task(serving_reconciliation.run())
+        if settings.serving_reconciliation_enabled
+        else None
+    )
+    observability_poller = ObservabilityPollingService(settings, engine)
+    if settings.observability_monitor_enabled:
+        with Session(engine) as session:
+            ObservabilityService(session, settings).ensure_defaults()
+    observability_task = (
+        asyncio.create_task(observability_poller.run())
+        if settings.observability_monitor_enabled
+        else None
+    )
+    yield
+    if observability_task:
+        observability_poller.stop()
+        await observability_task
+    if serving_task:
+        serving_reconciliation.stop()
+        await serving_task
+    if liveness_task:
+        liveness.stop()
+        await liveness_task
+    if hardware_task:
+        poller.stop()
+        await hardware_task
+
+
+interactive_api_enabled = interactive_api_enabled_for(settings.env)
+app = FastAPI(
+    title="ITWorx ModelForge API",
+    version=__version__,
+    description="Local AI ModelOps and GPU control-plane API",
+    lifespan=lifespan,
+    docs_url="/docs" if interactive_api_enabled else None,
+    redoc_url="/redoc" if interactive_api_enabled else None,
+    openapi_url="/openapi.json" if interactive_api_enabled else None,
+)
+app.include_router(health.router)
+app.include_router(lifecycle.router)
+app.include_router(migrations.router)
+app.include_router(observability.router)
+app.include_router(recovery.router)
+app.include_router(agent.router)
+app.include_router(hardware.router)
+app.include_router(catalog.router)
+app.include_router(registry.router)
+app.include_router(acquisition.router)
+app.include_router(runtime.router)
+app.include_router(serving.router)
+app.include_router(evaluation.router)
+
+
+@app.exception_handler(LifecycleError)
+async def lifecycle_error(request: Request, exc: LifecycleError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": exc.message,
+                "correlation_id": correlation_id,
+            }
+        },
+    )
+
+
+@app.exception_handler(MigrationEngineError)
+async def migration_engine_error(request: Request, exc: MigrationEngineError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": exc.message,
+                "correlation_id": correlation_id,
+                "details": {},
+            }
+        },
+    )
+
+
+@app.exception_handler(RecoveryError)
+async def recovery_error(request: Request, exc: RecoveryError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": exc.message,
+                "correlation_id": correlation_id,
+                "details": exc.details,
+            }
+        },
+    )
+
+
+@app.exception_handler(ObservabilityError)
+async def observability_error(request: Request, exc: ObservabilityError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": exc.message,
+                "correlation_id": correlation_id,
+                "details": {},
+            }
+        },
+    )
+
+
+@contextmanager
+def _authentication_session(application: FastAPI) -> Iterator[Session]:
+    """Use a test/application session override when present, otherwise a short owned session."""
+
+    override = application.dependency_overrides.get(get_session)
+    if override is None:
+        with Session(engine) as session:
+            yield session
+        return
+
+    provided = override()
+    if isinstance(provided, Session):
+        yield provided
+        return
+    iterator = iter(cast(Any, provided))
+    session = next(iterator)
+    if not isinstance(session, Session):
+        raise TypeError("get_session override did not provide a SQLAlchemy Session")
+    try:
+        yield session
+    finally:
+        with suppress(StopIteration):
+            next(iterator)
+
+
+def _authenticate_node_before_body(
+    application: FastAPI,
+    request_settings: Settings,
+    authorization: str | None,
+) -> NodeAuthenticationEvidence:
+    service_override = application.dependency_overrides.get(agent.get_agent_service)
+    if service_override is not None:
+        service = service_override()
+        if not isinstance(service, NodeAgentService):
+            raise TypeError("node authentication service override has an invalid type")
+        _identity, evidence = service.authenticate_with_evidence(authorization)
+        service.session.commit()
+        return evidence
+
+    with _authentication_session(application) as session:
+        service = NodeAgentService(session, request_settings)
+        _identity, evidence = service.authenticate_with_evidence(authorization)
+        session.commit()
+        return evidence
+
+
+def _authenticate_capability_before_body(
+    application: FastAPI,
+    request_settings: Settings,
+    authorization: str | None,
+    capability: str,
+) -> CapabilityAuthenticationEvidence:
+    service_override = application.dependency_overrides.get(serving.get_serving_service)
+    if service_override is not None:
+        service = service_override()
+        if not isinstance(service, ServingService):
+            raise TypeError("capability authentication service override has an invalid type")
+        _client, evidence = service.authenticate_with_evidence(authorization, capability)
+        return evidence
+
+    with _authentication_session(application) as session:
+        service = ServingService(
+            session,
+            request_settings,
+            ManifestRegistry(request_settings.config_root),
+            None,
+        )
+        _client, evidence = service.authenticate_with_evidence(authorization, capability)
+        return evidence
+
+
+def _authentication_error(
+    status_code: int,
+    code: str,
+    message: str,
+    correlation_id: str,
+    details: dict[str, Any] | None = None,
+) -> JSONResponse:
+    return JSONResponse(
+        status_code=status_code,
+        content={
+            "error": {
+                "code": code,
+                "message": message,
+                "correlation_id": correlation_id,
+                "details": details or {},
+            }
+        },
+    )
+
+
+@app.middleware("http")
+async def correlation_middleware(
+    request: Request, call_next: Callable[[Request], Awaitable[Response]]
+) -> Response:
+    started = time.perf_counter()
+    correlation_id = request.headers.get("x-correlation-id") or str(uuid.uuid4())
+    request.state.correlation_id = correlation_id
+    boundary = access_boundary_for_request(
+        request.method,
+        request.url.path,
+    )
+    settings_provider = cast(
+        Callable[[], Settings],
+        request.app.dependency_overrides.get(get_settings, get_settings),
+    )
+    request_settings = settings_provider()
+    request.state.request_body_limit_bytes = request_body_limit_bytes(
+        request_settings,
+        boundary,
+    )
+    response: Response | None = None
+    if boundary is AccessBoundary.CONTROL_PLANE:
+        try:
+            request.state.principal = authenticate_operator_token(
+                request_settings,
+                request.headers.get("x-modelforge-admin-token"),
+            )
+        except HTTPException as exc:
+            response = _authentication_error(
+                exc.status_code,
+                f"http_{exc.status_code}",
+                str(exc.detail),
+                correlation_id,
+            )
+    elif boundary is AccessBoundary.NODE:
+        try:
+            request.state.node_authentication = await run_in_threadpool(
+                _authenticate_node_before_body,
+                request.app,
+                request_settings,
+                request.headers.get("authorization"),
+            )
+        except AgentProtocolError as exc:
+            response = _authentication_error(
+                exc.status_code,
+                exc.code,
+                str(exc),
+                correlation_id,
+            )
+        except SQLAlchemyError:
+            response = _authentication_error(
+                503,
+                "machine_authentication_unavailable",
+                "node authentication backend is unavailable",
+                correlation_id,
+            )
+    elif boundary is AccessBoundary.CAPABILITY_CLIENT:
+        capability = required_capability_for_request(request.method, request.url.path)
+        if capability is None:
+            response = _authentication_error(
+                503,
+                "machine_authentication_unavailable",
+                "capability authentication policy is unavailable",
+                correlation_id,
+            )
+        else:
+            try:
+                request.state.capability_authentication = await run_in_threadpool(
+                    _authenticate_capability_before_body,
+                    request.app,
+                    request_settings,
+                    request.headers.get("authorization"),
+                    capability,
+                )
+            except ServingError as exc:
+                response = _authentication_error(
+                    exc.status_code,
+                    exc.code,
+                    str(exc),
+                    correlation_id,
+                    exc.details,
+                )
+            except SQLAlchemyError:
+                response = _authentication_error(
+                    503,
+                    "machine_authentication_unavailable",
+                    "capability authentication backend is unavailable",
+                    correlation_id,
+                )
+    if response is None:
+        response = await call_next(request)
+    duration_seconds = time.perf_counter() - started
+    response.headers["x-correlation-id"] = correlation_id
+    body_error_status_code = getattr(
+        request.state,
+        "request_body_error_status_code",
+        None,
+    )
+    effective_status_code = (
+        body_error_status_code
+        if isinstance(body_error_status_code, int)
+        else response.status_code
+    )
+    labels = {
+        "method": request.method,
+        "route_class": route_class(request.url.path),
+    }
+    metrics.increment(
+        "modelforge_api_requests_total",
+        {**labels, "status_class": f"{effective_status_code // 100}xx"},
+    )
+    metrics.observe("modelforge_api_request_duration_seconds", labels, duration_seconds)
+    logger.info(
+        "http_request",
+        correlation_id=correlation_id,
+        method=request.method,
+        path=request.url.path,
+        status_code=effective_status_code,
+        duration_ms=duration_seconds * 1000,
+    )
+    return response
+
+
+# Starlette wraps user middleware in reverse registration order. Register CORS after the
+# request limiter and correlation/authentication middleware so it remains outermost and decorates
+# pre-body errors as well as responses produced by routing and exception handlers. The pure ASGI
+# limiter stays outside BaseHTTPMiddleware so empty request frames cannot be normalized away before
+# its progress counters observe them; authentication still runs before the first wrapped receive.
+app.add_middleware(RequestBodyLimitMiddleware)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_origin_list,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+MAX_ECHOED_INPUT_CHARACTERS = 200
+
+
+def _renderable(value: object) -> object:
+    """Render a rejected value so the error response can always be serialised.
+
+    A body may legally contain values JSON cannot round-trip — `NaN` and `Infinity` are accepted by
+    Python's parser but rejected by the serialiser — and echoing one raw made the error handler
+    itself fail, turning a 422 into a server error. Rejected input is also unbounded by definition,
+    so it is truncated rather than mirrored back in full.
+    """
+
+    if isinstance(value, float) and not math.isfinite(value):
+        return repr(value)
+    if isinstance(value, str):
+        return value[:MAX_ECHOED_INPUT_CHARACTERS]
+    if isinstance(value, bytes):
+        return value[:MAX_ECHOED_INPUT_CHARACTERS].decode("utf-8", "replace")
+    if isinstance(value, list | tuple):
+        return [_renderable(item) for item in value[:20]]
+    if isinstance(value, dict):
+        return {str(key): _renderable(item) for key, item in list(value.items())[:20]}
+    if isinstance(value, bool | int | float | type(None)):
+        return value
+    return repr(value)[:MAX_ECHOED_INPUT_CHARACTERS]
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=422,
+        content={
+            "error": {
+                "code": "request_validation_failed",
+                "message": "Request validation failed",
+                "correlation_id": correlation_id,
+                "details": {
+                    "errors": [
+                        {key: _renderable(value) for key, value in error.items() if key != "ctx"}
+                        for error in exc.errors()
+                    ]
+                },
+            }
+        },
+    )
+
+
+@app.exception_handler(HTTPException)
+async def http_error(request: Request, exc: HTTPException) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": f"http_{exc.status_code}",
+                "message": str(exc.detail),
+                "correlation_id": correlation_id,
+                "details": {},
+            }
+        },
+    )
+
+
+@app.exception_handler(AgentProtocolError)
+async def agent_protocol_error(request: Request, exc: AgentProtocolError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": str(exc),
+                "correlation_id": correlation_id,
+                "details": {},
+            }
+        },
+    )
+
+
+@app.exception_handler(NodeDecommissionError)
+async def node_decommission_error(request: Request, exc: NodeDecommissionError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": exc.message,
+                "correlation_id": correlation_id,
+                "details": exc.details,
+            }
+        },
+    )
+
+
+@app.exception_handler(RegistryError)
+async def registry_error(request: Request, exc: RegistryError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": str(exc),
+                "correlation_id": correlation_id,
+                "details": exc.details,
+            }
+        },
+    )
+
+
+@app.exception_handler(ServingError)
+async def serving_error(request: Request, exc: ServingError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": str(exc),
+                "correlation_id": correlation_id,
+                "details": exc.details,
+            }
+        },
+    )
+
+
+@app.exception_handler(EvaluationError)
+async def evaluation_error(request: Request, exc: EvaluationError) -> JSONResponse:
+    correlation_id = getattr(request.state, "correlation_id", "unknown")
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "error": {
+                "code": exc.code,
+                "message": str(exc),
+                "correlation_id": correlation_id,
+                "details": exc.details,
+            }
+        },
+    )
+
+
+@app.get("/")
+def root() -> dict[str, str]:
+    return {
+        "name": "ITWorx ModelForge",
+        "version": __version__,
+        "docs": "/docs" if interactive_api_enabled else "disabled",
+        "api": "/api/v1",
+    }

backend/src/modelforge_api/persistence/__init__.py

@@ -0,0 +1,3 @@
+from .models import Base
+
+__all__ = ["Base"]

backend/src/modelforge_api/persistence/acquisition_repository.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import TypeVar
+
+from sqlalchemy import or_, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    ArtifactInspection,
+    ArtifactJob,
+    ArtifactSet,
+    DownloadPlan,
+    DownloadPlanFile,
+    UpstreamFile,
+    UpstreamSnapshot,
+)
+
+T = TypeVar("T")
+
+
+class AcquisitionRepository:
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def add(self, entity: T) -> T:
+        self.session.add(entity)
+        self.session.flush()
+        return entity
+
+    def latest_snapshot(self, model_id: uuid.UUID) -> UpstreamSnapshot | None:
+        return self.session.scalar(
+            select(UpstreamSnapshot)
+            .where(UpstreamSnapshot.model_id == model_id)
+            .order_by(UpstreamSnapshot.observed_at.desc(), UpstreamSnapshot.id.desc())
+            .limit(1)
+        )
+
+    def snapshot(self, snapshot_id: uuid.UUID) -> UpstreamSnapshot | None:
+        return self.session.get(UpstreamSnapshot, snapshot_id)
+
+    def files(self, snapshot_id: uuid.UUID) -> list[UpstreamFile]:
+        return list(
+            self.session.scalars(
+                select(UpstreamFile)
+                .where(UpstreamFile.snapshot_id == snapshot_id)
+                .order_by(UpstreamFile.path)
+            )
+        )
+
+    def artifact_set(self, artifact_set_id: uuid.UUID) -> ArtifactSet | None:
+        return self.session.get(ArtifactSet, artifact_set_id)
+
+    def artifact_set_for_variant(self, revision_id: uuid.UUID, variant: str) -> ArtifactSet | None:
+        return self.session.scalar(
+            select(ArtifactSet).where(
+                ArtifactSet.revision_id == revision_id, ArtifactSet.variant_key == variant
+            )
+        )
+
+    def artifact_sets(self, revision_id: uuid.UUID) -> list[ArtifactSet]:
+        return list(
+            self.session.scalars(
+                select(ArtifactSet)
+                .where(ArtifactSet.revision_id == revision_id)
+                .order_by(ArtifactSet.variant_key)
+            )
+        )
+
+    def plan(self, plan_id: uuid.UUID) -> DownloadPlan | None:
+        return self.session.get(DownloadPlan, plan_id)
+
+    def plan_by_key(self, key: str) -> DownloadPlan | None:
+        return self.session.scalar(select(DownloadPlan).where(DownloadPlan.idempotency_key == key))
+
+    def plan_files(self, plan_id: uuid.UUID) -> list[DownloadPlanFile]:
+        return list(
+            self.session.scalars(
+                select(DownloadPlanFile)
+                .where(DownloadPlanFile.plan_id == plan_id)
+                .order_by(DownloadPlanFile.ordinal)
+            )
+        )
+
+    def job(self, job_id: uuid.UUID) -> ArtifactJob | None:
+        return self.session.get(ArtifactJob, job_id)
+
+    def job_for_plan(self, plan_id: uuid.UUID) -> ArtifactJob | None:
+        return self.session.scalar(select(ArtifactJob).where(ArtifactJob.plan_id == plan_id))
+
+    def jobs(self, limit: int = 100) -> list[ArtifactJob]:
+        return list(
+            self.session.scalars(
+                select(ArtifactJob).order_by(ArtifactJob.created_at.desc()).limit(limit)
+            )
+        )
+
+    def claimable_job(self, node_id: uuid.UUID, now: datetime) -> ArtifactJob | None:
+        return self.session.scalar(
+            select(ArtifactJob)
+            .where(
+                ArtifactJob.compute_node_id == node_id,
+                ArtifactJob.cancel_requested.is_(False),
+                or_(
+                    ArtifactJob.status == "queued",
+                    (ArtifactJob.status.in_(("claimed", "downloading", "verifying", "promoting")))
+                    & (ArtifactJob.lease_expires_at < now),
+                ),
+            )
+            .order_by(ArtifactJob.created_at)
+            .with_for_update(skip_locked=True)
+            .limit(1)
+        )
+
+    def inspections(self, job_id: uuid.UUID) -> list[ArtifactInspection]:
+        return list(
+            self.session.scalars(
+                select(ArtifactInspection)
+                .where(ArtifactInspection.job_id == job_id)
+                .order_by(ArtifactInspection.file_path, ArtifactInspection.inspection_type)
+            )
+        )

backend/src/modelforge_api/persistence/audit_postgres.py

@@ -0,0 +1,37 @@
+"""Immutable names for the PostgreSQL audit privilege boundary.
+
+Migration 0024 owns the DDL. Runtime code keeps only the callable contract and catalog policy
+names here; it never receives the owner/migration credential or an arbitrary privileged SQL path.
+"""
+
+from __future__ import annotations
+
+AUDIT_OWNER_ROLE = "modelforge"
+AUDIT_RUNTIME_ROLE = "modelforge_runtime"
+AUDIT_FUNCTION_SCHEMA = "modelforge_audit"
+AUDIT_APPEND_FUNCTION = "append_event_v2"
+AUDIT_MUTATION_TRIGGER_FUNCTION = "enforce_owner_mutation"
+AUDIT_EVENT_MUTATION_TRIGGER = "trg_modelforge_audit_events_owner"
+AUDIT_EVENT_TRUNCATE_TRIGGER = "trg_modelforge_audit_events_truncate_owner"
+AUDIT_HEAD_MUTATION_TRIGGER = "trg_modelforge_audit_head_owner"
+AUDIT_HEAD_TRUNCATE_TRIGGER = "trg_modelforge_audit_head_truncate_owner"
+AUDIT_FUNCTION_SEARCH_PATH = "pg_catalog"
+
+# ``pg_proc.prosrc`` hashes for the exact migration-0024 function bodies. Production startup
+# attests these as well as owner/grants/search_path: a same-signature replacement is not the
+# canonical boundary. The static migration test derives both values from its immutable SQL so a
+# body edit cannot silently drift this copy.
+AUDIT_APPEND_BODY_SHA256 = "c9154911f1b1b70f77fe2642de156c98358cd0423f0c3e2a75bc4f1ee4409a86"
+AUDIT_GUARD_BODY_SHA256 = "8af5fc31f3b2c729445baaae902eccd9cb3d70b257d0a19e6399f538ee43c6bf"
+
+POSTGRES_APPEND_AUDIT_SQL = """
+select event_id, sequence, event_hash, occurred_at
+from modelforge_audit.append_event_v2(
+    cast(:event_id as uuid), cast(:occurred_at as timestamptz),
+    :correlation_id, :actor_type, :actor_id, :action, :resource_type,
+    :resource_id, :outcome, cast(:details_json as jsonb),
+    :expected_event_count, :expected_last_sequence, :expected_last_event_hash,
+    :expected_hash_format, :expected_v2_start_sequence,
+    :expected_legacy_prefix_count, :expected_legacy_prefix_seal
+)
+""".strip()

backend/src/modelforge_api/persistence/hardware_repository.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import uuid
+from typing import TypeVar
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AcceleratorTelemetryLatest,
+    ComputeNode,
+    HardwareInventoryRun,
+    HostTelemetryLatest,
+    NodeCredential,
+    NodeEnrollment,
+    StorageVolumeState,
+)
+
+T = TypeVar("T")
+
+
+class HardwareRepository:
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def node_by_key(self, key: str) -> ComputeNode | None:
+        return self.session.scalar(select(ComputeNode).where(ComputeNode.key == key))
+
+    def node_by_key_for_update(self, key: str) -> ComputeNode | None:
+        return self.session.scalar(
+            select(ComputeNode).where(ComputeNode.key == key).with_for_update()
+        )
+
+    def nodes(self) -> list[ComputeNode]:
+        return list(self.session.scalars(select(ComputeNode).order_by(ComputeNode.hostname)))
+
+    def node(self, node_id: uuid.UUID) -> ComputeNode | None:
+        return self.session.get(ComputeNode, node_id)
+
+    def node_for_update(self, node_id: uuid.UUID) -> ComputeNode | None:
+        return self.session.scalar(
+            select(ComputeNode).where(ComputeNode.id == node_id).with_for_update()
+        )
+
+    def accelerators(self, node_id: uuid.UUID | None = None) -> list[Accelerator]:
+        query = select(Accelerator).order_by(Accelerator.device_index)
+        if node_id is not None:
+            query = query.where(Accelerator.compute_node_id == node_id)
+        return list(self.session.scalars(query))
+
+    def accelerator(self, accelerator_id: uuid.UUID) -> Accelerator | None:
+        return self.session.get(Accelerator, accelerator_id)
+
+    def accelerator_by_uuid(self, node_id: uuid.UUID, device_uuid: str) -> Accelerator | None:
+        return self.session.scalar(
+            select(Accelerator).where(
+                Accelerator.compute_node_id == node_id, Accelerator.device_uuid == device_uuid
+            )
+        )
+
+    def host_telemetry(self, node_id: uuid.UUID) -> HostTelemetryLatest | None:
+        return self.session.scalar(
+            select(HostTelemetryLatest).where(HostTelemetryLatest.compute_node_id == node_id)
+        )
+
+    def accelerator_telemetry(self, accelerator_id: uuid.UUID) -> AcceleratorTelemetryLatest | None:
+        return self.session.scalar(
+            select(AcceleratorTelemetryLatest).where(
+                AcceleratorTelemetryLatest.accelerator_id == accelerator_id
+            )
+        )
+
+    def storage(self, node_id: uuid.UUID) -> list[StorageVolumeState]:
+        return list(
+            self.session.scalars(
+                select(StorageVolumeState)
+                .where(StorageVolumeState.compute_node_id == node_id)
+                .order_by(StorageVolumeState.purpose)
+            )
+        )
+
+    def storage_by_key(
+        self, node_id: uuid.UUID, purpose: str, path: str
+    ) -> StorageVolumeState | None:
+        return self.session.scalar(
+            select(StorageVolumeState).where(
+                StorageVolumeState.compute_node_id == node_id,
+                StorageVolumeState.purpose == purpose,
+                StorageVolumeState.path == path,
+            )
+        )
+
+    def latest_run(self) -> HardwareInventoryRun | None:
+        return self.session.scalar(
+            select(HardwareInventoryRun).order_by(HardwareInventoryRun.started_at.desc()).limit(1)
+        )
+
+    def enrollment(self, enrollment_id: uuid.UUID) -> NodeEnrollment | None:
+        return self.session.get(NodeEnrollment, enrollment_id)
+
+    def enrollment_by_hash(self, token_hash: str) -> NodeEnrollment | None:
+        return self.session.scalar(
+            select(NodeEnrollment).where(NodeEnrollment.token_hash == token_hash)
+        )
+
+    def enrollments(self) -> list[NodeEnrollment]:
+        return list(
+            self.session.scalars(select(NodeEnrollment).order_by(NodeEnrollment.created_at.desc()))
+        )
+
+    def credential(self, credential_id: uuid.UUID) -> NodeCredential | None:
+        return self.session.get(NodeCredential, credential_id)
+
+    def active_credential_for_node(self, node_id: uuid.UUID) -> NodeCredential | None:
+        return self.session.scalar(
+            select(NodeCredential).where(
+                NodeCredential.compute_node_id == node_id,
+                NodeCredential.revoked_at.is_(None),
+            )
+        )
+
+    def add(self, entity: T) -> T:
+        self.session.add(entity)
+        self.session.flush()
+        return entity

backend/src/modelforge_api/persistence/registry_repository.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any, TypeVar
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session, selectinload
+
+from modelforge_api.persistence.models import (
+    ArtifactLocation,
+    DerivedArtifact,
+    DerivedArtifactSource,
+    Model,
+    ModelArtifact,
+    ModelRevision,
+    RuntimeProfile,
+    StorageRoot,
+)
+
+T = TypeVar("T")
+
+
+class RegistryRepository:
+    """Persistence-only operations for the M2 registry aggregate."""
+
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def page_models(
+        self,
+        *,
+        page: int,
+        page_size: int,
+        search: str | None = None,
+        lifecycle: str | None = None,
+        source_type: str | None = None,
+    ) -> tuple[list[Model], int]:
+        query = select(Model)
+        count_query = select(func.count()).select_from(Model)
+        filters: list[Any] = []
+        if search:
+            pattern = f"%{search.lower()}%"
+            filters.append(
+                func.lower(Model.display_name).like(pattern)
+                | func.lower(Model.key).like(pattern)
+                | func.lower(Model.upstream_source).like(pattern)
+            )
+        if lifecycle:
+            filters.append(Model.lifecycle == lifecycle)
+        if source_type:
+            filters.append(Model.source_type == source_type)
+        if filters:
+            query = query.where(*filters)
+            count_query = count_query.where(*filters)
+        total = int(self.session.scalar(count_query) or 0)
+        items = list(
+            self.session.scalars(
+                query.options(selectinload(Model.revisions).selectinload(ModelRevision.artifacts))
+                .order_by(Model.display_name, Model.id)
+                .offset((page - 1) * page_size)
+                .limit(page_size)
+            )
+        )
+        return items, total
+
+    def model(self, model_id: uuid.UUID) -> Model | None:
+        return self.session.scalar(
+            select(Model)
+            .where(Model.id == model_id)
+            .options(selectinload(Model.revisions).selectinload(ModelRevision.artifacts))
+        )
+
+    def model_by_key(self, key: str) -> Model | None:
+        return self.session.scalar(select(Model).where(Model.key == key))
+
+    def model_by_source(self, source: str) -> Model | None:
+        return self.session.scalar(select(Model).where(Model.upstream_source == source))
+
+    def revision(self, revision_id: uuid.UUID) -> ModelRevision | None:
+        return self.session.get(ModelRevision, revision_id)
+
+    def revision_by_commit(self, model_id: uuid.UUID, commit: str) -> ModelRevision | None:
+        return self.session.scalar(
+            select(ModelRevision).where(
+                ModelRevision.model_id == model_id,
+                ModelRevision.resolved_commit_sha == commit,
+            )
+        )
+
+    def revisions(self, model_id: uuid.UUID) -> list[ModelRevision]:
+        return list(
+            self.session.scalars(
+                select(ModelRevision)
+                .where(ModelRevision.model_id == model_id)
+                .order_by(ModelRevision.discovered_at.desc())
+            )
+        )
+
+    def artifact(self, artifact_id: uuid.UUID) -> ModelArtifact | None:
+        return self.session.scalar(
+            select(ModelArtifact)
+            .where(ModelArtifact.id == artifact_id)
+            .options(selectinload(ModelArtifact.locations))
+        )
+
+    def artifact_by_digest(self, revision_id: uuid.UUID, digest: str) -> ModelArtifact | None:
+        return self.session.scalar(
+            select(ModelArtifact).where(
+                ModelArtifact.revision_id == revision_id, ModelArtifact.sha256 == digest
+            )
+        )
+
+    def artifacts(self, revision_id: uuid.UUID) -> list[ModelArtifact]:
+        return list(
+            self.session.scalars(
+                select(ModelArtifact)
+                .where(ModelArtifact.revision_id == revision_id)
+                .options(selectinload(ModelArtifact.locations))
+                .order_by(ModelArtifact.filename)
+            )
+        )
+
+    def derived(self, derived_id: uuid.UUID) -> DerivedArtifact | None:
+        return self.session.scalar(
+            select(DerivedArtifact)
+            .where(DerivedArtifact.id == derived_id)
+            .options(
+                selectinload(DerivedArtifact.source_artifacts),
+                selectinload(DerivedArtifact.locations),
+            )
+        )
+
+    def derived_for_revision(self, revision_id: uuid.UUID) -> list[DerivedArtifact]:
+        return list(
+            self.session.scalars(
+                select(DerivedArtifact)
+                .where(DerivedArtifact.revision_id == revision_id)
+                .options(
+                    selectinload(DerivedArtifact.source_artifacts),
+                    selectinload(DerivedArtifact.locations),
+                )
+                .order_by(DerivedArtifact.filename)
+            )
+        )
+
+    def derived_sources(self, derived_id: uuid.UUID) -> list[DerivedArtifactSource]:
+        return list(
+            self.session.scalars(
+                select(DerivedArtifactSource)
+                .where(DerivedArtifactSource.derived_artifact_id == derived_id)
+                .order_by(DerivedArtifactSource.ordinal)
+            )
+        )
+
+    def storage_root(self, root_id: uuid.UUID) -> StorageRoot | None:
+        return self.session.get(StorageRoot, root_id)
+
+    def storage_roots(self, node_id: uuid.UUID | None = None) -> list[StorageRoot]:
+        query = select(StorageRoot).order_by(StorageRoot.name)
+        if node_id:
+            query = query.where(StorageRoot.compute_node_id == node_id)
+        return list(self.session.scalars(query))
+
+    def location(self, location_id: uuid.UUID) -> ArtifactLocation | None:
+        return self.session.get(ArtifactLocation, location_id)
+
+    def add(self, entity: T) -> T:
+        self.session.add(entity)
+        self.session.flush()
+        return entity
+
+    def dependencies(self, resource_type: str, resource_id: uuid.UUID) -> list[dict[str, str]]:
+        dependencies: list[dict[str, str]] = []
+        if resource_type == "model":
+            ids = self.session.scalars(
+                select(ModelRevision.id).where(ModelRevision.model_id == resource_id)
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "model_revision",
+                    "resource_id": str(item),
+                    "relation": "revision",
+                }
+                for item in ids
+            )
+        elif resource_type == "model_revision":
+            ids = self.session.scalars(
+                select(ModelArtifact.id).where(ModelArtifact.revision_id == resource_id)
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "model_artifact",
+                    "resource_id": str(item),
+                    "relation": "artifact",
+                }
+                for item in ids
+            )
+            derived_ids = self.session.scalars(
+                select(DerivedArtifact.id).where(DerivedArtifact.revision_id == resource_id)
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "derived_artifact",
+                    "resource_id": str(item),
+                    "relation": "derived",
+                }
+                for item in derived_ids
+            )
+        elif resource_type == "model_artifact":
+            links = self.session.scalars(
+                select(DerivedArtifactSource.derived_artifact_id).where(
+                    DerivedArtifactSource.source_artifact_id == resource_id
+                )
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "derived_artifact",
+                    "resource_id": str(item),
+                    "relation": "source",
+                }
+                for item in links
+            )
+            locations = self.session.scalars(
+                select(ArtifactLocation.id).where(ArtifactLocation.artifact_id == resource_id)
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "artifact_location",
+                    "resource_id": str(item),
+                    "relation": "location",
+                }
+                for item in locations
+            )
+            artifact = self.session.get(ModelArtifact, resource_id)
+            if artifact:
+                profiles = self.session.scalars(
+                    select(RuntimeProfile.id).where(
+                        RuntimeProfile.artifact_sha256 == artifact.sha256
+                    )
+                )
+                dependencies.extend(
+                    {
+                        "resource_type": "runtime_profile",
+                        "resource_id": str(item),
+                        "relation": "digest",
+                    }
+                    for item in profiles
+                )
+        elif resource_type == "derived_artifact":
+            sources = self.session.scalars(
+                select(DerivedArtifactSource.source_artifact_id).where(
+                    DerivedArtifactSource.derived_artifact_id == resource_id
+                )
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "model_artifact",
+                    "resource_id": str(item),
+                    "relation": "source_lineage",
+                }
+                for item in sources
+            )
+            locations = self.session.scalars(
+                select(ArtifactLocation.id).where(
+                    ArtifactLocation.derived_artifact_id == resource_id
+                )
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "artifact_location",
+                    "resource_id": str(item),
+                    "relation": "location",
+                }
+                for item in locations
+            )
+        elif resource_type == "storage_root":
+            locations = self.session.scalars(
+                select(ArtifactLocation.id).where(ArtifactLocation.storage_root_id == resource_id)
+            )
+            dependencies.extend(
+                {
+                    "resource_type": "artifact_location",
+                    "resource_id": str(item),
+                    "relation": "location",
+                }
+                for item in locations
+            )
+        return dependencies

backend/src/modelforge_api/persistence/repositories.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.services.audit import AuditContext, AuditWriter
+
+from .models import AuditEvent, Base, Model
+
+
+class Repository[Entity: Base]:
+    def __init__(self, session: Session, entity_type: type[Entity]) -> None:
+        self.session = session
+        self.entity_type = entity_type
+
+    def get(self, entity_id: Any) -> Entity | None:
+        return self.session.get(self.entity_type, entity_id)
+
+    def list(self) -> list[Entity]:
+        return list(self.session.scalars(select(self.entity_type)))
+
+    def add(self, entity: Entity) -> Entity:
+        self.session.add(entity)
+        self.session.flush()
+        return entity
+
+
+class ModelRepository(Repository[Model]):
+    def __init__(self, session: Session) -> None:
+        super().__init__(session, Model)
+
+
+class AuditRepository:
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def append(
+        self,
+        *,
+        correlation_id: str,
+        actor_type: str,
+        actor_id: str,
+        action: str,
+        resource_type: str,
+        resource_id: str | None,
+        outcome: str,
+        details: dict[str, Any] | None = None,
+    ) -> AuditEvent:
+        return AuditWriter(
+            self.session,
+            context=AuditContext(
+                correlation_id=correlation_id,
+                actor_type=actor_type,
+                actor_id=actor_id,
+            ),
+        ).write(
+            action=action,
+            resource_type=resource_type,
+            resource_id=resource_id,
+            outcome=outcome,
+            details=details or {},
+        )

backend/src/modelforge_api/persistence/runtime_engine.py

@@ -0,0 +1,23 @@
+"""Identity registry for Engines constructed by the application process.
+
+Alembic creates its own Engine and is intentionally absent. This registry is only a SQLite/test
+and accidental-misuse defense; PostgreSQL permissions and triggers are the production boundary.
+"""
+
+from __future__ import annotations
+
+from weakref import WeakSet
+
+from sqlalchemy.engine import Connection, Engine
+
+_APPLICATION_ENGINES: WeakSet[Engine] = WeakSet()
+
+
+def register_application_engine(engine: Engine) -> Engine:
+    _APPLICATION_ENGINES.add(engine)
+    return engine
+
+
+def is_application_connection(connection: Connection) -> bool:
+    return connection.engine in _APPLICATION_ENGINES
+

backend/src/modelforge_api/persistence/runtime_repository.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import TypeVar
+
+from sqlalchemy import or_, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Accelerator,
+    ArtifactLocation,
+    ArtifactSet,
+    ArtifactSetMember,
+    ComputeNode,
+    DeploymentCandidate,
+    ExecutionApproval,
+    Model,
+    ModelArtifact,
+    ModelRevision,
+    RuntimeCompatibilityAssessment,
+    RuntimeEnvironment,
+    RuntimeProbe,
+    RuntimeProbeMetric,
+    RuntimeProfile,
+    StorageRoot,
+)
+
+T = TypeVar("T")
+
+
+class RuntimeRepository:
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def add(self, entity: T) -> T:
+        self.session.add(entity)
+        self.session.flush()
+        return entity
+
+    def environment(self, environment_id: uuid.UUID) -> RuntimeEnvironment | None:
+        return self.session.get(RuntimeEnvironment, environment_id)
+
+    def environment_by_fingerprint(self, fingerprint: str) -> RuntimeEnvironment | None:
+        return self.session.scalar(
+            select(RuntimeEnvironment).where(RuntimeEnvironment.fingerprint == fingerprint)
+        )
+
+    def environments(self) -> list[RuntimeEnvironment]:
+        return list(
+            self.session.scalars(
+                select(RuntimeEnvironment).order_by(RuntimeEnvironment.created_at.desc())
+            )
+        )
+
+    def profile(self, profile_id: uuid.UUID) -> RuntimeProfile | None:
+        return self.session.get(RuntimeProfile, profile_id)
+
+    def profile_by_fingerprint(self, fingerprint: str) -> RuntimeProfile | None:
+        return self.session.scalar(
+            select(RuntimeProfile).where(RuntimeProfile.fingerprint == fingerprint)
+        )
+
+    def profiles(self) -> list[RuntimeProfile]:
+        return list(
+            self.session.scalars(select(RuntimeProfile).order_by(RuntimeProfile.created_at.desc()))
+        )
+
+    def artifact_set(self, artifact_set_id: uuid.UUID) -> ArtifactSet | None:
+        return self.session.get(ArtifactSet, artifact_set_id)
+
+    def revision(self, revision_id: uuid.UUID) -> ModelRevision | None:
+        return self.session.get(ModelRevision, revision_id)
+
+    def model(self, model_id: uuid.UUID) -> Model | None:
+        return self.session.get(Model, model_id)
+
+    def set_artifacts(
+        self, artifact_set_id: uuid.UUID
+    ) -> list[tuple[ArtifactSetMember, ModelArtifact]]:
+        rows = self.session.execute(
+            select(ArtifactSetMember, ModelArtifact)
+            .join(ModelArtifact, ModelArtifact.id == ArtifactSetMember.artifact_id)
+            .where(ArtifactSetMember.artifact_set_id == artifact_set_id)
+            .order_by(ArtifactSetMember.ordinal)
+        )
+        return [(member, artifact) for member, artifact in rows]
+
+    def artifact_locations(self, artifact_ids: list[uuid.UUID]) -> list[ArtifactLocation]:
+        if not artifact_ids:
+            return []
+        return list(
+            self.session.scalars(
+                select(ArtifactLocation).where(ArtifactLocation.artifact_id.in_(artifact_ids))
+            )
+        )
+
+    def storage_root(self, root_id: uuid.UUID) -> StorageRoot | None:
+        return self.session.get(StorageRoot, root_id)
+
+    def node(self, node_id: uuid.UUID) -> ComputeNode | None:
+        return self.session.get(ComputeNode, node_id)
+
+    def accelerators(self, node_id: uuid.UUID) -> list[Accelerator]:
+        return list(
+            self.session.scalars(
+                select(Accelerator).where(
+                    Accelerator.compute_node_id == node_id,
+                    Accelerator.status == "active",
+                )
+            )
+        )
+
+    def assessment(self, assessment_id: uuid.UUID) -> RuntimeCompatibilityAssessment | None:
+        return self.session.get(RuntimeCompatibilityAssessment, assessment_id)
+
+    def assessments(self) -> list[RuntimeCompatibilityAssessment]:
+        return list(
+            self.session.scalars(
+                select(RuntimeCompatibilityAssessment).order_by(
+                    RuntimeCompatibilityAssessment.created_at.desc()
+                )
+            )
+        )
+
+    def approval(self, approval_id: uuid.UUID) -> ExecutionApproval | None:
+        return self.session.get(ExecutionApproval, approval_id)
+
+    def approvals(self, artifact_set_id: uuid.UUID | None = None) -> list[ExecutionApproval]:
+        query = select(ExecutionApproval).order_by(ExecutionApproval.approved_at.desc())
+        if artifact_set_id:
+            query = query.where(ExecutionApproval.artifact_set_id == artifact_set_id)
+        return list(self.session.scalars(query))
+
+    def probe(self, probe_id: uuid.UUID) -> RuntimeProbe | None:
+        return self.session.get(RuntimeProbe, probe_id)
+
+    def probe_by_key(self, key: str) -> RuntimeProbe | None:
+        return self.session.scalar(select(RuntimeProbe).where(RuntimeProbe.idempotency_key == key))
+
+    def probes(self) -> list[RuntimeProbe]:
+        return list(
+            self.session.scalars(select(RuntimeProbe).order_by(RuntimeProbe.created_at.desc()))
+        )
+
+    def claimable_probe(self, node_id: uuid.UUID, now: datetime) -> RuntimeProbe | None:
+        return self.session.scalar(
+            select(RuntimeProbe)
+            .where(
+                RuntimeProbe.compute_node_id == node_id,
+                RuntimeProbe.cancel_requested.is_(False),
+                or_(
+                    RuntimeProbe.status == "queued",
+                    RuntimeProbe.status.in_(
+                        ("preparing", "loading", "healthchecking", "ready", "unloading")
+                    )
+                    & (RuntimeProbe.lease_expires_at < now),
+                ),
+            )
+            .order_by(RuntimeProbe.created_at)
+            .with_for_update(skip_locked=True)
+            .limit(1)
+        )
+
+    def candidate_for_probe(self, probe_id: uuid.UUID) -> DeploymentCandidate | None:
+        return self.session.scalar(
+            select(DeploymentCandidate).where(DeploymentCandidate.runtime_probe_id == probe_id)
+        )
+
+    def candidate(self, candidate_id: uuid.UUID) -> DeploymentCandidate | None:
+        return self.session.get(DeploymentCandidate, candidate_id)
+
+    def candidates(self) -> list[DeploymentCandidate]:
+        return list(
+            self.session.scalars(
+                select(DeploymentCandidate).order_by(DeploymentCandidate.created_at.desc())
+            )
+        )
+
+    def metrics(self, probe_id: uuid.UUID) -> list[RuntimeProbeMetric]:
+        return list(
+            self.session.scalars(
+                select(RuntimeProbeMetric)
+                .where(RuntimeProbeMetric.runtime_probe_id == probe_id)
+                .order_by(RuntimeProbeMetric.observed_at)
+            )
+        )

backend/src/modelforge_api/persistence/serving_repository.py

@@ -0,0 +1,424 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timedelta
+from typing import TypeVar
+
+from sqlalchemy import case, delete, func, or_, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AcceleratorTelemetryLatest,
+    ArtifactLocation,
+    ArtifactSet,
+    ArtifactSetMember,
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    CapabilityExperimentRoute,
+    CapabilityResourceEnvelope,
+    ComputeNode,
+    CoResidencyEvidence,
+    DeploymentCandidate,
+    EmbeddingSpace,
+    ExecutionApproval,
+    GatewayRequest,
+    ModelArtifact,
+    ModelRevision,
+    PlacementPlanRecord,
+    ProductionExecutionApproval,
+    ResidencyAllocation,
+    RuntimeEnvironment,
+    RuntimeProbe,
+    RuntimeProfile,
+    SchedulerAcceleratorState,
+    SchedulerPolicyRevision,
+    ServiceClient,
+    ServiceCredential,
+    ServingGpuLease,
+    ServingJob,
+    StorageRoot,
+)
+
+T = TypeVar("T")
+
+
+class ServingRepository:
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def add(self, entity: T) -> T:
+        self.session.add(entity)
+        self.session.flush()
+        return entity
+
+    def capability(self, key: str) -> Capability | None:
+        return self.session.scalar(select(Capability).where(Capability.key == key))
+
+    def contract(self, key: str, version: int) -> CapabilityContract | None:
+        return self.session.scalar(
+            select(CapabilityContract)
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
+            .where(Capability.key == key, CapabilityContract.version == version)
+        )
+
+    def candidate(self, candidate_id: uuid.UUID) -> DeploymentCandidate | None:
+        return self.session.get(DeploymentCandidate, candidate_id)
+
+    def probe(self, probe_id: uuid.UUID) -> RuntimeProbe | None:
+        return self.session.get(RuntimeProbe, probe_id)
+
+    def profile(self, profile_id: uuid.UUID) -> RuntimeProfile | None:
+        return self.session.get(RuntimeProfile, profile_id)
+
+    def environment(self, environment_id: uuid.UUID) -> RuntimeEnvironment | None:
+        return self.session.get(RuntimeEnvironment, environment_id)
+
+    def artifact_set(self, artifact_set_id: uuid.UUID) -> ArtifactSet | None:
+        return self.session.get(ArtifactSet, artifact_set_id)
+
+    def revision(self, revision_id: uuid.UUID) -> ModelRevision | None:
+        return self.session.get(ModelRevision, revision_id)
+
+    def node(self, node_id: uuid.UUID) -> ComputeNode | None:
+        return self.session.get(ComputeNode, node_id)
+
+    def accelerator(self, accelerator_id: uuid.UUID) -> Accelerator | None:
+        return self.session.get(Accelerator, accelerator_id)
+
+    def node_accelerators(self, node_id: uuid.UUID) -> list[Accelerator]:
+        return list(
+            self.session.scalars(
+                select(Accelerator).where(
+                    Accelerator.compute_node_id == node_id,
+                    Accelerator.status == "active",
+                )
+            )
+        )
+
+    def lock_accelerator(self, accelerator_id: uuid.UUID) -> Accelerator | None:
+        return self.session.scalar(
+            select(Accelerator).where(Accelerator.id == accelerator_id).with_for_update()
+        )
+
+    def telemetry(self, accelerator_id: uuid.UUID) -> AcceleratorTelemetryLatest | None:
+        return self.session.scalar(
+            select(AcceleratorTelemetryLatest).where(
+                AcceleratorTelemetryLatest.accelerator_id == accelerator_id
+            )
+        )
+
+    def set_artifacts(self, artifact_set_id: uuid.UUID) -> list[ModelArtifact]:
+        return list(
+            self.session.scalars(
+                select(ModelArtifact)
+                .join(ArtifactSetMember, ArtifactSetMember.artifact_id == ModelArtifact.id)
+                .where(ArtifactSetMember.artifact_set_id == artifact_set_id)
+                .order_by(ArtifactSetMember.ordinal)
+            )
+        )
+
+    def artifact_locations(self, artifact_ids: list[uuid.UUID]) -> list[ArtifactLocation]:
+        return list(
+            self.session.scalars(
+                select(ArtifactLocation).where(ArtifactLocation.artifact_id.in_(artifact_ids))
+            )
+        )
+
+    def storage_root(self, root_id: uuid.UUID) -> StorageRoot | None:
+        return self.session.get(StorageRoot, root_id)
+
+    def approval_by_fingerprint(self, fingerprint: str) -> ProductionExecutionApproval | None:
+        return self.session.scalar(
+            select(ProductionExecutionApproval).where(
+                ProductionExecutionApproval.evidence_fingerprint == fingerprint
+            )
+        )
+
+    def approval(self, approval_id: uuid.UUID) -> ProductionExecutionApproval | None:
+        return self.session.get(ProductionExecutionApproval, approval_id)
+
+    def execution_approval(self, approval_id: uuid.UUID) -> ExecutionApproval | None:
+        return self.session.get(ExecutionApproval, approval_id)
+
+    def approvals(self) -> list[ProductionExecutionApproval]:
+        return list(
+            self.session.scalars(
+                select(ProductionExecutionApproval).order_by(
+                    ProductionExecutionApproval.approved_at.desc()
+                )
+            )
+        )
+
+    def embedding_space_by_digest(self, digest: str) -> EmbeddingSpace | None:
+        return self.session.scalar(
+            select(EmbeddingSpace).where(EmbeddingSpace.identity_digest == digest)
+        )
+
+    def embedding_space(self, space_id: uuid.UUID | None) -> EmbeddingSpace | None:
+        if space_id is None:
+            return None
+        return self.session.get(EmbeddingSpace, space_id)
+
+    def deployment_by_fingerprint(self, fingerprint: str) -> CapabilityDeployment | None:
+        return self.session.scalar(
+            select(CapabilityDeployment).where(
+                CapabilityDeployment.config_fingerprint == fingerprint
+            )
+        )
+
+    def deployment(self, deployment_id: uuid.UUID) -> CapabilityDeployment | None:
+        return self.session.get(CapabilityDeployment, deployment_id)
+
+    def deployments(self) -> list[CapabilityDeployment]:
+        return list(
+            self.session.scalars(
+                select(CapabilityDeployment).order_by(CapabilityDeployment.created_at.desc())
+            )
+        )
+
+    def experiment_route(self, route_key: str) -> CapabilityExperimentRoute | None:
+        return self.session.scalar(
+            select(CapabilityExperimentRoute).where(
+                CapabilityExperimentRoute.route_key == route_key
+            )
+        )
+
+    def experiment_route_by_id(self, route_id: uuid.UUID) -> CapabilityExperimentRoute | None:
+        return self.session.get(CapabilityExperimentRoute, route_id)
+
+    def experiment_routes(self) -> list[CapabilityExperimentRoute]:
+        return list(
+            self.session.scalars(
+                select(CapabilityExperimentRoute).order_by(
+                    CapabilityExperimentRoute.created_at.desc()
+                )
+            )
+        )
+
+    def stable_deployment(self, contract_id: uuid.UUID) -> CapabilityDeployment | None:
+        return self.session.scalar(
+            select(CapabilityDeployment).where(
+                CapabilityDeployment.capability_contract_id == contract_id,
+                CapabilityDeployment.channel == "stable",
+                CapabilityDeployment.status == "stable",
+                CapabilityDeployment.production.is_(True),
+            )
+        )
+
+    def envelope(self, deployment_id: uuid.UUID) -> CapabilityResourceEnvelope | None:
+        return self.session.scalar(
+            select(CapabilityResourceEnvelope).where(
+                CapabilityResourceEnvelope.capability_deployment_id == deployment_id
+            )
+        )
+
+    def residency(
+        self, deployment_id: uuid.UUID, *, lock: bool = False
+    ) -> ResidencyAllocation | None:
+        query = select(ResidencyAllocation).where(
+            ResidencyAllocation.capability_deployment_id == deployment_id
+        )
+        if lock:
+            query = query.with_for_update()
+        return self.session.scalar(query)
+
+    def accelerator_residencies(self, accelerator_id: uuid.UUID) -> list[ResidencyAllocation]:
+        return list(
+            self.session.scalars(
+                select(ResidencyAllocation)
+                .join(
+                    CapabilityDeployment,
+                    CapabilityDeployment.id == ResidencyAllocation.capability_deployment_id,
+                )
+                .where(CapabilityDeployment.accelerator_id == accelerator_id)
+                .order_by(ResidencyAllocation.created_at)
+            )
+        )
+
+    def client_by_name(self, name: str) -> ServiceClient | None:
+        return self.session.scalar(select(ServiceClient).where(ServiceClient.name == name))
+
+    def client(self, client_id: uuid.UUID) -> ServiceClient | None:
+        return self.session.get(ServiceClient, client_id)
+
+    def clients(self) -> list[ServiceClient]:
+        return list(self.session.scalars(select(ServiceClient).order_by(ServiceClient.created_at)))
+
+    def active_credential(self, client_id: uuid.UUID) -> ServiceCredential | None:
+        return self.session.scalar(
+            select(ServiceCredential).where(
+                ServiceCredential.service_client_id == client_id,
+                ServiceCredential.revoked_at.is_(None),
+            )
+        )
+
+    def credential_by_hash(
+        self, secret_hash: str
+    ) -> tuple[ServiceCredential, ServiceClient] | None:
+        row = self.session.execute(
+            select(ServiceCredential, ServiceClient)
+            .join(ServiceClient, ServiceClient.id == ServiceCredential.service_client_id)
+            .where(ServiceCredential.secret_hash == secret_hash)
+        ).first()
+        return (row[0], row[1]) if row else None
+
+    def recent_request_count(self, client_id: uuid.UUID, since: datetime) -> int:
+        return int(
+            self.session.scalar(
+                select(func.count(GatewayRequest.id)).where(
+                    GatewayRequest.service_client_id == client_id,
+                    GatewayRequest.created_at >= since,
+                )
+            )
+            or 0
+        )
+
+    def active_client_requests(self, client_id: uuid.UUID) -> int:
+        return int(
+            self.session.scalar(
+                select(func.count(GatewayRequest.id)).where(
+                    GatewayRequest.service_client_id == client_id,
+                    GatewayRequest.status.in_(("queued", "running")),
+                )
+            )
+            or 0
+        )
+
+    def active_gateway_requests(self) -> int:
+        return int(
+            self.session.scalar(
+                select(func.count(GatewayRequest.id)).where(
+                    GatewayRequest.status.in_(("queued", "running"))
+                )
+            )
+            or 0
+        )
+
+    def gateway_requests(self, limit: int = 100) -> list[GatewayRequest]:
+        return list(
+            self.session.scalars(
+                select(GatewayRequest).order_by(GatewayRequest.created_at.desc()).limit(limit)
+            )
+        )
+
+    def queued_job_count(self, deployment_id: uuid.UUID) -> int:
+        return int(
+            self.session.scalar(
+                select(func.count(ServingJob.id)).where(
+                    ServingJob.capability_deployment_id == deployment_id,
+                    ServingJob.operation == "invoke",
+                    ServingJob.status.in_(("queued", "running")),
+                )
+            )
+            or 0
+        )
+
+    def job(self, job_id: uuid.UUID) -> ServingJob | None:
+        return self.session.get(ServingJob, job_id)
+
+    def job_by_key(self, key: str) -> ServingJob | None:
+        return self.session.scalar(select(ServingJob).where(ServingJob.idempotency_key == key))
+
+    def claimable_job(self, node_id: uuid.UUID, now: datetime) -> ServingJob | None:
+        aging_cutoff = now - timedelta(minutes=5)
+        priority_order = case(
+            (ServingJob.priority == "production", 0),
+            ((ServingJob.priority == "background") & (ServingJob.created_at < aging_cutoff), 1),
+            (
+                (ServingJob.priority.in_(("lab", "benchmark")))
+                & (ServingJob.created_at < aging_cutoff),
+                2,
+            ),
+            (ServingJob.priority == "interactive", 1),
+            (ServingJob.priority == "background", 2),
+            (ServingJob.priority.in_(("lab", "benchmark")), 3),
+            else_=4,
+        )
+        return self.session.scalar(
+            select(ServingJob)
+            .where(
+                ServingJob.compute_node_id == node_id,
+                or_(
+                    ServingJob.status == "queued",
+                    (ServingJob.status == "running") & (ServingJob.lease_expires_at < now),
+                ),
+            )
+            .order_by(priority_order, ServingJob.created_at)
+            .with_for_update(skip_locked=True)
+            .limit(1)
+        )
+
+    def active_leases(self, accelerator_id: uuid.UUID, now: datetime) -> list[ServingGpuLease]:
+        return list(
+            self.session.scalars(
+                select(ServingGpuLease).where(
+                    ServingGpuLease.accelerator_id == accelerator_id,
+                    ServingGpuLease.state.in_(("pending", "granted", "active", "releasing")),
+                    ServingGpuLease.expires_at > now,
+                )
+            )
+        )
+
+    def lease_for_request(self, request_id: uuid.UUID) -> ServingGpuLease | None:
+        return self.session.scalar(
+            select(ServingGpuLease).where(ServingGpuLease.request_id == request_id)
+        )
+
+    def expired_leases(self, now: datetime) -> list[ServingGpuLease]:
+        return list(
+            self.session.scalars(
+                select(ServingGpuLease).where(
+                    ServingGpuLease.state.in_(("pending", "granted", "active", "releasing")),
+                    ServingGpuLease.expires_at <= now,
+                )
+            )
+        )
+
+    def active_scheduler_policy(self) -> SchedulerPolicyRevision | None:
+        return self.session.scalar(
+            select(SchedulerPolicyRevision)
+            .where(SchedulerPolicyRevision.active.is_(True))
+            .order_by(SchedulerPolicyRevision.created_at.desc())
+            .limit(1)
+        )
+
+    def scheduler_accelerator_state(
+        self, accelerator_id: uuid.UUID
+    ) -> SchedulerAcceleratorState | None:
+        return self.session.scalar(
+            select(SchedulerAcceleratorState).where(
+                SchedulerAcceleratorState.accelerator_id == accelerator_id
+            )
+        )
+
+    def placement_plans(self, limit: int = 100) -> list[PlacementPlanRecord]:
+        return list(
+            self.session.scalars(
+                select(PlacementPlanRecord)
+                .order_by(PlacementPlanRecord.created_at.desc())
+                .limit(limit)
+            )
+        )
+
+    def trim_placement_plans(self, keep: int) -> None:
+        retained = (
+            select(PlacementPlanRecord.id)
+            .order_by(PlacementPlanRecord.created_at.desc())
+            .limit(keep)
+        )
+        self.session.execute(
+            delete(PlacementPlanRecord).where(PlacementPlanRecord.id.not_in(retained))
+        )
+
+    def co_residency_evidence(
+        self, left_id: uuid.UUID, right_id: uuid.UUID
+    ) -> CoResidencyEvidence | None:
+        low, high = sorted((left_id, right_id), key=str)
+        return self.session.scalar(
+            select(CoResidencyEvidence).where(
+                CoResidencyEvidence.left_deployment_id == low,
+                CoResidencyEvidence.right_deployment_id == high,
+            )
+        )

backend/src/modelforge_api/providers/__init__.py

@@ -0,0 +1 @@
+"""External provider boundaries."""

backend/src/modelforge_api/providers/huggingface.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import PurePosixPath
+from typing import Any, Protocol
+
+import httpx
+from huggingface_hub import HfApi
+from huggingface_hub.errors import (
+    GatedRepoError,
+    HfHubHTTPError,
+    RepositoryNotFoundError,
+    RevisionNotFoundError,
+)
+
+
+class HuggingFaceProviderError(RuntimeError):
+    code = "huggingface_error"
+    access_state = "error"
+
+
+class HuggingFaceNotFound(HuggingFaceProviderError):
+    code = "repository_not_found"
+    access_state = "not_found"
+
+
+class HuggingFaceGated(HuggingFaceProviderError):
+    code = "repository_gated"
+    access_state = "gated"
+
+
+class HuggingFaceRevisionNotFound(HuggingFaceProviderError):
+    code = "revision_not_found"
+    access_state = "revision_not_found"
+
+
+class HuggingFaceUnavailable(HuggingFaceProviderError):
+    code = "provider_unavailable"
+    access_state = "unavailable"
+
+
+@dataclass(frozen=True)
+class ProviderFile:
+    path: str
+    size_bytes: int | None
+    blob_id: str | None
+    upstream_sha256: str | None
+    file_format: str
+    role: str
+    risk_flags: tuple[str, ...] = ()
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class ProviderSnapshot:
+    repository_id: str
+    requested_revision: str
+    resolved_commit_sha: str
+    access_state: str
+    metadata: dict[str, Any]
+    card_metadata: dict[str, Any]
+    security_metadata: dict[str, Any]
+    source_updated_at: datetime | None
+    files: tuple[ProviderFile, ...]
+
+
+class HuggingFaceProvider(Protocol):
+    def search(
+        self, query: str, *, limit: int, sort: str, pipeline_tag: str | None
+    ) -> list[dict[str, Any]]: ...
+
+    def snapshot(self, repository_id: str, revision: str) -> ProviderSnapshot: ...
+
+
+def classify_file(path: str) -> tuple[str, str, tuple[str, ...]]:
+    suffix = PurePosixPath(path.lower()).suffix
+    name = PurePosixPath(path.lower()).name
+    risks: list[str] = []
+    if suffix == ".safetensors":
+        return "safetensors", "weights", ()
+    if suffix == ".gguf":
+        return "gguf", "weights", ()
+    if suffix in {".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle"}:
+        risks.append("pickle_or_executable_serialization")
+        return suffix.lstrip("."), "weights", tuple(risks)
+    if suffix == ".py":
+        return "python", "repository_code", ("remote_code",)
+    if name.endswith(".safetensors.index.json"):
+        return "json", "weight_index", ()
+    if suffix in {".json", ".model", ".txt", ".tiktoken"}:
+        role = "configuration" if "config" in name else "tokenizer"
+        return suffix.lstrip("."), role, ()
+    if suffix in {".md", ".rst"}:
+        return suffix.lstrip("."), "documentation", ()
+    return suffix.lstrip(".") or "unknown", "other", ()
+
+
+def _card_dict(value: Any) -> dict[str, Any]:
+    if value is None:
+        return {}
+    if hasattr(value, "to_dict"):
+        result = value.to_dict()
+        return dict(result) if isinstance(result, dict) else {}
+    return dict(value) if isinstance(value, dict) else {}
+
+
+def _lfs_value(lfs: Any, name: str) -> Any:
+    if lfs is None:
+        return None
+    return lfs.get(name) if isinstance(lfs, dict) else getattr(lfs, name, None)
+
+
+class OfficialHuggingFaceProvider:
+    """Narrow adapter over the official huggingface_hub client; no HTML scraping."""
+
+    def __init__(self, *, token: str | None = None, timeout: float = 30) -> None:
+        self.api = HfApi(token=token)
+        self.token = token
+        self.timeout = timeout
+
+    @staticmethod
+    def _search_item(item: Any) -> dict[str, Any]:
+        return {
+            "repository_id": item.id,
+            "resolved_commit_sha": getattr(item, "sha", None),
+            "access_state": "gated"
+            if getattr(item, "gated", False)
+            else "private"
+            if getattr(item, "private", False)
+            else "public",
+            "pipeline_tag": getattr(item, "pipeline_tag", None),
+            "library_name": getattr(item, "library_name", None),
+            "tags": list(getattr(item, "tags", None) or []),
+            "downloads": getattr(item, "downloads", None),
+            "likes": getattr(item, "likes", None),
+            "last_modified": getattr(item, "last_modified", None),
+        }
+
+    def search(
+        self, query: str, *, limit: int, sort: str, pipeline_tag: str | None
+    ) -> list[dict[str, Any]]:
+        try:
+            items: Iterable[Any] = self.api.list_models(
+                search=query,
+                pipeline_tag=pipeline_tag,
+                sort=sort,
+                limit=limit,
+                full=True,
+                cardData=True,
+                token=self.token,
+            )
+            return [self._search_item(item) for item in items]
+        except (HfHubHTTPError, httpx.TimeoutException) as exc:
+            raise HuggingFaceUnavailable("Hugging Face search is unavailable") from exc
+
+    def snapshot(self, repository_id: str, revision: str) -> ProviderSnapshot:
+        try:
+            info = self.api.model_info(
+                repository_id,
+                revision=revision,
+                timeout=self.timeout,
+                securityStatus=True,
+                files_metadata=True,
+                token=self.token,
+            )
+        except GatedRepoError as exc:
+            raise HuggingFaceGated(f"repository {repository_id} is gated") from exc
+        except RevisionNotFoundError as exc:
+            raise HuggingFaceRevisionNotFound(
+                f"revision {revision} was not found for {repository_id}"
+            ) from exc
+        except RepositoryNotFoundError as exc:
+            raise HuggingFaceNotFound(
+                f"repository {repository_id} was not found or is private"
+            ) from exc
+        except (HfHubHTTPError, httpx.TimeoutException) as exc:
+            raise HuggingFaceUnavailable(
+                f"Hugging Face metadata is unavailable for {repository_id}"
+            ) from exc
+        files: list[ProviderFile] = []
+        for sibling in info.siblings or []:
+            file_format, role, risks = classify_file(sibling.rfilename)
+            lfs = getattr(sibling, "lfs", None)
+            files.append(
+                ProviderFile(
+                    path=sibling.rfilename,
+                    size_bytes=getattr(sibling, "size", None) or _lfs_value(lfs, "size"),
+                    blob_id=getattr(sibling, "blob_id", None),
+                    upstream_sha256=_lfs_value(lfs, "sha256"),
+                    file_format=file_format,
+                    role=role,
+                    risk_flags=risks,
+                    metadata={
+                        "lfs_pointer_size": _lfs_value(lfs, "pointer_size"),
+                        "source": "huggingface_hub.model_info(files_metadata=true)",
+                    },
+                )
+            )
+        card = _card_dict(getattr(info, "card_data", None))
+        if not info.sha:
+            raise HuggingFaceUnavailable(
+                f"Hugging Face did not resolve an exact commit for {repository_id}"
+            )
+        access = (
+            "gated"
+            if getattr(info, "gated", False)
+            else "private"
+            if getattr(info, "private", False)
+            else "public"
+        )
+        return ProviderSnapshot(
+            repository_id=info.id,
+            requested_revision=revision,
+            resolved_commit_sha=info.sha,
+            access_state=access,
+            metadata={
+                "repository_id": info.id,
+                "author": getattr(info, "author", None),
+                "pipeline_tag": getattr(info, "pipeline_tag", None),
+                "library_name": getattr(info, "library_name", None),
+                "tags": list(getattr(info, "tags", None) or []),
+                "downloads": getattr(info, "downloads", None),
+                "likes": getattr(info, "likes", None),
+                "gated": getattr(info, "gated", False),
+                "private": getattr(info, "private", False),
+                "source": "huggingface_hub.model_info",
+            },
+            card_metadata=card,
+            security_metadata={
+                "upstream_scanner": getattr(info, "security_repo_status", None),
+                "evidence_only": True,
+                "local_static_inspection": "not_run",
+            },
+            source_updated_at=getattr(info, "last_modified", None),
+            files=tuple(sorted(files, key=lambda item: item.path)),
+        )

backend/src/modelforge_api/py.typed

@@ -0,0 +1 @@
+

backend/src/modelforge_api/services/acquisition.py

@@ -0,0 +1,948 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import secrets
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.acquisition import (
+    AgentArtifactJobFile,
+    AgentArtifactJobLease,
+    AgentJobComplete,
+    AgentJobControl,
+    AgentJobFailure,
+    AgentJobProgress,
+    ArtifactJobResponse,
+    ArtifactSetResponse,
+    DiscoveryCandidate,
+    DiscoverySearchRequest,
+    DownloadPlanCreate,
+    DownloadPlanFileResponse,
+    DownloadPlanResponse,
+    UpstreamFileResponse,
+    UpstreamSnapshotResponse,
+)
+from modelforge_api.domain.enums import ArtifactStatus, NodeLiveness
+from modelforge_api.persistence.acquisition_repository import AcquisitionRepository
+from modelforge_api.persistence.models import (
+    ArtifactInspection,
+    ArtifactJob,
+    ArtifactJobAttempt,
+    ArtifactLocation,
+    ArtifactSet,
+    ArtifactSetMember,
+    ComputeNode,
+    DownloadPlan,
+    DownloadPlanFile,
+    Model,
+    ModelArtifact,
+    ModelRevision,
+    StorageRoot,
+    UpstreamFile,
+    UpstreamSnapshot,
+)
+from modelforge_api.providers.huggingface import (
+    HuggingFaceProvider,
+    HuggingFaceProviderError,
+    ProviderFile,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.registry import RegistryConflict, RegistryNotFound, RegistryService
+from modelforge_api.settings import Settings
+
+
+class AcquisitionError(RegistryConflict):
+    code = "acquisition_error"
+
+
+class ProviderBoundaryError(AcquisitionError):
+    status_code = 502
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def _aware(value: datetime) -> datetime:
+    return value if value.tzinfo else value.replace(tzinfo=UTC)
+
+
+def _canonical_hash(payload: dict[str, Any]) -> str:
+    return hashlib.sha256(
+        json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
+
+
+def _safe_key(repository_id: str) -> str:
+    return repository_id.lower().replace("/", "--").replace("_", "-")[:120]
+
+
+def _select_artifact_files(files: list[ProviderFile]) -> tuple[str, str, list[ProviderFile]]:
+    safe_weights = [
+        item for item in files if item.role == "weights" and item.file_format == "safetensors"
+    ]
+    gguf_weights = [item for item in files if item.role == "weights" and item.file_format == "gguf"]
+    other_weights = [
+        item
+        for item in files
+        if item.role == "weights" and item not in safe_weights and item not in gguf_weights
+    ]
+    support = [
+        item
+        for item in files
+        if item.role in {"configuration", "tokenizer", "weight_index"}
+        and "remote_code" not in item.risk_flags
+    ]
+    if safe_weights:
+        return (
+            "safetensors-default",
+            "Safetensors files were preferred; duplicate pickle/GGUF weight variants were excluded.",
+            sorted(safe_weights + support, key=lambda item: item.path),
+        )
+    if gguf_weights:
+        return (
+            "gguf-default",
+            "No safetensors were advertised; the GGUF variant was selected without pickle weights.",
+            sorted(gguf_weights + support, key=lambda item: item.path),
+        )
+    return (
+        "upstream-default",
+        "No safe tensor variant was advertised; upstream weights retain explicit risk flags.",
+        sorted(other_weights + support, key=lambda item: item.path),
+    )
+
+
+class AcquisitionService:
+    def __init__(
+        self,
+        session: Session,
+        settings: Settings,
+        provider: HuggingFaceProvider,
+        *,
+        actor_type: str = "operator",
+        actor_id: str = "local-operator",
+    ) -> None:
+        self.session = session
+        self.settings = settings
+        self.provider = provider
+        self.repo = AcquisitionRepository(session)
+        self.audit = AuditWriter(session, actor_type, actor_id)
+
+    def _commit(self) -> None:
+        self.session.commit()
+
+    def search(self, request: DiscoverySearchRequest) -> list[DiscoveryCandidate]:
+        try:
+            results = self.provider.search(
+                request.query,
+                limit=request.limit,
+                sort=request.sort,
+                pipeline_tag=request.pipeline_tag,
+            )
+        except HuggingFaceProviderError as exc:
+            raise ProviderBoundaryError(str(exc), {"provider_code": exc.code}) from exc
+        sources = {
+            item.upstream_source: item.id
+            for item in self.session.scalars(
+                select(Model).where(Model.source_type == "huggingface")
+            )
+        }
+        response: list[DiscoveryCandidate] = []
+        for item in results:
+            repository_id = str(item["repository_id"])
+            response.append(
+                DiscoveryCandidate(
+                    **item,
+                    matched_model_id=sources.get(repository_id),
+                    upstream_facts={
+                        "source": "huggingface_hub.list_models",
+                        "pipeline_tag": item.get("pipeline_tag"),
+                        "tags": item.get("tags", []),
+                    },
+                    local_interpretation={
+                        "candidate_only": True,
+                        "approval": "not_evaluated",
+                        "hardware_compatibility": "not_measured",
+                    },
+                )
+            )
+        self.audit.write(
+            "HF_DISCOVERY_SEARCHED",
+            "upstream_provider",
+            "huggingface",
+            {"query": request.query, "result_count": len(response)},
+        )
+        self._commit()
+        return response
+
+    def refresh_model(self, model_id: uuid.UUID, revision: str) -> UpstreamSnapshotResponse:
+        model = self.session.get(Model, model_id)
+        if not model:
+            raise RegistryNotFound("model not found")
+        if model.source_type != "huggingface":
+            raise AcquisitionError("only Hugging Face models can use this provider boundary")
+        try:
+            source = self.provider.snapshot(model.upstream_source, revision)
+        except HuggingFaceProviderError as exc:
+            self.audit.write(
+                "HF_METADATA_REFRESH_FAILED",
+                "model",
+                str(model.id),
+                {"provider_code": exc.code, "access_state": exc.access_state},
+                outcome="blocked",
+            )
+            self._commit()
+            raise ProviderBoundaryError(
+                str(exc), {"provider_code": exc.code, "access_state": exc.access_state}
+            ) from exc
+        now = _utcnow()
+        snapshot = self.repo.add(
+            UpstreamSnapshot(
+                model_id=model.id,
+                provider="huggingface",
+                repository_id=source.repository_id,
+                requested_revision=revision,
+                resolved_commit_sha=source.resolved_commit_sha,
+                access_state=source.access_state,
+                metadata_snapshot=source.metadata,
+                card_metadata=source.card_metadata,
+                security_metadata=source.security_metadata,
+                source_updated_at=source.source_updated_at,
+                observed_at=now,
+                stale_after=now + timedelta(seconds=self.settings.hf_snapshot_ttl_seconds),
+            )
+        )
+        for item in source.files:
+            self.repo.add(
+                UpstreamFile(
+                    snapshot_id=snapshot.id,
+                    path=item.path,
+                    size_bytes=item.size_bytes,
+                    blob_id=item.blob_id,
+                    upstream_sha256=item.upstream_sha256,
+                    file_format=item.file_format,
+                    role=item.role,
+                    risk_flags=list(item.risk_flags),
+                    metadata_snapshot=item.metadata,
+                )
+            )
+        existing_revision = self.session.scalar(
+            select(ModelRevision).where(
+                ModelRevision.model_id == model.id,
+                ModelRevision.resolved_commit_sha == source.resolved_commit_sha,
+            )
+        )
+        exact_revision = existing_revision or self.repo.add(
+            ModelRevision(
+                model_id=model.id,
+                upstream_revision=revision,
+                resolved_commit_sha=source.resolved_commit_sha,
+                metadata_snapshot={
+                    "upstream_snapshot_id": str(snapshot.id),
+                    "provider": "huggingface",
+                    "observed_at": now.isoformat(),
+                },
+                discovered_at=now,
+                immutable_at=now,
+            )
+        )
+        variant, reason, selected = _select_artifact_files(list(source.files))
+        artifact_set = self.repo.artifact_set_for_variant(exact_revision.id, variant)
+        if artifact_set is None and selected:
+            missing_sizes = [item.path for item in selected if item.size_bytes is None]
+            artifact_set = self.repo.add(
+                ArtifactSet(
+                    revision_id=exact_revision.id,
+                    snapshot_id=snapshot.id,
+                    variant_key=variant,
+                    label=variant.replace("-", " ").title(),
+                    selection_reason=reason,
+                    selected_paths=[item.path for item in selected],
+                    total_size_bytes=sum(item.size_bytes or 0 for item in selected),
+                    file_count=len(selected),
+                    availability="remote",
+                    status="remote",
+                    completeness="incomplete_metadata" if missing_sizes else "planned",
+                    security_status="risk_detected"
+                    if any(item.risk_flags for item in selected)
+                    else "unverified",
+                    license_status="captured_unreviewed"
+                    if source.card_metadata.get("license")
+                    else "unknown",
+                    immutable_at=now,
+                )
+            )
+        model.upstream_metadata = source.metadata | {
+            "resolved_commit_sha": source.resolved_commit_sha,
+            "latest_snapshot_id": str(snapshot.id),
+            "metadata_observed_at": now.isoformat(),
+        }
+        model.license_metadata = {
+            "status": "captured_unreviewed" if source.card_metadata.get("license") else "unknown",
+            "declared": source.card_metadata.get("license"),
+            "source": "huggingface_model_card",
+            "approval": "not_evaluated",
+        }
+        model.interpretation_metadata = model.interpretation_metadata | {
+            "discovery_interpretation": {
+                "pipeline_tag": source.metadata.get("pipeline_tag"),
+                "safe_default_variant": variant if selected else None,
+                "hardware_compatibility": "not_measured",
+                "evidence_status": "upstream_metadata_only",
+            }
+        }
+        self.audit.write(
+            "HF_METADATA_REFRESHED",
+            "model",
+            str(model.id),
+            {
+                "snapshot_id": str(snapshot.id),
+                "resolved_commit_sha": source.resolved_commit_sha,
+                "file_count": len(source.files),
+                "artifact_set_id": str(artifact_set.id) if artifact_set else None,
+            },
+        )
+        self.audit.write(
+            "REVISION_RESOLVED",
+            "model_revision",
+            str(exact_revision.id),
+            {
+                "requested_revision": revision,
+                "resolved_commit_sha": source.resolved_commit_sha,
+                "new_revision": existing_revision is None,
+            },
+        )
+        self._commit()
+        return self.snapshot_response(snapshot.id)
+
+    def snapshot_response(self, snapshot_id: uuid.UUID) -> UpstreamSnapshotResponse:
+        snapshot = self.repo.snapshot(snapshot_id)
+        if not snapshot:
+            raise RegistryNotFound("upstream snapshot not found")
+        return UpstreamSnapshotResponse(
+            id=snapshot.id,
+            model_id=snapshot.model_id,
+            provider=snapshot.provider,
+            repository_id=snapshot.repository_id,
+            requested_revision=snapshot.requested_revision,
+            resolved_commit_sha=snapshot.resolved_commit_sha,
+            access_state=snapshot.access_state,
+            metadata_snapshot=snapshot.metadata_snapshot,
+            card_metadata=snapshot.card_metadata,
+            security_metadata=snapshot.security_metadata,
+            source_updated_at=snapshot.source_updated_at,
+            observed_at=snapshot.observed_at,
+            stale=_aware(snapshot.stale_after) <= _utcnow(),
+            stale_after=snapshot.stale_after,
+            files=[
+                UpstreamFileResponse.model_validate(item) for item in self.repo.files(snapshot.id)
+            ],
+        )
+
+    def latest_snapshot(self, model_id: uuid.UUID) -> UpstreamSnapshotResponse:
+        if not self.session.get(Model, model_id):
+            raise RegistryNotFound("model not found")
+        snapshot = self.repo.latest_snapshot(model_id)
+        if not snapshot:
+            raise RegistryNotFound("model has no upstream metadata snapshot")
+        return self.snapshot_response(snapshot.id)
+
+    def artifact_sets(self, revision_id: uuid.UUID) -> list[ArtifactSetResponse]:
+        if not self.session.get(ModelRevision, revision_id):
+            raise RegistryNotFound("model revision not found")
+        return [
+            ArtifactSetResponse.model_validate(item)
+            for item in self.repo.artifact_sets(revision_id)
+        ]
+
+    def create_plan(self, request: DownloadPlanCreate) -> DownloadPlanResponse:
+        artifact_set = self.repo.artifact_set(request.artifact_set_id)
+        root = self.session.get(StorageRoot, request.storage_root_id)
+        node = self.session.get(ComputeNode, request.compute_node_id)
+        if not artifact_set:
+            raise RegistryNotFound("artifact set not found")
+        if not root:
+            raise RegistryNotFound("storage root not found")
+        if not node:
+            raise RegistryNotFound("compute node not found")
+        if root.compute_node_id != node.id:
+            raise AcquisitionError("storage root belongs to a different compute node")
+        if not root.agent_path:
+            raise AcquisitionError("storage root has no node-agent path mapping")
+        if (
+            node.decommissioned_at is not None
+            or not node.enabled
+            or node.liveness_state != NodeLiveness.ONLINE
+        ):
+            raise AcquisitionError("target compute node is not online and enabled")
+        if "artifact.acquire.v1" not in node.agent_capabilities:
+            raise AcquisitionError("target agent does not advertise artifact.acquire.v1")
+        snapshot = self.repo.snapshot(artifact_set.snapshot_id)
+        revision = self.session.get(ModelRevision, artifact_set.revision_id)
+        if not snapshot or not revision:
+            raise AcquisitionError("artifact set provenance is incomplete")
+        if snapshot.access_state != "public" and not self.settings.hf_token:
+            raise AcquisitionError(
+                f"repository access is {snapshot.access_state}; Hugging Face authentication is required"
+            )
+        source_files = {item.path: item for item in self.repo.files(snapshot.id)}
+        files = [source_files[path] for path in artifact_set.selected_paths if path in source_files]
+        if len(files) != artifact_set.file_count or any(item.size_bytes is None for item in files):
+            raise AcquisitionError("artifact set has incomplete file-size metadata")
+        decision = RegistryService.capacity_decision(root, artifact_set.total_size_bytes)
+        if not decision.allowed:
+            self.audit.write(
+                "DOWNLOAD_PLAN_BLOCKED",
+                "artifact_set",
+                str(artifact_set.id),
+                decision.model_dump(mode="json"),
+                outcome="blocked",
+            )
+            self._commit()
+            raise AcquisitionError(decision.reason, decision.model_dump(mode="json"))
+        now = _utcnow()
+        payload = {
+            "artifact_set_id": str(artifact_set.id),
+            "repository_id": snapshot.repository_id,
+            "resolved_commit_sha": revision.resolved_commit_sha,
+            "compute_node_id": str(node.id),
+            "storage_root_id": str(root.id),
+            "target_root": root.agent_path,
+            "files": [
+                {
+                    "path": item.path,
+                    "size_bytes": item.size_bytes,
+                    "upstream_sha256": item.upstream_sha256,
+                    "file_format": item.file_format,
+                    "role": item.role,
+                    "risk_flags": item.risk_flags,
+                }
+                for item in files
+            ],
+        }
+        key = _canonical_hash(payload)
+        existing = self.repo.plan_by_key(key)
+        if existing:
+            return self.plan_response(existing.id)
+        preflight = {
+            "capacity": decision.model_dump(mode="json"),
+            "node_liveness": node.liveness_state,
+            "agent_capability": "artifact.acquire.v1",
+            "storage_agent_path": root.agent_path,
+            "exact_revision": True,
+            "trust_remote_code": False,
+        }
+        plan = self.repo.add(
+            DownloadPlan(
+                artifact_set_id=artifact_set.id,
+                compute_node_id=node.id,
+                storage_root_id=root.id,
+                repository_id=snapshot.repository_id,
+                resolved_commit_sha=revision.resolved_commit_sha,
+                total_size_bytes=artifact_set.total_size_bytes,
+                file_count=len(files),
+                status="planned",
+                idempotency_key=key,
+                preflight=preflight,
+                immutable_payload=payload,
+                planned_at=now,
+                expires_at=now + timedelta(seconds=request.expires_in_seconds),
+                immutable_at=now,
+            )
+        )
+        for ordinal, item in enumerate(files):
+            self.repo.add(
+                DownloadPlanFile(
+                    plan_id=plan.id,
+                    ordinal=ordinal,
+                    path=item.path,
+                    size_bytes=int(item.size_bytes or 0),
+                    upstream_sha256=item.upstream_sha256,
+                    file_format=item.file_format,
+                    role=item.role,
+                    risk_flags=item.risk_flags,
+                )
+            )
+        self.audit.write(
+            "DOWNLOAD_PLAN_CREATED",
+            "download_plan",
+            str(plan.id),
+            {"idempotency_key": key, "total_size_bytes": plan.total_size_bytes},
+        )
+        self._commit()
+        return self.plan_response(plan.id)
+
+    def approve_plan(self, plan_id: uuid.UUID) -> DownloadPlanResponse:
+        plan = self.repo.plan(plan_id)
+        if not plan:
+            raise RegistryNotFound("download plan not found")
+        if plan.status == "ready":
+            return self.plan_response(plan.id)
+        if plan.status != "planned":
+            raise AcquisitionError("only a planned download plan can be approved")
+        if _aware(plan.expires_at) <= _utcnow():
+            plan.status = "stale"
+            self._commit()
+            raise AcquisitionError("download plan is stale; create a new immutable plan")
+        artifact_set = self.repo.artifact_set(plan.artifact_set_id)
+        if not artifact_set:
+            raise AcquisitionError("download plan lost its artifact set")
+        if artifact_set.security_status == "risk_detected":
+            self.audit.write(
+                "DOWNLOAD_PLAN_APPROVAL_BLOCKED",
+                "download_plan",
+                str(plan.id),
+                {"reason": "artifact selection contains blocked security risks"},
+                outcome="blocked",
+            )
+            self._commit()
+            raise AcquisitionError("artifact selection contains blocked security risks")
+        plan.status = "ready"
+        artifact_set.status = "ready"
+        self.audit.write(
+            "DOWNLOAD_PLAN_APPROVED",
+            "download_plan",
+            str(plan.id),
+            {
+                "resolved_commit_sha": plan.resolved_commit_sha,
+                "file_count": plan.file_count,
+                "security_preflight": "passed",
+            },
+        )
+        self._commit()
+        return self.plan_response(plan.id)
+
+    def plan_response(self, plan_id: uuid.UUID) -> DownloadPlanResponse:
+        plan = self.repo.plan(plan_id)
+        if not plan:
+            raise RegistryNotFound("download plan not found")
+        return DownloadPlanResponse(
+            **{
+                name: getattr(plan, name)
+                for name in DownloadPlanResponse.model_fields
+                if name not in {"files", "stale"}
+            },
+            stale=_aware(plan.expires_at) <= _utcnow(),
+            files=[
+                DownloadPlanFileResponse.model_validate(item)
+                for item in self.repo.plan_files(plan.id)
+            ],
+        )
+
+    def execute_plan(self, plan_id: uuid.UUID) -> ArtifactJobResponse:
+        plan = self.repo.plan(plan_id)
+        if not plan:
+            raise RegistryNotFound("download plan not found")
+        existing = self.repo.job_for_plan(plan.id)
+        if existing:
+            return ArtifactJobResponse.model_validate(existing)
+        if plan.status != "ready":
+            raise AcquisitionError("download plan requires explicit approval before execution")
+        if _aware(plan.expires_at) <= _utcnow():
+            plan.status = "stale"
+            self._commit()
+            raise AcquisitionError("download plan is stale; create a new immutable plan")
+        root = self.session.get(StorageRoot, plan.storage_root_id)
+        node = self.session.get(ComputeNode, plan.compute_node_id)
+        if not root or not node:
+            raise AcquisitionError("download target no longer exists")
+        decision = RegistryService.capacity_decision(root, plan.total_size_bytes)
+        if not decision.allowed:
+            self.audit.write(
+                "ARTIFACT_DOWNLOAD_BLOCKED",
+                "download_plan",
+                str(plan.id),
+                decision.model_dump(mode="json"),
+                outcome="blocked",
+            )
+            self._commit()
+            raise AcquisitionError(
+                "execution capacity preflight failed", decision.model_dump(mode="json")
+            )
+        if node.liveness_state != NodeLiveness.ONLINE or not node.enabled:
+            raise AcquisitionError("target compute node is not online and enabled")
+        job = self.repo.add(
+            ArtifactJob(
+                plan_id=plan.id,
+                compute_node_id=plan.compute_node_id,
+                storage_root_id=plan.storage_root_id,
+                status="queued",
+                idempotency_key=_canonical_hash({"job_for_plan": str(plan.id)}),
+                total_bytes=plan.total_size_bytes,
+            )
+        )
+        plan.status = "queued"
+        artifact_set = self.repo.artifact_set(plan.artifact_set_id)
+        if artifact_set:
+            artifact_set.status = "queued"
+        self.audit.write(
+            "ARTIFACT_DOWNLOAD_QUEUED",
+            "artifact_job",
+            str(job.id),
+            {"plan_id": str(plan.id), "compute_node_id": str(job.compute_node_id)},
+        )
+        self._commit()
+        return ArtifactJobResponse.model_validate(job)
+
+    def jobs(self) -> list[ArtifactJobResponse]:
+        return [ArtifactJobResponse.model_validate(item) for item in self.repo.jobs()]
+
+    def job(self, job_id: uuid.UUID) -> ArtifactJobResponse:
+        job = self.repo.job(job_id)
+        if not job:
+            raise RegistryNotFound("artifact job not found")
+        return ArtifactJobResponse.model_validate(job)
+
+    def cancel(self, job_id: uuid.UUID) -> ArtifactJobResponse:
+        job = self.repo.job(job_id)
+        if not job:
+            raise RegistryNotFound("artifact job not found")
+        if job.status in {"completed", "failed", "cancelled"}:
+            raise AcquisitionError("terminal artifact jobs cannot be cancelled")
+        job.cancel_requested = True
+        if job.status == "queued":
+            job.status = "cancelled"
+            job.completed_at = _utcnow()
+        self.audit.write(
+            "DOWNLOAD_CANCELLED" if job.status == "cancelled" else "ARTIFACT_JOB_CANCEL_REQUESTED",
+            "artifact_job",
+            str(job.id),
+            {},
+        )
+        self._commit()
+        return ArtifactJobResponse.model_validate(job)
+
+    def retry(self, job_id: uuid.UUID) -> ArtifactJobResponse:
+        job = self.repo.job(job_id)
+        if not job:
+            raise RegistryNotFound("artifact job not found")
+        if job.status != "failed":
+            raise AcquisitionError("only failed artifact jobs can be retried explicitly")
+        plan = self.repo.plan(job.plan_id)
+        node = self.session.get(ComputeNode, job.compute_node_id)
+        root = self.session.get(StorageRoot, job.storage_root_id)
+        if not plan or not node or not root:
+            raise AcquisitionError("artifact job target or immutable plan no longer exists")
+        if not node.enabled or node.liveness_state != NodeLiveness.ONLINE:
+            raise AcquisitionError("target compute node is not online and enabled")
+        decision = RegistryService.capacity_decision(root, plan.total_size_bytes)
+        if not decision.allowed:
+            raise AcquisitionError(
+                "retry capacity preflight failed", decision.model_dump(mode="json")
+            )
+        previous_error = {"code": job.error_code, "message": job.error_message}
+        job.status = "queued"
+        job.cancel_requested = False
+        job.progress_bytes = 0
+        job.current_file = None
+        job.error_code = None
+        job.error_message = None
+        job.completed_at = None
+        job.lease_token_hash = None
+        job.lease_expires_at = None
+        job.result = {
+            "operator_retry": True,
+            "previous_error": previous_error,
+            "resume_from_quarantine": bool(job.quarantine_relative_path),
+        }
+        plan.status = "queued"
+        artifact_set = self.repo.artifact_set(plan.artifact_set_id)
+        if artifact_set:
+            artifact_set.status = "queued"
+        self.audit.write(
+            "ARTIFACT_JOB_OPERATOR_RETRY",
+            "artifact_job",
+            str(job.id),
+            {
+                "previous_attempts": job.attempt_count,
+                "resume_from_quarantine": bool(job.quarantine_relative_path),
+            },
+        )
+        self._commit()
+        return ArtifactJobResponse.model_validate(job)
+
+    @staticmethod
+    def _lease_hash(token: str) -> str:
+        return hashlib.sha256(token.encode()).hexdigest()
+
+    def claim_next(self, node: ComputeNode) -> AgentArtifactJobLease | None:
+        now = _utcnow()
+        job = self.repo.claimable_job(node.id, now)
+        if not job:
+            return None
+        plan = self.repo.plan(job.plan_id)
+        root = self.session.get(StorageRoot, job.storage_root_id)
+        if not plan or not root or not root.agent_path:
+            raise AcquisitionError("claimable job has an invalid target")
+        token = f"mflease_{secrets.token_urlsafe(48)}"
+        job.lease_token_hash = self._lease_hash(token)
+        job.lease_expires_at = now + timedelta(seconds=120)
+        job.status = "claimed"
+        job.attempt_count += 1
+        job.started_at = job.started_at or now
+        self.repo.add(
+            ArtifactJobAttempt(job_id=job.id, attempt=job.attempt_count, status="claimed")
+        )
+        self.audit.write(
+            "ARTIFACT_DOWNLOAD_STARTED",
+            "artifact_job",
+            str(job.id),
+            {"attempt": job.attempt_count, "compute_node_id": str(node.id)},
+        )
+        self._commit()
+        return AgentArtifactJobLease(
+            job_id=job.id,
+            lease_token=token,
+            lease_expires_at=job.lease_expires_at,
+            repository_id=plan.repository_id,
+            resolved_commit_sha=plan.resolved_commit_sha,
+            storage_root_id=root.id,
+            target_root=root.agent_path,
+            total_size_bytes=plan.total_size_bytes,
+            reserve_bytes=root.reserve_bytes,
+            reserve_percent=root.reserve_percent,
+            files=[
+                AgentArtifactJobFile.model_validate(item) for item in self.repo.plan_files(plan.id)
+            ],
+        )
+
+    def _leased_job(self, job_id: uuid.UUID, node: ComputeNode, token: str) -> ArtifactJob:
+        job = self.repo.job(job_id)
+        if not job:
+            raise RegistryNotFound("artifact job not found")
+        if job.compute_node_id != node.id:
+            raise AcquisitionError("artifact job belongs to a different node")
+        if not job.lease_token_hash or not secrets.compare_digest(
+            job.lease_token_hash, self._lease_hash(token)
+        ):
+            raise AcquisitionError("invalid artifact-job lease")
+        if not job.lease_expires_at or _aware(job.lease_expires_at) <= _utcnow():
+            raise AcquisitionError("artifact-job lease expired")
+        return job
+
+    def progress(
+        self, job_id: uuid.UUID, node: ComputeNode, request: AgentJobProgress
+    ) -> AgentJobControl:
+        job = self._leased_job(job_id, node, request.lease_token)
+        if request.progress_bytes < job.progress_bytes or request.progress_bytes > job.total_bytes:
+            raise AcquisitionError("artifact-job progress is out of bounds")
+        job.status = request.status
+        job.progress_bytes = request.progress_bytes
+        job.current_file = request.current_file
+        first_quarantine_observation = (
+            job.quarantine_relative_path is None and request.quarantine_relative_path is not None
+        )
+        job.quarantine_relative_path = request.quarantine_relative_path
+        job.lease_expires_at = _utcnow() + timedelta(seconds=120)
+        if first_quarantine_observation:
+            self.audit.write(
+                "ARTIFACT_QUARANTINED",
+                "artifact_job",
+                str(job.id),
+                {"quarantine_relative_path": request.quarantine_relative_path},
+            )
+        self._commit()
+        return AgentJobControl(
+            accepted=True,
+            cancel_requested=job.cancel_requested,
+            lease_expires_at=job.lease_expires_at,
+        )
+
+    def complete(
+        self, job_id: uuid.UUID, node: ComputeNode, request: AgentJobComplete
+    ) -> ArtifactJobResponse:
+        job = self._leased_job(job_id, node, request.lease_token)
+        if job.cancel_requested:
+            raise AcquisitionError("artifact job was cancelled")
+        plan = self.repo.plan(job.plan_id)
+        if not plan:
+            raise AcquisitionError("artifact job lost its immutable plan")
+        expected = {item.path: item for item in self.repo.plan_files(plan.id)}
+        if set(expected) != {item.path for item in request.files}:
+            raise AcquisitionError("completed file inventory does not match immutable plan")
+        for item in request.files:
+            planned = expected[item.path]
+            if planned.size_bytes != item.size_bytes:
+                raise AcquisitionError(f"size mismatch for {item.path}")
+            if planned.upstream_sha256 and planned.upstream_sha256 != item.sha256:
+                raise AcquisitionError(f"upstream checksum mismatch for {item.path}")
+            for finding in item.inspections:
+                self.repo.add(
+                    ArtifactInspection(
+                        job_id=job.id,
+                        file_path=item.path,
+                        inspection_type=str(finding.get("type", "static"))[:64],
+                        status=str(finding.get("status", "observed"))[:32],
+                        severity=str(finding.get("severity", "info"))[:32],
+                        evidence=dict(finding.get("evidence", {})),
+                    )
+                )
+                if finding.get("severity") == "block":
+                    raise AcquisitionError(f"blocking static inspection for {item.path}")
+        artifact_set = self.repo.artifact_set(plan.artifact_set_id)
+        if not artifact_set:
+            raise AcquisitionError("artifact set not found")
+        for ordinal, item in enumerate(request.files):
+            planned = expected[item.path]
+            artifact = self.session.scalar(
+                select(ModelArtifact).where(
+                    ModelArtifact.revision_id == artifact_set.revision_id,
+                    ModelArtifact.sha256 == item.sha256,
+                )
+            )
+            if artifact is None:
+                artifact = self.repo.add(
+                    ModelArtifact(
+                        revision_id=artifact_set.revision_id,
+                        filename=item.path,
+                        artifact_type=planned.role,
+                        serialization_format=planned.file_format,
+                        sha256=item.sha256,
+                        size_bytes=item.size_bytes,
+                        security_status="static_checks_passed_unapproved",
+                        license_status=artifact_set.license_status,
+                        status=ArtifactStatus.VERIFIED,
+                        verification_details={
+                            "job_id": str(job.id),
+                            "integrity": "streaming_sha256",
+                            "upstream_checksum": planned.upstream_sha256,
+                            "supply_chain_approval": "not_evaluated_m4",
+                        },
+                        quarantined=False,
+                        verified_at=_utcnow(),
+                        immutable_at=_utcnow(),
+                    )
+                )
+            location = self.session.scalar(
+                select(ArtifactLocation).where(
+                    ArtifactLocation.storage_root_id == job.storage_root_id,
+                    ArtifactLocation.relative_path == item.relative_path,
+                )
+            )
+            if location is None:
+                self.repo.add(
+                    ArtifactLocation(
+                        artifact_id=artifact.id,
+                        storage_root_id=job.storage_root_id,
+                        relative_path=item.relative_path,
+                        status=ArtifactStatus.VERIFIED,
+                        size_bytes=item.size_bytes,
+                        observed_sha256=item.sha256,
+                        last_checked_at=_utcnow(),
+                    )
+                )
+            if self.session.get(ArtifactSetMember, (artifact_set.id, artifact.id)) is None:
+                self.repo.add(
+                    ArtifactSetMember(
+                        artifact_set_id=artifact_set.id,
+                        artifact_id=artifact.id,
+                        ordinal=ordinal,
+                        required=True,
+                    )
+                )
+        artifact_set.status = "verified"
+        artifact_set.availability = "local"
+        artifact_set.completeness = "complete"
+        artifact_set.security_status = "static_checks_passed_unapproved"
+        plan.status = "completed"
+        job.status = "completed"
+        job.progress_bytes = job.total_bytes
+        job.current_file = None
+        job.promoted_relative_path = request.promoted_relative_path
+        job.completed_at = _utcnow()
+        job.lease_token_hash = None
+        job.lease_expires_at = None
+        job.result = {
+            "files": len(request.files),
+            "capacity_observation": request.capacity_observation,
+            "integrity_verified": True,
+            "supply_chain_approved": False,
+        }
+        root = self.session.get(StorageRoot, job.storage_root_id)
+        capacity = request.capacity_observation
+        if root:
+            observed_capacity = capacity.get("capacity_bytes")
+            observed_free = capacity.get("free_bytes")
+            if isinstance(observed_capacity, int) and isinstance(observed_free, int):
+                root.capacity_bytes = observed_capacity
+                root.free_bytes = observed_free
+                root.capacity_observed_at = _utcnow()
+                root.status = RegistryService.capacity_decision(root, 0).status
+                root.validation_details = {
+                    "source": "artifact_job_completion",
+                    "job_id": str(job.id),
+                    "agent_path": capacity.get("agent_path"),
+                }
+        attempt = self.session.scalar(
+            select(ArtifactJobAttempt)
+            .where(ArtifactJobAttempt.job_id == job.id)
+            .order_by(ArtifactJobAttempt.attempt.desc(), ArtifactJobAttempt.started_at.desc())
+            .limit(1)
+        )
+        if attempt:
+            attempt.status = "completed"
+            attempt.completed_at = _utcnow()
+        self.audit.write(
+            "ARTIFACT_SET_ACQUIRED",
+            "artifact_set",
+            str(artifact_set.id),
+            {"job_id": str(job.id), "file_count": len(request.files), "approved": False},
+        )
+        for action in ("ARTIFACT_DOWNLOADED", "ARTIFACT_VERIFIED", "ARTIFACT_PROMOTED"):
+            self.audit.write(
+                action,
+                "artifact_set",
+                str(artifact_set.id),
+                {"job_id": str(job.id), "file_count": len(request.files)},
+            )
+        self._commit()
+        return ArtifactJobResponse.model_validate(job)
+
+    def fail(
+        self, job_id: uuid.UUID, node: ComputeNode, request: AgentJobFailure
+    ) -> ArtifactJobResponse:
+        job = self._leased_job(job_id, node, request.lease_token)
+        retry = request.retryable and job.attempt_count < 3 and not job.cancel_requested
+        job.status = "queued" if retry else "cancelled" if job.cancel_requested else "failed"
+        job.error_code = request.error_code
+        job.error_message = request.error_message
+        job.result = {"failure_details": request.details, "retry_scheduled": retry}
+        job.lease_token_hash = None
+        job.lease_expires_at = None
+        if not retry:
+            job.completed_at = _utcnow()
+        attempt = self.session.scalar(
+            select(ArtifactJobAttempt)
+            .where(ArtifactJobAttempt.job_id == job.id)
+            .order_by(ArtifactJobAttempt.attempt.desc(), ArtifactJobAttempt.started_at.desc())
+            .limit(1)
+        )
+        if attempt:
+            attempt.status = "retry_scheduled" if retry else job.status
+            attempt.completed_at = _utcnow()
+        plan = self.repo.plan(job.plan_id)
+        if plan:
+            plan.status = job.status
+        action = (
+            "DOWNLOAD_CANCELLED"
+            if job.status == "cancelled"
+            else "ARTIFACT_JOB_RETRY_SCHEDULED"
+            if retry
+            else "ARTIFACT_REJECTED"
+            if request.error_code in {"static_inspection_blocked", "verification_digest_mismatch"}
+            else "ARTIFACT_DOWNLOAD_FAILED"
+        )
+        self.audit.write(
+            action,
+            "artifact_job",
+            str(job.id),
+            {"error_code": request.error_code, "retryable": request.retryable},
+            outcome="blocked",
+        )
+        self._commit()
+        return ArtifactJobResponse.model_validate(job)

backend/src/modelforge_api/services/audit.py

@@ -0,0 +1,558 @@
+"""Canonical, append-only audit writes, checkpoints and verification.
+
+The chain head is a single serialization point. PostgreSQL writers use a transaction-scoped
+advisory lock; SQLite test databases use an engine-scoped process lock held until the owning
+transaction ends. An event and the durable singleton checkpoint advance in the same transaction.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import uuid
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Protocol
+from weakref import WeakKeyDictionary
+
+from sqlalchemy import event, select, text
+from sqlalchemy.engine import Connection, Engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.audit import (
+    AUDIT_CHAIN_SINGLETON_ID,
+    AUDIT_CURRENT_HASH_FORMAT,
+    AUDIT_EMPTY_LEGACY_PREFIX_SEAL,
+    AUDIT_HASH_FORMAT_V1,
+    AUDIT_HASH_FORMAT_V2,
+    AUDIT_LEGACY_PREFIX_DOMAIN,
+    canonical_audit_payload_and_hash,
+    canonical_audit_payload_text_and_hash,
+    normalise_audit_event_id,
+    normalise_audit_timestamp,
+)
+from modelforge_api.persistence.models import (
+    AuditChainHead,
+    AuditEvent,
+    _append_canonical_audit_event,
+)
+
+AUDIT_CHAIN_POSTGRES_LOCK_KEY = int.from_bytes(b"MF_AUDIT", byteorder="big", signed=False)
+_SQLITE_LOCK_INFO_KEY = "modelforge_audit_chain_lock"
+_SQLITE_LOCKS: WeakKeyDictionary[Engine, threading.Lock] = WeakKeyDictionary()
+_SQLITE_LOCKS_GUARD = threading.Lock()
+
+
+class AuditChainIntegrityError(RuntimeError):
+    """The writer cannot safely append to the authoritative audit chain."""
+
+
+class AuditEventLike(Protocol):
+    id: Any
+    sequence: int
+    occurred_at: Any
+    correlation_id: str
+    actor_type: str
+    actor_id: str
+    action: str
+    resource_type: str
+    resource_id: str | None
+    outcome: str
+    details: dict[str, Any]
+    previous_event_hash: str | None
+    event_hash: str
+    hash_format: str
+    canonical_payload: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class AuditContext:
+    """Authenticated/request identity seam for audit-producing services.
+
+    Existing callers can keep passing their actor fields directly. The API authentication boundary
+    can instead construct this context with its request correlation id without changing chain code.
+    """
+
+    actor_type: str
+    actor_id: str
+    correlation_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class AuditChainCheckpoint:
+    """Transport-neutral checkpoint used by local and restored-database verification."""
+
+    singleton_id: int
+    event_count: int
+    last_sequence: int
+    last_event_hash: str | None
+    hash_format: str
+    v2_start_sequence: int
+    legacy_prefix_count: int
+    legacy_prefix_seal: str
+
+
+@dataclass(frozen=True, slots=True)
+class AuditEventRecord:
+    """Transport-neutral event used when recovery reads a database through psql."""
+
+    id: Any
+    sequence: int
+    occurred_at: Any
+    correlation_id: str
+    actor_type: str
+    actor_id: str
+    action: str
+    resource_type: str
+    resource_id: str | None
+    outcome: str
+    details: dict[str, Any]
+    previous_event_hash: str | None
+    event_hash: str
+    hash_format: str
+    canonical_payload: str | None = None
+
+
+def _decode_canonical_audit_payload(value: str) -> dict[str, Any]:
+    def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, item in pairs:
+            if key in result:
+                raise ValueError(f"duplicate canonical audit payload key {key!r}")
+            result[key] = item
+        return result
+
+    decoded = json.loads(value, object_pairs_hook=reject_duplicate_keys)
+    if not isinstance(decoded, dict):
+        raise ValueError("canonical audit payload is not an object")
+    return decoded
+
+
+def _event_payload_violations(audit_event: AuditEventLike) -> list[str]:
+    """Verify one event's semantic payload and exact stored hash bytes."""
+
+    identity = str(audit_event.id)
+    if audit_event.hash_format == AUDIT_HASH_FORMAT_V1:
+        violations: list[str] = []
+        if audit_event.canonical_payload is not None:
+            violations.append(f"legacy audit event {identity} unexpectedly stores a v2 payload")
+        try:
+            _, expected_hash = canonical_audit_payload_and_hash(
+                correlation_id=audit_event.correlation_id,
+                actor_type=audit_event.actor_type,
+                actor_id=audit_event.actor_id,
+                action=audit_event.action,
+                resource_type=audit_event.resource_type,
+                resource_id=audit_event.resource_id,
+                outcome=audit_event.outcome,
+                details=audit_event.details,
+                previous_event_hash=audit_event.previous_event_hash,
+                hash_format=audit_event.hash_format,
+                event_id=audit_event.id,
+                occurred_at=audit_event.occurred_at,
+            )
+        except (TypeError, ValueError):
+            violations.append(f"audit event {identity} has a non-canonical payload")
+        else:
+            if audit_event.event_hash != expected_hash:
+                violations.append(
+                    f"audit event {identity} content hash does not match its payload"
+                )
+        return violations
+
+    if audit_event.hash_format != AUDIT_HASH_FORMAT_V2:
+        return [f"audit event {identity} has an unsupported hash format"]
+    canonical_payload = audit_event.canonical_payload
+    if not isinstance(canonical_payload, str):
+        return [f"audit event {identity} has no exact v2 canonical payload"]
+    violations = []
+    if hashlib.sha256(canonical_payload.encode("utf-8")).hexdigest() != audit_event.event_hash:
+        violations.append(f"audit event {identity} content hash does not match stored bytes")
+    try:
+        decoded_payload = _decode_canonical_audit_payload(canonical_payload)
+        _, expected_text, _expected_hash = canonical_audit_payload_text_and_hash(
+            correlation_id=audit_event.correlation_id,
+            actor_type=audit_event.actor_type,
+            actor_id=audit_event.actor_id,
+            action=audit_event.action,
+            resource_type=audit_event.resource_type,
+            resource_id=audit_event.resource_id,
+            outcome=audit_event.outcome,
+            details=audit_event.details,
+            previous_event_hash=audit_event.previous_event_hash,
+            hash_format=audit_event.hash_format,
+            event_id=audit_event.id,
+            occurred_at=audit_event.occurred_at,
+        )
+        expected_payload = _decode_canonical_audit_payload(expected_text)
+    except (json.JSONDecodeError, TypeError, ValueError):
+        violations.append(f"audit event {identity} has a non-canonical payload")
+    else:
+        canonical_decoded = json.dumps(
+            decoded_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+        )
+        canonical_expected = json.dumps(
+            expected_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+        )
+        if canonical_decoded != canonical_expected:
+            violations.append(
+                f"audit event {identity} content hash bytes do not describe its event columns"
+            )
+    return violations
+
+
+def _legacy_prefix_entry(event_row: AuditEventLike) -> bytes:
+    payload = {
+        "sequence": int(event_row.sequence),
+        "id": normalise_audit_event_id(event_row.id),
+        "occurred_at": normalise_audit_timestamp(event_row.occurred_at),
+        "event_hash": str(event_row.event_hash),
+    }
+    return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
+
+
+def legacy_audit_prefix_seal(events: Iterable[AuditEventLike]) -> str:
+    """Seal immutable v1 identities/timestamps without rewriting approved legacy events."""
+
+    digest = hashlib.sha256()
+    digest.update(AUDIT_LEGACY_PREFIX_DOMAIN)
+    for audit_event in events:
+        digest.update(_legacy_prefix_entry(audit_event))
+    return digest.hexdigest()
+
+
+EMPTY_LEGACY_PREFIX_SEAL = AUDIT_EMPTY_LEGACY_PREFIX_SEAL
+
+
+def audit_chain_violations(
+    events: Iterable[AuditEventLike], checkpoint: AuditChainHead | AuditChainCheckpoint | None
+) -> list[str]:
+    """Verify sequence, links, versioned hashes, the prefix seal and durable checkpoint."""
+
+    violations: list[str] = []
+    if checkpoint is None:
+        violations.append("audit chain checkpoint is missing")
+        cutover = 1
+        prefix_count = 0
+    else:
+        cutover = int(checkpoint.v2_start_sequence)
+        prefix_count = int(checkpoint.legacy_prefix_count)
+        if checkpoint.singleton_id != AUDIT_CHAIN_SINGLETON_ID:
+            violations.append("audit chain checkpoint has an invalid singleton id")
+        if checkpoint.hash_format != AUDIT_CURRENT_HASH_FORMAT:
+            violations.append("audit chain checkpoint names an unsupported current hash format")
+        if cutover < 1 or prefix_count != cutover - 1:
+            violations.append("audit chain checkpoint has an invalid v2 cutover")
+
+    previous: AuditEventLike | None = None
+    seen_sequences: set[int] = set()
+    seen_event_ids: set[str] = set()
+    observed = 0
+    prefix_observed = 0
+    prefix_digest = hashlib.sha256()
+    prefix_digest.update(AUDIT_LEGACY_PREFIX_DOMAIN)
+    for expected_sequence, audit_event in enumerate(events, start=1):
+        observed += 1
+        identity = str(audit_event.id)
+        try:
+            normalised_identity = normalise_audit_event_id(audit_event.id)
+        except ValueError:
+            violations.append(f"audit event {identity} has a non-canonical UUID")
+        else:
+            if normalised_identity in seen_event_ids:
+                violations.append(f"audit event UUID {normalised_identity} is duplicated")
+            seen_event_ids.add(normalised_identity)
+        sequence = int(audit_event.sequence)
+        if sequence in seen_sequences:
+            violations.append(f"audit sequence {sequence} is duplicated at event {identity}")
+        seen_sequences.add(sequence)
+        if sequence != expected_sequence:
+            violations.append(
+                f"audit event {identity} has sequence {sequence}; expected {expected_sequence}"
+            )
+
+        expected_link = previous.event_hash if previous is not None else None
+        if audit_event.previous_event_hash != expected_link:
+            position = "first event" if previous is None else f"event after {previous.id}"
+            violations.append(
+                f"audit event {identity} has an invalid previous hash for the {position}"
+            )
+
+        expected_format = AUDIT_HASH_FORMAT_V1 if sequence < cutover else AUDIT_HASH_FORMAT_V2
+        if audit_event.hash_format != expected_format:
+            violations.append(
+                f"audit event {identity} uses {audit_event.hash_format!r}; "
+                f"expected {expected_format!r} at sequence {sequence}"
+            )
+        violations.extend(_event_payload_violations(audit_event))
+
+        if sequence <= prefix_count:
+            prefix_observed += 1
+            try:
+                prefix_digest.update(_legacy_prefix_entry(audit_event))
+            except (TypeError, ValueError):
+                violations.append(
+                    f"legacy audit event {identity} has a non-canonical identity or timestamp"
+                )
+        previous = audit_event
+
+    if checkpoint is not None:
+        observed_last_sequence = int(previous.sequence) if previous is not None else 0
+        observed_last_hash = previous.event_hash if previous is not None else None
+        if int(checkpoint.event_count) != observed:
+            violations.append(
+                f"audit checkpoint records {checkpoint.event_count} events; observed {observed}"
+            )
+        if int(checkpoint.last_sequence) != observed_last_sequence:
+            violations.append(
+                "audit checkpoint last sequence does not match the retained event suffix"
+            )
+        if checkpoint.last_event_hash != observed_last_hash:
+            violations.append("audit checkpoint last hash does not match the retained event suffix")
+        if prefix_observed != prefix_count:
+            violations.append(
+                f"audit checkpoint seals {prefix_count} legacy events; observed {prefix_observed}"
+            )
+        if checkpoint.legacy_prefix_seal != prefix_digest.hexdigest():
+            violations.append("audit legacy-prefix seal does not match immutable legacy history")
+    return violations
+
+
+def _audit_engine(session: Session) -> Engine:
+    bind = session.get_bind()
+    if isinstance(bind, Connection):
+        return bind.engine
+    return bind
+
+
+def _acquire_audit_write_lock(session: Session) -> None:
+    """Hold the chain-head lock until the current root transaction completes."""
+
+    engine = _audit_engine(session)
+    dialect = engine.dialect.name
+    if dialect == "postgresql":
+        session.execute(
+            text("SELECT pg_advisory_xact_lock(:lock_key)"),
+            {"lock_key": AUDIT_CHAIN_POSTGRES_LOCK_KEY},
+        )
+        return
+    if dialect != "sqlite":
+        raise RuntimeError(f"audit-chain writes do not support the {dialect!r} database dialect")
+
+    if _SQLITE_LOCK_INFO_KEY in session.info:
+        return
+    if not session.in_transaction():
+        session.begin()
+    with _SQLITE_LOCKS_GUARD:
+        lock = _SQLITE_LOCKS.setdefault(engine, threading.Lock())
+    lock.acquire()
+    session.info[_SQLITE_LOCK_INFO_KEY] = lock
+
+
+@event.listens_for(Session, "after_transaction_end")
+def _release_sqlite_audit_write_lock(session: Session, transaction: Any) -> None:
+    if transaction.parent is not None:
+        return
+    lock = session.info.pop(_SQLITE_LOCK_INFO_KEY, None)
+    if lock is not None:
+        lock.release()
+
+
+def _checkpoint_from_mapping(row: Any) -> AuditChainCheckpoint:
+    return AuditChainCheckpoint(
+        singleton_id=int(row.singleton_id),
+        event_count=int(row.event_count),
+        last_sequence=int(row.last_sequence),
+        last_event_hash=row.last_event_hash,
+        hash_format=str(row.hash_format),
+        v2_start_sequence=int(row.v2_start_sequence),
+        legacy_prefix_count=int(row.legacy_prefix_count),
+        legacy_prefix_seal=str(row.legacy_prefix_seal),
+    )
+
+
+def _load_checkpoint(session: Session) -> AuditChainCheckpoint:
+    row = session.execute(
+        select(
+            AuditChainHead.singleton_id,
+            AuditChainHead.event_count,
+            AuditChainHead.last_sequence,
+            AuditChainHead.last_event_hash,
+            AuditChainHead.hash_format,
+            AuditChainHead.v2_start_sequence,
+            AuditChainHead.legacy_prefix_count,
+            AuditChainHead.legacy_prefix_seal,
+        ).where(AuditChainHead.singleton_id == AUDIT_CHAIN_SINGLETON_ID)
+    ).one_or_none()
+    if row is not None:
+        return _checkpoint_from_mapping(row)
+    raise AuditChainIntegrityError(
+        "audit chain checkpoint is missing; only schema creation or the audited migration may seed it"
+    )
+
+
+def _assert_checkpoint_is_appendable(
+    session: Session, checkpoint: AuditChainCheckpoint
+) -> AuditEvent | None:
+    """Validate the locked checkpoint and constant-size retained tail before append.
+
+    Migration and recovery seal/verify the complete immutable prefix. Runtime append therefore
+    proves the checkpoint shape, current tail hash/link and compare-and-set predecessor in O(1).
+    A privileged edit in older middle history is intentionally the responsibility of the explicit
+    strict invariant/recovery gates; ordinary SQLAlchemy audit DML is blocked separately.
+    """
+
+    violations: list[str] = []
+    if checkpoint.singleton_id != AUDIT_CHAIN_SINGLETON_ID:
+        violations.append("checkpoint singleton id is invalid")
+    if checkpoint.hash_format != AUDIT_CURRENT_HASH_FORMAT:
+        violations.append("checkpoint hash format is unsupported")
+    if checkpoint.event_count < 0 or checkpoint.last_sequence < 0:
+        violations.append("checkpoint counts cannot be negative")
+    if checkpoint.event_count != checkpoint.last_sequence:
+        violations.append("checkpoint event count and last sequence disagree")
+    if checkpoint.v2_start_sequence < 1:
+        violations.append("checkpoint v2 cutover is invalid")
+    if checkpoint.legacy_prefix_count != checkpoint.v2_start_sequence - 1:
+        violations.append("checkpoint legacy-prefix count and v2 cutover disagree")
+    if checkpoint.legacy_prefix_count > checkpoint.event_count:
+        violations.append("checkpoint legacy prefix exceeds the retained event count")
+    if (
+        len(checkpoint.legacy_prefix_seal) != 64
+        or any(character not in "0123456789abcdef" for character in checkpoint.legacy_prefix_seal)
+    ):
+        violations.append("checkpoint legacy-prefix seal is malformed")
+    if (
+        checkpoint.legacy_prefix_count == 0
+        and checkpoint.legacy_prefix_seal != EMPTY_LEGACY_PREFIX_SEAL
+    ):
+        violations.append("empty legacy-prefix checkpoint has the wrong seal")
+
+    tail = list(
+        session.scalars(
+        select(AuditEvent)
+            .order_by(AuditEvent.sequence.desc(), AuditEvent.id.desc())
+            .limit(2)
+            .execution_options(populate_existing=True)
+        )
+    )
+    latest = tail[0] if tail else None
+    if checkpoint.event_count == 0:
+        if latest is not None:
+            violations.append("empty checkpoint has a retained audit tail")
+        if checkpoint.last_event_hash is not None:
+            violations.append("empty checkpoint carries a last-event hash")
+    else:
+        if latest is None:
+            violations.append("non-empty checkpoint has no retained audit tail")
+        elif (
+            latest.sequence != checkpoint.last_sequence
+            or latest.event_hash != checkpoint.last_event_hash
+        ):
+            violations.append("checkpoint does not identify the current retained audit tail")
+        if checkpoint.last_event_hash is None:
+            violations.append("non-empty checkpoint has no last-event hash")
+        elif len(checkpoint.last_event_hash) != 64 or any(
+            character not in "0123456789abcdef"
+            for character in checkpoint.last_event_hash
+        ):
+            violations.append("checkpoint last-event hash is malformed")
+
+    if latest is not None:
+        expected_format = (
+            AUDIT_HASH_FORMAT_V1
+            if latest.sequence < checkpoint.v2_start_sequence
+            else AUDIT_HASH_FORMAT_V2
+        )
+        if latest.hash_format != expected_format:
+            violations.append("retained audit tail uses the wrong hash format")
+        payload_violations = _event_payload_violations(latest)
+        violations.extend(
+            f"retained audit tail: {violation}" for violation in payload_violations
+        )
+
+        if latest.sequence == 1:
+            if latest.previous_event_hash is not None:
+                violations.append("first retained audit event has a previous hash")
+            if len(tail) != 1:
+                violations.append("checkpoint count one has more than one retained event")
+        elif latest.sequence > 1:
+            if len(tail) != 2:
+                violations.append("retained audit tail has no predecessor")
+            else:
+                predecessor = tail[1]
+                if predecessor.sequence != latest.sequence - 1:
+                    violations.append("retained audit tail predecessor is not contiguous")
+                if latest.previous_event_hash != predecessor.event_hash:
+                    violations.append("retained audit tail link does not match its predecessor")
+
+    if violations:
+        raise AuditChainIntegrityError(
+            "audit checkpoint/tail failed pre-append verification: "
+            + "; ".join(violations[:5])
+        )
+    return latest
+
+
+class AuditWriter:
+    def __init__(
+        self,
+        session: Session,
+        actor_type: str | None = None,
+        actor_id: str | None = None,
+        *,
+        context: AuditContext | None = None,
+    ) -> None:
+        if context is not None:
+            if actor_type is not None or actor_id is not None:
+                raise ValueError("pass either an audit context or actor fields, not both")
+            resolved = context
+        else:
+            if actor_type is None or actor_id is None:
+                raise ValueError("audit actor type and id are required")
+            resolved = AuditContext(actor_type=actor_type, actor_id=actor_id)
+        self.session = session
+        self.context = resolved
+
+    def write(
+        self,
+        action: str,
+        resource_type: str,
+        resource_id: str | None,
+        details: dict[str, Any],
+        outcome: str = "success",
+    ) -> AuditEvent:
+        _acquire_audit_write_lock(self.session)
+        try:
+            checkpoint = _load_checkpoint(self.session)
+            previous = _assert_checkpoint_is_appendable(self.session, checkpoint)
+            event_id = uuid.uuid4()
+            occurred_at = datetime.now(UTC)
+            return _append_canonical_audit_event(
+                self.session,
+                event_id=event_id,
+                occurred_at=occurred_at,
+                correlation_id=self.context.correlation_id or str(uuid.uuid4()),
+                actor_type=self.context.actor_type,
+                actor_id=self.context.actor_id,
+                action=action,
+                resource_type=resource_type,
+                resource_id=resource_id,
+                outcome=outcome,
+                details=details,
+                previous_event_hash=previous.event_hash if previous else None,
+                expected_event_count=checkpoint.event_count,
+                expected_last_sequence=checkpoint.last_sequence,
+                expected_last_event_hash=checkpoint.last_event_hash,
+                expected_hash_format=checkpoint.hash_format,
+                expected_v2_start_sequence=checkpoint.v2_start_sequence,
+                expected_legacy_prefix_count=checkpoint.legacy_prefix_count,
+                expected_legacy_prefix_seal=checkpoint.legacy_prefix_seal,
+            )
+        except Exception:
+            # A caller must never be able to catch an audit failure and commit an unaudited domain
+            # mutation or a detached event. Roll back the complete owning transaction fail-closed.
+            self.session.rollback()
+            raise

backend/src/modelforge_api/services/capability_evaluation.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any, cast
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.capability_evaluation import (
+    ALLOWED_METRICS,
+    CapabilityAdvisorResponse,
+    CapabilityCaseResult,
+    CapabilityEvaluationCase,
+    CapabilityEvaluationRunCreate,
+    CapabilityEvaluationRunResponse,
+    CapabilityEvaluationSuiteCreate,
+    CapabilityEvaluationSuiteResponse,
+    CapabilityRecommendationState,
+    EvaluationType,
+    MetricDefinition,
+)
+from modelforge_api.persistence.models import (
+    Capability,
+    CapabilityContract,
+    CapabilityDeployment,
+    CapabilityEvaluationRun,
+    CapabilityEvaluationSuite,
+)
+from modelforge_api.services.serving import ServingError
+
+
+def _digest(value: Any) -> str:
+    raw = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(raw.encode()).hexdigest()
+
+
+class CapabilityEvaluationService:
+    def __init__(self, session: Session) -> None:
+        self.session = session
+
+    def _contract(self, key: str, version: int) -> CapabilityContract | None:
+        return self.session.scalar(
+            select(CapabilityContract)
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
+            .where(Capability.key == key, CapabilityContract.version == version)
+        )
+
+    def create_suite(
+        self, request: CapabilityEvaluationSuiteCreate
+    ) -> CapabilityEvaluationSuiteResponse:
+        contract = self._contract(request.capability, request.contract_version)
+        if not contract:
+            raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability contract was not found")
+        contract_type = str(contract.contract.get("estate", {}).get("evaluation_type", ""))
+        if contract_type != request.evaluation_type:
+            raise ServingError(409, "EVALUATION_TYPE_MISMATCH", "suite type differs from capability contract")
+        definition = request.model_dump(mode="json")
+        digest = _digest(definition)
+        existing = self.session.scalar(
+            select(CapabilityEvaluationSuite).where(
+                CapabilityEvaluationSuite.definition_digest == digest
+            )
+        )
+        if existing:
+            return self.suite_response(existing)
+        suite = CapabilityEvaluationSuite(
+            capability_contract_id=contract.id,
+            key=request.key,
+            evaluation_type=request.evaluation_type,
+            revision=request.revision,
+            dataset_revision=request.dataset_revision,
+            definition_digest=digest,
+            metric_definitions={item.name: item.model_dump(mode="json") for item in request.metrics},
+            case_definitions=[item.model_dump(mode="json") for item in request.cases],
+            thresholds=request.thresholds,
+        )
+        self.session.add(suite)
+        self.session.commit()
+        self.session.refresh(suite)
+        return self.suite_response(suite)
+
+    def suite_response(self, suite: CapabilityEvaluationSuite) -> CapabilityEvaluationSuiteResponse:
+        contract = self.session.get(CapabilityContract, suite.capability_contract_id)
+        capability = self.session.get(Capability, contract.capability_id) if contract else None
+        if not contract or not capability:
+            raise ServingError(409, "EVALUATION_PROVENANCE_INCOMPLETE", "suite contract is missing")
+        return CapabilityEvaluationSuiteResponse(
+            id=suite.id,
+            capability=capability.key,
+            contract_version=contract.version,
+            key=suite.key,
+            evaluation_type=cast(EvaluationType, suite.evaluation_type),
+            revision=suite.revision,
+            dataset_revision=suite.dataset_revision,
+            definition_digest=suite.definition_digest,
+            metrics=[
+                MetricDefinition.model_validate(item)
+                for item in suite.metric_definitions.values()
+            ],
+            cases=[
+                CapabilityEvaluationCase.model_validate(item)
+                for item in suite.case_definitions
+            ],
+            thresholds={key: float(value) for key, value in suite.thresholds.items()},
+            created_at=suite.created_at,
+        )
+
+    def suites(self) -> list[CapabilityEvaluationSuiteResponse]:
+        rows = self.session.scalars(
+            select(CapabilityEvaluationSuite).order_by(CapabilityEvaluationSuite.created_at)
+        ).all()
+        return [self.suite_response(item) for item in rows]
+
+    def create_run(self, request: CapabilityEvaluationRunCreate) -> CapabilityEvaluationRunResponse:
+        suite = self.session.get(CapabilityEvaluationSuite, request.suite_id)
+        deployment = self.session.get(CapabilityDeployment, request.capability_deployment_id)
+        if not suite or not deployment:
+            raise ServingError(404, "EVALUATION_TARGET_NOT_FOUND", "suite or deployment was not found")
+        if suite.capability_contract_id != deployment.capability_contract_id:
+            raise ServingError(409, "EVALUATION_TARGET_MISMATCH", "suite and deployment capabilities differ")
+        declared = set(suite.metric_definitions)
+        if set(request.metrics) != declared or declared - ALLOWED_METRICS[suite.evaluation_type]:
+            raise ServingError(422, "INVALID_METRICS", "aggregate metrics do not match the suite")
+        expected_cases = {str(item["key"]) for item in suite.case_definitions}
+        actual_cases = {item.case_key for item in request.cases}
+        if actual_cases != expected_cases:
+            raise ServingError(422, "INVALID_CASE_SET", "run must contain every immutable suite case exactly once")
+        evidence_payload = request.model_dump(mode="json")
+        digest = _digest(evidence_payload)
+        existing = self.session.scalar(
+            select(CapabilityEvaluationRun).where(CapabilityEvaluationRun.evidence_digest == digest)
+        )
+        if existing:
+            return self.run_response(existing)
+        run = CapabilityEvaluationRun(
+            suite_id=suite.id,
+            capability_deployment_id=deployment.id,
+            status=request.status,
+            metric_values=request.metrics,
+            case_results=[item.model_dump(mode="json") for item in request.cases],
+            resource_metrics=request.resource_metrics,
+            environment_fingerprint=request.environment_fingerprint,
+            evidence_digest=digest,
+            evidence={**request.evidence, "payload_persisted": False, "content_logged": False},
+            started_at=request.started_at,
+            completed_at=request.completed_at,
+        )
+        self.session.add(run)
+        self.session.commit()
+        self.session.refresh(run)
+        return self.run_response(run)
+
+    def run_response(self, run: CapabilityEvaluationRun) -> CapabilityEvaluationRunResponse:
+        suite = self.session.get(CapabilityEvaluationSuite, run.suite_id)
+        if not suite:
+            raise ServingError(409, "EVALUATION_PROVENANCE_INCOMPLETE", "run suite is missing")
+        return CapabilityEvaluationRunResponse(
+            id=run.id,
+            suite_id=run.suite_id,
+            capability_deployment_id=run.capability_deployment_id,
+            evaluation_type=cast(EvaluationType, suite.evaluation_type),
+            status=run.status,
+            metrics=run.metric_values,
+            cases=[CapabilityCaseResult.model_validate(item) for item in run.case_results],
+            resource_metrics=run.resource_metrics,
+            environment_fingerprint=run.environment_fingerprint,
+            evidence_digest=run.evidence_digest,
+            evidence=run.evidence,
+            started_at=run.started_at,
+            completed_at=run.completed_at,
+            created_at=run.created_at,
+        )
+
+    def runs(self) -> list[CapabilityEvaluationRunResponse]:
+        rows = self.session.scalars(
+            select(CapabilityEvaluationRun).order_by(CapabilityEvaluationRun.created_at)
+        ).all()
+        return [self.run_response(item) for item in rows]
+
+    def advisor(self) -> list[CapabilityAdvisorResponse]:
+        contracts = self.session.execute(
+            select(CapabilityContract, Capability)
+            .join(Capability, Capability.id == CapabilityContract.capability_id)
+            .order_by(Capability.key, CapabilityContract.version)
+        ).all()
+        result: list[CapabilityAdvisorResponse] = []
+        for contract, capability in contracts:
+            deployment = self.session.scalar(
+                select(CapabilityDeployment)
+                .where(
+                    CapabilityDeployment.capability_contract_id == contract.id,
+                    CapabilityDeployment.status.in_(["candidate", "approved", "active", "stable"]),
+                )
+                .order_by(CapabilityDeployment.production.desc(), CapabilityDeployment.created_at.desc())
+            )
+            if not deployment:
+                result.append(CapabilityAdvisorResponse(
+                    capability=capability.key,
+                    contract_version=contract.version,
+                    state="REQUIRES_MORE_EVIDENCE",
+                    reasons=["no active capability deployment has proven this contract"],
+                ))
+                continue
+            run = self.session.scalar(
+                select(CapabilityEvaluationRun)
+                .join(CapabilityEvaluationSuite, CapabilityEvaluationSuite.id == CapabilityEvaluationRun.suite_id)
+                .where(
+                    CapabilityEvaluationRun.capability_deployment_id == deployment.id,
+                    CapabilityEvaluationSuite.capability_contract_id == contract.id,
+                )
+                .order_by(CapabilityEvaluationRun.completed_at.desc())
+            )
+            if not run:
+                if deployment.production:
+                    result.append(CapabilityAdvisorResponse(
+                        capability=capability.key,
+                        contract_version=contract.version,
+                        state="KEEP_CURRENT",
+                        deployment_id=deployment.id,
+                        reasons=["current production deployment remains active; no comparable new typed run supersedes it"],
+                    ))
+                    continue
+                result.append(CapabilityAdvisorResponse(
+                    capability=capability.key,
+                    contract_version=contract.version,
+                    state="LAB_READY",
+                    deployment_id=deployment.id,
+                    reasons=["runtime deployment is available; modality evaluation is still required"],
+                ))
+                continue
+            suite = self.session.get(CapabilityEvaluationSuite, run.suite_id)
+            if suite is None:
+                raise ServingError(
+                    409, "EVALUATION_PROVENANCE_INCOMPLETE", "run suite is missing"
+                )
+            failures = [item["case_key"] for item in run.case_results if item["status"] != "passed"]
+            threshold_failures: list[str] = []
+            for name, threshold in suite.thresholds.items():
+                definition = suite.metric_definitions[name]
+                value = float(run.metric_values[name])
+                direction = definition["direction"]
+                if (direction == "higher_is_better" and value < float(threshold)) or (
+                    direction == "lower_is_better" and value > float(threshold)
+                ):
+                    threshold_failures.append(name)
+            if run.status != "completed" or failures or threshold_failures:
+                reasons = []
+                if run.status != "completed":
+                    reasons.append("latest evaluation run did not complete")
+                if failures:
+                    reasons.append("case failures: " + ",".join(failures))
+                if threshold_failures:
+                    reasons.append("threshold failures: " + ",".join(threshold_failures))
+                state: CapabilityRecommendationState = "BLOCKED"
+            else:
+                reasons = ["completed typed evaluation satisfies every declared threshold"]
+                state = "PROMOTION_ELIGIBLE"
+            result.append(CapabilityAdvisorResponse(
+                capability=capability.key,
+                contract_version=contract.version,
+                state=state,
+                deployment_id=deployment.id,
+                evaluation_run_id=run.id,
+                reasons=reasons,
+            ))
+        return result

backend/src/modelforge_api/services/gpu_scheduler.py

@@ -0,0 +1,395 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime, timedelta
+from enum import StrEnum
+from typing import Any
+
+
+class AttributionConfidence(StrEnum):
+    KNOWN = "KNOWN"
+    ESTIMATED = "ESTIMATED"
+    UNKNOWN = "UNKNOWN"
+
+
+class PressureState(StrEnum):
+    NORMAL = "NORMAL"
+    ELEVATED = "ELEVATED"
+    HIGH = "HIGH"
+    CRITICAL = "CRITICAL"
+
+
+class PlacementVerdict(StrEnum):
+    ADMIT = "ADMIT"
+    ADMIT_AFTER_EVICTION = "ADMIT_AFTER_EVICTION"
+    QUEUE = "QUEUE"
+    REJECT_CAPACITY = "REJECT_CAPACITY"
+    REJECT_HEALTH = "REJECT_HEALTH"
+    REJECT_POLICY = "REJECT_POLICY"
+
+
+class ResidencyPolicy(StrEnum):
+    ALWAYS_WARM = "always_warm"
+    KEEP_WARM = "keep_warm"
+    LOAD_ON_DEMAND = "load_on_demand"
+    LAB_ONLY = "lab_only"
+
+
+PRIORITY_RANK = {"production": 0, "interactive": 1, "background": 2, "lab": 3, "benchmark": 3}
+
+
+@dataclass(frozen=True)
+class SchedulerPolicy:
+    revision: str = "m10-v1"
+    reserve_minimum_bytes: int = 1_073_741_824
+    reserve_percentage: float = 0.05
+    runtime_margin_bytes: int = 268_435_456
+    deployment_margin_minimum_bytes: int = 134_217_728
+    deployment_margin_percentage: float = 0.10
+    request_execution_floor_bytes: int = 67_108_864
+    elevated_headroom_percentage: float = 0.20
+    high_headroom_percentage: float = 0.10
+    critical_headroom_percentage: float = 0.03
+    recovery_extra_percentage: float = 0.03
+    pressure_stable_seconds: int = 30
+    eviction_cooldown_seconds: int = 60
+    global_queue_limit: int = 128
+    placement_history_limit: int = 500
+    lab_paused: bool = False
+
+
+@dataclass(frozen=True)
+class LeaseAllocation:
+    reserved_bytes: int
+    materialized_bytes: int = 0
+
+    @property
+    def future_bytes(self) -> int:
+        return max(0, self.reserved_bytes - self.materialized_bytes)
+
+
+@dataclass(frozen=True)
+class AccountingSnapshot:
+    total_bytes: int
+    observed_used_bytes: int | None
+    resident_bytes: int
+    future_lease_bytes: int
+    reserve_bytes: int
+    external_bytes: int
+    schedulable_bytes: int
+    attribution: AttributionConfidence
+    invariant_delta_bytes: int
+    pressure: PressureState
+
+
+@dataclass(frozen=True)
+class ResidentCandidate:
+    deployment_id: str
+    capability: str
+    resident_bytes: int
+    active_requests: int
+    priority: str
+    policy: str
+    idle_since: datetime | None
+    cold_load_ms: float
+    last_used_at: datetime | None
+    pinned: bool = False
+
+
+@dataclass(frozen=True)
+class PlacementInput:
+    deployment_id: str
+    capability: str
+    node_id: str
+    accelerator_id: str
+    priority: str
+    required_bytes: int
+    cold_load_ms: float
+    is_resident: bool
+    envelope_stale: bool
+    runtime_healthy: bool
+    node_eligible: bool
+    deadline_remaining_ms: float | None = None
+
+
+@dataclass(frozen=True)
+class PlannedEviction:
+    deployment_id: str
+    capability: str
+    expected_reclaimed_bytes: int
+    reason: str
+
+
+@dataclass(frozen=True)
+class PlacementDecision:
+    verdict: PlacementVerdict
+    reason_codes: tuple[str, ...]
+    required_bytes: int
+    headroom_before_bytes: int
+    headroom_after_bytes: int
+    evictions: tuple[PlannedEviction, ...] = field(default_factory=tuple)
+    fingerprint: str = ""
+
+    def evidence(self) -> dict[str, Any]:
+        result = asdict(self)
+        result["verdict"] = self.verdict.value
+        result["evictions"] = [asdict(item) for item in self.evictions]
+        return result
+
+
+def dynamic_reserve(total_bytes: int, policy: SchedulerPolicy) -> int:
+    return max(
+        policy.reserve_minimum_bytes,
+        int(total_bytes * policy.reserve_percentage),
+        policy.runtime_margin_bytes,
+    )
+
+
+def required_envelope(peak_bytes: int, policy: SchedulerPolicy) -> int:
+    margin = max(
+        policy.deployment_margin_minimum_bytes,
+        int(peak_bytes * policy.deployment_margin_percentage),
+    )
+    return max(0, peak_bytes) + margin
+
+
+def pressure_for(
+    headroom: int, total: int, previous: PressureState = PressureState.NORMAL
+) -> PressureState:
+    ratio = headroom / total if total > 0 else 0.0
+    thresholds = {
+        PressureState.CRITICAL: 0.03,
+        PressureState.HIGH: 0.10,
+        PressureState.ELEVATED: 0.20,
+    }
+    if ratio <= thresholds[PressureState.CRITICAL]:
+        return PressureState.CRITICAL
+    if ratio <= thresholds[PressureState.HIGH]:
+        return PressureState.HIGH
+    if ratio <= thresholds[PressureState.ELEVATED]:
+        return PressureState.ELEVATED
+    # Recovery has a 3%-of-total gap. Persistence of the candidate state is
+    # handled by the accelerator pressure record in the service.
+    if previous == PressureState.CRITICAL and ratio <= 0.06:
+        return previous
+    if previous == PressureState.HIGH and ratio <= 0.13:
+        return previous
+    if previous == PressureState.ELEVATED and ratio <= 0.23:
+        return previous
+    return PressureState.NORMAL
+
+
+def calculate_accounting(
+    *,
+    total_bytes: int,
+    observed_used_bytes: int | None,
+    resident_bytes: int,
+    leases: list[LeaseAllocation],
+    policy: SchedulerPolicy,
+    telemetry_fresh: bool = True,
+    previous_pressure: PressureState = PressureState.NORMAL,
+) -> AccountingSnapshot:
+    total = max(0, total_bytes)
+    resident = max(0, resident_bytes)
+    reserve = dynamic_reserve(total, policy)
+    future = sum(item.future_bytes for item in leases)
+    if observed_used_bytes is None or not telemetry_fresh:
+        return AccountingSnapshot(
+            total,
+            observed_used_bytes,
+            resident,
+            future,
+            reserve,
+            max(0, total - resident),
+            0,
+            AttributionConfidence.UNKNOWN,
+            0,
+            PressureState.CRITICAL,
+        )
+    observed = min(total, max(0, observed_used_bytes))
+    external = max(0, observed - resident)
+    attributed = external + resident
+    delta = observed - attributed
+    tolerance = max(67_108_864, int(total * 0.01))
+    confidence = (
+        AttributionConfidence.KNOWN if abs(delta) <= tolerance else AttributionConfidence.ESTIMATED
+    )
+    # NVML used already contains resident allocations. Only the unmaterialized
+    # portion of leases is additionally reserved, preventing double counting.
+    schedulable = max(0, total - max(observed, resident) - future - reserve)
+    pressure = pressure_for(schedulable, total, previous_pressure)
+    return AccountingSnapshot(
+        total,
+        observed,
+        resident,
+        future,
+        reserve,
+        external,
+        schedulable,
+        confidence,
+        delta,
+        pressure,
+    )
+
+
+def _eviction_key(candidate: ResidentCandidate) -> tuple[int, int, float, float]:
+    policy_penalty = {
+        ResidencyPolicy.LAB_ONLY.value: 0,
+        ResidencyPolicy.LOAD_ON_DEMAND.value: 1,
+        ResidencyPolicy.KEEP_WARM.value: 2,
+        ResidencyPolicy.ALWAYS_WARM.value: 10,
+    }.get(candidate.policy, 5)
+    last_use = candidate.last_used_at or candidate.idle_since or datetime.min.replace(tzinfo=UTC)
+    # Safety/priority precede reload cost; among equivalent candidates evict
+    # the cheaper and least-recently-used residency first.
+    return (
+        policy_penalty,
+        -PRIORITY_RANK.get(candidate.priority, 4),
+        candidate.cold_load_ms,
+        last_use.timestamp(),
+    )
+
+
+def plan_placement(
+    request: PlacementInput,
+    snapshot: AccountingSnapshot,
+    residents: list[ResidentCandidate],
+    policy: SchedulerPolicy,
+    *,
+    global_queue_depth: int = 0,
+) -> PlacementDecision:
+    reasons: list[str] = []
+    verdict: PlacementVerdict
+    evictions: list[PlannedEviction] = []
+    needed = max(policy.request_execution_floor_bytes, request.required_bytes)
+    if not request.node_eligible or not request.runtime_healthy:
+        verdict = PlacementVerdict.REJECT_HEALTH
+        reasons.append("NODE_UNAVAILABLE" if not request.node_eligible else "RUNTIME_UNHEALTHY")
+    elif request.envelope_stale or snapshot.attribution is AttributionConfidence.UNKNOWN:
+        verdict = PlacementVerdict.REJECT_HEALTH
+        reasons.append("SCHEDULER_STATE_STALE")
+    elif policy.lab_paused and PRIORITY_RANK.get(request.priority, 4) >= PRIORITY_RANK["lab"]:
+        verdict = PlacementVerdict.REJECT_POLICY
+        reasons.append("POLICY_BLOCKED")
+    elif (
+        request.deadline_remaining_ms is not None
+        and not request.is_resident
+        and request.cold_load_ms > request.deadline_remaining_ms
+    ):
+        verdict = PlacementVerdict.REJECT_POLICY
+        reasons.append("DEADLINE_CANNOT_BE_MET")
+    elif snapshot.pressure is PressureState.CRITICAL:
+        verdict = PlacementVerdict.REJECT_CAPACITY
+        reasons.append("EXTERNAL_GPU_PRESSURE")
+    elif (
+        snapshot.pressure in {PressureState.HIGH, PressureState.ELEVATED}
+        and PRIORITY_RANK.get(request.priority, 4) >= PRIORITY_RANK["background"]
+        and not request.is_resident
+    ):
+        verdict = (
+            PlacementVerdict.QUEUE
+            if global_queue_depth < policy.global_queue_limit
+            else PlacementVerdict.REJECT_CAPACITY
+        )
+        reasons.append(
+            "EXTERNAL_GPU_PRESSURE" if verdict is PlacementVerdict.QUEUE else "QUEUE_FULL"
+        )
+    elif needed <= snapshot.schedulable_bytes:
+        verdict = PlacementVerdict.ADMIT
+        reasons.append(
+            "RESIDENT_WARM_HIT" if request.is_resident else "MEASURED_CAPACITY_AVAILABLE"
+        )
+    else:
+        reclaim = 0
+        for candidate in sorted(residents, key=_eviction_key):
+            if (
+                candidate.deployment_id == request.deployment_id
+                or candidate.active_requests > 0
+                or candidate.pinned
+                or candidate.policy == ResidencyPolicy.ALWAYS_WARM.value
+                or PRIORITY_RANK.get(candidate.priority, 4) < PRIORITY_RANK.get(request.priority, 4)
+            ):
+                continue
+            reason = (
+                "IDLE_LAB_EVICTED_FOR_PRODUCTION"
+                if PRIORITY_RANK.get(request.priority, 4) <= PRIORITY_RANK["interactive"]
+                and PRIORITY_RANK.get(candidate.priority, 4) >= PRIORITY_RANK["lab"]
+                else "CAPACITY_REBALANCE"
+            )
+            evictions.append(
+                PlannedEviction(
+                    candidate.deployment_id, candidate.capability, candidate.resident_bytes, reason
+                )
+            )
+            reclaim += candidate.resident_bytes
+            if needed <= snapshot.schedulable_bytes + reclaim:
+                break
+        if needed <= snapshot.schedulable_bytes + reclaim:
+            verdict = PlacementVerdict.ADMIT_AFTER_EVICTION
+            reasons.append("MANAGED_IDLE_EVICTION_REQUIRED")
+        elif any(item.active_requests > 0 for item in residents):
+            verdict = PlacementVerdict.QUEUE
+            reasons.append("RESIDENCY_CONFLICT")
+            evictions = []
+        else:
+            verdict = PlacementVerdict.REJECT_CAPACITY
+            reasons.append("INSUFFICIENT_SCHEDULABLE_VRAM")
+            evictions = []
+    after = max(
+        0,
+        snapshot.schedulable_bytes
+        + sum(item.expected_reclaimed_bytes for item in evictions)
+        - (
+            needed
+            if verdict in {PlacementVerdict.ADMIT, PlacementVerdict.ADMIT_AFTER_EVICTION}
+            else 0
+        ),
+    )
+    evidence = {
+        "policy_revision": policy.revision,
+        "request": asdict(request),
+        "snapshot": asdict(snapshot),
+        "verdict": verdict.value,
+        "reasons": reasons,
+        "evictions": [asdict(item) for item in evictions],
+    }
+    fingerprint = hashlib.sha256(
+        json.dumps(evidence, sort_keys=True, default=str, separators=(",", ":")).encode()
+    ).hexdigest()
+    return PlacementDecision(
+        verdict,
+        tuple(reasons),
+        needed,
+        snapshot.schedulable_bytes,
+        after,
+        tuple(evictions),
+        fingerprint,
+    )
+
+
+@dataclass
+class PressureHysteresis:
+    state: PressureState = PressureState.NORMAL
+    candidate: PressureState | None = None
+    candidate_since: datetime | None = None
+    changed_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+
+    def observe(
+        self, candidate: PressureState, now: datetime, stable_seconds: int = 30
+    ) -> PressureState:
+        if candidate == self.state:
+            self.candidate = None
+            self.candidate_since = None
+            return self.state
+        if self.candidate != candidate:
+            self.candidate = candidate
+            self.candidate_since = now
+            return self.state
+        if self.candidate_since and now - self.candidate_since >= timedelta(seconds=stable_seconds):
+            self.state = candidate
+            self.changed_at = now
+            self.candidate = None
+            self.candidate_since = None
+        return self.state

backend/src/modelforge_api/services/hardware_factory.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.hardware.collectors import (
+    NodeIdentityProvider,
+    SystemHostCollector,
+    build_nvml_collector,
+)
+from modelforge_api.services.hardware_inventory import HardwareInventoryService
+from modelforge_api.settings import Settings
+
+
+def build_hardware_service(session: Session, settings: Settings) -> HardwareInventoryService:
+    host = SystemHostCollector(
+        NodeIdentityProvider(
+            settings.node_identity_file,
+            settings.node_identity,
+            force_persisted=settings.node_identity_mode == "persisted",
+        ),
+        {
+            "model_cache": Path(settings.hf_home),
+            "artifacts": Path(settings.artifact_root),
+            "quarantine": Path(settings.quarantine_root),
+        },
+    )
+    return HardwareInventoryService(session, host, build_nvml_collector())

backend/src/modelforge_api/services/hardware_inventory.py

@@ -0,0 +1,557 @@
+from __future__ import annotations
+
+import threading
+from datetime import UTC, datetime
+from typing import Any, TypeVar
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.agent_protocol import AGENT_PROTOCOL_VERSION
+from modelforge_api.domain.enums import (
+    AgentHealth,
+    Availability,
+    HardwareStatus,
+    InventoryRunStatus,
+    NodeLiveness,
+    ObservationSource,
+)
+from modelforge_api.domain.hardware import (
+    AcceleratorCollector,
+    AcceleratorState,
+    AcceleratorTelemetry,
+    HardwareOverview,
+    HardwareSnapshot,
+    HardwareState,
+    HostCollector,
+    NodeState,
+    ObservedValue,
+    StorageState,
+)
+from modelforge_api.persistence.hardware_repository import HardwareRepository
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AcceleratorTelemetryLatest,
+    ComputeNode,
+    HardwareInventoryRun,
+    HostTelemetryLatest,
+    StorageVolumeState,
+)
+from modelforge_api.services.audit import AuditWriter
+
+
+class HardwareRefreshBusy(RuntimeError):
+    pass
+
+
+T = TypeVar("T")
+
+
+def _aware(value: datetime) -> datetime:
+    return value if value.tzinfo else value.replace(tzinfo=UTC)
+
+
+class HardwareInventoryService:
+    _refresh_lock = threading.Lock()
+
+    def __init__(
+        self,
+        session: Session,
+        host_collector: HostCollector,
+        accelerator_collector: AcceleratorCollector,
+        source: ObservationSource = ObservationSource.LOCAL_CONTROL_PLANE,
+        received_at: datetime | None = None,
+        observation_sequence: int | None = None,
+    ) -> None:
+        self.session = session
+        self.repository = HardwareRepository(session)
+        self.host_collector = host_collector
+        self.accelerator_collector = accelerator_collector
+        self.source = source
+        self.received_at = received_at
+        self.observation_sequence = observation_sequence
+
+    def _audit(
+        self,
+        action: str,
+        resource_type: str,
+        resource_id: str | None,
+        details: dict[str, Any],
+    ) -> None:
+        AuditWriter(self.session, "system", "hardware-inventory").write(
+            action, resource_type, resource_id, details
+        )
+
+    @staticmethod
+    def _value(observation: ObservedValue[T]) -> T | None:
+        return observation.value if observation.availability is Availability.KNOWN else None
+
+    def refresh(self) -> HardwareState:
+        if not self._refresh_lock.acquire(blocking=False):
+            raise HardwareRefreshBusy("hardware refresh already in progress")
+        started = datetime.now(UTC)
+        received_at = self.received_at or datetime.now(UTC)
+        run = HardwareInventoryRun(
+            status=InventoryRunStatus.RUNNING,
+            source=self.source,
+            summary={},
+            started_at=started,
+            observation_sequence=self.observation_sequence,
+            received_at=received_at,
+        )
+        self.repository.add(run)
+        try:
+            host = self.host_collector.collect()
+            nvidia = self.accelerator_collector.collect()
+            snapshot = HardwareSnapshot(host=host, nvidia=nvidia)
+            node = self.repository.node_by_key(host.identity_key)
+            created = node is None
+            if node is None:
+                node = ComputeNode(key=host.identity_key, hostname=host.hostname)
+                self.repository.add(node)
+            elif node.decommissioned_at is not None:
+                raise RuntimeError(
+                    "decommissioned compute-node identity requires explicit recovery enrollment"
+                )
+            previous_fingerprint = node.hardware_fingerprint
+            node.hostname = host.hostname
+            if created or not node.display_name:
+                node.display_name = host.display_name
+            node.identity_source = host.identity_source
+            node.os_name = host.os_name
+            node.os_version = self._value(host.os_version)
+            node.architecture = host.architecture
+            node.kernel_version = self._value(host.kernel_version)
+            node.cpu_model = self._value(host.cpu_model)
+            node.logical_cpu_count = self._value(host.logical_cpu_count)
+            node.physical_core_count = self._value(host.physical_core_count)
+            node.total_ram_bytes = self._value(host.total_ram_bytes)
+            node.agent_version = host.agent_version
+            node.status = HardwareStatus.ACTIVE
+            node.status_reason = None
+            node.hardware_fingerprint = snapshot.fingerprint
+            node.inventory = host.model_dump(
+                mode="json", exclude={"storage", "available_ram_bytes"}
+            )
+            node.last_seen_at = host.inventory_at
+            node.inventory_at = host.inventory_at
+            node.observation_source = self.source
+            node.last_inventory_observed_at = host.inventory_at
+            node.last_inventory_received_at = received_at
+            node.inventory_sequence = self.observation_sequence or node.inventory_sequence
+            if self.source is ObservationSource.LOCAL_CONTROL_PLANE:
+                node.liveness_state = NodeLiveness.ONLINE
+                node.last_heartbeat_at = received_at
+            self.session.flush()
+            if created:
+                self._audit(
+                    "NODE_DISCOVERED", "compute_node", str(node.id), {"hostname": node.hostname}
+                )
+            elif previous_fingerprint and previous_fingerprint != snapshot.fingerprint:
+                self._audit(
+                    "NODE_INVENTORY_CHANGED"
+                    if self.source is ObservationSource.REMOTE_AGENT
+                    else "HARDWARE_CHANGED",
+                    "compute_node",
+                    str(node.id),
+                    {
+                        "previous_fingerprint": previous_fingerprint,
+                        "fingerprint": snapshot.fingerprint,
+                    },
+                )
+
+            host_latest = self.repository.host_telemetry(node.id)
+            if host_latest is None:
+                host_latest = HostTelemetryLatest(
+                    compute_node_id=node.id, observed_at=host.inventory_at
+                )
+                self.repository.add(host_latest)
+            host_latest.available_ram_bytes = self._value(host.available_ram_bytes)
+            host_latest.availability = {
+                "available_ram_bytes": host.available_ram_bytes.model_dump(mode="json")
+            }
+            host_latest.observed_at = host.inventory_at
+            host_latest.received_at = received_at
+
+            for storage in host.storage:
+                state = self.repository.storage_by_key(node.id, storage.purpose, storage.path)
+                if state is None:
+                    state = StorageVolumeState(
+                        compute_node_id=node.id,
+                        purpose=storage.purpose,
+                        path=storage.path,
+                        observed_at=host.inventory_at,
+                    )
+                    self.repository.add(state)
+                state.total_bytes = self._value(storage.total_bytes)
+                state.used_bytes = self._value(storage.used_bytes)
+                state.free_bytes = self._value(storage.free_bytes)
+                state.availability = {
+                    key: getattr(storage, key).model_dump(mode="json")
+                    for key in ("total_bytes", "used_bytes", "free_bytes")
+                }
+                state.observed_at = host.inventory_at
+                state.received_at = received_at
+
+            seen: set[str] = set()
+            telemetry_by_uuid = {item.device_uuid: item for item in nvidia.telemetry}
+            for observed in nvidia.inventory:
+                seen.add(observed.device_uuid)
+                accelerator = self.repository.accelerator_by_uuid(node.id, observed.device_uuid)
+                new_accelerator = accelerator is None
+                if accelerator is None:
+                    accelerator = Accelerator(
+                        compute_node_id=node.id,
+                        device_index=observed.device_index,
+                        device_uuid=observed.device_uuid,
+                        name=observed.name,
+                    )
+                    self.repository.add(accelerator)
+                accelerator.device_index = observed.device_index
+                accelerator.pci_bus_id = self._value(observed.pci_bus_id)
+                accelerator.name = observed.name
+                accelerator.vendor = observed.vendor
+                accelerator.architecture = self._value(observed.architecture)
+                accelerator.compute_capability_major = self._value(
+                    observed.compute_capability_major
+                )
+                accelerator.compute_capability_minor = self._value(
+                    observed.compute_capability_minor
+                )
+                accelerator.total_vram_bytes = self._value(observed.total_vram_bytes)
+                accelerator.memory_total_mb = (
+                    round(accelerator.total_vram_bytes / 1024 / 1024)
+                    if accelerator.total_vram_bytes is not None
+                    else None
+                )
+                accelerator.driver_version = self._value(observed.driver_version)
+                accelerator.cuda_version = self._value(observed.cuda_driver_version)
+                accelerator.mig_mode_current = self._value(observed.mig_mode_current)
+                accelerator.status = HardwareStatus.ACTIVE
+                accelerator.status_reason = None
+                accelerator.inventory_source = observed.source
+                accelerator.last_seen_at = observed.inventory_at
+                accelerator.inventory_at = observed.inventory_at
+                accelerator.capabilities = observed.model_dump(mode="json")
+                self.session.flush()
+                if new_accelerator:
+                    self._audit(
+                        "ACCELERATOR_DISCOVERED",
+                        "accelerator",
+                        str(accelerator.id),
+                        {"device_uuid": accelerator.device_uuid, "name": accelerator.name},
+                    )
+                telemetry = telemetry_by_uuid.get(observed.device_uuid)
+                if telemetry:
+                    latest = self.repository.accelerator_telemetry(accelerator.id)
+                    if latest is None:
+                        latest = AcceleratorTelemetryLatest(
+                            accelerator_id=accelerator.id, observed_at=telemetry.observed_at
+                        )
+                        self.repository.add(latest)
+                    for field in (
+                        "used_vram_bytes",
+                        "free_vram_bytes",
+                        "gpu_utilization_percent",
+                        "memory_utilization_percent",
+                        "temperature_c",
+                        "power_draw_w",
+                        "power_limit_w",
+                        "graphics_clock_mhz",
+                        "memory_clock_mhz",
+                        "fan_speed_percent",
+                        "performance_state",
+                    ):
+                        setattr(latest, field, self._value(getattr(telemetry, field)))
+                    latest.availability = {
+                        field: getattr(telemetry, field).model_dump(mode="json")
+                        for field in (
+                            "used_vram_bytes",
+                            "free_vram_bytes",
+                            "gpu_utilization_percent",
+                            "memory_utilization_percent",
+                            "temperature_c",
+                            "power_draw_w",
+                            "power_limit_w",
+                            "graphics_clock_mhz",
+                            "memory_clock_mhz",
+                            "fan_speed_percent",
+                            "performance_state",
+                        )
+                    }
+                    latest.observed_at = telemetry.observed_at
+                    latest.received_at = received_at
+
+            if nvidia.availability is Availability.KNOWN:
+                for accelerator in self.repository.accelerators(node.id):
+                    if (
+                        accelerator.device_uuid not in seen
+                        and accelerator.status != HardwareStatus.MISSING
+                    ):
+                        accelerator.status = HardwareStatus.MISSING
+                        accelerator.status_reason = (
+                            "not observed in latest successful NVML enumeration"
+                        )
+                        self._audit(
+                            "ACCELERATOR_MISSING",
+                            "accelerator",
+                            str(accelerator.id),
+                            {"device_uuid": accelerator.device_uuid},
+                        )
+
+            run.compute_node_id = node.id
+            run.status = (
+                InventoryRunStatus.SUCCEEDED
+                if nvidia.availability is Availability.KNOWN
+                else InventoryRunStatus.DEGRADED
+            )
+            run.fingerprint = snapshot.fingerprint
+            run.summary = {
+                "accelerator_count": len(nvidia.inventory),
+                "nvidia_availability": nvidia.availability,
+                "nvidia_reason": nvidia.reason,
+            }
+            run.completed_at = datetime.now(UTC)
+            run.observed_at = host.inventory_at
+            self.session.commit()
+            return self.state()
+        except Exception as exc:
+            self.session.rollback()
+            failed = HardwareInventoryRun(
+                status=InventoryRunStatus.FAILED,
+                source="local_collectors",
+                summary={},
+                error=f"{type(exc).__name__}: {exc}",
+                started_at=started,
+                completed_at=datetime.now(UTC),
+                observation_sequence=self.observation_sequence,
+                received_at=received_at,
+            )
+            self.session.add(failed)
+            self.session.flush()
+            self._audit(
+                "INVENTORY_FAILED",
+                "hardware_inventory_run",
+                str(failed.id),
+                {"error": failed.error},
+            )
+            self.session.commit()
+            raise
+        finally:
+            self._refresh_lock.release()
+
+    @staticmethod
+    def _observed(
+        payload: dict[str, Any],
+        key: str,
+        value: Any = None,
+        default: Availability = Availability.UNKNOWN,
+    ) -> ObservedValue[Any]:
+        data = payload.get(key)
+        if isinstance(data, dict) and "availability" in data:
+            return ObservedValue.model_validate(data)
+        return ObservedValue.known(value) if value is not None else ObservedValue.absent(default)
+
+    def _accelerator_state(self, accelerator: Accelerator) -> AcceleratorState:
+        facts = accelerator.capabilities or {}
+        latest = self.repository.accelerator_telemetry(accelerator.id)
+        telemetry = None
+        if latest:
+            telemetry = AcceleratorTelemetry(
+                device_uuid=accelerator.device_uuid,
+                observed_at=latest.observed_at,
+                **{
+                    field: self._observed(latest.availability, field, getattr(latest, field))
+                    for field in (
+                        "used_vram_bytes",
+                        "free_vram_bytes",
+                        "gpu_utilization_percent",
+                        "memory_utilization_percent",
+                        "temperature_c",
+                        "power_draw_w",
+                        "power_limit_w",
+                        "graphics_clock_mhz",
+                        "memory_clock_mhz",
+                        "fan_speed_percent",
+                        "performance_state",
+                    )
+                },
+            )
+        return AcceleratorState(
+            id=str(accelerator.id),
+            node_id=str(accelerator.compute_node_id),
+            status=HardwareStatus(accelerator.status),
+            status_reason=accelerator.status_reason,
+            device_index=accelerator.device_index,
+            device_uuid=accelerator.device_uuid,
+            pci_bus_id=self._observed(facts, "pci_bus_id", accelerator.pci_bus_id),
+            name=accelerator.name,
+            vendor=accelerator.vendor,
+            architecture=self._observed(facts, "architecture", accelerator.architecture),
+            compute_capability_major=self._observed(
+                facts, "compute_capability_major", accelerator.compute_capability_major
+            ),
+            compute_capability_minor=self._observed(
+                facts, "compute_capability_minor", accelerator.compute_capability_minor
+            ),
+            total_vram_bytes=self._observed(
+                facts, "total_vram_bytes", accelerator.total_vram_bytes
+            ),
+            driver_version=self._observed(facts, "driver_version", accelerator.driver_version),
+            cuda_driver_version=self._observed(
+                facts, "cuda_driver_version", accelerator.cuda_version
+            ),
+            mig_mode_current=self._observed(
+                facts, "mig_mode_current", accelerator.mig_mode_current
+            ),
+            first_seen_at=accelerator.first_seen_at,
+            last_seen_at=accelerator.last_seen_at,
+            inventory_at=accelerator.inventory_at,
+            telemetry=telemetry,
+        )
+
+    def state(self) -> HardwareState:
+        nodes: list[NodeState] = []
+        for node in self.repository.nodes():
+            active_credential = self.repository.active_credential_for_node(node.id)
+            facts = node.inventory or {}
+            host_latest = self.repository.host_telemetry(node.id)
+            available = self._observed(
+                host_latest.availability if host_latest else {},
+                "available_ram_bytes",
+                host_latest.available_ram_bytes if host_latest else None,
+            )
+            storage = [
+                StorageState(
+                    id=str(item.id),
+                    purpose=item.purpose,
+                    path=item.path,
+                    total_bytes=self._observed(item.availability, "total_bytes", item.total_bytes),
+                    used_bytes=self._observed(item.availability, "used_bytes", item.used_bytes),
+                    free_bytes=self._observed(item.availability, "free_bytes", item.free_bytes),
+                    observed_at=item.observed_at,
+                )
+                for item in self.repository.storage(node.id)
+            ]
+            accelerators = [
+                self._accelerator_state(item) for item in self.repository.accelerators(node.id)
+            ]
+            nodes.append(
+                NodeState(
+                    id=str(node.id),
+                    identity_key=node.key,
+                    identity_source=node.identity_source,
+                    hostname=node.hostname,
+                    display_name=node.display_name or node.hostname,
+                    status=HardwareStatus(node.status),
+                    status_reason=node.status_reason,
+                    os_name=node.os_name,
+                    os_version=self._observed(facts, "os_version", node.os_version),
+                    architecture=node.architecture,
+                    kernel_version=self._observed(facts, "kernel_version", node.kernel_version),
+                    cpu_model=self._observed(facts, "cpu_model", node.cpu_model),
+                    logical_cpu_count=self._observed(
+                        facts, "logical_cpu_count", node.logical_cpu_count
+                    ),
+                    physical_core_count=self._observed(
+                        facts, "physical_core_count", node.physical_core_count
+                    ),
+                    total_ram_bytes=self._observed(facts, "total_ram_bytes", node.total_ram_bytes),
+                    available_ram_bytes=available,
+                    agent_version=node.agent_version,
+                    first_seen_at=node.first_seen_at,
+                    last_seen_at=node.last_seen_at,
+                    inventory_at=node.inventory_at,
+                    hardware_fingerprint=node.hardware_fingerprint,
+                    enabled=node.enabled,
+                    liveness=NodeLiveness(node.liveness_state),
+                    agent_health=(
+                        AgentHealth.REVOKED
+                        if node.observation_source == ObservationSource.REMOTE_AGENT
+                        and active_credential is None
+                        else (
+                            AgentHealth.HEALTHY
+                            if node.agent_protocol_version == AGENT_PROTOCOL_VERSION
+                            and node.enabled
+                            else (
+                                AgentHealth.INCOMPATIBLE
+                                if node.agent_protocol_version is not None
+                                and node.agent_protocol_version != AGENT_PROTOCOL_VERSION
+                                else AgentHealth.UNKNOWN
+                            )
+                        )
+                    ),
+                    observation_source=ObservationSource(node.observation_source),
+                    protocol_version=node.agent_protocol_version,
+                    supported_capabilities=node.agent_capabilities or [],
+                    agent_started_at=node.agent_started_at,
+                    last_heartbeat_at=node.last_heartbeat_at,
+                    last_inventory_received_at=node.last_inventory_received_at,
+                    last_telemetry_received_at=node.last_telemetry_received_at,
+                    inventory_age_seconds=(
+                        max(
+                            0,
+                            int(
+                                (
+                                    datetime.now(UTC) - _aware(node.last_inventory_received_at)
+                                ).total_seconds()
+                            ),
+                        )
+                        if node.last_inventory_received_at
+                        else None
+                    ),
+                    telemetry_age_seconds=(
+                        max(
+                            0,
+                            int(
+                                (
+                                    datetime.now(UTC) - _aware(node.last_telemetry_received_at)
+                                ).total_seconds()
+                            ),
+                        )
+                        if node.last_telemetry_received_at
+                        else None
+                    ),
+                    last_connection_error=node.last_connection_error,
+                    role=node.role,
+                    labels=node.labels or {},
+                    production_eligible=node.production_eligible,
+                    lab_eligible=node.lab_eligible,
+                    benchmark_eligible=node.benchmark_eligible,
+                    generation=node.generation,
+                    decommissioned_at=node.decommissioned_at,
+                    decommission_reason=node.decommission_reason,
+                    decommissioned_by=node.decommissioned_by,
+                    environment=(facts.get("metadata") or {}).get("environment"),
+                    storage=storage,
+                    accelerators=accelerators,
+                )
+            )
+        latest = self.repository.latest_run()
+        accelerators = [item for node in nodes for item in node.accelerators]
+        inventory_state = HardwareStatus.PENDING
+        if latest is not None:
+            inventory_state = {
+                InventoryRunStatus.RUNNING: HardwareStatus.PENDING,
+                InventoryRunStatus.SUCCEEDED: HardwareStatus.ACTIVE,
+                InventoryRunStatus.DEGRADED: HardwareStatus.DEGRADED,
+                InventoryRunStatus.FAILED: HardwareStatus.UNAVAILABLE,
+            }[InventoryRunStatus(latest.status)]
+        status = HardwareStatus.PENDING if not nodes else HardwareStatus.ACTIVE
+        reason = (
+            latest.error
+            if latest and latest.status == InventoryRunStatus.FAILED
+            else (latest.summary.get("nvidia_reason") if latest else "inventory has not run")
+        )
+        return HardwareState(
+            overview=HardwareOverview(
+                status=status,
+                inventory_state=inventory_state,
+                node_count=len(nodes),
+                accelerator_count=len(
+                    [item for item in accelerators if item.status is HardwareStatus.ACTIVE]
+                ),
+                last_inventory_at=latest.completed_at if latest else None,
+                reason=reason,
+            ),
+            nodes=nodes,
+        )

backend/src/modelforge_api/services/hardware_polling.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.db import engine
+from modelforge_api.services.hardware_factory import build_hardware_service
+from modelforge_api.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+
+class HardwarePollingService:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+        self._stop = asyncio.Event()
+
+    def _refresh_once(self) -> None:
+        with Session(engine) as session:
+            build_hardware_service(session, self.settings).refresh()
+
+    async def run(self) -> None:
+        while not self._stop.is_set():
+            try:
+                await asyncio.to_thread(self._refresh_once)
+            except Exception:
+                logger.exception("hardware inventory poll failed")
+            try:
+                await asyncio.wait_for(
+                    self._stop.wait(), self.settings.hardware_poll_interval_seconds
+                )
+            except TimeoutError:
+                continue
+
+    def stop(self) -> None:
+        self._stop.set()

backend/src/modelforge_api/services/invariants.py

@@ -0,0 +1,810 @@
+"""M16 executable platform invariants.
+
+An invariant is a property that must hold no matter what fault was injected. Writing them down in
+prose is not enough for a release gate: each one here is a query the platform can run against its
+own authoritative state, before a chaos scenario, after it, and after recovery.
+
+These checks are read-only. They never mutate state, never call an external system, and never
+depend on the observability database being healthy, so they stay usable exactly when monitoring is
+the thing that failed.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Accelerator,
+    AcceleratorTelemetryLatest,
+    ArtifactJob,
+    ArtifactLocation,
+    AuditChainHead,
+    AuditEvent,
+    BackupSet,
+    CapabilityContract,
+    CapabilityDeployment,
+    ComputeNode,
+    Deployment,
+    EmbeddingSpace,
+    GatewayRequest,
+    GpuLease,
+    HostTelemetryLatest,
+    LifecycleOperation,
+    MigrationCutoverOperation,
+    ModelArtifact,
+    NodeCredential,
+    ResidencyAllocation,
+    RestoreOperation,
+    RestorePlan,
+    RuntimeProbe,
+    SchedulerAcceleratorState,
+    ServiceClient,
+    ServiceCredential,
+    ServingGpuLease,
+    ServingJob,
+    StorageRoot,
+    StorageVolumeState,
+)
+from modelforge_api.services.audit import audit_chain_violations
+
+# Lease and residency states that claim a live GPU reservation right now.
+ACTIVE_LEASE_STATES = ("pending", "granted", "active", "held")
+ACTIVE_RESIDENCY_STATES = ("loading", "resident", "unloading")
+TERMINAL_LIFECYCLE_STAGES = ("COMMITTED", "ROLLED_BACK", "FAILED")
+TERMINAL_CUTOVER_STAGES = ("COMMITTED", "ROLLED_BACK", "FAILED")
+
+
+class InvariantSeverity(StrEnum):
+    CRITICAL = "CRITICAL"
+    HIGH = "HIGH"
+
+
+class InvariantStatus(StrEnum):
+    HOLDS = "HOLDS"
+    VIOLATED = "VIOLATED"
+    NOT_APPLICABLE = "NOT_APPLICABLE"
+
+
+class InvariantResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    key: str
+    name: str
+    severity: InvariantSeverity
+    status: InvariantStatus
+    observed: int
+    detail: str
+    violations: list[str] = Field(default_factory=list)
+
+
+class InvariantReport(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    observed_at: datetime
+    checked: int
+    holding: int
+    violated: int
+    results: list[InvariantResult]
+
+    @property
+    def ok(self) -> bool:
+        return self.violated == 0
+
+
+def _result(
+    key: str,
+    name: str,
+    severity: InvariantSeverity,
+    violations: list[str],
+    holds_detail: str,
+    violated_detail: str,
+) -> InvariantResult:
+    if violations:
+        return InvariantResult(
+            key=key,
+            name=name,
+            severity=severity,
+            status=InvariantStatus.VIOLATED,
+            observed=len(violations),
+            detail=violated_detail,
+            violations=violations[:25],
+        )
+    return InvariantResult(
+        key=key,
+        name=name,
+        severity=severity,
+        status=InvariantStatus.HOLDS,
+        observed=0,
+        detail=holds_detail,
+    )
+
+
+def _no_duplicate_production_stable(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(CapabilityDeployment.capability_contract_id, func.count())
+        .where(
+            CapabilityDeployment.production.is_(True),
+            CapabilityDeployment.status == "stable",
+        )
+        .group_by(CapabilityDeployment.capability_contract_id)
+        .having(func.count() > 1)
+    ).all()
+    return _result(
+        "single_production_stable",
+        "At most one stable production deployment per capability contract",
+        InvariantSeverity.CRITICAL,
+        [
+            f"contract {contract} has {count} stable production deployments"
+            for contract, count in rows
+        ],
+        "every capability contract has at most one stable production deployment",
+        "a capability contract has more than one stable production deployment",
+    )
+
+
+def _no_duplicate_node_identity(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(ComputeNode.key, func.count())
+        .where(ComputeNode.enabled.is_(True))
+        .group_by(ComputeNode.key)
+        .having(func.count() > 1)
+    ).all()
+    hardware = session.execute(
+        select(ComputeNode.hardware_fingerprint, func.count())
+        .where(
+            ComputeNode.enabled.is_(True),
+            ComputeNode.hardware_fingerprint.is_not(None),
+        )
+        .group_by(ComputeNode.hardware_fingerprint)
+        .having(func.count() > 1)
+    ).all()
+    violations = [f"identity key {key} is claimed by {count} enabled nodes" for key, count in rows]
+    violations += [
+        f"hardware fingerprint {fingerprint} is claimed by {count} enabled nodes"
+        for fingerprint, count in hardware
+    ]
+    return _result(
+        "single_node_identity",
+        "No two enabled nodes claim the same identity or hardware",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every enabled node holds a distinct identity and hardware fingerprint",
+        "more than one enabled node claims the same hardware",
+    )
+
+
+def _single_active_node_credential(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(NodeCredential.compute_node_id, func.count())
+        .where(NodeCredential.revoked_at.is_(None))
+        .group_by(NodeCredential.compute_node_id)
+        .having(func.count() > 1)
+    ).all()
+    return _result(
+        "single_active_node_credential",
+        "A node has at most one unrevoked credential",
+        InvariantSeverity.CRITICAL,
+        [f"node {node} has {count} active credentials" for node, count in rows],
+        "no node holds more than one active credential",
+        "a node holds more than one active credential",
+    )
+
+
+def _no_stale_gpu_lease(session: Session, now: datetime | None = None) -> InvariantResult:
+    moment = now or datetime.now(UTC)
+    rows = session.scalars(
+        select(ServingGpuLease).where(
+            ServingGpuLease.state.in_(ACTIVE_LEASE_STATES),
+            ServingGpuLease.released_at.is_(None),
+            ServingGpuLease.expires_at.is_not(None),
+        )
+    )
+    violations = []
+    for lease in rows:
+        expires = lease.expires_at
+        if expires is None:
+            continue
+        aware = expires if expires.tzinfo else expires.replace(tzinfo=UTC)
+        if aware < moment:
+            violations.append(
+                f"lease {lease.id} is {lease.state} but expired at {aware.isoformat()}"
+            )
+    return _result(
+        "no_stale_gpu_lease",
+        "No expired GPU lease is still presented as active",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every active GPU lease is within its expiry",
+        "an expired GPU lease is still held as current truth",
+    )
+
+
+def _no_mixed_embedding_space(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(
+            EmbeddingSpace.capability_contract_id,
+            func.count(func.distinct(EmbeddingSpace.identity_digest)),
+        ).group_by(EmbeddingSpace.capability_contract_id)
+    ).all()
+    deployments = session.execute(
+        select(
+            CapabilityDeployment.capability_contract_id,
+            func.count(func.distinct(CapabilityDeployment.embedding_space_id)),
+        )
+        .where(
+            CapabilityDeployment.production.is_(True),
+            CapabilityDeployment.status == "stable",
+            CapabilityDeployment.embedding_space_id.is_not(None),
+        )
+        .group_by(CapabilityDeployment.capability_contract_id)
+        .having(func.count(func.distinct(CapabilityDeployment.embedding_space_id)) > 1)
+    ).all()
+    violations = [
+        f"contract {contract} serves {count} embedding spaces from stable production"
+        for contract, count in deployments
+    ]
+    detail = (
+        f"{len(rows)} contracts carry embedding-space identities; stable production serves one each"
+    )
+    return _result(
+        "no_mixed_embedding_space",
+        "Stable production never serves two embedding spaces for one contract",
+        InvariantSeverity.CRITICAL,
+        violations,
+        detail,
+        "stable production serves more than one embedding space for a capability contract",
+    )
+
+
+def _no_lifecycle_commit_without_evidence(session: Session) -> InvariantResult:
+    rows = session.scalars(
+        select(LifecycleOperation).where(LifecycleOperation.stage == "COMMITTED")
+    )
+    violations = []
+    for operation in rows:
+        plan = operation.promotion_plan_id
+        if plan is None:
+            violations.append(f"operation {operation.id} committed without a promotion plan")
+            continue
+        if not operation.approver or not operation.executor:
+            violations.append(
+                f"operation {operation.id} committed without recorded approver and executor"
+            )
+    return _result(
+        "lifecycle_commit_has_evidence",
+        "No lifecycle operation commits without its plan, approver and executor",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every committed lifecycle operation carries its approval evidence",
+        "a lifecycle operation committed without approval evidence",
+    )
+
+
+def _no_cutover_commit_without_validation(session: Session) -> InvariantResult:
+    rows = session.scalars(
+        select(MigrationCutoverOperation).where(MigrationCutoverOperation.stage == "COMMITTED")
+    )
+    violations = [
+        f"cutover {item.id} committed without an external state fingerprint"
+        for item in rows
+        if not item.external_state_fingerprint
+    ]
+    return _result(
+        "cutover_commit_has_validation",
+        "No migration cutover commits without an external-state fingerprint",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every committed cutover records the external state it observed",
+        "a cutover committed without observing external truth",
+    )
+
+
+def _no_restore_from_unverified_backup(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(RestoreOperation.id, BackupSet.backup_id, BackupSet.state)
+        .join(BackupSet, BackupSet.id == RestoreOperation.backup_set_id)
+        .where(RestoreOperation.state != "PLANNED")
+    ).all()
+    violations = [
+        f"restore {operation} ran against backup {backup} in state {state}"
+        for operation, backup, state in rows
+        if state not in ("VERIFIED", "EXPIRED")
+    ]
+    plans = session.execute(
+        select(RestorePlan.id, BackupSet.backup_id, BackupSet.state).join(
+            BackupSet, BackupSet.id == RestorePlan.backup_set_id
+        )
+    ).all()
+    violations += [
+        f"restore plan {plan} targets backup {backup} that was never verified"
+        for plan, backup, state in plans
+        if state in ("PLANNED", "CREATING", "CREATED", "FAILED")
+    ]
+    return _result(
+        "restore_requires_verified_backup",
+        "No restore is planned or executed from an unverified backup",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every restore plan and operation references a verified backup",
+        "a restore referenced a backup that was never verified",
+    )
+
+
+def _revoked_credentials_stay_revoked(session: Session) -> InvariantResult:
+    violations: list[str] = []
+    for label, revoked_rows in (
+        (
+            "node",
+            session.scalars(
+                select(NodeCredential).where(NodeCredential.revoked_at.is_not(None))
+            ).all(),
+        ),
+        (
+            "service",
+            session.scalars(
+                select(ServiceCredential).where(ServiceCredential.revoked_at.is_not(None))
+            ).all(),
+        ),
+    ):
+        for credential in revoked_rows:
+            used = credential.last_used_at
+            revoked = credential.revoked_at
+            if used is None or revoked is None:
+                continue
+            used_aware = used if used.tzinfo else used.replace(tzinfo=UTC)
+            revoked_aware = revoked if revoked.tzinfo else revoked.replace(tzinfo=UTC)
+            if used_aware > revoked_aware:
+                violations.append(
+                    f"{label} credential {credential.id} was used after it was revoked"
+                )
+    return _result(
+        "revoked_credentials_stay_revoked",
+        "No revoked credential is ever accepted again",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "no revoked node or service credential records a later successful use",
+        "a revoked credential was accepted after revocation",
+    )
+
+
+def _capability_clients_are_not_operators(session: Session) -> InvariantResult:
+    """A gateway client may only name capabilities; operator and node scopes are a different plane."""
+
+    violations: list[str] = []
+    for client in session.scalars(select(ServiceClient)):
+        allowed = client.allowed_capabilities
+        capabilities = allowed if isinstance(allowed, list) else []
+        forbidden = [
+            capability
+            for capability in capabilities
+            if isinstance(capability, str)
+            and (
+                capability.startswith(("admin", "operator", "node."))
+                or capability in {"recovery", "lifecycle", "migration"}
+            )
+        ]
+        if forbidden:
+            violations.append(
+                f"service client {client.id} claims non-capability scopes {forbidden}"
+            )
+    return _result(
+        "capability_clients_are_not_operators",
+        "No capability client holds operator or node scopes",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every service client is scoped to capabilities only",
+        "a capability client carries operator or node scopes",
+    )
+
+
+def _no_unsafe_artifact_promoted(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(ModelArtifact.id, ModelArtifact.filename, ModelArtifact.security_status)
+        .join(ArtifactLocation, ArtifactLocation.artifact_id == ModelArtifact.id)
+        .where(
+            ArtifactLocation.status == "verified",
+            ModelArtifact.security_status.in_(("blocked", "quarantined", "unverified")),
+        )
+        .distinct()
+    ).all()
+    return _result(
+        "no_unsafe_artifact_promoted",
+        "No artifact is promoted to a verified location while its security status is unsafe",
+        InvariantSeverity.CRITICAL,
+        [
+            f"artifact {artifact} ({filename}) is verified on disk but {status}"
+            for artifact, filename, status in rows
+        ],
+        "every verified artifact location holds an artifact that passed its security checks",
+        "an unsafe artifact occupies a verified location",
+    )
+
+
+def _no_orphan_serving_work(session: Session, now: datetime | None = None) -> InvariantResult:
+    moment = now or datetime.now(UTC)
+    jobs = session.scalars(
+        select(ServingJob).where(ServingJob.status.in_(("queued", "leased", "running")))
+    )
+    violations = []
+    for job in jobs:
+        expires = job.lease_expires_at
+        if expires is None:
+            continue
+        aware = expires if expires.tzinfo else expires.replace(tzinfo=UTC)
+        if aware < moment:
+            violations.append(
+                f"serving job {job.id} is {job.status} with a lease expired at {aware.isoformat()}"
+            )
+    residency = session.scalars(
+        select(ResidencyAllocation).where(ResidencyAllocation.state.in_(ACTIVE_RESIDENCY_STATES))
+    )
+    for allocation in residency:
+        node = session.get(ComputeNode, allocation.compute_node_id)
+        if node is not None and not node.enabled:
+            violations.append(
+                f"residency {allocation.id} is {allocation.state} on disabled node {node.id}"
+            )
+    return _result(
+        "no_orphan_serving_work",
+        "No queued or leased serving work survives its lease, and no residency sits on a disabled node",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "no orphaned serving job or residency allocation exists",
+        "orphaned serving work survived recovery",
+    )
+
+
+def _no_hidden_auto_promotion(session: Session) -> InvariantResult:
+    rows = session.scalars(
+        select(CapabilityDeployment).where(
+            CapabilityDeployment.production.is_(True),
+            CapabilityDeployment.status.in_(("stable", "draining")),
+        )
+    )
+    violations = [
+        f"production deployment {item.id} has no recorded production approval"
+        for item in rows
+        if item.production_approval_id is None
+    ]
+    return _result(
+        "no_hidden_auto_promotion",
+        "Every production deployment carries an explicit production approval",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every production deployment references its production approval",
+        "a production deployment exists without an approval",
+    )
+
+
+def _audit_chain_is_intact(session: Session) -> InvariantResult:
+    events = list(
+        session.scalars(select(AuditEvent).order_by(AuditEvent.sequence, AuditEvent.id))
+    )
+    checkpoint = session.get(AuditChainHead, 1)
+    violations = audit_chain_violations(events, checkpoint)
+    return _result(
+        "audit_chain_intact",
+        "The audit trail has a strict sequence and canonical content-hash chain",
+        InvariantSeverity.CRITICAL,
+        violations,
+        f"{len(events)} audit events form one contiguous, content-verified chain",
+        "the audit trail contains a gap, fork, invalid link or content-hash mismatch",
+    )
+
+
+def _decommissioned_nodes_are_terminal(session: Session) -> InvariantResult:
+    violations: list[str] = []
+    nodes = list(
+        session.scalars(select(ComputeNode).where(ComputeNode.decommissioned_at.is_not(None)))
+    )
+    for node in nodes:
+        if (
+            node.enabled
+            or node.status != "decommissioned"
+            or node.liveness_state != "decommissioned"
+        ):
+            violations.append(
+                f"node {node.id} is decommissioned but enabled={node.enabled}, "
+                f"status={node.status} and liveness={node.liveness_state}"
+            )
+        if node.production_eligible or node.lab_eligible or node.benchmark_eligible:
+            violations.append(f"node {node.id} is decommissioned but retains scheduler eligibility")
+        if node.inventory or node.agent_capabilities:
+            violations.append(f"node {node.id} is decommissioned but retains current inventory")
+        active_credentials = (
+            session.scalar(
+                select(func.count())
+                .select_from(NodeCredential)
+                .where(
+                    NodeCredential.compute_node_id == node.id,
+                    NodeCredential.revoked_at.is_(None),
+                )
+            )
+            or 0
+        )
+        if active_credentials:
+            violations.append(f"node {node.id} retains {active_credentials} active credentials")
+        for model, status_column, terminal in (
+            (ArtifactJob, ArtifactJob.status, ("completed", "failed", "cancelled")),
+            (RuntimeProbe, RuntimeProbe.status, ("completed", "failed", "cancelled")),
+            (ServingJob, ServingJob.status, ("completed", "failed", "cancelled")),
+        ):
+            active = (
+                session.scalar(
+                    select(func.count())
+                    .select_from(model)
+                    .where(
+                        model.compute_node_id == node.id,
+                        status_column.not_in(terminal),
+                    )
+                )
+                or 0
+            )
+            if active:
+                violations.append(f"node {node.id} retains {active} active {model.__tablename__}")
+        residency = (
+            session.scalar(
+                select(func.count())
+                .select_from(ResidencyAllocation)
+                .where(ResidencyAllocation.compute_node_id == node.id)
+            )
+            or 0
+        )
+        if residency:
+            violations.append(f"node {node.id} retains {residency} active residencies")
+        accelerator_ids = list(
+            session.scalars(select(Accelerator.id).where(Accelerator.compute_node_id == node.id))
+        )
+        if accelerator_ids:
+            active_serving_leases = (
+                session.scalar(
+                    select(func.count())
+                    .select_from(ServingGpuLease)
+                    .where(
+                        ServingGpuLease.accelerator_id.in_(accelerator_ids),
+                        ServingGpuLease.state.not_in(
+                            ("released", "expired", "failed", "cancelled")
+                        ),
+                    )
+                )
+                or 0
+            )
+            active_legacy_leases = (
+                session.scalar(
+                    select(func.count())
+                    .select_from(GpuLease)
+                    .where(
+                        GpuLease.accelerator_id.in_(accelerator_ids),
+                        GpuLease.state.not_in(("released", "expired", "failed", "cancelled")),
+                    )
+                )
+                or 0
+            )
+            current_accelerator_state = sum(
+                int(
+                    session.scalar(
+                        select(func.count()).select_from(model).where(column.in_(accelerator_ids))
+                    )
+                    or 0
+                )
+                for model, column in (
+                    (AcceleratorTelemetryLatest, AcceleratorTelemetryLatest.accelerator_id),
+                    (SchedulerAcceleratorState, SchedulerAcceleratorState.accelerator_id),
+                )
+            )
+            if active_serving_leases + active_legacy_leases:
+                violations.append(
+                    f"node {node.id} retains {active_serving_leases + active_legacy_leases} "
+                    "active GPU leases"
+                )
+            if current_accelerator_state:
+                violations.append(
+                    f"node {node.id} retains {current_accelerator_state} current accelerator rows"
+                )
+            active_accelerators = (
+                session.scalar(
+                    select(func.count())
+                    .select_from(Accelerator)
+                    .where(
+                        Accelerator.id.in_(accelerator_ids),
+                        Accelerator.status != "decommissioned",
+                    )
+                )
+                or 0
+            )
+            if active_accelerators:
+                violations.append(
+                    f"node {node.id} retains {active_accelerators} non-decommissioned accelerators"
+                )
+        active_deployments = sum(
+            int(
+                session.scalar(
+                    select(func.count())
+                    .select_from(model)
+                    .where(
+                        model.compute_node_id == node.id,
+                        status.not_in(("retired", "deprecated", "failed", "archived")),
+                    )
+                )
+                or 0
+            )
+            for model, status in (
+                (CapabilityDeployment, CapabilityDeployment.status),
+                (Deployment, Deployment.status),
+            )
+        )
+        active_gateway_requests = (
+            session.scalar(
+                select(func.count())
+                .select_from(GatewayRequest)
+                .where(
+                    GatewayRequest.compute_node_id == node.id,
+                    GatewayRequest.status.in_(("queued", "running")),
+                )
+            )
+            or 0
+        )
+        if active_deployments or active_gateway_requests:
+            violations.append(
+                f"node {node.id} retains {active_deployments} active deployments and "
+                f"{active_gateway_requests} gateway requests"
+            )
+        unsafe_roots = (
+            session.scalar(
+                select(func.count())
+                .select_from(StorageRoot)
+                .where(
+                    StorageRoot.compute_node_id == node.id,
+                    (StorageRoot.writable.is_(True) | (StorageRoot.status != "unavailable")),
+                )
+            )
+            or 0
+        )
+        if unsafe_roots:
+            violations.append(f"node {node.id} retains {unsafe_roots} writable storage roots")
+        current_node_state = sum(
+            int(
+                session.scalar(select(func.count()).select_from(model).where(column == node.id))
+                or 0
+            )
+            for model, column in (
+                (HostTelemetryLatest, HostTelemetryLatest.compute_node_id),
+                (StorageVolumeState, StorageVolumeState.compute_node_id),
+            )
+        )
+        if current_node_state:
+            violations.append(f"node {node.id} retains {current_node_state} current telemetry rows")
+    return _result(
+        "decommissioned_nodes_are_terminal",
+        "Decommissioned nodes cannot authenticate, schedule, or retain current runtime truth",
+        InvariantSeverity.CRITICAL,
+        violations,
+        "every decommissioned node is a terminal tombstone with no active work or current state",
+        "a decommissioned node retained mutable current state",
+    )
+
+
+def _contracts_have_one_stable_identity(session: Session) -> InvariantResult:
+    rows = session.execute(
+        select(
+            CapabilityContract.id,
+            func.count(func.distinct(CapabilityDeployment.artifact_set_id)),
+        )
+        .join(
+            CapabilityDeployment,
+            CapabilityDeployment.capability_contract_id == CapabilityContract.id,
+        )
+        .where(
+            CapabilityDeployment.production.is_(True),
+            CapabilityDeployment.status == "stable",
+        )
+        .group_by(CapabilityContract.id)
+        .having(func.count(func.distinct(CapabilityDeployment.artifact_set_id)) > 1)
+    ).all()
+    return _result(
+        "stable_identity_is_singular",
+        "Stable production serves exactly one artifact set per contract",
+        InvariantSeverity.CRITICAL,
+        [
+            f"contract {contract} serves {count} artifact sets from stable production"
+            for contract, count in rows
+        ],
+        "every contract serves a single stable production artifact set",
+        "a contract serves more than one stable production artifact set",
+    )
+
+
+CHECKS = (
+    _no_duplicate_production_stable,
+    _contracts_have_one_stable_identity,
+    _no_duplicate_node_identity,
+    _single_active_node_credential,
+    _no_stale_gpu_lease,
+    _no_mixed_embedding_space,
+    _no_lifecycle_commit_without_evidence,
+    _no_cutover_commit_without_validation,
+    _no_restore_from_unverified_backup,
+    _revoked_credentials_stay_revoked,
+    _capability_clients_are_not_operators,
+    _no_unsafe_artifact_promoted,
+    _no_orphan_serving_work,
+    _no_hidden_auto_promotion,
+    _audit_chain_is_intact,
+    _decommissioned_nodes_are_terminal,
+)
+
+
+def check_invariants(session: Session, now: datetime | None = None) -> InvariantReport:
+    """Run every platform invariant against authoritative state and report each outcome."""
+
+    moment = now or datetime.now(UTC)
+    results: list[InvariantResult] = []
+    for check in CHECKS:
+        try:
+            results.append(check(session, moment))  # type: ignore[call-arg]
+        except TypeError:
+            results.append(check(session))
+    violated = sum(item.status is InvariantStatus.VIOLATED for item in results)
+    return InvariantReport(
+        observed_at=moment,
+        checked=len(results),
+        holding=sum(item.status is InvariantStatus.HOLDS for item in results),
+        violated=violated,
+        results=results,
+    )
+
+
+def invariant_keys() -> tuple[str, ...]:
+    """Stable ordering used by the chaos harness and the release gate."""
+
+    return tuple(
+        result.key
+        for result in (
+            InvariantResult(
+                key=key,
+                name=key,
+                severity=InvariantSeverity.CRITICAL,
+                status=InvariantStatus.HOLDS,
+                observed=0,
+                detail="",
+            )
+            for key in (
+                "single_production_stable",
+                "stable_identity_is_singular",
+                "single_node_identity",
+                "single_active_node_credential",
+                "no_stale_gpu_lease",
+                "no_mixed_embedding_space",
+                "lifecycle_commit_has_evidence",
+                "cutover_commit_has_validation",
+                "restore_requires_verified_backup",
+                "revoked_credentials_stay_revoked",
+                "capability_clients_are_not_operators",
+                "no_unsafe_artifact_promoted",
+                "no_orphan_serving_work",
+                "no_hidden_auto_promotion",
+                "audit_chain_intact",
+                "decommissioned_nodes_are_terminal",
+            )
+        )
+    )
+
+
+def summarise(report: InvariantReport) -> dict[str, Any]:
+    return {
+        "observed_at": report.observed_at.isoformat(),
+        "checked": report.checked,
+        "holding": report.holding,
+        "violated": report.violated,
+        "violations": [
+            {"key": item.key, "detail": item.detail, "examples": item.violations}
+            for item in report.results
+            if item.status is InvariantStatus.VIOLATED
+        ],
+    }

backend/src/modelforge_api/services/manifest_registry.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, cast
+
+import yaml  # type: ignore[import-untyped]
+
+from modelforge_api.domain.contracts import (
+    BenchmarkSuiteManifest,
+    CandidateRegistryManifest,
+    CapabilityContractManifest,
+    PolicyDefaults,
+    ProjectManifest,
+)
+from modelforge_api.settings import get_settings
+
+PROJECT_ROOT = Path(__file__).resolve().parents[4]
+
+
+class ManifestRegistry:
+    """Validated, read-only M0 registry backed by version-controlled manifests.
+
+    Parsed manifests are memoised for the lifetime of the instance. They are read-only
+    configuration mounted into the container and cannot change without restarting the process, so
+    re-reading them per call bought nothing and cost a great deal: one readiness probe used to
+    perform 77 YAML reads across 14 files — one file ten times — and take 528 ms. That was the whole
+    of the 638 ms p50 M16 measured on /api/v1/health/ready under load, which is a real operational
+    risk for an orchestrator with a tight probe timeout.
+
+    The cache is per-instance rather than global so a test pointing at a temporary directory still
+    gets what is on disk when it builds its own registry.
+    """
+
+    def __init__(self, config_root: Path | None = None) -> None:
+        self.config_root = config_root or get_settings().config_root
+        self._cache: dict[str, Any] = {}
+
+    def _memoised(self, key: str, produce: Callable[[], Any]) -> Any:
+        if key not in self._cache:
+            self._cache[key] = produce()
+        return self._cache[key]
+
+    @staticmethod
+    def _read_yaml(path: Path) -> dict[str, Any]:
+        with path.open("r", encoding="utf-8") as handle:
+            payload = yaml.safe_load(handle)
+        if not isinstance(payload, dict):
+            raise TypeError(f"manifest must be an object: {path}")
+        return payload
+
+    def capabilities(self) -> list[CapabilityContractManifest]:
+        return cast(
+            "list[CapabilityContractManifest]", self._memoised("capabilities", self._capabilities)
+        )
+
+    def _capabilities(self) -> list[CapabilityContractManifest]:
+        manifests = [
+            CapabilityContractManifest.model_validate(self._read_yaml(path))
+            for path in sorted((self.config_root / "capabilities").glob("*.yaml"))
+        ]
+        identities = [(item.capability, item.version) for item in manifests]
+        if len(identities) != len(set(identities)):
+            raise ValueError("duplicate capability contract identity")
+        return manifests
+
+    def projects(self) -> list[ProjectManifest]:
+        return cast("list[ProjectManifest]", self._memoised("projects", self._projects))
+
+    def _projects(self) -> list[ProjectManifest]:
+        manifests = [
+            ProjectManifest.model_validate(self._read_yaml(path))
+            for path in sorted((self.config_root / "projects").glob("*.yaml"))
+        ]
+        capabilities = self.capabilities()
+        by_identity = {(item.capability, item.version): item for item in capabilities}
+        capability_versions = set(by_identity)
+        for project in manifests:
+            for key, binding in project.bindings.items():
+                if (key, binding.contract_version) not in capability_versions:
+                    raise ValueError(
+                        f"{project.project.id} binds missing capability contract {key}@{binding.contract_version}"
+                    )
+                contract = by_identity[key, binding.contract_version]
+                if (
+                    contract.upgrade_class.value == "requires_reindex"
+                    and binding.migration_support != "reindex"
+                ):
+                    raise ValueError(
+                        f"{project.project.id}/{key} must declare reindex migration support"
+                    )
+        return manifests
+
+    def candidates(self) -> CandidateRegistryManifest:
+        return cast("CandidateRegistryManifest", self._memoised("candidates", self._candidates))
+
+    def _candidates(self) -> CandidateRegistryManifest:
+        path = self.config_root / "models" / "initial-candidates.yaml"
+        return CandidateRegistryManifest.model_validate(self._read_yaml(path))
+
+    def benchmarks(self) -> list[BenchmarkSuiteManifest]:
+        return cast("list[BenchmarkSuiteManifest]", self._memoised("benchmarks", self._benchmarks))
+
+    def _benchmarks(self) -> list[BenchmarkSuiteManifest]:
+        return [
+            BenchmarkSuiteManifest.model_validate(self._read_yaml(path))
+            for path in sorted((self.config_root / "benchmarks").glob("*.yaml"))
+        ]
+
+    def policies(self) -> PolicyDefaults:
+        return cast("PolicyDefaults", self._memoised("policies", self._policies))
+
+    def _policies(self) -> PolicyDefaults:
+        return PolicyDefaults.model_validate(
+            self._read_yaml(self.config_root / "policies" / "defaults.yaml")
+        )
+
+
+@lru_cache
+def get_manifest_registry() -> ManifestRegistry:
+    return ManifestRegistry()

backend/src/modelforge_api/services/migration_adapters.py

@@ -0,0 +1,86 @@
+"""Closed, typed boundary for project-owned stateful migrations.
+
+The control plane stores intent and evidence. Concrete adapters implement these
+operations in their project data plane; neither plans nor API clients can
+submit code, SQL, shell commands, paths, or arbitrary operation names.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Sequence
+from dataclasses import dataclass
+from typing import Protocol
+
+from modelforge_api.domain.migration_contracts import (
+    AdapterContract,
+    BatchReport,
+    CutoverReport,
+    PreflightReport,
+    RollbackReport,
+    ShadowReport,
+    ValidationReport,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class SourceItemIdentity:
+    item_id: str
+    content_hash: str
+    source_revision: str
+
+
+@dataclass(frozen=True, slots=True)
+class ExternalMigrationTruth:
+    active_target: str
+    source_exists: bool
+    target_exists: bool
+    target_healthy: bool
+    fingerprint: str
+
+
+class MigrationAdapter(Protocol):
+    """Adapter contract; implementations are packaged code, never user payloads."""
+
+    contract: AdapterContract
+
+    async def preflight(self) -> PreflightReport: ...
+
+    async def create_shadow_target(self) -> ExternalMigrationTruth: ...
+
+    def enumerate_source_items(self) -> AsyncIterator[SourceItemIdentity]: ...
+
+    async def transform_batch(self, items: Sequence[SourceItemIdentity]) -> BatchReport: ...
+
+    async def validate_target(self) -> ValidationReport: ...
+
+    async def shadow_compare(self) -> ShadowReport: ...
+
+    async def cutover(self) -> CutoverReport: ...
+
+    async def rollback(self) -> RollbackReport: ...
+
+    async def inspect_external_state(self) -> ExternalMigrationTruth: ...
+
+
+REQUIRED_REINDEX_OPERATIONS = frozenset(
+    {
+        "preflight",
+        "create_shadow_target",
+        "enumerate_source_items",
+        "transform_batch",
+        "write_batch",
+        "validate_batch",
+        "finalize_backfill",
+        "validate_target",
+        "shadow_compare",
+        "cutover",
+        "rollback",
+        "inspect_external_state",
+    }
+)
+
+
+def assert_reindex_adapter(contract: AdapterContract) -> None:
+    missing = REQUIRED_REINDEX_OPERATIONS - contract.operations
+    if missing:
+        raise ValueError(f"migration adapter lacks required typed operations: {sorted(missing)}")

backend/src/modelforge_api/services/node_agent.py

@@ -0,0 +1,744 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import secrets
+import uuid
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any, NoReturn, TypeVar
+
+from sqlalchemy import update
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.exc import StaleDataError
+
+from modelforge_api.db import rows_affected
+from modelforge_api.domain.agent_protocol import (
+    EnrollmentRequest,
+    EnrollmentResponse,
+    EnrollmentTokenCreate,
+    EnrollmentTokenCreated,
+    HeartbeatRequest,
+    InventoryReport,
+    NodeCredentialCreated,
+    NodeManagementUpdate,
+    ObservationAck,
+    TelemetryReport,
+)
+from modelforge_api.domain.enums import NodeLiveness, ObservationSource
+from modelforge_api.domain.hardware import NvidiaCollection, ObservedValue
+from modelforge_api.persistence.hardware_repository import HardwareRepository
+from modelforge_api.persistence.models import (
+    AcceleratorTelemetryLatest,
+    ComputeNode,
+    HostTelemetryLatest,
+    NodeCredential,
+    NodeEnrollment,
+    StorageVolumeState,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.hardware_inventory import HardwareInventoryService
+from modelforge_api.settings import Settings
+
+
+class AgentProtocolError(RuntimeError):
+    status_code = 400
+    code = "agent_protocol_error"
+
+
+class AgentAuthenticationError(AgentProtocolError):
+    status_code = 401
+    code = "agent_authentication_failed"
+
+
+class AgentAuthorizationError(AgentProtocolError):
+    status_code = 403
+    code = "agent_not_authorized"
+
+
+class AgentConflictError(AgentProtocolError):
+    status_code = 409
+    code = "agent_conflict"
+
+
+class AgentNotFoundError(AgentProtocolError):
+    status_code = 404
+    code = "agent_resource_not_found"
+
+
+@dataclass(frozen=True, slots=True)
+class NodeAuthenticationEvidence:
+    """Secret-free proof that pre-body authentication accepted one exact node credential."""
+
+    credential_id: uuid.UUID
+    node_id: uuid.UUID
+    credential_sha256: str
+
+
+def secret_hash(secret: str) -> str:
+    return hashlib.sha256(secret.encode()).hexdigest()
+
+
+def _aware(value: datetime) -> datetime:
+    return value if value.tzinfo else value.replace(tzinfo=UTC)
+
+
+T = TypeVar("T")
+
+
+class _StaticCollector[T]:
+    def __init__(self, value: T) -> None:
+        self.value = value
+
+    def collect(self) -> T:
+        return self.value
+
+
+class NodeAgentService:
+    def __init__(self, session: Session, settings: Settings) -> None:
+        self.session = session
+        self.settings = settings
+        self.repository = HardwareRepository(session)
+
+    def _audit(
+        self,
+        action: str,
+        resource_type: str,
+        resource_id: str | None,
+        details: dict[str, Any],
+    ) -> None:
+        AuditWriter(self.session, "operator", "node-management").write(
+            action, resource_type, resource_id, details
+        )
+
+    def create_enrollment(self, request: EnrollmentTokenCreate) -> EnrollmentTokenCreated:
+        now = datetime.now(UTC)
+        enrollment_id = uuid.uuid4()
+        raw = f"mfenroll_{enrollment_id}_{secrets.token_urlsafe(32)}"
+        enrollment = NodeEnrollment(
+            id=enrollment_id,
+            token_hash=secret_hash(raw),
+            scope="node.enroll",
+            requested_metadata=request.model_dump(mode="json"),
+            expires_at=now + timedelta(seconds=request.expires_in_seconds),
+        )
+        self.session.add(enrollment)
+        self.session.flush()
+        self._audit(
+            "NODE_ENROLLMENT_TOKEN_CREATED",
+            "node_enrollment",
+            str(enrollment.id),
+            {"expires_at": enrollment.expires_at.isoformat()},
+        )
+        self.session.commit()
+        return EnrollmentTokenCreated(
+            id=str(enrollment.id),
+            enrollment_token=raw,
+            expires_at=enrollment.expires_at,
+            setup_environment={
+                "MODELFORGE_AGENT_ENROLLMENT_TOKEN": raw,
+                "MODELFORGE_AGENT_CONTROL_PLANE_URL": "<control-plane-url>",
+            },
+        )
+
+    def revoke_enrollment(self, enrollment_id: uuid.UUID) -> None:
+        # Revocation and enrolment claim the same row with mutually exclusive predicates. On
+        # PostgreSQL a concurrent UPDATE waits for the row owner, then re-evaluates its predicate
+        # against the committed row: either revocation sets `revoked_at`, or enrolment sets
+        # `used_at` and mints a credential in that same transaction. Both cannot report success.
+        revoked = rows_affected(
+            self.session.execute(
+                update(NodeEnrollment)
+                .where(
+                    NodeEnrollment.id == enrollment_id,
+                    NodeEnrollment.used_at.is_(None),
+                    NodeEnrollment.revoked_at.is_(None),
+                )
+                .values(revoked_at=datetime.now(UTC))
+                .execution_options(synchronize_session=False)
+            )
+        )
+        if revoked == 1:
+            self._audit(
+                "NODE_ENROLLMENT_TOKEN_REVOKED",
+                "node_enrollment",
+                str(enrollment_id),
+                {},
+            )
+            self.session.commit()
+            return
+
+        # A failed conditional write may leave an identity-mapped object stale. Roll back the
+        # read transaction and force a database refresh before classifying the losing outcome.
+        self.session.rollback()
+        self.session.expire_all()
+        enrollment = self.session.get(NodeEnrollment, enrollment_id, populate_existing=True)
+        if enrollment is None:
+            raise AgentNotFoundError("enrollment token not found")
+        if enrollment.used_at is not None:
+            raise AgentConflictError("used enrollment token cannot be revoked")
+        if enrollment.revoked_at is not None:
+            # DELETE is idempotent, but only the UPDATE winner emitted a revocation audit event.
+            return
+        raise AgentConflictError("enrollment token changed during revocation")
+
+    def _claim_enrollment(
+        self,
+        *,
+        enrollment_id: uuid.UUID,
+        token_hash: str,
+        claim_now: datetime,
+    ) -> bool:
+        return (
+            rows_affected(
+                self.session.execute(
+                    update(NodeEnrollment)
+                    .where(
+                        NodeEnrollment.id == enrollment_id,
+                        NodeEnrollment.token_hash == token_hash,
+                        NodeEnrollment.used_at.is_(None),
+                        NodeEnrollment.revoked_at.is_(None),
+                        NodeEnrollment.scope == "node.enroll",
+                        NodeEnrollment.expires_at > claim_now,
+                    )
+                    .values(used_at=claim_now)
+                    .execution_options(synchronize_session=False)
+                )
+            )
+            == 1
+        )
+
+    def _raise_failed_enrollment_claim(
+        self,
+        *,
+        enrollment_id: uuid.UUID,
+        token_hash: str,
+        claim_now: datetime,
+    ) -> NoReturn:
+        # The conditional UPDATE is the authorization decision. Discard its transaction and
+        # refresh explicitly so a cached pre-race row cannot determine the typed losing result.
+        self.session.rollback()
+        self.session.expire_all()
+        enrollment = self.session.get(NodeEnrollment, enrollment_id, populate_existing=True)
+        if enrollment is None or not hmac.compare_digest(enrollment.token_hash, token_hash):
+            raise AgentAuthenticationError("invalid enrollment token")
+        if enrollment.scope != "node.enroll":
+            raise AgentAuthorizationError("enrollment token scope is not authorized")
+        if enrollment.revoked_at is not None:
+            raise AgentAuthenticationError("enrollment token revoked")
+        if enrollment.used_at is not None:
+            raise AgentAuthenticationError("enrollment token already used")
+        if _aware(enrollment.expires_at) <= claim_now:
+            raise AgentAuthenticationError("enrollment token expired")
+        raise AgentAuthenticationError("enrollment token is no longer claimable")
+
+    def enroll(self, request: EnrollmentRequest) -> EnrollmentResponse:
+        if request.metadata.protocol_version != self.settings.agent_protocol_version:
+            raise AgentConflictError(
+                f"agent protocol {request.metadata.protocol_version} is incompatible; "
+                f"control plane supports {self.settings.agent_protocol_version}"
+            )
+        token_hash = secret_hash(request.enrollment_token)
+        enrollment = self.repository.enrollment_by_hash(token_hash)
+        validation_now = datetime.now(UTC)
+        if enrollment is None or not hmac.compare_digest(
+            enrollment.token_hash, token_hash
+        ):
+            raise AgentAuthenticationError("invalid enrollment token")
+        if enrollment.scope != "node.enroll":
+            raise AgentAuthorizationError("enrollment token scope is not authorized")
+        if enrollment.revoked_at is not None:
+            raise AgentAuthenticationError("enrollment token revoked")
+        if enrollment.used_at is not None:
+            raise AgentAuthenticationError("enrollment token already used")
+        if _aware(enrollment.expires_at) <= validation_now:
+            raise AgentAuthenticationError("enrollment token expired")
+
+        node = self.repository.node_by_key_for_update(request.identity_key)
+        if node is not None and node.decommissioned_at is not None:
+            raise AgentConflictError(
+                "decommissioned node identity requires an explicit recovery enrollment flow"
+            )
+
+        # Re-sample time immediately before the authorization write. The claim predicate binds
+        # every mutable authorization fact, including token identity and expiry, so a validation-
+        # to-use interleaving cannot mint a credential from a token that is no longer valid.
+        claim_now = datetime.now(UTC)
+        claimed = self._claim_enrollment(
+            enrollment_id=enrollment.id,
+            token_hash=token_hash,
+            claim_now=claim_now,
+        )
+        if not claimed:
+            self._raise_failed_enrollment_claim(
+                enrollment_id=enrollment.id,
+                token_hash=token_hash,
+                claim_now=claim_now,
+            )
+
+        # `synchronize_session=False` keeps the authorization UPDATE free of ORM-side inference.
+        # Refresh the winning row before using the same identity-mapped instance below.
+        self.session.refresh(enrollment)
+        now = claim_now
+
+        requested = enrollment.requested_metadata or {}
+        created = node is None
+        if node is None:
+            node = ComputeNode(key=request.identity_key, hostname=request.hostname)
+            self.session.add(node)
+            self.session.flush()
+        node.hostname = request.hostname
+        node.display_name = (
+            requested.get("display_name") or node.display_name or request.display_name
+        )
+        node.identity_source = request.identity_source
+        node.enabled = True
+        node.role = requested.get("role")
+        node.labels = requested.get("labels") or {}
+        node.production_eligible = bool(requested.get("production_eligible", False))
+        node.lab_eligible = bool(requested.get("lab_eligible", True))
+        node.benchmark_eligible = bool(requested.get("benchmark_eligible", False))
+        node.observation_source = ObservationSource.REMOTE_AGENT
+        node.liveness_state = NodeLiveness.ONLINE
+        node.agent_version = request.metadata.agent_version
+        node.agent_protocol_version = request.metadata.protocol_version
+        node.agent_capabilities = request.metadata.supported_capabilities
+        node.agent_started_at = request.metadata.started_at
+        node.last_heartbeat_at = now
+
+        try:
+            self.session.flush()
+        except StaleDataError as exc:
+            self.session.rollback()
+            raise AgentConflictError(
+                "compute node changed during enrollment; retry with the unused enrollment token"
+            ) from exc
+
+        existing = self.repository.active_credential_for_node(node.id)
+        if existing is not None:
+            existing.revoked_at = now
+        credential_id = uuid.uuid4()
+        raw_credential = f"mfnode_{credential_id}_{secrets.token_urlsafe(48)}"
+        credential = NodeCredential(
+            id=credential_id,
+            compute_node_id=node.id,
+            secret_hash=secret_hash(raw_credential),
+            scope="node.publish",
+        )
+        self.session.add(credential)
+        self.session.flush()
+        enrollment.enrolled_node_id = node.id
+        self._audit(
+            "NODE_ENROLLED",
+            "compute_node",
+            str(node.id),
+            {"created": created, "credential_id": str(credential.id)},
+        )
+        self.session.commit()
+        return EnrollmentResponse(
+            node_id=str(node.id),
+            credential_id=str(credential.id),
+            node_credential=raw_credential,
+        )
+
+    @staticmethod
+    def _parse_credential(authorization: str | None) -> tuple[uuid.UUID, str]:
+        if not authorization or not authorization.startswith("Bearer "):
+            raise AgentAuthenticationError("node credential required")
+        raw = authorization.removeprefix("Bearer ").strip()
+        parts = raw.split("_", 2)
+        if len(parts) != 3 or parts[0] != "mfnode":
+            raise AgentAuthenticationError("invalid node credential")
+        try:
+            credential_id = uuid.UUID(parts[1])
+        except ValueError as exc:
+            raise AgentAuthenticationError("invalid node credential") from exc
+        return credential_id, raw
+
+    def _resolve_authenticated_node(
+        self,
+        credential_id: uuid.UUID,
+        raw: str,
+        *,
+        update_last_used: bool,
+    ) -> tuple[NodeCredential, ComputeNode]:
+        credential = self.repository.credential(credential_id)
+        if credential is None or credential.revoked_at is not None:
+            raise AgentAuthenticationError("node credential revoked or unknown")
+        if not hmac.compare_digest(credential.secret_hash, secret_hash(raw)):
+            raise AgentAuthenticationError("invalid node credential")
+        if credential.scope != "node.publish":
+            raise AgentAuthorizationError("node credential scope is not authorized for publishing")
+        node = self.repository.node(credential.compute_node_id)
+        if node is None:
+            raise AgentAuthenticationError("credential node no longer exists")
+        if node.decommissioned_at is not None:
+            raise AgentAuthenticationError("node credential belongs to a decommissioned node")
+        if not node.enabled:
+            raise AgentAuthorizationError("compute node is disabled")
+        if update_last_used:
+            credential.last_used_at = datetime.now(UTC)
+        return credential, node
+
+    def authenticate_with_evidence(
+        self, authorization: str | None
+    ) -> tuple[tuple[NodeCredential, ComputeNode], NodeAuthenticationEvidence]:
+        credential_id, raw = self._parse_credential(authorization)
+        credential, node = self._resolve_authenticated_node(
+            credential_id,
+            raw,
+            update_last_used=True,
+        )
+        return (credential, node), NodeAuthenticationEvidence(
+            credential_id=credential.id,
+            node_id=node.id,
+            credential_sha256=secret_hash(raw),
+        )
+
+    def authenticate(self, authorization: str | None) -> tuple[NodeCredential, ComputeNode]:
+        identity, _evidence = self.authenticate_with_evidence(authorization)
+        return identity
+
+    def reuse_authentication(
+        self,
+        evidence: NodeAuthenticationEvidence,
+        authorization: str | None,
+    ) -> tuple[NodeCredential, ComputeNode]:
+        """Revalidate current node state while reusing the pre-body credential proof."""
+
+        credential_id, raw = self._parse_credential(authorization)
+        if credential_id != evidence.credential_id or not hmac.compare_digest(
+            secret_hash(raw), evidence.credential_sha256
+        ):
+            raise AgentAuthenticationError("invalid node credential")
+        credential, node = self._resolve_authenticated_node(
+            credential_id,
+            raw,
+            update_last_used=False,
+        )
+        if node.id != evidence.node_id:
+            raise AgentAuthenticationError("credential node identity changed")
+        return credential, node
+
+    @staticmethod
+    def _assert_identity(node: ComputeNode, identity_key: str) -> None:
+        if node.key != identity_key:
+            raise AgentAuthorizationError("credential cannot publish for another node")
+
+    @staticmethod
+    def _assert_not_decommissioned(node: ComputeNode) -> None:
+        if node.decommissioned_at is not None:
+            raise AgentConflictError("decommissioned compute node cannot publish agent state")
+
+    def heartbeat(self, node: ComputeNode, request: HeartbeatRequest) -> ObservationAck:
+        self._assert_not_decommissioned(node)
+        self._assert_identity(node, request.identity_key)
+        if request.metadata.protocol_version != self.settings.agent_protocol_version:
+            raise AgentConflictError("agent protocol version is incompatible")
+        now = datetime.now(UTC)
+        previous = NodeLiveness(node.liveness_state)
+        node.agent_version = request.metadata.agent_version
+        node.agent_protocol_version = request.metadata.protocol_version
+        node.agent_capabilities = request.metadata.supported_capabilities
+        node.agent_started_at = request.metadata.started_at
+        node.last_heartbeat_at = now
+        node.last_connection_error = request.last_error
+        node.liveness_state = NodeLiveness.ONLINE
+        if previous in {NodeLiveness.STALE, NodeLiveness.OFFLINE}:
+            self._audit("NODE_RETURNED_ONLINE", "compute_node", str(node.id), {"from": previous})
+        self.session.commit()
+        return ObservationAck(accepted=True, received_at=now)
+
+    def publish_inventory(self, node: ComputeNode, report: InventoryReport) -> ObservationAck:
+        self._assert_not_decommissioned(node)
+        self._assert_identity(node, report.identity_key)
+        if report.host.identity_key != node.key:
+            raise AgentAuthorizationError("host inventory identity does not match credential node")
+        if report.protocol_version != self.settings.agent_protocol_version:
+            raise AgentConflictError("agent protocol version is incompatible")
+        if report.sequence <= node.inventory_sequence:
+            return ObservationAck(
+                accepted=False,
+                reason="replayed inventory sequence",
+                received_at=datetime.now(UTC),
+            )
+        received_at = datetime.now(UTC)
+        if _aware(report.observed_at) > received_at + timedelta(
+            seconds=self.settings.agent_max_clock_skew_seconds
+        ):
+            node.inventory_sequence = report.sequence
+            node.last_inventory_received_at = received_at
+            node.last_connection_error = "inventory observation exceeds clock-skew limit"
+            self.session.commit()
+            return ObservationAck(
+                accepted=False,
+                reason="future inventory observation retained last-good state",
+                received_at=received_at,
+            )
+        if node.last_inventory_observed_at is not None and _aware(report.observed_at) <= _aware(
+            node.last_inventory_observed_at
+        ):
+            node.inventory_sequence = report.sequence
+            node.last_inventory_received_at = received_at
+            node.last_connection_error = "inventory observation clock moved backwards"
+            self.session.commit()
+            return ObservationAck(
+                accepted=False,
+                reason="older inventory observation retained last-good state",
+                received_at=received_at,
+            )
+        nvidia = NvidiaCollection(
+            availability=report.nvidia.availability,
+            reason=report.nvidia.reason,
+            inventory=report.nvidia.inventory,
+            telemetry=[],
+            observed_at=report.observed_at,
+        )
+        HardwareInventoryService(
+            self.session,
+            _StaticCollector(report.host),
+            _StaticCollector(nvidia),
+            source=ObservationSource.REMOTE_AGENT,
+            received_at=received_at,
+            observation_sequence=report.sequence,
+        ).refresh()
+        refreshed_node = self.repository.node(node.id)
+        if refreshed_node is None:
+            raise AgentNotFoundError("compute node disappeared during reconciliation")
+        refreshed_node.inventory_sequence = report.sequence
+        refreshed_node.last_inventory_observed_at = report.observed_at
+        refreshed_node.last_inventory_received_at = received_at
+        refreshed_node.last_connection_error = None
+        self.session.commit()
+        return ObservationAck(accepted=True, received_at=received_at)
+
+    @staticmethod
+    def _value(observation: ObservedValue[T]) -> T | None:
+        return observation.value if observation.availability.value == "known" else None
+
+    def publish_telemetry(self, node: ComputeNode, report: TelemetryReport) -> ObservationAck:
+        self._assert_not_decommissioned(node)
+        self._assert_identity(node, report.identity_key)
+        if report.protocol_version != self.settings.agent_protocol_version:
+            raise AgentConflictError("agent protocol version is incompatible")
+        if report.sequence <= node.telemetry_sequence:
+            return ObservationAck(
+                accepted=False,
+                reason="replayed telemetry sequence",
+                received_at=datetime.now(UTC),
+            )
+        received_at = datetime.now(UTC)
+        if _aware(report.observed_at) > received_at + timedelta(
+            seconds=self.settings.agent_max_clock_skew_seconds
+        ):
+            node.telemetry_sequence = report.sequence
+            node.last_telemetry_received_at = received_at
+            node.last_connection_error = "telemetry observation exceeds clock-skew limit"
+            self.session.commit()
+            return ObservationAck(
+                accepted=False,
+                reason="future telemetry observation retained last-good state",
+                received_at=received_at,
+            )
+        if node.last_telemetry_observed_at is not None and _aware(report.observed_at) <= _aware(
+            node.last_telemetry_observed_at
+        ):
+            node.telemetry_sequence = report.sequence
+            node.last_telemetry_received_at = received_at
+            node.last_connection_error = "telemetry observation clock moved backwards"
+            self.session.commit()
+            return ObservationAck(
+                accepted=False,
+                reason="older telemetry observation retained last-good state",
+                received_at=received_at,
+            )
+        host = self.repository.host_telemetry(node.id)
+        if host is None:
+            host = HostTelemetryLatest(compute_node_id=node.id, observed_at=report.observed_at)
+            self.session.add(host)
+        host.available_ram_bytes = self._value(report.available_ram_bytes)
+        host.availability = {
+            "available_ram_bytes": report.available_ram_bytes.model_dump(mode="json")
+        }
+        host.observed_at = report.observed_at
+        host.received_at = received_at
+        for storage in report.storage:
+            current = self.repository.storage_by_key(node.id, storage.purpose, storage.path)
+            if current is None:
+                current = StorageVolumeState(
+                    compute_node_id=node.id,
+                    purpose=storage.purpose,
+                    path=storage.path,
+                    observed_at=report.observed_at,
+                )
+                self.session.add(current)
+            current.total_bytes = self._value(storage.total_bytes)
+            current.used_bytes = self._value(storage.used_bytes)
+            current.free_bytes = self._value(storage.free_bytes)
+            current.availability = {
+                field: getattr(storage, field).model_dump(mode="json")
+                for field in ("total_bytes", "used_bytes", "free_bytes")
+            }
+            current.observed_at = report.observed_at
+            current.received_at = received_at
+        known = {item.device_uuid: item for item in self.repository.accelerators(node.id)}
+        for telemetry in report.accelerators:
+            accelerator = known.get(telemetry.device_uuid)
+            if accelerator is None:
+                raise AgentConflictError(
+                    f"telemetry references unknown accelerator {telemetry.device_uuid}"
+                )
+            latest = self.repository.accelerator_telemetry(accelerator.id)
+            if latest is None:
+                latest = AcceleratorTelemetryLatest(
+                    accelerator_id=accelerator.id, observed_at=telemetry.observed_at
+                )
+                self.session.add(latest)
+            for field in (
+                "used_vram_bytes",
+                "free_vram_bytes",
+                "gpu_utilization_percent",
+                "memory_utilization_percent",
+                "temperature_c",
+                "power_draw_w",
+                "power_limit_w",
+                "graphics_clock_mhz",
+                "memory_clock_mhz",
+                "fan_speed_percent",
+                "performance_state",
+            ):
+                setattr(latest, field, self._value(getattr(telemetry, field)))
+            latest.availability = {
+                field: getattr(telemetry, field).model_dump(mode="json")
+                for field in (
+                    "used_vram_bytes",
+                    "free_vram_bytes",
+                    "gpu_utilization_percent",
+                    "memory_utilization_percent",
+                    "temperature_c",
+                    "power_draw_w",
+                    "power_limit_w",
+                    "graphics_clock_mhz",
+                    "memory_clock_mhz",
+                    "fan_speed_percent",
+                    "performance_state",
+                )
+            }
+            latest.observed_at = telemetry.observed_at
+            latest.received_at = received_at
+        node.telemetry_sequence = report.sequence
+        node.last_telemetry_observed_at = report.observed_at
+        node.last_telemetry_received_at = received_at
+        node.last_connection_error = None
+        self.session.commit()
+        return ObservationAck(accepted=True, received_at=received_at)
+
+    def update_node(self, node_id: uuid.UUID, update: NodeManagementUpdate) -> ComputeNode:
+        node = self.repository.node_for_update(node_id)
+        if node is None:
+            raise AgentNotFoundError("compute node not found")
+        if node.decommissioned_at is not None:
+            raise AgentConflictError("decommissioned compute node cannot be updated")
+        values = update.model_dump(exclude_unset=True)
+        was_enabled = node.enabled
+        for key, value in values.items():
+            setattr(node, key, value)
+        if "enabled" in values and values["enabled"] != was_enabled:
+            node.liveness_state = (
+                NodeLiveness.OFFLINE if values["enabled"] else NodeLiveness.DISABLED
+            )
+            self._audit(
+                "NODE_ENABLED" if values["enabled"] else "NODE_DISABLED",
+                "compute_node",
+                str(node.id),
+                {},
+            )
+        self.session.commit()
+        return node
+
+    def revoke_credential(self, node_id: uuid.UUID) -> None:
+        credential = self.repository.active_credential_for_node(node_id)
+        if credential is None:
+            raise AgentNotFoundError("active node credential not found")
+        credential.revoked_at = datetime.now(UTC)
+        node = self.repository.node(node_id)
+        if node is not None:
+            node.liveness_state = NodeLiveness.OFFLINE
+            node.last_connection_error = "node credential revoked"
+        self._audit(
+            "NODE_CREDENTIAL_REVOKED",
+            "node_credential",
+            str(credential.id),
+            {"node_id": str(node_id)},
+        )
+        self.session.commit()
+
+    def rotate_credential(self, node_id: uuid.UUID) -> NodeCredentialCreated:
+        node = self.repository.node_for_update(node_id)
+        if node is None:
+            raise AgentNotFoundError("compute node not found")
+        if node.decommissioned_at is not None:
+            raise AgentConflictError("decommissioned compute node cannot rotate credentials")
+        current = self.repository.active_credential_for_node(node_id)
+        if current is not None:
+            current.revoked_at = datetime.now(UTC)
+            self.session.flush()
+        credential_id = uuid.uuid4()
+        raw = f"mfnode_{credential_id}_{secrets.token_urlsafe(48)}"
+        credential = NodeCredential(
+            id=credential_id,
+            compute_node_id=node_id,
+            secret_hash=secret_hash(raw),
+            scope="node.publish",
+        )
+        self.session.add(credential)
+        node.liveness_state = NodeLiveness.STALE
+        node.last_connection_error = "credential rotated; awaiting agent reconnect"
+        self._audit(
+            "NODE_CREDENTIAL_ROTATED",
+            "node_credential",
+            str(credential_id),
+            {
+                "node_id": str(node_id),
+                "replaced_credential_id": str(current.id) if current else None,
+            },
+        )
+        self.session.commit()
+        return NodeCredentialCreated(
+            node_id=str(node_id), credential_id=str(credential_id), node_credential=raw
+        )
+
+    def evaluate_liveness(self, now: datetime | None = None) -> None:
+        current = now or datetime.now(UTC)
+        for node in self.repository.nodes():
+            if node.decommissioned_at is not None:
+                if node.enabled or node.liveness_state != NodeLiveness.DECOMMISSIONED:
+                    node.enabled = False
+                    node.liveness_state = NodeLiveness.DECOMMISSIONED
+                continue
+            previous = NodeLiveness(node.liveness_state)
+            if not node.enabled:
+                target = NodeLiveness.DISABLED
+            elif node.observation_source == ObservationSource.LOCAL_CONTROL_PLANE:
+                target = previous
+            elif node.last_heartbeat_at is None:
+                target = NodeLiveness.OFFLINE
+            else:
+                age = (current - _aware(node.last_heartbeat_at)).total_seconds()
+                target = (
+                    NodeLiveness.OFFLINE
+                    if age >= self.settings.node_offline_after_seconds
+                    else (
+                        NodeLiveness.STALE
+                        if age >= self.settings.node_stale_after_seconds
+                        else NodeLiveness.ONLINE
+                    )
+                )
+            if target == previous:
+                continue
+            node.liveness_state = target
+            if target == NodeLiveness.STALE:
+                self._audit("NODE_BECAME_STALE", "compute_node", str(node.id), {})
+            elif target == NodeLiveness.OFFLINE:
+                self._audit("NODE_BECAME_OFFLINE", "compute_node", str(node.id), {})
+        self.session.commit()

backend/src/modelforge_api/services/node_liveness.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from sqlalchemy.orm import Session
+
+from modelforge_api.db import engine
+from modelforge_api.services.node_agent import NodeAgentService
+from modelforge_api.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+
+class NodeLivenessPollingService:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+        self._stop = asyncio.Event()
+
+    def _evaluate_once(self) -> None:
+        with Session(engine) as session:
+            NodeAgentService(session, self.settings).evaluate_liveness()
+
+    async def run(self) -> None:
+        while not self._stop.is_set():
+            try:
+                await asyncio.to_thread(self._evaluate_once)
+            except Exception:
+                logger.exception("node liveness evaluation failed")
+            try:
+                await asyncio.wait_for(
+                    self._stop.wait(), self.settings.liveness_poll_interval_seconds
+                )
+            except TimeoutError:
+                continue
+
+    def stop(self) -> None:
+        self._stop.set()

backend/src/modelforge_api/services/observability_polling.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+
+import structlog
+from sqlalchemy import Engine
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.observability import metrics
+from modelforge_api.services.observability import ObservabilityService
+from modelforge_api.settings import Settings
+
+logger = structlog.get_logger()
+
+
+class ObservabilityPollingService:
+    """Minute-scale aggregation; it never participates in inference admission or completion."""
+
+    def __init__(self, settings: Settings, engine: Engine) -> None:
+        self.settings = settings
+        self.engine = engine
+        self._stop = threading.Event()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def run_once(self) -> None:
+        try:
+            with Session(self.engine) as session:
+                service = ObservabilityService(session, self.settings)
+                service.ensure_defaults()
+                service.collect_capacity()
+                service.evaluate_slos()
+                service.evaluate_alerts()
+                service.apply_retention()
+            metrics.gauge("modelforge_observability_degraded", {}, 0)
+        except (SQLAlchemyError, OSError, RuntimeError, ValueError) as error:
+            metrics.gauge("modelforge_observability_degraded", {}, 1)
+            logger.error(
+                "observability_collection_failed",
+                error_type=type(error).__name__,
+            )
+
+    async def run(self) -> None:
+        while not self._stop.is_set():
+            await asyncio.to_thread(self.run_once)
+            for _ in range(self.settings.observability_poll_interval_seconds):
+                if self._stop.is_set():
+                    break
+                await asyncio.sleep(1)

backend/src/modelforge_api/services/project_registry.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import (
+    Capability,
+    CapabilityContract,
+    Project,
+    ProjectBinding,
+)
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.manifest_registry import ManifestRegistry
+
+
+@dataclass(frozen=True)
+class ProjectRegistrySyncResult:
+    projects_created: int
+    bindings_created: int
+    unavailable_contracts: tuple[str, ...]
+
+
+def sync_project_registry(
+    session: Session, manifests: ManifestRegistry
+) -> ProjectRegistrySyncResult:
+    """Materialize generic manifest projects against contracts that exist in the DB.
+
+    A capability contract is lifecycle-owned and may not exist yet on a fresh control plane.
+    Such bindings remain in the validated manifest and are reported, never fabricated.
+    Existing bindings are operator-owned and are not overwritten by startup reconciliation.
+    """
+    audit = AuditWriter(session, "system", "project-manifest-sync")
+    projects_created = 0
+    bindings_created = 0
+    unavailable: list[str] = []
+    for item in manifests.projects():
+        project = session.scalar(select(Project).where(Project.key == item.project.id))
+        if project is None:
+            project = Project(
+                key=item.project.id,
+                name=item.project.name,
+                description=item.project.description,
+                active=True,
+            )
+            session.add(project)
+            session.flush()
+            audit.write(
+                "PROJECT_MATERIALIZED",
+                "project",
+                str(project.id),
+                {"project_key": project.key, "source": "versioned_manifest"},
+            )
+            projects_created += 1
+        for capability_key, binding in item.bindings.items():
+            contract = session.scalar(
+                select(CapabilityContract)
+                .join(Capability, Capability.id == CapabilityContract.capability_id)
+                .where(
+                    Capability.key == capability_key,
+                    CapabilityContract.version == binding.contract_version,
+                )
+            )
+            identity = f"{item.project.id}:{capability_key}@{binding.contract_version}"
+            if contract is None:
+                unavailable.append(identity)
+                continue
+            existing = session.scalar(
+                select(ProjectBinding).where(
+                    ProjectBinding.project_id == project.id,
+                    ProjectBinding.capability_contract_id == contract.id,
+                    ProjectBinding.channel == binding.channel.value,
+                )
+            )
+            if existing is not None:
+                continue
+            project_binding = ProjectBinding(
+                project_id=project.id,
+                capability_contract_id=contract.id,
+                channel=binding.channel.value,
+                priority=binding.priority.value,
+                optional=binding.optional,
+                fallback_policy=binding.fallback.model_dump(mode="json"),
+                migration_support=binding.migration_support,
+                slo=binding.slo.model_dump(mode="json"),
+                benchmark_requirements=binding.benchmark_suites,
+            )
+            session.add(project_binding)
+            session.flush()
+            audit.write(
+                "PROJECT_BINDING_MATERIALIZED",
+                "project_binding",
+                str(project_binding.id),
+                {
+                    "project_key": project.key,
+                    "capability": f"{capability_key}@{binding.contract_version}",
+                    "channel": binding.channel.value,
+                },
+            )
+            bindings_created += 1
+    session.commit()
+    return ProjectRegistrySyncResult(
+        projects_created=projects_created,
+        bindings_created=bindings_created,
+        unavailable_contracts=tuple(sorted(unavailable)),
+    )

backend/src/modelforge_api/services/recovery_crypto.py

@@ -0,0 +1,145 @@
+"""Backup-at-rest protection.
+
+ModelForge designs no cryptography of its own: this is a thin adapter over AES-256-GCM from
+`cryptography`, with a bounded streamed chunk framing so a multi-gigabyte dump never has to be
+held in memory. A wrong key fails closed and leaves no partial plaintext behind.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import os
+import struct
+from pathlib import Path
+from typing import Protocol
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ALGORITHM = "AES-256-GCM"
+MAGIC = b"MFBK1\x00"
+CHUNK_BYTES = 4 * 1024 * 1024
+_NONCE_BYTES = 12
+_HEADER = struct.Struct("!I")
+
+
+class BackupCryptoError(Exception):
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+class BackupCipher(Protocol):
+    """Adapter boundary so a future KMS/HSM provider can replace the local key."""
+
+    algorithm: str
+    key_id: str
+
+    def encrypt_file(self, source: Path, destination: Path) -> int: ...
+
+    def decrypt_file(self, source: Path, destination: Path) -> int: ...
+
+
+def derive_key(secret: str) -> bytes:
+    """Accept either a base64 32-byte key or a passphrase hardened into one."""
+
+    candidate = secret.strip()
+    try:
+        raw = base64.b64decode(candidate, validate=True)
+    except (ValueError, TypeError):
+        raw = b""
+    if len(raw) == 32:
+        return raw
+    return hashlib.sha256(candidate.encode("utf-8")).digest()
+
+
+class AesGcmBackupCipher:
+    algorithm = ALGORITHM
+
+    def __init__(self, secret: str, key_id: str) -> None:
+        if not secret.strip():
+            raise BackupCryptoError(
+                "ENCRYPTION_KEY_UNAVAILABLE", "a backup encryption key is required"
+            )
+        self._key = derive_key(secret)
+        self.key_id = key_id
+
+    @property
+    def key_fingerprint(self) -> str:
+        """Identifies which key sealed a payload without exposing the key itself."""
+
+        return hashlib.sha256(b"modelforge-backup-key" + self._key).hexdigest()[:32]
+
+    def encrypt_file(self, source: Path, destination: Path) -> int:
+        aead = AESGCM(self._key)
+        written = 0
+        temporary = destination.with_name(destination.name + ".partial")
+        try:
+            with source.open("rb") as reader, temporary.open("wb") as writer:
+                writer.write(MAGIC)
+                writer.write(self.key_id.encode("utf-8")[:64].ljust(64, b"\x00"))
+                index = 0
+                while chunk := reader.read(CHUNK_BYTES):
+                    nonce = os.urandom(_NONCE_BYTES)
+                    sealed = aead.encrypt(nonce, chunk, _associated(self.key_id, index))
+                    writer.write(nonce)
+                    writer.write(_HEADER.pack(len(sealed)))
+                    writer.write(sealed)
+                    written += len(chunk)
+                    index += 1
+            temporary.replace(destination)
+        except Exception:
+            temporary.unlink(missing_ok=True)
+            raise
+        return written
+
+    def decrypt_file(self, source: Path, destination: Path) -> int:
+        aead = AESGCM(self._key)
+        written = 0
+        temporary = destination.with_name(destination.name + ".partial")
+        try:
+            with source.open("rb") as reader, temporary.open("wb") as writer:
+                if reader.read(len(MAGIC)) != MAGIC:
+                    raise BackupCryptoError(
+                        "DECRYPTION_FAILED", "payload is not a ModelForge encrypted backup"
+                    )
+                stored_key_id = reader.read(64).rstrip(b"\x00").decode("utf-8", "replace")
+                index = 0
+                while True:
+                    nonce = reader.read(_NONCE_BYTES)
+                    if not nonce:
+                        break
+                    if len(nonce) != _NONCE_BYTES:
+                        raise BackupCryptoError("DECRYPTION_FAILED", "truncated encrypted payload")
+                    header = reader.read(_HEADER.size)
+                    if len(header) != _HEADER.size:
+                        raise BackupCryptoError("DECRYPTION_FAILED", "truncated encrypted payload")
+                    (length,) = _HEADER.unpack(header)
+                    sealed = reader.read(length)
+                    if len(sealed) != length:
+                        raise BackupCryptoError("DECRYPTION_FAILED", "truncated encrypted payload")
+                    try:
+                        plaintext = aead.decrypt(
+                            nonce, sealed, _associated(stored_key_id, index)
+                        )
+                    except InvalidTag as error:
+                        raise BackupCryptoError(
+                            "DECRYPTION_FAILED",
+                            "encrypted backup could not be authenticated with the supplied key",
+                        ) from error
+                    writer.write(plaintext)
+                    written += len(plaintext)
+                    index += 1
+            temporary.replace(destination)
+        except Exception:
+            # Fail closed: a wrong key must never leave readable partial plaintext behind.
+            temporary.unlink(missing_ok=True)
+            destination.unlink(missing_ok=True)
+            raise
+        return written
+
+
+def _associated(key_id: str, index: int) -> bytes:
+    return f"modelforge:{key_id}:{index}".encode()

backend/src/modelforge_api/services/recovery_fingerprint.py

@@ -0,0 +1,698 @@
+"""Bounded semantic fingerprints over authoritative control-plane identity.
+
+A raw binary hash of a database file proves nothing useful about a restore: two correct
+PostgreSQL clusters holding the same logical state differ byte for byte. Instead, every
+authoritative table is hashed over its logical columns, secrets excluded, so a restored control
+plane can be compared with the state that was backed up and every difference can be explained.
+
+Current-truth tables (NVML telemetry, leases, residency, inventory runs) are deliberately excluded:
+they must be re-measured after a restore, never resurrected from a snapshot.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from dataclasses import dataclass
+from datetime import UTC, date, datetime
+from decimal import Decimal
+from typing import Any
+
+from sqlalchemy import Table, func, select
+from sqlalchemy.engine import Connection
+from sqlalchemy.orm import Session
+
+from modelforge_api.persistence.models import Base
+
+MAX_ROWS_PER_TABLE = 250_000
+FINGERPRINT_VERSION = "m15.2"
+LEGACY_FINGERPRINT_VERSION = "m15.1"
+AUDIT_CHECKPOINT_SOURCE_SCHEMA = "20260828_0022"
+AUDIT_CHECKPOINT_CURRENT_SCHEMA = "20260830_0024"
+NON_FINGERPRINT_INFRASTRUCTURE_TABLES = frozenset({"alembic_version"})
+
+# Column names carrying secret material never enter a fingerprint, even as a hash.
+SENSITIVE_COLUMN_TOKENS = (
+    "secret",
+    "token",
+    "password",
+    "passphrase",
+    "api_key",
+    "private_key",
+)
+
+FINGERPRINT_GROUPS: dict[str, tuple[str, ...]] = {
+    "provenance": (
+        "models",
+        "model_revisions",
+        "model_artifacts",
+        "derived_artifacts",
+        "derived_artifact_sources",
+        "artifact_sets",
+        "artifact_set_members",
+        "upstream_snapshots",
+        "upstream_files",
+    ),
+    "artifact_placement": (
+        "storage_roots",
+        "artifact_locations",
+        "artifact_location_removal_records",
+    ),
+    "acquisition": (
+        "download_plans",
+        "download_plan_files",
+        "artifact_jobs",
+        "artifact_job_attempts",
+        "artifact_inspections",
+        "discovery_candidate_assessments",
+    ),
+    "registry": (
+        "capabilities",
+        "capability_contracts",
+        "projects",
+        "project_bindings",
+        "project_evaluation_bindings",
+        "embedding_spaces",
+    ),
+    "runtime": (
+        "runtime_environments",
+        "runtime_profiles",
+        "runtime_compatibility_assessments",
+        "execution_approvals",
+        "deployment_candidates",
+        "deployments",
+        "resource_envelopes",
+    ),
+    "serving": (
+        "capability_deployments",
+        "capability_experiment_routes",
+        "capability_resource_envelopes",
+        "service_clients",
+        "production_execution_approvals",
+        "promotions",
+    ),
+    "evaluation": (
+        "evaluation_suites",
+        "evaluation_suite_revisions",
+        "evaluation_cases",
+        "evaluation_runs",
+        "evaluation_case_results",
+        "capability_evaluation_suites",
+        "capability_evaluation_runs",
+        "evaluation_comparisons",
+        "model_comparisons",
+        "advisor_policies",
+        "advisor_recommendations",
+        "retrieval_candidate_pools",
+        "retrieval_pipeline_identities",
+        "reranking_evaluation_runs",
+        "reranking_case_results",
+        "project_fit_evidence",
+        "benchmark_suites",
+        "benchmark_runs",
+        "experiments",
+        "recommendations",
+    ),
+    "lifecycle": (
+        "lifecycle_subjects",
+        "lifecycle_policy_revisions",
+        "lifecycle_approval_requests",
+        "lifecycle_approval_evidence",
+        "lifecycle_promotion_plans",
+        "lifecycle_rollback_snapshots",
+        "lifecycle_operations",
+        "lifecycle_canary_runs",
+        "lifecycle_events",
+        "retention_policy_revisions",
+        "lifecycle_retention_records",
+        "lifecycle_cleanup_plans",
+    ),
+    "migration": (
+        "migration_validation_policy_revisions",
+        "migration_plans",
+        "migration_batch_checkpoints",
+        "migration_validation_snapshots",
+        "migration_shadow_sessions",
+        "migration_rollback_snapshots",
+        "migration_cutover_operations",
+        "migration_events",
+        "embedding_migrations",
+        "migrations",
+    ),
+    "observability": (
+        "service_level_indicators",
+        "slo_policy_revisions",
+        "slo_evaluations",
+        "alert_rule_revisions",
+        "operational_alerts",
+        "alert_history_events",
+        "maintenance_windows",
+        "operational_incidents",
+        "incident_timeline_events",
+        "capacity_snapshots",
+        "capacity_aggregates",
+    ),
+    "identity": (
+        "compute_nodes",
+        "accelerators",
+        "node_enrollments",
+        "node_credentials",
+        "node_decommission_operations",
+        "service_credentials",
+    ),
+    "scheduler_history": (
+        "scheduler_policy_revisions",
+        "placement_plans",
+        "co_residency_evidence",
+        "scheduler_evictions",
+    ),
+    "traffic_history": (
+        "gateway_requests",
+        "serving_jobs",
+        "runtime_probes",
+        "runtime_probe_metrics",
+    ),
+    "audit": ("audit_events", "audit_chain_heads"),
+    "recovery": (
+        "recovery_policy_revisions",
+        "recovery_asset_records",
+        "backup_sets",
+        "backup_manifest_entries",
+        "restore_plans",
+        "restore_operations",
+        "restore_operation_events",
+        "artifact_recovery_operations",
+    ),
+}
+
+# Current truth, not history: these must be re-measured by a running control plane after a
+# restore. Restoring them would present a stale NVML or lease snapshot as present-day fact.
+CURRENT_TRUTH_TABLES: frozenset[str] = frozenset(
+    {
+        "host_telemetry_latest",
+        "accelerator_telemetry_latest",
+        "storage_volume_states",
+        "hardware_inventory_runs",
+        "scheduler_accelerator_states",
+        "residency_allocations",
+        "serving_gpu_leases",
+        "gpu_leases",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class FingerprintCompatibility:
+    """Explicit comparison mode for fingerprints spanning schema 0022 -> 0024."""
+
+    compatible: bool
+    mode: str
+    allow_audit_checkpoint_introduction: bool
+    reason: str
+
+
+def _fingerprint_table_contract(
+    fingerprint: dict[str, Any],
+    *,
+    expected_tables: frozenset[str],
+    allow_legacy_unexpected_field_absent: bool = False,
+) -> bool:
+    tables = fingerprint.get("tables")
+    digest = fingerprint.get("digest")
+    unexpected_tables = fingerprint.get("unexpected_tables")
+    physical_tables = fingerprint.get("physical_tables")
+    if allow_legacy_unexpected_field_absent and "unexpected_tables" not in fingerprint:
+        unexpected_tables = []
+    if allow_legacy_unexpected_field_absent and "physical_tables" not in fingerprint:
+        physical_tables = sorted(expected_tables | CURRENT_TRUTH_TABLES)
+    expected_physical = expected_tables | CURRENT_TRUTH_TABLES
+    if (
+        not isinstance(tables, dict)
+        or set(tables) != set(expected_tables)
+        or unexpected_tables != []
+        or not isinstance(physical_tables, list)
+        or any(not isinstance(name, str) for name in physical_tables)
+        or physical_tables != sorted(set(physical_tables))
+        or set(physical_tables) - NON_FINGERPRINT_INFRASTRUCTURE_TABLES
+        != set(expected_physical)
+        or not isinstance(digest, str)
+        or len(digest) != 64
+        or any(character not in "0123456789abcdef" for character in digest)
+    ):
+        return False
+    for entry in tables.values():
+        if not isinstance(entry, dict):
+            return False
+        status = entry.get("status")
+        row_count = entry.get("row_count")
+        table_digest = entry.get("digest")
+        redacted_columns = entry.get("redacted_columns")
+        if (
+            not isinstance(redacted_columns, list)
+            or any(not isinstance(column, str) for column in redacted_columns)
+            or redacted_columns != sorted(set(redacted_columns))
+        ):
+            return False
+        if status == "COMPLETE":
+            if (
+                not isinstance(row_count, int)
+                or isinstance(row_count, bool)
+                or row_count < 0
+                or row_count > MAX_ROWS_PER_TABLE
+                or not isinstance(table_digest, str)
+                or len(table_digest) != 64
+                or any(character not in "0123456789abcdef" for character in table_digest)
+            ):
+                return False
+        else:
+            # A capped or content-redacted digest is useful operational evidence, but it is not
+            # a complete count+digest proof and can never authorize READY.
+            return False
+    return True
+
+
+def _physical_table_transition_is_exact(
+    source: dict[str, Any],
+    restored: dict[str, Any],
+    *,
+    introduce_checkpoint: bool,
+) -> bool:
+    source_physical = source.get("physical_tables")
+    restored_physical = restored.get("physical_tables")
+    if not isinstance(restored_physical, list):
+        return False
+    if source_physical is None and introduce_checkpoint:
+        # Genuine m15.1 fingerprints predate this field. Their exact schema-0022 presence is
+        # inferred from the versioned table contract and checked on the migrated target.
+        return True
+    if not isinstance(source_physical, list):
+        return False
+    expected = set(source_physical)
+    if introduce_checkpoint:
+        expected.add("audit_chain_heads")
+    return set(restored_physical) == expected
+
+
+def _has_single_complete_checkpoint(fingerprint: dict[str, Any]) -> bool:
+    checkpoint = fingerprint["tables"]["audit_chain_heads"]
+    return (
+        isinstance(checkpoint, dict)
+        and checkpoint.get("status") == "COMPLETE"
+        and checkpoint.get("row_count") == 1
+    )
+
+
+def fingerprint_compatibility(
+    source: dict[str, Any],
+    restored: dict[str, Any],
+    *,
+    source_schema_revision: str | None,
+) -> FingerprintCompatibility:
+    """Return the only supported semantic-fingerprint comparison modes.
+
+    ``m15.2`` is strict and requires the singleton checkpoint on both sides. The one cross-version
+    exception is a genuine schema-0022 ``m15.1`` backup: 0022 had no checkpoint table, while the
+    0024 migration must introduce exactly one verified head. No other missing/new table or version
+    pairing is interpreted as a migration side effect.
+    """
+
+    source_version = source.get("version")
+    restored_version = restored.get("version")
+    current_tables = classified_tables()
+    legacy_tables = current_tables - {"audit_chain_heads"}
+    if (
+        source_schema_revision == AUDIT_CHECKPOINT_CURRENT_SCHEMA
+        and source_version == FINGERPRINT_VERSION
+        and restored_version == FINGERPRINT_VERSION
+        and _fingerprint_table_contract(source, expected_tables=current_tables)
+        and _fingerprint_table_contract(restored, expected_tables=current_tables)
+        and _has_single_complete_checkpoint(source)
+        and _has_single_complete_checkpoint(restored)
+        and _physical_table_transition_is_exact(
+            source, restored, introduce_checkpoint=False
+        )
+    ):
+        return FingerprintCompatibility(
+            compatible=True,
+            mode="CURRENT_STRICT",
+            allow_audit_checkpoint_introduction=False,
+            reason="both schema-0024 fingerprints use m15.2 and include one checkpoint",
+        )
+    if (
+        source_schema_revision == AUDIT_CHECKPOINT_SOURCE_SCHEMA
+        and source_version == LEGACY_FINGERPRINT_VERSION
+        and restored_version == FINGERPRINT_VERSION
+        and _fingerprint_table_contract(
+            source,
+            expected_tables=legacy_tables,
+            allow_legacy_unexpected_field_absent=True,
+        )
+        and _fingerprint_table_contract(restored, expected_tables=current_tables)
+        and _has_single_complete_checkpoint(restored)
+        and _physical_table_transition_is_exact(
+            source, restored, introduce_checkpoint=True
+        )
+    ):
+        return FingerprintCompatibility(
+            compatible=True,
+            mode="SCHEMA_0022_TO_0024",
+            allow_audit_checkpoint_introduction=True,
+            reason="schema 0022 m15.1 legitimately gains one audit checkpoint at migration",
+        )
+    return FingerprintCompatibility(
+        compatible=False,
+        mode="INCOMPATIBLE",
+        allow_audit_checkpoint_introduction=False,
+        reason=(
+            "unsupported fingerprint/schema pairing: "
+            f"source_schema={source_schema_revision!r}, source_version={source_version!r}, "
+            f"restored_version={restored_version!r}"
+        ),
+    )
+
+
+def audit_fingerprint_differences_are_compatible(
+    source: dict[str, Any],
+    restored: dict[str, Any],
+    contract: FingerprintCompatibility,
+) -> bool:
+    """Require exact authoritative tables except the enumerated reconciliation mutations."""
+
+    if not contract.compatible:
+        return False
+    current_tables = classified_tables()
+    if contract.mode == "CURRENT_STRICT":
+        if (
+            contract.allow_audit_checkpoint_introduction
+            or source.get("version") != FINGERPRINT_VERSION
+            or restored.get("version") != FINGERPRINT_VERSION
+            or not _fingerprint_table_contract(source, expected_tables=current_tables)
+            or not _fingerprint_table_contract(restored, expected_tables=current_tables)
+            or not _has_single_complete_checkpoint(source)
+            or not _has_single_complete_checkpoint(restored)
+            or not _physical_table_transition_is_exact(
+                source, restored, introduce_checkpoint=False
+            )
+        ):
+            return False
+    elif contract.mode == "SCHEMA_0022_TO_0024":
+        if (
+            not contract.allow_audit_checkpoint_introduction
+            or source.get("version") != LEGACY_FINGERPRINT_VERSION
+            or restored.get("version") != FINGERPRINT_VERSION
+            or not _fingerprint_table_contract(
+                source,
+                expected_tables=current_tables - {"audit_chain_heads"},
+                allow_legacy_unexpected_field_absent=True,
+            )
+            or not _fingerprint_table_contract(restored, expected_tables=current_tables)
+            or not _has_single_complete_checkpoint(restored)
+            or not _physical_table_transition_is_exact(
+                source, restored, introduce_checkpoint=True
+            )
+        ):
+            return False
+    else:
+        return False
+    source_tables: dict[str, Any] = source["tables"]
+    restored_tables: dict[str, Any] = restored["tables"]
+    if source_tables == restored_tables:
+        return not contract.allow_audit_checkpoint_introduction
+
+    def same_static_fields(left: dict[str, Any], right: dict[str, Any]) -> bool:
+        return {
+            key: value
+            for key, value in left.items()
+            if key not in {"row_count", "digest"}
+        } == {
+            key: value
+            for key, value in right.items()
+            if key not in {"row_count", "digest"}
+        }
+
+    changed: set[str] = set()
+    for table in sorted(set(source_tables) | set(restored_tables)):
+        left = source_tables.get(table)
+        right = restored_tables.get(table)
+        if left == right:
+            continue
+        changed.add(table)
+        if table == "audit_events":
+            if (
+                not isinstance(left, dict)
+                or not isinstance(right, dict)
+                or not isinstance(left.get("row_count"), int)
+                or right.get("row_count") != left["row_count"] + 1
+                or right.get("digest") == left.get("digest")
+                or not same_static_fields(left, right)
+            ):
+                return False
+            continue
+        if table == "audit_chain_heads":
+            if contract.allow_audit_checkpoint_introduction:
+                if left is not None or not isinstance(right, dict) or right.get("row_count") != 1:
+                    return False
+            elif (
+                not isinstance(left, dict)
+                or not isinstance(right, dict)
+                or left.get("row_count") != 1
+                or right.get("row_count") != 1
+                or right.get("digest") == left.get("digest")
+                or not same_static_fields(left, right)
+            ):
+                return False
+            continue
+        if table == "operational_alerts":
+            if (
+                not isinstance(left, dict)
+                or not isinstance(right, dict)
+                or left.get("row_count") != right.get("row_count")
+                or left.get("digest") == right.get("digest")
+                or not same_static_fields(left, right)
+            ):
+                return False
+            continue
+        return False
+    return {"audit_events", "audit_chain_heads"} <= changed
+
+
+def classified_tables() -> frozenset[str]:
+    return frozenset(name for tables in FINGERPRINT_GROUPS.values() for name in tables)
+
+
+def unclassified_tables() -> frozenset[str]:
+    """Any persisted table must be either fingerprinted or explicitly current-truth."""
+
+    return frozenset(Base.metadata.tables) - classified_tables() - CURRENT_TRUTH_TABLES
+
+
+def _sensitive(column_name: str) -> bool:
+    lowered = column_name.lower()
+    return any(marker in lowered for marker in SENSITIVE_COLUMN_TOKENS)
+
+
+def _canonical_cell_payload(value: Any) -> tuple[bytes, bytes]:
+    if value is None:
+        return b"n", b""
+    if isinstance(value, bool):
+        return b"b", b"1" if value else b"0"
+    if isinstance(value, datetime):
+        moment = value if value.tzinfo else value.replace(tzinfo=UTC)
+        payload = moment.astimezone(UTC).isoformat(timespec="microseconds").encode()
+        return b"t", payload
+    if isinstance(value, date):
+        return b"d", value.isoformat().encode()
+    if isinstance(value, uuid.UUID):
+        return b"u", str(value).encode("ascii")
+    if isinstance(value, Decimal):
+        return b"m", format(value.normalize(), "f").encode("ascii")
+    if isinstance(value, float):
+        return b"f", repr(round(value, 9)).encode("ascii")
+    if isinstance(value, int):
+        return b"i", str(value).encode("ascii")
+    if isinstance(value, bytes | bytearray | memoryview):
+        return b"y", bytes(value)
+    if isinstance(value, dict | list | tuple):
+        payload = json.dumps(
+            value, sort_keys=True, separators=(",", ":"), default=str
+        ).encode("utf-8")
+        return b"j", payload
+    if isinstance(value, str):
+        return b"s", value.encode("utf-8")
+    return b"x", str(value).encode("utf-8")
+
+
+def _canonical_cell(value: Any) -> bytes:
+    tag, payload = _canonical_cell_payload(value)
+    return tag + len(payload).to_bytes(8, byteorder="big") + payload
+
+
+def _canonical_row(values: Any) -> bytes:
+    cells = b"".join(_canonical_cell(value) for value in values)
+    payload = len(values).to_bytes(4, byteorder="big") + cells
+    return b"r" + len(payload).to_bytes(8, byteorder="big") + payload
+
+
+def _table_digest(connection: Connection, table: Table) -> dict[str, Any]:
+    columns = [
+        column
+        for column in sorted(table.columns, key=lambda item: item.name)
+        if not _sensitive(column.name)
+    ]
+    redacted = sorted(column.name for column in table.columns if _sensitive(column.name))
+    if not columns:
+        row_count = int(
+            connection.scalar(select(func.count()).select_from(table)) or 0
+        )
+        return {
+            "row_count": row_count,
+            "digest": hashlib.sha256(_canonical_row((row_count,))).hexdigest(),
+            "status": "FULLY_REDACTED",
+            "redacted_columns": redacted,
+        }
+    statement = select(*columns).select_from(table)
+    hashes: list[bytes] = []
+    truncated = False
+    for row in connection.execute(statement).yield_per(2_000):
+        if len(hashes) >= MAX_ROWS_PER_TABLE:
+            truncated = True
+            break
+        hashes.append(hashlib.sha256(_canonical_row(row)).digest())
+    hashes.sort()
+    aggregate = hashlib.sha256()
+    for item in hashes:
+        aggregate.update(len(item).to_bytes(4, byteorder="big"))
+        aggregate.update(item)
+    return {
+        "row_count": len(hashes),
+        "digest": aggregate.hexdigest(),
+        "status": "BOUNDED" if truncated else "COMPLETE",
+        "redacted_columns": redacted,
+    }
+
+
+def semantic_fingerprint(connection: Connection) -> dict[str, Any]:
+    """Compute the bounded semantic fingerprint of one control-plane database."""
+
+    tables: dict[str, Any] = {}
+    groups: dict[str, Any] = {}
+    inspector_tables = set(connection.dialect.get_table_names(connection))
+    unexpected_tables = sorted(
+        inspector_tables
+        - classified_tables()
+        - CURRENT_TRUTH_TABLES
+        - NON_FINGERPRINT_INFRASTRUCTURE_TABLES
+    )
+    for group, names in FINGERPRINT_GROUPS.items():
+        group_hash = hashlib.sha256()
+        group_rows = 0
+        for name in sorted(names):
+            table = Base.metadata.tables.get(name)
+            if table is None or name not in inspector_tables:
+                tables[name] = {"row_count": None, "digest": None, "status": "ABSENT"}
+                group_hash.update(_canonical_row(("table", name, "ABSENT", None, None)))
+                continue
+            entry = _table_digest(connection, table)
+            tables[name] = entry
+            group_rows += int(entry["row_count"] or 0)
+            group_hash.update(
+                _canonical_row(
+                    (
+                        "table",
+                        name,
+                        entry["status"],
+                        entry["row_count"],
+                        entry["digest"],
+                        entry["redacted_columns"],
+                    )
+                )
+            )
+        groups[group] = {
+            "digest": group_hash.hexdigest(),
+            "row_count": group_rows,
+            "table_count": len(names),
+        }
+    overall = hashlib.sha256()
+    for group in sorted(groups):
+        overall.update(
+            _canonical_row(
+                (
+                    "group",
+                    group,
+                    groups[group]["digest"],
+                    groups[group]["row_count"],
+                    groups[group]["table_count"],
+                )
+            )
+        )
+    for name in sorted(inspector_tables):
+        overall.update(_canonical_row(("physical_table", name)))
+    for name in unexpected_tables:
+        overall.update(_canonical_row(("unexpected_table", name)))
+    return {
+        "version": FINGERPRINT_VERSION,
+        "digest": overall.hexdigest(),
+        "groups": groups,
+        "tables": tables,
+        "physical_tables": sorted(inspector_tables),
+        "unexpected_tables": unexpected_tables,
+        "excluded_current_truth_tables": sorted(CURRENT_TRUTH_TABLES),
+        "row_total": sum(int(item["row_count"] or 0) for item in tables.values()),
+    }
+
+
+def fingerprint_session(session: Session) -> dict[str, Any]:
+    return semantic_fingerprint(session.connection())
+
+
+def diff_fingerprints(source: dict[str, Any], restored: dict[str, Any]) -> dict[str, Any]:
+    """Explain exactly which subjects differ, so a restore is never declared good by hand-wave."""
+
+    source_tables: dict[str, Any] = source.get("tables", {})
+    restored_tables: dict[str, Any] = restored.get("tables", {})
+    identical: list[str] = []
+    differences: list[dict[str, Any]] = []
+    for name in sorted(set(source_tables) | set(restored_tables)):
+        left = source_tables.get(name, {"digest": None, "row_count": None, "status": "ABSENT"})
+        right = restored_tables.get(name, {"digest": None, "row_count": None, "status": "ABSENT"})
+        if left.get("digest") == right.get("digest"):
+            identical.append(name)
+            continue
+        differences.append(
+            {
+                "table": name,
+                "source_rows": left.get("row_count"),
+                "restored_rows": right.get("row_count"),
+                "row_delta": _delta(left.get("row_count"), right.get("row_count")),
+                "source_digest": left.get("digest"),
+                "restored_digest": right.get("digest"),
+            }
+        )
+    group_differences = sorted(
+        group
+        for group in set(source.get("groups", {})) | set(restored.get("groups", {}))
+        if source.get("groups", {}).get(group, {}).get("digest")
+        != restored.get("groups", {}).get(group, {}).get("digest")
+    )
+    return {
+        "identical": source.get("digest") == restored.get("digest"),
+        "source_version": source.get("version"),
+        "restored_version": restored.get("version"),
+        "version_match": source.get("version") == restored.get("version"),
+        "source_digest": source.get("digest"),
+        "restored_digest": restored.get("digest"),
+        "source_physical_tables": source.get("physical_tables"),
+        "restored_physical_tables": restored.get("physical_tables"),
+        "physical_tables_match": source.get("physical_tables")
+        == restored.get("physical_tables"),
+        "identical_table_count": len(identical),
+        "differing_groups": group_differences,
+        "differences": differences,
+    }
+
+
+def _delta(left: Any, right: Any) -> int | None:
+    if isinstance(left, int) and isinstance(right, int):
+        return right - left
+    return None

backend/src/modelforge_api/services/recovery_paths.py

@@ -0,0 +1,170 @@
+"""Allowlisted path and archive handling for backup and restore payloads.
+
+Nothing in the recovery plane may resolve to a host path outside a configured root, follow a
+symlink out of it, or extract an archive member that escapes its destination.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import tarfile
+from pathlib import Path, PurePosixPath
+
+MAX_ARCHIVE_MEMBERS = 10_000
+MAX_ARCHIVE_BYTES = 64 * 1024**3
+_HASH_CHUNK = 1024 * 1024
+_DRIVE_QUALIFIED = re.compile(r"^[A-Za-z]:")
+
+
+class RecoveryPathError(Exception):
+    """Raised when a path or archive member is not provably inside an allowlisted root."""
+
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+def normalise_root(root: Path) -> Path:
+    """Resolve an allowlisted root, creating it when missing so callers get a stable anchor."""
+
+    resolved = Path(root).expanduser()
+    if not resolved.is_absolute():
+        raise RecoveryPathError("PATH_NOT_ALLOWED", "recovery roots must be absolute paths")
+    resolved.mkdir(parents=True, exist_ok=True)
+    return resolved.resolve(strict=True)
+
+
+def _reject_traversal(relative: str) -> PurePosixPath:
+    candidate = PurePosixPath(relative.replace("\\", "/"))
+    if candidate.is_absolute() or not relative.strip():
+        raise RecoveryPathError("PATH_NOT_ALLOWED", f"relative path required, got {relative!r}")
+    if any(part in {"..", ""} for part in candidate.parts):
+        raise RecoveryPathError("PATH_NOT_ALLOWED", f"path traversal rejected: {relative!r}")
+    if any(part.startswith("~") for part in candidate.parts):
+        raise RecoveryPathError("PATH_NOT_ALLOWED", f"home expansion rejected: {relative!r}")
+    # A drive-qualified component means different things on Windows and POSIX; refuse it on both.
+    if any(_DRIVE_QUALIFIED.match(part) for part in candidate.parts):
+        raise RecoveryPathError("PATH_NOT_ALLOWED", f"drive-qualified path rejected: {relative!r}")
+    return candidate
+
+
+def resolve_within(root: Path, relative: str, *, must_exist: bool = False) -> Path:
+    """Return ``root/relative`` proven to stay inside ``root`` after symlink resolution."""
+
+    anchor = normalise_root(root)
+    candidate = _reject_traversal(relative)
+    target = anchor.joinpath(*candidate.parts)
+    if must_exist and not target.exists():
+        raise RecoveryPathError("PAYLOAD_MISSING", f"{relative} does not exist under the root")
+
+    # Resolve the deepest component that exists, so a not-yet-created backup directory is still
+    # checked against the allowlisted root and a symlinked ancestor cannot smuggle a path out.
+    existing = target
+    remaining: list[str] = []
+    while not existing.exists():
+        if existing == existing.parent:
+            raise RecoveryPathError("PATH_NOT_ALLOWED", f"{relative} has no resolvable parent")
+        remaining.append(existing.name)
+        existing = existing.parent
+    resolved = existing.resolve().joinpath(*reversed(remaining))
+    if resolved != anchor and anchor not in resolved.parents:
+        raise RecoveryPathError(
+            "PATH_NOT_ALLOWED", f"{relative} resolves outside the allowlisted recovery root"
+        )
+    if target.is_symlink():
+        raise RecoveryPathError("PATH_NOT_ALLOWED", f"{relative} is a symlink and is rejected")
+    return target
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(_HASH_CHUNK):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def sha256_bytes(payload: bytes) -> str:
+    return hashlib.sha256(payload).hexdigest()
+
+
+def free_bytes(path: Path) -> int | None:
+    try:
+        usage = os.statvfs(path) if hasattr(os, "statvfs") else None
+    except OSError:
+        return None
+    if usage is not None:
+        return int(usage.f_bavail) * int(usage.f_frsize)
+    try:
+        import shutil
+
+        return int(shutil.disk_usage(path).free)
+    except OSError:
+        return None
+
+
+def capacity_bytes(path: Path) -> int | None:
+    try:
+        import shutil
+
+        return int(shutil.disk_usage(path).total)
+    except OSError:
+        return None
+
+
+def validate_archive(archive: Path, destination_root: Path) -> list[str]:
+    """Validate every member of a tar archive before any extraction happens.
+
+    Bounded member count, bounded total size, no absolute members, no traversal, no symlinks,
+    hardlinks, devices or FIFOs.
+    """
+
+    anchor = normalise_root(destination_root)
+    names: list[str] = []
+    total = 0
+    with tarfile.open(archive, "r:*") as handle:
+        for member in handle:
+            if len(names) >= MAX_ARCHIVE_MEMBERS:
+                raise RecoveryPathError(
+                    "ARCHIVE_UNSAFE", f"archive exceeds {MAX_ARCHIVE_MEMBERS} members"
+                )
+            if member.issym() or member.islnk():
+                raise RecoveryPathError(
+                    "ARCHIVE_UNSAFE", f"archive member {member.name} is a link"
+                )
+            if member.isdev() or member.isfifo():
+                raise RecoveryPathError(
+                    "ARCHIVE_UNSAFE", f"archive member {member.name} is a special file"
+                )
+            if not (member.isfile() or member.isdir()):
+                raise RecoveryPathError(
+                    "ARCHIVE_UNSAFE", f"archive member {member.name} has an unsupported type"
+                )
+            try:
+                resolved = resolve_within(anchor, member.name)
+            except RecoveryPathError as error:
+                raise RecoveryPathError(
+                    "ARCHIVE_UNSAFE", f"archive member {member.name} escapes destination"
+                ) from error
+            if anchor not in resolved.parents and resolved != anchor:
+                raise RecoveryPathError(
+                    "ARCHIVE_UNSAFE", f"archive member {member.name} escapes destination"
+                )
+            total += max(0, member.size)
+            if total > MAX_ARCHIVE_BYTES:
+                raise RecoveryPathError("ARCHIVE_UNSAFE", "archive exceeds the extraction budget")
+            names.append(member.name)
+    return names
+
+
+def extract_archive(archive: Path, destination_root: Path) -> list[str]:
+    """Validate first, extract second; a failed validation never writes a byte."""
+
+    names = validate_archive(archive, destination_root)
+    anchor = normalise_root(destination_root)
+    with tarfile.open(archive, "r:*") as handle:
+        handle.extractall(anchor, filter="data")
+    return names

backend/src/modelforge_api/services/recovery_postgres.py

@@ -0,0 +1,392 @@
+"""PostgreSQL logical backup and restore.
+
+ModelForge uses `pg_dump --format=custom`, a supported online-consistent logical backup: the dump
+runs in one repeatable-read snapshot, so concurrent control-plane writes cannot tear it. Copying
+a live data directory is never done. Restore uses `pg_restore` into a genuinely separate database.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]{0,62}$")
+
+
+class PostgresToolError(Exception):
+    def __init__(self, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+
+
+@dataclass(frozen=True)
+class PostgresTarget:
+    host: str
+    port: int
+    database: str
+    user: str
+    password: str | None
+
+    @property
+    def redacted(self) -> str:
+        return f"postgresql://{self.user}@{self.host}:{self.port}/{self.database}"
+
+    def environment(self, base: dict[str, str]) -> dict[str, str]:
+        env = dict(base)
+        if self.password:
+            env["PGPASSWORD"] = self.password
+        env.setdefault("PGCONNECT_TIMEOUT", "15")
+        return env
+
+    def connection_arguments(self, *, database: str | None = None) -> list[str]:
+        return [
+            "--host",
+            self.host,
+            "--port",
+            str(self.port),
+            "--username",
+            self.user,
+            "--dbname",
+            database or self.database,
+        ]
+
+
+def parse_target(database_url: str) -> PostgresTarget:
+    parsed = urlparse(database_url)
+    if not parsed.scheme.startswith("postgresql"):
+        raise PostgresToolError(
+            "POSTGRES_VERSION_INCOMPATIBLE", "only PostgreSQL destinations are supported"
+        )
+    database = parsed.path.lstrip("/")
+    if not database or not _SAFE_IDENTIFIER.match(database):
+        raise PostgresToolError(
+            "DESTINATION_NOT_ISOLATED", f"unsupported database identifier {database!r}"
+        )
+    if not parsed.hostname:
+        raise PostgresToolError("DESTINATION_NOT_ISOLATED", "a database host is required")
+    return PostgresTarget(
+        host=parsed.hostname,
+        port=parsed.port or 5432,
+        database=database,
+        user=unquote(parsed.username or "postgres"),
+        password=unquote(parsed.password) if parsed.password else None,
+    )
+
+
+@dataclass
+class CommandResult:
+    command: str
+    returncode: int
+    stdout: str
+    stderr: str
+    duration_seconds: float
+
+
+@dataclass
+class PostgresEngine:
+    pg_dump_path: str = "pg_dump"
+    pg_restore_path: str = "pg_restore"
+    psql_path: str = "psql"
+    timeout_seconds: int = 1800
+    base_environment: dict[str, str] = field(default_factory=dict)
+
+    def _run(
+        self, argv: list[str], target: PostgresTarget, *, stdin: str | None = None
+    ) -> CommandResult:
+        import time
+
+        started = time.perf_counter()
+        try:
+            completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+                argv,
+                capture_output=True,
+                text=True,
+                timeout=self.timeout_seconds,
+                env=target.environment(self.base_environment),
+                input=stdin,
+                check=False,
+            )
+        except FileNotFoundError as error:
+            raise PostgresToolError(
+                "BACKUP_TOOL_UNAVAILABLE", f"{argv[0]} is not available in this environment"
+            ) from error
+        except subprocess.TimeoutExpired as error:
+            raise PostgresToolError(
+                "BACKUP_TOOL_FAILED", f"{argv[0]} exceeded {self.timeout_seconds}s"
+            ) from error
+        return CommandResult(
+            command=Path(argv[0]).name,
+            returncode=completed.returncode,
+            stdout=completed.stdout or "",
+            stderr=_redact(completed.stderr or ""),
+            duration_seconds=time.perf_counter() - started,
+        )
+
+    def tool_version(self, tool: str) -> str:
+        try:
+            completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+                [tool, "--version"],
+                capture_output=True,
+                text=True,
+                timeout=30,
+                check=False,
+            )
+        except (FileNotFoundError, subprocess.TimeoutExpired) as error:
+            raise PostgresToolError(
+                "BACKUP_TOOL_UNAVAILABLE", f"{tool} is not available in this environment"
+            ) from error
+        if completed.returncode != 0:
+            raise PostgresToolError("BACKUP_TOOL_FAILED", f"{tool} --version failed")
+        return (completed.stdout or "").strip()
+
+    def server_identity(self, target: PostgresTarget) -> dict[str, object]:
+        result = self._run(
+            [
+                self.psql_path,
+                *target.connection_arguments(),
+                "--no-align",
+                "--tuples-only",
+                "--no-psqlrc",
+                "--command",
+                (
+                    "select current_setting('server_version_num'), "
+                    "current_setting('server_version'), current_database(), "
+                    "coalesce((select string_agg(extname || ' ' || extversion, ',' "
+                    "order by extname) from pg_extension), '')"
+                ),
+            ],
+            target,
+        )
+        if result.returncode != 0:
+            raise PostgresToolError(
+                "BACKUP_TOOL_FAILED", f"psql identity query failed: {result.stderr[:400]}"
+            )
+        parts = (result.stdout.strip().splitlines() or [""])[0].split("|")
+        version_num = int(parts[0]) if parts and parts[0].strip().isdigit() else 0
+        extensions = [item for item in (parts[3].split(",") if len(parts) > 3 else []) if item]
+        return {
+            "server_version_num": version_num,
+            "server_version": parts[1] if len(parts) > 1 else "",
+            "major_version": version_num // 10_000,
+            "database": parts[2] if len(parts) > 2 else target.database,
+            "extensions": extensions,
+        }
+
+    def alembic_revision(self, target: PostgresTarget) -> str | None:
+        result = self._run(
+            [
+                self.psql_path,
+                *target.connection_arguments(),
+                "--no-align",
+                "--tuples-only",
+                "--no-psqlrc",
+                "--command",
+                "select version_num from alembic_version limit 1",
+            ],
+            target,
+        )
+        if result.returncode != 0:
+            return None
+        value = result.stdout.strip()
+        return value or None
+
+    def dump(self, target: PostgresTarget, destination: Path) -> CommandResult:
+        """Consistent single-snapshot custom-format dump including schema, data and sequences."""
+
+        argv = [
+            self.pg_dump_path,
+            *target.connection_arguments(),
+            "--format=custom",
+            "--compress=6",
+            "--no-owner",
+            "--no-privileges",
+            "--serializable-deferrable",
+            "--file",
+            str(destination),
+        ]
+        result = self._run(argv, target)
+        if result.returncode != 0:
+            destination.unlink(missing_ok=True)
+            raise PostgresToolError(
+                "BACKUP_TOOL_FAILED", f"pg_dump failed: {result.stderr[:600]}"
+            )
+        return result
+
+    def table_is_empty(self, target: PostgresTarget) -> bool:
+        result = self._run(
+            [
+                self.psql_path,
+                *target.connection_arguments(),
+                "--no-align",
+                "--tuples-only",
+                "--no-psqlrc",
+                "--command",
+                "select count(*) from information_schema.tables where table_schema='public'",
+            ],
+            target,
+        )
+        if result.returncode != 0:
+            raise PostgresToolError(
+                "DESTINATION_NOT_ISOLATED", f"destination probe failed: {result.stderr[:400]}"
+            )
+        return result.stdout.strip() in {"0", ""}
+
+    def database_exists(self, target: PostgresTarget) -> bool:
+        probe = PostgresTarget(
+            host=target.host,
+            port=target.port,
+            database="postgres",
+            user=target.user,
+            password=target.password,
+        )
+        result = self._run(
+            [
+                self.psql_path,
+                *probe.connection_arguments(),
+                "--no-align",
+                "--tuples-only",
+                "--no-psqlrc",
+                "--command",
+                "select 1 from pg_database where datname = :name".replace(
+                    ":name", _quote_literal(target.database)
+                ),
+            ],
+            probe,
+        )
+        if result.returncode != 0:
+            raise PostgresToolError(
+                "DESTINATION_NOT_ISOLATED", f"database probe failed: {result.stderr[:400]}"
+            )
+        return result.stdout.strip() == "1"
+
+    def create_database(self, target: PostgresTarget) -> CommandResult:
+        probe = PostgresTarget(
+            host=target.host,
+            port=target.port,
+            database="postgres",
+            user=target.user,
+            password=target.password,
+        )
+        result = self._run(
+            [
+                self.psql_path,
+                *probe.connection_arguments(),
+                "--no-psqlrc",
+                "--command",
+                f'create database "{target.database}"',
+            ],
+            probe,
+        )
+        if result.returncode != 0:
+            raise PostgresToolError(
+                "DESTINATION_NOT_ISOLATED",
+                f"could not create restore database: {result.stderr[:400]}",
+            )
+        return result
+
+    def restore(self, target: PostgresTarget, payload: Path) -> CommandResult:
+        argv = [
+            self.pg_restore_path,
+            *target.connection_arguments(),
+            "--no-owner",
+            "--no-privileges",
+            "--exit-on-error",
+            "--single-transaction",
+            str(payload),
+        ]
+        result = self._run(argv, target)
+        if result.returncode != 0:
+            raise PostgresToolError(
+                "BACKUP_TOOL_FAILED", f"pg_restore failed: {result.stderr[:600]}"
+            )
+        return result
+
+    def list_dump_contents(self, payload: Path) -> int:
+        """`pg_restore --list` proves a custom-format archive is structurally readable."""
+
+        try:
+            completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
+                [self.pg_restore_path, "--list", str(payload)],
+                capture_output=True,
+                text=True,
+                timeout=self.timeout_seconds,
+                check=False,
+            )
+        except FileNotFoundError as error:
+            raise PostgresToolError(
+                "BACKUP_TOOL_UNAVAILABLE", "pg_restore is not available in this environment"
+            ) from error
+        except subprocess.TimeoutExpired as error:
+            raise PostgresToolError("BACKUP_TOOL_FAILED", "pg_restore --list timed out") from error
+        if completed.returncode != 0:
+            raise PostgresToolError(
+                "HASH_MISMATCH",
+                f"backup payload is not a readable archive: {_redact(completed.stderr)[:400]}",
+            )
+        return sum(1 for line in (completed.stdout or "").splitlines() if line and line[0] != ";")
+
+    def scalar(self, target: PostgresTarget, sql: str) -> str:
+        result = self._run(
+            [
+                self.psql_path,
+                *target.connection_arguments(),
+                "--no-align",
+                "--tuples-only",
+                "--no-psqlrc",
+                "--command",
+                sql,
+            ],
+            target,
+        )
+        if result.returncode != 0:
+            raise PostgresToolError("BACKUP_TOOL_FAILED", f"query failed: {result.stderr[:400]}")
+        return result.stdout.strip()
+
+    def query_rows(
+        self,
+        target: PostgresTarget,
+        sql: str,
+        *,
+        max_rows: int = 1000,
+    ) -> list[dict[str, str]]:
+        """Return one bounded CSV page without delimiter ambiguity in JSON/text fields."""
+
+        if max_rows < 1:
+            raise ValueError("max_rows must be positive")
+        result = self._run(
+            [
+                self.psql_path,
+                *target.connection_arguments(),
+                "--csv",
+                "--no-psqlrc",
+                "--command",
+                sql,
+            ],
+            target,
+        )
+        if result.returncode != 0:
+            raise PostgresToolError("BACKUP_TOOL_FAILED", f"query failed: {result.stderr[:400]}")
+        reader = csv.DictReader(io.StringIO(result.stdout))
+        if reader.fieldnames is None:
+            return []
+        rows = [dict(row) for row in reader]
+        if len(rows) > max_rows:
+            raise PostgresToolError(
+                "BACKUP_TOOL_FAILED",
+                f"bounded query returned more than {max_rows} rows",
+            )
+        return rows
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _redact(text: str) -> str:
+    return re.sub(r"(password=)\S+", r"\1***", text)

backend/src/modelforge_api/services/registry.py

@@ -0,0 +1,663 @@
+from __future__ import annotations
+
+import hashlib
+import math
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal, cast
+
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.enums import ArtifactStatus, ModelLifecycle, StorageRootStatus
+from modelforge_api.domain.lifecycle import InvalidTransition, assert_model_transition
+from modelforge_api.domain.registry import (
+    ArtifactCreate,
+    ArtifactLocationResponse,
+    ArtifactResponse,
+    CapacityDecision,
+    DerivedArtifactCreate,
+    DerivedArtifactResponse,
+    DerivedSourceResponse,
+    ModelCreate,
+    ModelResponse,
+    ModelUpdate,
+    Page,
+    RevisionCreate,
+    RevisionResponse,
+    StorageRootCreate,
+    StorageRootObservation,
+    StorageRootResponse,
+    StorageRootUpdate,
+    VerifyResponse,
+)
+from modelforge_api.persistence.models import (
+    ArtifactLocation,
+    ComputeNode,
+    DerivedArtifact,
+    DerivedArtifactSource,
+    Model,
+    ModelArtifact,
+    ModelRevision,
+    StorageRoot,
+)
+from modelforge_api.persistence.registry_repository import RegistryRepository
+from modelforge_api.services.audit import AuditWriter
+from modelforge_api.services.manifest_registry import ManifestRegistry
+
+
+class RegistryError(Exception):
+    status_code = 400
+    code = "registry_error"
+
+    def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
+        super().__init__(message)
+        self.details = details or {}
+
+
+class RegistryNotFound(RegistryError):
+    status_code = 404
+    code = "registry_not_found"
+
+
+class RegistryConflict(RegistryError):
+    status_code = 409
+    code = "registry_conflict"
+
+
+class RegistryService:
+    def __init__(self, session: Session, *, actor_id: str = "local-operator") -> None:
+        self.session = session
+        self.repo = RegistryRepository(session)
+        self.audit = AuditWriter(session, "operator", actor_id)
+
+    def _commit(self) -> None:
+        try:
+            self.session.commit()
+        except IntegrityError as exc:
+            self.session.rollback()
+            raise RegistryConflict("registry identity already exists") from exc
+
+    @staticmethod
+    def _model_response(model: Model) -> ModelResponse:
+        revisions = model.revisions
+        artifacts = [artifact for revision in revisions for artifact in revision.artifacts]
+        verification_status = (
+            "verified"
+            if artifacts and all(item.status == ArtifactStatus.VERIFIED for item in artifacts)
+            else "quarantined"
+            if any(item.quarantined for item in artifacts)
+            else "unverified"
+        )
+        return ModelResponse(
+            id=model.id,
+            key=model.key,
+            display_name=model.display_name,
+            description=model.description,
+            source_type=cast(Literal["huggingface", "local", "custom"], model.source_type),
+            upstream_provider=model.upstream_provider,
+            upstream_source=model.upstream_source,
+            upstream_metadata=model.upstream_metadata,
+            local_metadata=model.local_metadata,
+            interpretation_metadata=model.interpretation_metadata,
+            family=model.family,
+            modalities=model.modalities,
+            parameter_metadata=model.parameter_metadata,
+            license_metadata=model.license_metadata,
+            lifecycle=ModelLifecycle(model.lifecycle),
+            deprecated_at=model.deprecated_at,
+            created_at=model.created_at,
+            updated_at=model.updated_at,
+            revision_count=len(revisions),
+            artifact_count=sum(len(item.artifacts) for item in revisions),
+            verification_status=verification_status,
+            deployment_status="not_deployed",
+        )
+
+    @staticmethod
+    def _revision_response(revision: ModelRevision) -> RevisionResponse:
+        return RevisionResponse.model_validate(revision)
+
+    @staticmethod
+    def _location_response(location: ArtifactLocation) -> ArtifactLocationResponse:
+        return ArtifactLocationResponse.model_validate(location)
+
+    def _artifact_response(self, artifact: ModelArtifact) -> ArtifactResponse:
+        return ArtifactResponse(
+            id=artifact.id,
+            revision_id=artifact.revision_id,
+            filename=artifact.filename,
+            artifact_type=artifact.artifact_type,
+            serialization_format=artifact.serialization_format,
+            sha256=artifact.sha256,
+            size_bytes=artifact.size_bytes,
+            status=ArtifactStatus(artifact.status),
+            security_status=artifact.security_status,
+            license_status=artifact.license_status,
+            quarantined=artifact.quarantined,
+            verification_details=artifact.verification_details,
+            verified_at=artifact.verified_at,
+            immutable_at=artifact.immutable_at,
+            deprecated_at=artifact.deprecated_at,
+            archived_at=artifact.archived_at,
+            created_at=artifact.created_at,
+            updated_at=artifact.updated_at,
+            locations=[self._location_response(item) for item in artifact.locations],
+        )
+
+    def _derived_response(self, artifact: DerivedArtifact) -> DerivedArtifactResponse:
+        sources = self.repo.derived_sources(artifact.id)
+        return DerivedArtifactResponse(
+            id=artifact.id,
+            revision_id=artifact.revision_id,
+            filename=artifact.filename,
+            artifact_type=artifact.artifact_type,
+            sha256=artifact.sha256,
+            size_bytes=artifact.size_bytes,
+            transformation_type=artifact.transformation_type,
+            tool=artifact.derivation_tool,
+            tool_version=artifact.derivation_tool_version,
+            configuration=artifact.derivation_arguments,
+            environment_snapshot=artifact.environment_snapshot,
+            status=ArtifactStatus(artifact.status),
+            immutable_at=artifact.immutable_at,
+            created_at=artifact.created_at,
+            updated_at=artifact.updated_at,
+            sources=[
+                DerivedSourceResponse(
+                    artifact_id=item.source_artifact_id,
+                    sha256=item.source_sha256,
+                    ordinal=item.ordinal,
+                )
+                for item in sources
+            ],
+            locations=[self._location_response(item) for item in artifact.locations],
+        )
+
+    @staticmethod
+    def _storage_response(root: StorageRoot) -> StorageRootResponse:
+        return StorageRootResponse.model_validate(root)
+
+    def list_models(
+        self,
+        *,
+        page: int,
+        page_size: int,
+        search: str | None,
+        lifecycle: str | None,
+        source_type: str | None,
+    ) -> Page[ModelResponse]:
+        items, total = self.repo.page_models(
+            page=page,
+            page_size=page_size,
+            search=search,
+            lifecycle=lifecycle,
+            source_type=source_type,
+        )
+        return Page(
+            items=[self._model_response(item) for item in items],
+            page=page,
+            page_size=page_size,
+            total=total,
+            pages=math.ceil(total / page_size) if total else 0,
+        )
+
+    def get_model(self, model_id: uuid.UUID) -> ModelResponse:
+        model = self.repo.model(model_id)
+        if not model:
+            raise RegistryNotFound("model not found")
+        return self._model_response(model)
+
+    def create_model(self, request: ModelCreate) -> ModelResponse:
+        model = Model(**request.model_dump(mode="json"))
+        self.repo.add(model)
+        self.audit.write("MODEL_CREATED", "model", str(model.id), {"key": model.key})
+        self._commit()
+        return self.get_model(model.id)
+
+    def update_model(self, model_id: uuid.UUID, request: ModelUpdate) -> ModelResponse:
+        model = self.repo.model(model_id)
+        if not model:
+            raise RegistryNotFound("model not found")
+        changes = request.model_dump(exclude_unset=True, mode="json")
+        if "lifecycle" in changes and changes["lifecycle"] != model.lifecycle:
+            try:
+                assert_model_transition(
+                    ModelLifecycle(model.lifecycle), ModelLifecycle(changes["lifecycle"])
+                )
+            except InvalidTransition as exc:
+                raise RegistryConflict(str(exc)) from exc
+        for field, value in changes.items():
+            setattr(model, field, value)
+        self.audit.write("MODEL_UPDATED", "model", str(model.id), {"fields": sorted(changes)})
+        self._commit()
+        return self.get_model(model.id)
+
+    def deprecate_model(self, model_id: uuid.UUID) -> ModelResponse:
+        model = self.repo.model(model_id)
+        if not model:
+            raise RegistryNotFound("model not found")
+        if model.lifecycle != ModelLifecycle.DEPRECATED:
+            try:
+                assert_model_transition(ModelLifecycle(model.lifecycle), ModelLifecycle.DEPRECATED)
+            except InvalidTransition as exc:
+                raise RegistryConflict(str(exc)) from exc
+        model.lifecycle = ModelLifecycle.DEPRECATED
+        model.deprecated_at = datetime.now(UTC)
+        self.audit.write("MODEL_DEPRECATED", "model", str(model.id), {})
+        self._commit()
+        return self.get_model(model.id)
+
+    def archive_model(self, model_id: uuid.UUID) -> ModelResponse:
+        model = self.repo.model(model_id)
+        if not model:
+            raise RegistryNotFound("model not found")
+        try:
+            assert_model_transition(ModelLifecycle(model.lifecycle), ModelLifecycle.ARCHIVED)
+        except InvalidTransition as exc:
+            raise RegistryConflict(str(exc)) from exc
+        model.lifecycle = ModelLifecycle.ARCHIVED
+        self.audit.write("MODEL_ARCHIVED", "model", str(model.id), {})
+        self._commit()
+        return self.get_model(model.id)
+
+    def delete(self, resource_type: str, resource_id: uuid.UUID) -> None:
+        entity_types = {
+            "model": Model,
+            "model_revision": ModelRevision,
+            "model_artifact": ModelArtifact,
+            "derived_artifact": DerivedArtifact,
+            "storage_root": StorageRoot,
+        }
+        entity_type = entity_types[resource_type]
+        entity = self.session.get(entity_type, resource_id)
+        if not entity:
+            raise RegistryNotFound(f"{resource_type.replace('_', ' ')} not found")
+        dependencies = self.repo.dependencies(resource_type, resource_id)
+        if dependencies:
+            self.audit.write(
+                "REGISTRY_DELETE_BLOCKED",
+                resource_type,
+                str(resource_id),
+                {"dependencies": dependencies},
+                outcome="blocked",
+            )
+            self.session.commit()
+            raise RegistryConflict(
+                "resource has provenance or operational dependencies",
+                {"dependencies": dependencies},
+            )
+        self.session.delete(entity)
+        self.audit.write(f"{resource_type.upper()}_DELETED", resource_type, str(resource_id), {})
+        self._commit()
+
+    def create_revision(self, model_id: uuid.UUID, request: RevisionCreate) -> RevisionResponse:
+        if not self.repo.model(model_id):
+            raise RegistryNotFound("model not found")
+        if self.repo.revision_by_commit(model_id, request.resolved_commit_sha):
+            raise RegistryConflict("exact revision is already registered")
+        now = datetime.now(UTC)
+        revision = ModelRevision(
+            model_id=model_id,
+            immutable_at=now,
+            **request.model_dump(mode="json"),
+        )
+        self.repo.add(revision)
+        self.audit.write(
+            "MODEL_REVISION_REGISTERED",
+            "model_revision",
+            str(revision.id),
+            {"model_id": str(model_id), "resolved_commit_sha": revision.resolved_commit_sha},
+        )
+        self._commit()
+        return self._revision_response(revision)
+
+    def revisions(self, model_id: uuid.UUID) -> list[RevisionResponse]:
+        if not self.repo.model(model_id):
+            raise RegistryNotFound("model not found")
+        return [self._revision_response(item) for item in self.repo.revisions(model_id)]
+
+    def create_artifact(self, revision_id: uuid.UUID, request: ArtifactCreate) -> ArtifactResponse:
+        if not self.repo.revision(revision_id):
+            raise RegistryNotFound("revision not found")
+        if self.repo.artifact_by_digest(revision_id, request.sha256):
+            raise RegistryConflict("artifact digest already exists for this revision")
+        now = datetime.now(UTC)
+        values = request.model_dump(exclude={"locations"}, mode="json")
+        artifact = ModelArtifact(
+            revision_id=revision_id,
+            storage_uri=None,
+            quarantined=request.status != ArtifactStatus.VERIFIED,
+            immutable_at=now,
+            **values,
+        )
+        self.repo.add(artifact)
+        for location_request in request.locations:
+            self._add_location(artifact.id, None, location_request.model_dump(mode="json"))
+        self.audit.write(
+            "ARTIFACT_REGISTERED",
+            "model_artifact",
+            str(artifact.id),
+            {"revision_id": str(revision_id), "sha256": artifact.sha256},
+        )
+        self._commit()
+        return self._artifact_response(self.repo.artifact(artifact.id))  # type: ignore[arg-type]
+
+    def artifacts(self, revision_id: uuid.UUID) -> list[ArtifactResponse]:
+        if not self.repo.revision(revision_id):
+            raise RegistryNotFound("revision not found")
+        return [self._artifact_response(item) for item in self.repo.artifacts(revision_id)]
+
+    def create_derived(self, request: DerivedArtifactCreate) -> DerivedArtifactResponse:
+        revision = self.repo.revision(request.revision_id)
+        if not revision:
+            raise RegistryNotFound("revision not found")
+        sources: list[ModelArtifact] = []
+        for source_id in request.source_artifact_ids:
+            source = self.repo.artifact(source_id)
+            if not source:
+                raise RegistryNotFound(f"source artifact not found: {source_id}")
+            if source.revision_id != request.revision_id:
+                raise RegistryConflict("all source artifacts must belong to the derived revision")
+            sources.append(source)
+        if len(set(request.source_artifact_ids)) != len(request.source_artifact_ids):
+            raise RegistryConflict("source artifacts must be unique")
+        now = datetime.now(UTC)
+        artifact = DerivedArtifact(
+            source_artifact_id=sources[0].id,
+            revision_id=request.revision_id,
+            filename=request.filename,
+            artifact_type=request.artifact_type,
+            transformation_type=request.transformation_type,
+            storage_uri=None,
+            sha256=request.sha256,
+            size_bytes=request.size_bytes,
+            quantization=None,
+            derivation_tool=request.tool,
+            derivation_tool_version=request.tool_version,
+            derivation_arguments=request.configuration,
+            environment_snapshot=request.environment_snapshot,
+            status=request.status,
+            verification_details={},
+            immutable_at=now,
+        )
+        self.repo.add(artifact)
+        for ordinal, source in enumerate(sources):
+            self.repo.add(
+                DerivedArtifactSource(
+                    derived_artifact_id=artifact.id,
+                    source_artifact_id=source.id,
+                    source_sha256=source.sha256,
+                    ordinal=ordinal,
+                )
+            )
+        for location_request in request.locations:
+            self._add_location(None, artifact.id, location_request.model_dump(mode="json"))
+        self.audit.write(
+            "DERIVED_ARTIFACT_REGISTERED",
+            "derived_artifact",
+            str(artifact.id),
+            {
+                "source_artifact_ids": [str(item.id) for item in sources],
+                "source_hashes": [item.sha256 for item in sources],
+                "output_hash": artifact.sha256,
+                "transformation_type": artifact.transformation_type,
+                "tool": artifact.derivation_tool,
+                "tool_version": artifact.derivation_tool_version,
+                "configuration": artifact.derivation_arguments,
+                "environment": artifact.environment_snapshot,
+            },
+        )
+        self._commit()
+        return self._derived_response(self.repo.derived(artifact.id))  # type: ignore[arg-type]
+
+    def derived(self, revision_id: uuid.UUID) -> list[DerivedArtifactResponse]:
+        if not self.repo.revision(revision_id):
+            raise RegistryNotFound("revision not found")
+        return [
+            self._derived_response(item) for item in self.repo.derived_for_revision(revision_id)
+        ]
+
+    def _add_location(
+        self,
+        artifact_id: uuid.UUID | None,
+        derived_id: uuid.UUID | None,
+        values: dict[str, Any],
+    ) -> ArtifactLocation:
+        root = self.repo.storage_root(uuid.UUID(str(values["storage_root_id"])))
+        if not root:
+            raise RegistryNotFound("storage root not found")
+        values["storage_root_id"] = root.id
+        return self.repo.add(
+            ArtifactLocation(
+                artifact_id=artifact_id,
+                derived_artifact_id=derived_id,
+                **values,
+            )
+        )
+
+    def create_storage_root(self, request: StorageRootCreate) -> StorageRootResponse:
+        node = self.session.get(ComputeNode, request.compute_node_id)
+        if not node:
+            raise RegistryNotFound("compute node not found")
+        if node.decommissioned_at is not None:
+            raise RegistryConflict("storage roots cannot be added to a decommissioned node")
+        root = StorageRoot(
+            **request.model_dump(),
+            status=StorageRootStatus.UNKNOWN,
+            writable=False,
+            validation_details={"reason": "capacity not observed"},
+        )
+        self.repo.add(root)
+        self.audit.write(
+            "STORAGE_ROOT_REGISTERED",
+            "storage_root",
+            str(root.id),
+            {"compute_node_id": str(root.compute_node_id), "path": root.path},
+        )
+        self._commit()
+        return self._storage_response(root)
+
+    def storage_roots(self, node_id: uuid.UUID | None = None) -> list[StorageRootResponse]:
+        return [self._storage_response(item) for item in self.repo.storage_roots(node_id)]
+
+    def update_storage_root(
+        self, root_id: uuid.UUID, request: StorageRootUpdate
+    ) -> StorageRootResponse:
+        root = self.repo.storage_root(root_id)
+        if not root:
+            raise RegistryNotFound("storage root not found")
+        normalized = request.agent_path.replace("\\", "/")
+        if not normalized.startswith("/data/artifacts/") or ".." in normalized.split("/"):
+            raise RegistryConflict("agent_path must be confined below /data/artifacts")
+        root.agent_path = normalized.rstrip("/")
+        self.audit.write(
+            "STORAGE_ROOT_AGENT_PATH_MAPPED",
+            "storage_root",
+            str(root.id),
+            {"agent_path": root.agent_path},
+        )
+        self._commit()
+        return self._storage_response(root)
+
+    def observe_storage_root(
+        self, root_id: uuid.UUID, observation: StorageRootObservation
+    ) -> StorageRootResponse:
+        root = self.repo.storage_root(root_id)
+        if not root:
+            raise RegistryNotFound("storage root not found")
+        root.writable = observation.writable
+        root.capacity_bytes = observation.capacity_bytes
+        root.free_bytes = observation.free_bytes
+        root.capacity_observed_at = datetime.now(UTC)
+        root.validation_details = observation.details
+        root.status = self.capacity_decision(root, 0).status
+        self.audit.write(
+            "STORAGE_ROOT_VALIDATED",
+            "storage_root",
+            str(root.id),
+            {
+                "status": root.status,
+                "writable": root.writable,
+                "capacity_bytes": root.capacity_bytes,
+                "free_bytes": root.free_bytes,
+            },
+        )
+        self._commit()
+        return self._storage_response(root)
+
+    @staticmethod
+    def capacity_decision(root: StorageRoot, requested_bytes: int) -> CapacityDecision:
+        if root.capacity_bytes is None or root.free_bytes is None:
+            return CapacityDecision(
+                allowed=False,
+                status=StorageRootStatus.UNKNOWN,
+                requested_bytes=requested_bytes,
+                usable_bytes=None,
+                reason="capacity is unknown; placement is blocked",
+            )
+        if not root.writable:
+            return CapacityDecision(
+                allowed=False,
+                status=StorageRootStatus.READ_ONLY,
+                requested_bytes=requested_bytes,
+                usable_bytes=0,
+                reason="storage root is not writable",
+            )
+        reserve = max(root.reserve_bytes, root.capacity_bytes * root.reserve_percent // 100)
+        usable = max(0, root.free_bytes - reserve)
+        allowed = requested_bytes <= usable
+        return CapacityDecision(
+            allowed=allowed,
+            status=StorageRootStatus.READY if allowed else StorageRootStatus.CAPACITY_BLOCKED,
+            requested_bytes=requested_bytes,
+            usable_bytes=usable,
+            reason="capacity guard passed"
+            if allowed
+            else "reserved free-space threshold would be crossed",
+        )
+
+    def check_capacity(self, root_id: uuid.UUID, requested_bytes: int) -> CapacityDecision:
+        root = self.repo.storage_root(root_id)
+        if not root:
+            raise RegistryNotFound("storage root not found")
+        decision = self.capacity_decision(root, requested_bytes)
+        self.audit.write(
+            "STORAGE_CAPACITY_CHECKED",
+            "storage_root",
+            str(root.id),
+            decision.model_dump(mode="json"),
+            outcome="success" if decision.allowed else "blocked",
+        )
+        self._commit()
+        return decision
+
+    def verify_artifact(self, artifact_id: uuid.UUID, location_id: uuid.UUID) -> VerifyResponse:
+        artifact = self.repo.artifact(artifact_id)
+        location = self.repo.location(location_id)
+        if not artifact:
+            raise RegistryNotFound("artifact not found")
+        if not location or location.artifact_id != artifact.id:
+            raise RegistryNotFound("artifact location not found")
+        root = self.repo.storage_root(location.storage_root_id)
+        if not root:
+            raise RegistryNotFound("storage root not found")
+        base = Path(root.path).resolve()
+        target = (base / location.relative_path).resolve()
+        if target != base and base not in target.parents:
+            raise RegistryConflict("artifact path escapes its storage root")
+        checked_at = datetime.now(UTC)
+        observed_sha: str | None = None
+        observed_size: int | None = None
+        if not target.is_file():
+            status = ArtifactStatus.MISSING
+        else:
+            digest = hashlib.sha256()
+            observed_size = 0
+            with target.open("rb") as handle:
+                for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                    observed_size += len(chunk)
+                    digest.update(chunk)
+            observed_sha = digest.hexdigest()
+            status = (
+                ArtifactStatus.VERIFIED
+                if observed_sha == artifact.sha256 and observed_size == artifact.size_bytes
+                else ArtifactStatus.CORRUPT
+            )
+        location.status = status
+        location.observed_sha256 = observed_sha
+        location.size_bytes = observed_size
+        location.last_checked_at = checked_at
+        artifact.status = status
+        artifact.verified_at = checked_at if status == ArtifactStatus.VERIFIED else None
+        artifact.quarantined = status != ArtifactStatus.VERIFIED
+        artifact.verification_details = {
+            "location_id": str(location.id),
+            "expected_size_bytes": artifact.size_bytes,
+            "observed_size_bytes": observed_size,
+            "observed_sha256": observed_sha,
+            "inspection": "streaming_sha256_only",
+        }
+        self.audit.write(
+            "ARTIFACT_VERIFIED"
+            if status == ArtifactStatus.VERIFIED
+            else "ARTIFACT_VERIFICATION_FAILED",
+            "model_artifact",
+            str(artifact.id),
+            artifact.verification_details | {"status": status},
+            outcome="success" if status == ArtifactStatus.VERIFIED else "blocked",
+        )
+        self._commit()
+        return VerifyResponse(
+            artifact_id=artifact.id,
+            status=status,
+            expected_sha256=artifact.sha256,
+            observed_sha256=observed_sha,
+            observed_size_bytes=observed_size,
+            checked_at=checked_at,
+            location_id=location.id,
+        )
+
+
+def seed_candidate_registry(session: Session, manifests: ManifestRegistry) -> int:
+    """Idempotently materialize M0 candidates without overwriting operator-owned metadata."""
+    repo = RegistryRepository(session)
+    audit = AuditWriter(session, "system", "candidate-seed")
+    inserted = 0
+    for candidate in manifests.candidates().candidates:
+        provider = candidate.source.split("/", 1)[0] if "/" in candidate.source else "unknown"
+        existing = repo.model_by_key(candidate.id) or repo.model_by_source(candidate.source)
+        interpretation = {
+            "intended_capabilities": candidate.intended_capabilities,
+            "proposed_role": candidate.proposed_role,
+            "preferred_runtime": candidate.preferred_runtime,
+            "evidence_status": "unverified_seed_claim",
+        }
+        if existing:
+            existing.display_name = candidate.display_name
+            existing.upstream_provider = provider
+            existing.upstream_source = candidate.source
+            existing.interpretation_metadata = interpretation
+            continue
+        model = Model(
+            key=candidate.id,
+            display_name=candidate.display_name,
+            description=None,
+            source_type="huggingface",
+            upstream_provider=provider,
+            upstream_source=candidate.source,
+            upstream_metadata={"repository_id": candidate.source, "verification": "unverified"},
+            local_metadata={},
+            interpretation_metadata=interpretation,
+            family=None,
+            modalities=[],
+            parameter_metadata={},
+            license_metadata={"status": "unknown", "spdx_id": None, "source": None},
+            lifecycle=ModelLifecycle.CANDIDATE,
+        )
+        repo.add(model)
+        audit.write("MODEL_SEEDED", "model", str(model.id), {"key": model.key})
+        inserted += 1
+    session.commit()
+    return inserted

backend/src/modelforge_api/services/runtime_compatibility.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import abc
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class InspectionResult:
+    blockers: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    requires_probe: bool = False
+
+
+class CompatibilityInspector(abc.ABC):
+    @abc.abstractmethod
+    def inspect(
+        self,
+        artifact: dict[str, Any],
+        supported_model_types: list[str],
+    ) -> InspectionResult: ...
+
+
+class TransformersInspector(CompatibilityInspector):
+    def __init__(self, *, sentence_transformers: bool) -> None:
+        self.sentence_transformers = sentence_transformers
+
+    def inspect(
+        self,
+        artifact: dict[str, Any],
+        supported_model_types: list[str],
+    ) -> InspectionResult:
+        files = set(artifact["files"])
+        required = {"config.json", "model.safetensors"}
+        if self.sentence_transformers:
+            required |= {"modules.json", "1_Pooling/config.json"}
+        missing = sorted(required - files)
+        blockers = ["MISSING_REQUIRED_FILES:" + ",".join(missing)] if missing else []
+        tokenizer_evidence = {
+            "tokenizer.json",
+            "tokenizer_config.json",
+            "sentencepiece.bpe.model",
+            "spiece.model",
+            "vocab.json",
+            "vocab.txt",
+        }
+        if not files.intersection(tokenizer_evidence):
+            blockers.append("MISSING_TOKENIZER_EVIDENCE")
+        if "safetensors" not in artifact["formats"]:
+            blockers.append("UNSUPPORTED_WEIGHT_FORMAT")
+        model_types = set(artifact["model_types"])
+        if model_types and not model_types.intersection(supported_model_types):
+            blockers.append("MODEL_ARCHITECTURE_UNSUPPORTED")
+        warnings = [] if model_types else ["MODEL_ARCHITECTURE_EVIDENCE_UNKNOWN"]
+        return InspectionResult(
+            blockers=blockers,
+            warnings=warnings,
+            requires_probe=not model_types,
+        )
+
+
+class VllmInspector(TransformersInspector):
+    def __init__(self) -> None:
+        super().__init__(sentence_transformers=False)
+
+    def inspect(
+        self,
+        artifact: dict[str, Any],
+        supported_model_types: list[str],
+    ) -> InspectionResult:
+        result = super().inspect(artifact, supported_model_types)
+        result.warnings.append("VLLM_TASK_SUPPORT_REQUIRES_LOCAL_PROBE")
+        result.requires_probe = True
+        return result
+
+
+class LlamaCppInspector(CompatibilityInspector):
+    def inspect(
+        self,
+        artifact: dict[str, Any],
+        _supported_model_types: list[str],
+    ) -> InspectionResult:
+        if not any(name.lower().endswith(".gguf") for name in artifact["files"]):
+            return InspectionResult(blockers=["NO_GGUF_ARTIFACT_VARIANT"])
+        return InspectionResult(requires_probe=True)
+
+
+class DiffusersInspector(CompatibilityInspector):
+    def inspect(
+        self,
+        artifact: dict[str, Any],
+        _supported_model_types: list[str],
+    ) -> InspectionResult:
+        if "model_index.json" not in artifact["files"]:
+            return InspectionResult(blockers=["NOT_A_DIFFUSION_PIPELINE"])
+        return InspectionResult(requires_probe=True)
+
+
+class CustomInspector(CompatibilityInspector):
+    def inspect(
+        self,
+        _artifact: dict[str, Any],
+        _supported_model_types: list[str],
+    ) -> InspectionResult:
+        return InspectionResult(
+            warnings=["CUSTOM_ADAPTER_SUPPORT_NOT_DECLARED"],
+            requires_probe=True,
+        )
+
+
+INSPECTORS: dict[str, CompatibilityInspector] = {
+    "sentence_transformers": TransformersInspector(sentence_transformers=True),
+    "qwen3_reranker": TransformersInspector(sentence_transformers=False),
+    "transformers_trocr": TransformersInspector(sentence_transformers=False),
+    "transformers_siglip2": TransformersInspector(sentence_transformers=False),
+    "transformers_whisper": TransformersInspector(sentence_transformers=False),
+    "transformers": TransformersInspector(sentence_transformers=False),
+    "vllm": VllmInspector(),
+    "llama_cpp": LlamaCppInspector(),
+    "diffusers": DiffusersInspector(),
+    "custom": CustomInspector(),
+}
+
+
+def inspect_runtime_support(
+    adapter: str,
+    artifact: dict[str, Any],
+    supported_model_types: list[str],
+) -> InspectionResult:
+    return INSPECTORS.get(adapter, CustomInspector()).inspect(artifact, supported_model_types)

backend/src/modelforge_api/services/serving_reconciliation.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import asyncio
+
+import redis
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError, ServingService
+from modelforge_api.services.transient_payloads import RedisPayloadStore
+from modelforge_api.settings import Settings
+
+
+class ServingReconciliationService:
+    def __init__(self, settings: Settings, engine: Engine) -> None:
+        self.settings = settings
+        self.engine = engine
+        self._stop = asyncio.Event()
+
+    async def run(self) -> None:
+        while not self._stop.is_set():
+            try:
+                with Session(self.engine) as session:
+                    ServingService(
+                        session,
+                        self.settings,
+                        ManifestRegistry(self.settings.config_root),
+                        RedisPayloadStore(self.settings.redis_url),
+                        actor_type="system",
+                        actor_id="serving-reconciler",
+                    ).reconcile()
+            except (OSError, RuntimeError, ValueError, ServingError, redis.RedisError):
+                # Readiness and structured state expose persistent failures; the loop stays bounded.
+                pass
+            try:
+                await asyncio.wait_for(
+                    self._stop.wait(), self.settings.serving_reconciliation_interval_seconds
+                )
+            except TimeoutError:
+                continue
+
+    def stop(self) -> None:
+        self._stop.set()

backend/src/modelforge_api/services/startup_validation.py

@@ -0,0 +1,505 @@
+"""Fail-fast configuration and compatibility validation.
+
+A control plane that starts against a schema it does not understand, or in production with a
+development secret, is worse than one that refuses to start: it looks healthy while writing state
+nobody can trust. Everything here is checked before the first request is served, and every failure
+is typed and names the setting at fault.
+
+The rules are deliberately asymmetric. Development is permissive because that is what development
+is for; production is strict, and a production process is never allowed to fall back to a
+development default silently.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+from sqlalchemy import Engine, text
+
+from modelforge_api.domain.release import (
+    MINIMUM_POSTGRES_MAJOR,
+    Compatibility,
+    schema_compatibility,
+    upgrade_required,
+)
+from modelforge_api.settings import Settings
+
+#: Credentials that ship in the development compose file. A production process using one of these
+#: is not configured, it is unconfigured, and saying so at startup costs nothing.
+DEVELOPMENT_CREDENTIALS = frozenset({"modelforge", "postgres", "password", "changeme", "admin"})
+
+MINIMUM_OPERATOR_KEY_LENGTH = 32
+AUDIT_RUNTIME_ROLE = "modelforge_runtime"
+AUDIT_OWNER_ROLE = "modelforge"
+
+AUDIT_RUNTIME_BOUNDARY_SQL = """
+with append_function as (
+    select procedure.oid, procedure.proowner, procedure.prosecdef,
+           procedure.prosrc,
+           coalesce(pg_catalog.array_to_string(procedure.proconfig, ','), '') as proconfig
+    from pg_catalog.pg_proc as procedure
+    where procedure.oid = pg_catalog.to_regprocedure(
+        'modelforge_audit.append_event_v2(uuid,timestamptz,text,text,text,text,text,text,text,jsonb,bigint,bigint,text,text,bigint,bigint,text)'
+    )
+      and procedure.prokind = 'f'
+), guard_function as (
+    select procedure.oid, procedure.proowner, procedure.prosecdef,
+           procedure.prosrc,
+           coalesce(pg_catalog.array_to_string(procedure.proconfig, ','), '') as proconfig
+    from pg_catalog.pg_proc as procedure
+    join pg_catalog.pg_namespace as namespace on namespace.oid = procedure.pronamespace
+    where namespace.nspname = 'modelforge_audit'
+      and procedure.proname = 'enforce_owner_mutation'
+      and pg_catalog.pg_get_function_identity_arguments(procedure.oid) = ''
+      and procedure.prokind = 'f'
+)
+select current_user as current_role,
+       session_user as session_role,
+       role.rolsuper as current_role_superuser,
+       role.rolcreaterole as current_role_createrole,
+       role.rolcreatedb as current_role_createdb,
+       role.rolreplication as current_role_replication,
+       role.rolbypassrls as current_role_bypassrls,
+       role.rolinherit as current_role_inherit,
+       pg_catalog.pg_has_role(current_user, 'modelforge', 'member') as owner_member,
+       (select count(*) from pg_catalog.pg_auth_members as membership
+        join pg_catalog.pg_roles as member on member.oid = membership.member
+        where member.rolname = current_user) as runtime_membership_count,
+       (select count(*) from pg_catalog.pg_class as relation
+        join pg_catalog.pg_namespace as namespace on namespace.oid = relation.relnamespace
+        where namespace.nspname = 'public'
+          and relation.relname in ('audit_events', 'audit_chain_heads')
+          and pg_catalog.pg_get_userbyid(relation.relowner) = 'modelforge') as owner_table_count,
+       (select count(*) from pg_catalog.pg_class as relation
+        join pg_catalog.pg_namespace as namespace on namespace.oid = relation.relnamespace
+        where namespace.nspname = 'public'
+          and relation.relname in ('audit_events', 'audit_chain_heads')
+          and pg_catalog.pg_get_userbyid(relation.relowner) = current_user) as runtime_owned_table_count,
+       pg_catalog.pg_get_userbyid(database.datdba) = current_user as runtime_owns_database,
+       pg_catalog.pg_get_userbyid(public_schema.nspowner) = current_user as runtime_owns_public_schema,
+       pg_catalog.has_table_privilege(current_user, 'public.audit_events', 'SELECT')
+         and pg_catalog.has_table_privilege(current_user, 'public.audit_chain_heads', 'SELECT')
+         as audit_select,
+       pg_catalog.has_table_privilege(
+           current_user, 'public.audit_events',
+           'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER'
+       ) or pg_catalog.has_table_privilege(
+           current_user, 'public.audit_chain_heads',
+           'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER'
+       ) as forbidden_audit_table_privilege,
+       pg_catalog.has_schema_privilege(current_user, 'public', 'CREATE')
+         or pg_catalog.has_schema_privilege(current_user, 'modelforge_audit', 'CREATE')
+         as forbidden_schema_create,
+       pg_catalog.has_database_privilege(current_user, pg_catalog.current_database(), 'CREATE,TEMP')
+         as forbidden_database_privilege,
+       (select count(*) = 1 from append_function) as append_exists,
+       coalesce((select pg_catalog.pg_get_userbyid(proowner) = 'modelforge'
+                 from append_function), false) as append_owner,
+       coalesce((select prosecdef from append_function), false) as append_security_definer,
+       coalesce((select proconfig = 'search_path=pg_catalog' from append_function), false)
+         as append_fixed_search_path,
+       coalesce((select pg_catalog.encode(
+                    pg_catalog.sha256(pg_catalog.convert_to(prosrc, 'UTF8')), 'hex'
+                 ) = 'c9154911f1b1b70f77fe2642de156c98358cd0423f0c3e2a75bc4f1ee4409a86'
+                 from append_function), false)
+         as append_body_exact,
+       coalesce((select pg_catalog.has_function_privilege(current_user, oid, 'EXECUTE')
+                 from append_function), false) as runtime_append_execute,
+       coalesce((select exists (
+           select 1
+           from pg_catalog.aclexplode(
+               coalesce(procedure.proacl, pg_catalog.acldefault('f', procedure.proowner))
+           ) as acl
+           where acl.grantee = 0 and acl.privilege_type = 'EXECUTE'
+       ) from pg_catalog.pg_proc as procedure
+       where procedure.oid = (select oid from append_function)), true) as public_append_execute,
+       (select count(*) = 1 from guard_function) as guard_exists,
+       coalesce((select pg_catalog.pg_get_userbyid(proowner) = 'modelforge'
+                 from guard_function), false) as guard_owner,
+       coalesce((select not prosecdef from guard_function), false) as guard_security_invoker,
+       coalesce((select proconfig = 'search_path=pg_catalog' from guard_function), false)
+         as guard_fixed_search_path,
+       coalesce((select pg_catalog.encode(
+                    pg_catalog.sha256(pg_catalog.convert_to(prosrc, 'UTF8')), 'hex'
+                 ) = '8af5fc31f3b2c729445baaae902eccd9cb3d70b257d0a19e6399f538ee43c6bf'
+                 from guard_function), false)
+         as guard_body_exact,
+       coalesce((select not pg_catalog.has_function_privilege(current_user, oid, 'EXECUTE')
+                 from guard_function), false) as runtime_cannot_execute_guard,
+       (select count(*) from pg_catalog.pg_trigger as trigger
+        join pg_catalog.pg_class as relation on relation.oid = trigger.tgrelid
+        join pg_catalog.pg_namespace as namespace on namespace.oid = relation.relnamespace
+        join pg_catalog.pg_proc as procedure on procedure.oid = trigger.tgfoid
+        join pg_catalog.pg_namespace as function_namespace
+          on function_namespace.oid = procedure.pronamespace
+        where not trigger.tgisinternal and trigger.tgenabled = 'O'
+          and namespace.nspname = 'public'
+          and trigger.tgqual is null and trigger.tgnargs = 0
+          and ((relation.relname = 'audit_events'
+                and trigger.tgname = 'trg_modelforge_audit_events_owner'
+                and trigger.tgtype = 31)
+            or (relation.relname = 'audit_events'
+                and trigger.tgname = 'trg_modelforge_audit_events_truncate_owner'
+                and trigger.tgtype = 34)
+            or (relation.relname = 'audit_chain_heads'
+                and trigger.tgname = 'trg_modelforge_audit_head_owner'
+                and trigger.tgtype = 31)
+            or (relation.relname = 'audit_chain_heads'
+                and trigger.tgname = 'trg_modelforge_audit_head_truncate_owner'
+                and trigger.tgtype = 34))
+          and function_namespace.nspname = 'modelforge_audit'
+          and procedure.proname = 'enforce_owner_mutation') as protected_trigger_count,
+       (select count(*) from pg_catalog.pg_proc as procedure
+        join pg_catalog.pg_namespace as namespace on namespace.oid = procedure.pronamespace
+        where namespace.nspname not in ('pg_catalog', 'information_schema')
+          and procedure.oid <> coalesce((select oid from append_function), 0)
+          and pg_catalog.has_function_privilege(current_user, procedure.oid, 'EXECUTE'))
+         as unexpected_function_execute_count
+from pg_catalog.pg_roles as role
+join pg_catalog.pg_database as database on database.datname = pg_catalog.current_database()
+join pg_catalog.pg_namespace as public_schema on public_schema.nspname = 'public'
+where role.rolname = current_user
+"""
+
+
+class StartupFailureCode(StrEnum):
+    MISSING_REQUIRED_SETTING = "MISSING_REQUIRED_SETTING"
+    INSECURE_PRODUCTION_SETTING = "INSECURE_PRODUCTION_SETTING"
+    INVALID_URL = "INVALID_URL"
+    INVALID_STORAGE_ROOT = "INVALID_STORAGE_ROOT"
+    INCOMPATIBLE_DATABASE = "INCOMPATIBLE_DATABASE"
+    IMPOSSIBLE_POLICY = "IMPOSSIBLE_POLICY"
+
+
+@dataclass(frozen=True, slots=True)
+class StartupProblem:
+    code: StartupFailureCode
+    setting: str
+    message: str
+
+    def __str__(self) -> str:
+        return f"[{self.code}] {self.setting}: {self.message}"
+
+
+class StartupValidationError(RuntimeError):
+    """Raised when configuration cannot support a correct start."""
+
+    def __init__(self, problems: list[StartupProblem]) -> None:
+        self.problems = problems
+        detail = "\n".join(f"  - {problem}" for problem in problems)
+        super().__init__(
+            f"ModelForge refused to start: {len(problems)} configuration "
+            f"problem(s) must be resolved.\n{detail}"
+        )
+
+
+@dataclass
+class ValidationReport:
+    problems: list[StartupProblem] = field(default_factory=list)
+    warnings: list[StartupProblem] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return not self.problems
+
+    def fail(self, code: StartupFailureCode, setting: str, message: str) -> None:
+        self.problems.append(StartupProblem(code, setting, message))
+
+    def warn(self, code: StartupFailureCode, setting: str, message: str) -> None:
+        self.warnings.append(StartupProblem(code, setting, message))
+
+
+def audit_runtime_boundary_violations(facts: Mapping[str, Any]) -> list[str]:
+    """Validate the catalog facts required of the production runtime connection."""
+
+    violations: list[str] = []
+
+    def integer_fact(key: str) -> int:
+        try:
+            return int(facts.get(key, -1))
+        except (TypeError, ValueError):
+            return -1
+
+    if facts.get("current_role") != AUDIT_RUNTIME_ROLE:
+        violations.append("current database role is not modelforge_runtime")
+    if facts.get("session_role") != AUDIT_RUNTIME_ROLE:
+        violations.append("authenticated database session role is not modelforge_runtime")
+    for key in (
+        "current_role_superuser",
+        "current_role_createrole",
+        "current_role_createdb",
+        "current_role_replication",
+        "current_role_bypassrls",
+        "current_role_inherit",
+        "owner_member",
+        "runtime_owns_database",
+        "runtime_owns_public_schema",
+        "forbidden_audit_table_privilege",
+        "forbidden_schema_create",
+        "forbidden_database_privilege",
+        "public_append_execute",
+    ):
+        if facts.get(key) is not False:
+            violations.append(f"catalog policy {key} must be false")
+    for key in (
+        "audit_select",
+        "append_exists",
+        "append_owner",
+        "append_security_definer",
+        "append_fixed_search_path",
+        "append_body_exact",
+        "runtime_append_execute",
+        "guard_exists",
+        "guard_owner",
+        "guard_security_invoker",
+        "guard_fixed_search_path",
+        "guard_body_exact",
+        "runtime_cannot_execute_guard",
+    ):
+        if facts.get(key) is not True:
+            violations.append(f"catalog policy {key} must be true")
+    if integer_fact("owner_table_count") != 2:
+        violations.append("modelforge must own both protected audit tables")
+    if integer_fact("runtime_owned_table_count") != 0:
+        violations.append("the runtime role must own no protected audit table")
+    if integer_fact("runtime_membership_count") != 0:
+        violations.append("the runtime role must have no SET ROLE-capable memberships")
+    if integer_fact("protected_trigger_count") != 4:
+        violations.append("all four protected audit mutation/truncate triggers must be enabled")
+    if integer_fact("unexpected_function_execute_count") != 0:
+        violations.append(
+            "runtime can execute a non-system function other than canonical audit append"
+        )
+    return violations
+
+
+def _validate_url(report: ValidationReport, setting: str, value: str, schemes: set[str]) -> None:
+    parsed = urlparse(value)
+    if parsed.scheme.split("+")[0] not in schemes:
+        report.fail(
+            StartupFailureCode.INVALID_URL,
+            setting,
+            f"expected one of {sorted(schemes)}, got {parsed.scheme or '(none)'!r}",
+        )
+        return
+    if not parsed.hostname:
+        report.fail(StartupFailureCode.INVALID_URL, setting, "no host in the URL")
+
+
+def _validate_writable_root(report: ValidationReport, setting: str, value: Path) -> None:
+    """A storage root must exist and be writable, or the first write is the one that finds out."""
+
+    if not value.exists():
+        report.fail(
+            StartupFailureCode.INVALID_STORAGE_ROOT, setting, f"{value} does not exist"
+        )
+        return
+    if not value.is_dir():
+        report.fail(
+            StartupFailureCode.INVALID_STORAGE_ROOT, setting, f"{value} is not a directory"
+        )
+        return
+    probe = value / ".modelforge-write-probe"
+    try:
+        probe.touch()
+        probe.unlink()
+    except OSError as error:
+        report.fail(
+            StartupFailureCode.INVALID_STORAGE_ROOT, setting, f"{value} is not writable: {error}"
+        )
+
+
+def validate_settings(settings: Settings) -> ValidationReport:
+    """Everything checkable without touching the database."""
+
+    report = ValidationReport()
+    production = settings.env == "production"
+
+    _validate_url(report, "MODELFORGE_DATABASE_URL", settings.database_url, {"postgresql"})
+    _validate_url(report, "MODELFORGE_REDIS_URL", settings.redis_url, {"redis", "rediss"})
+
+    if settings.operator_api_key is None:
+        if production:
+            report.fail(
+                StartupFailureCode.MISSING_REQUIRED_SETTING,
+                "MODELFORGE_OPERATOR_API_KEY",
+                "production requires an operator API key; every admin route depends on it",
+            )
+    else:
+        secret = settings.operator_api_key.get_secret_value()
+        if production and len(secret) < MINIMUM_OPERATOR_KEY_LENGTH:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_OPERATOR_API_KEY",
+                f"must be at least {MINIMUM_OPERATOR_KEY_LENGTH} characters in production",
+            )
+        if production and secret.lower() in DEVELOPMENT_CREDENTIALS:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_OPERATOR_API_KEY",
+                "this is a well-known development value and cannot be used in production",
+            )
+
+    if production:
+        parsed = urlparse(settings.database_url)
+        if unquote(parsed.username or "") != AUDIT_RUNTIME_ROLE:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_DATABASE_URL",
+                "production API connections must use the non-owner modelforge_runtime role",
+            )
+        if not parsed.password:
+            report.fail(
+                StartupFailureCode.MISSING_REQUIRED_SETTING,
+                "MODELFORGE_DATABASE_URL",
+                "the production runtime database URL must contain its externally supplied secret",
+            )
+        if (parsed.password or "").lower() in DEVELOPMENT_CREDENTIALS:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_DATABASE_URL",
+                "the database password is a well-known development value",
+            )
+        if settings.migration_database_url is not None:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_MIGRATION_DATABASE_URL",
+                "the migration/owner credential must not be present in the API process",
+            )
+        if settings.allow_remote_code:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_ALLOW_REMOTE_CODE",
+                "executing remote model code is never permitted in production",
+            )
+        if settings.backup_encryption_key is None:
+            report.fail(
+                StartupFailureCode.MISSING_REQUIRED_SETTING,
+                "MODELFORGE_BACKUP_ENCRYPTION_KEY",
+                "production backups are encrypted; without a key no backup can be produced",
+            )
+        if "*" in settings.cors_origin_list:
+            report.fail(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_CORS_ORIGINS",
+                "a wildcard origin with credentialed requests is not permitted in production",
+            )
+        if settings.restore_allow_production_target:
+            report.warn(
+                StartupFailureCode.INSECURE_PRODUCTION_SETTING,
+                "MODELFORGE_RESTORE_ALLOW_PRODUCTION_TARGET",
+                "restores may overwrite the live database; keep this off outside a rehearsal",
+            )
+
+    for setting, value in (
+        ("MODELFORGE_ARTIFACT_ROOT", Path(settings.artifact_root)),
+        ("MODELFORGE_QUARANTINE_ROOT", Path(settings.quarantine_root)),
+        ("MODELFORGE_BACKUP_ROOT", settings.backup_root),
+    ):
+        _validate_writable_root(report, setting, value)
+
+    # A reserve that consumes the whole device leaves nothing schedulable, which presents as
+    # "no eligible node" forever rather than as the misconfiguration it is.
+    if settings.scheduler_safety_reserve_percentage >= 0.5:
+        report.fail(
+            StartupFailureCode.IMPOSSIBLE_POLICY,
+            "MODELFORGE_SCHEDULER_SAFETY_RESERVE_PERCENTAGE",
+            "reserving half of every accelerator leaves no schedulable capacity",
+        )
+    if settings.node_offline_after_seconds <= settings.node_stale_after_seconds:
+        report.fail(
+            StartupFailureCode.IMPOSSIBLE_POLICY,
+            "MODELFORGE_NODE_OFFLINE_AFTER_SECONDS",
+            "a node cannot become offline before it becomes stale",
+        )
+    if settings.gateway_queue_timeout_seconds >= settings.gateway_request_timeout_seconds:
+        report.fail(
+            StartupFailureCode.IMPOSSIBLE_POLICY,
+            "MODELFORGE_GATEWAY_QUEUE_TIMEOUT_SECONDS",
+            "a request that times out before its queue wait can never be served",
+        )
+    return report
+
+
+def validate_database(
+    engine: Engine,
+    report: ValidationReport,
+    *,
+    require_audit_boundary: bool = False,
+) -> ValidationReport:
+    """Schema revision and server version, read from the database rather than assumed."""
+
+    with engine.connect() as connection:
+        server_version = int(connection.exec_driver_sql("show server_version_num").scalar_one())
+        revision = connection.execute(
+            text("select version_num from alembic_version")
+        ).scalar_one_or_none()
+        boundary_facts = (
+            dict(connection.execute(text(AUDIT_RUNTIME_BOUNDARY_SQL)).mappings().one())
+            if require_audit_boundary
+            else None
+        )
+
+    major = server_version // 10000
+    if major < MINIMUM_POSTGRES_MAJOR:
+        report.fail(
+            StartupFailureCode.INCOMPATIBLE_DATABASE,
+            "PostgreSQL",
+            f"server major version {major} is below the supported minimum "
+            f"{MINIMUM_POSTGRES_MAJOR}",
+        )
+
+    compatibility = schema_compatibility(revision)
+    if compatibility is not Compatibility.COMPATIBLE:
+        report.fail(
+            StartupFailureCode.INCOMPATIBLE_DATABASE,
+            "alembic_version",
+            f"schema revision {revision or '(none)'} is {compatibility}; "
+            f"{upgrade_required(compatibility)}",
+        )
+    if boundary_facts is not None:
+        boundary_violations = audit_runtime_boundary_violations(boundary_facts)
+        if boundary_violations:
+            report.fail(
+                StartupFailureCode.INCOMPATIBLE_DATABASE,
+                "PostgreSQL audit runtime boundary",
+                "; ".join(boundary_violations),
+            )
+    return report
+
+
+def validate_startup(settings: Settings, engine: Engine | None = None) -> ValidationReport:
+    report = validate_settings(settings)
+    if engine is not None:
+        try:
+            validate_database(
+                engine,
+                report,
+                require_audit_boundary=settings.env == "production",
+            )
+        except Exception as error:  # noqa: BLE001 - any failure to read the schema is fatal
+            report.fail(
+                StartupFailureCode.INCOMPATIBLE_DATABASE,
+                "alembic_version",
+                f"the schema revision could not be read: {type(error).__name__}",
+            )
+    return report
+
+
+def enforce_startup(settings: Settings, engine: Engine | None = None) -> ValidationReport:
+    """Validate and refuse to continue when production configuration cannot support a correct run.
+
+    Outside production the same problems are reported but do not stop the process: a developer with
+    no backup key should still be able to run the API, and finding out at startup rather than at
+    first use is the point.
+    """
+
+    report = validate_startup(settings, engine)
+    if report.problems and settings.env == "production":
+        raise StartupValidationError(report.problems)
+    return report

backend/src/modelforge_api/services/transient_payloads.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import json
+import secrets
+from typing import Any, Protocol, cast
+
+import redis
+
+
+class PayloadStore(Protocol):
+    def put(self, payload: dict[str, Any], ttl_seconds: int) -> str: ...
+    def get(self, reference: str) -> dict[str, Any] | None: ...
+    def delete(self, reference: str) -> None: ...
+
+
+class RedisPayloadStore:
+    """Short-lived request/result exchange; payloads never enter PostgreSQL or audit logs."""
+
+    def __init__(self, redis_url: str) -> None:
+        self.client = redis.Redis.from_url(redis_url, decode_responses=True)
+
+    @staticmethod
+    def _key(reference: str) -> str:
+        return f"modelforge:serving-payload:{reference}"
+
+    def put(self, payload: dict[str, Any], ttl_seconds: int) -> str:
+        reference = secrets.token_urlsafe(32)
+        self.client.setex(
+            self._key(reference),
+            ttl_seconds,
+            json.dumps(payload, separators=(",", ":"), allow_nan=False),
+        )
+        return reference
+
+    def get(self, reference: str) -> dict[str, Any] | None:
+        value = self.client.get(self._key(reference))
+        if value is None:
+            return None
+        decoded = json.loads(cast(str, value))
+        return dict(decoded) if isinstance(decoded, dict) else None
+
+    def delete(self, reference: str) -> None:
+        self.client.delete(self._key(reference))
+
+
+class MemoryPayloadStore:
+    def __init__(self) -> None:
+        self.values: dict[str, dict[str, Any]] = {}
+
+    def put(self, payload: dict[str, Any], ttl_seconds: int) -> str:
+        del ttl_seconds
+        reference = secrets.token_urlsafe(24)
+        self.values[reference] = payload
+        return reference
+
+    def get(self, reference: str) -> dict[str, Any] | None:
+        return self.values.get(reference)
+
+    def delete(self, reference: str) -> None:
+        self.values.pop(reference, None)

backend/src/modelforge_api/settings.py

@@ -0,0 +1,116 @@
+from functools import lru_cache
+from pathlib import Path
+from typing import Literal
+
+from pydantic import Field, SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+def default_config_root() -> Path:
+    current = Path.cwd()
+    local = current / "config"
+    return local if local.is_dir() else current.parent / "config"
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="MODELFORGE_",
+        env_file=".env",
+        extra="ignore",
+    )
+
+    env: Literal["development", "test", "production"] = "development"
+    api_host: str = "0.0.0.0"
+    api_port: int = 8000
+    database_url: str = "postgresql+psycopg://modelforge_runtime@localhost:5432/modelforge"
+    migration_database_url: SecretStr | None = None
+    redis_url: str = "redis://localhost:6379/0"
+    cors_origins: str = "http://localhost:3000"
+    log_level: str = "INFO"
+    hf_home: str = "/data/hf-cache"
+    hf_token: SecretStr | None = None
+    hf_timeout_seconds: float = Field(default=30, gt=0, le=300)
+    hf_snapshot_ttl_seconds: int = Field(default=3600, ge=60, le=86400)
+    artifact_root: str = "/data/artifacts"
+    quarantine_root: str = "/data/quarantine"
+    runtime_artifact_root: str = "/models/model-registry"
+    allow_remote_code: bool = False
+    enable_gpu_telemetry: bool = True
+    hardware_refresh_on_startup: bool = False
+    hardware_poll_interval_seconds: int = Field(default=30, ge=5)
+    node_identity: str | None = None
+    node_identity_mode: Literal["auto", "persisted"] = "auto"
+    node_identity_file: Path = Path("/data/state/node-id")
+    operator_api_key: SecretStr | None = None
+    node_stale_after_seconds: int = Field(default=30, ge=10)
+    node_offline_after_seconds: int = Field(default=90, ge=20)
+    liveness_poll_interval_seconds: int = Field(default=5, ge=1)
+    agent_max_clock_skew_seconds: int = Field(default=300, ge=0, le=86400)
+    node_agent_max_payload_bytes: int = Field(
+        default=4_194_304,
+        ge=65_536,
+        le=67_108_864,
+    )
+    node_liveness_monitor_enabled: bool = False
+    agent_protocol_version: int = 1
+    service_name: str = Field(default="modelforge-api")
+    config_root: Path = Field(default_factory=default_config_root)
+    registry_seed_on_startup: bool = False
+    gateway_max_batch_size: int = Field(default=8, ge=1, le=128)
+    gateway_max_input_characters: int = Field(default=8192, ge=1, le=1000000)
+    gateway_max_payload_bytes: int = Field(default=65536, ge=1024, le=10000000)
+    control_plane_max_payload_bytes: int = Field(
+        default=1_048_576,
+        ge=65_536,
+        le=16_777_216,
+    )
+    gateway_request_timeout_seconds: int = Field(default=45, ge=5, le=900)
+    gateway_queue_timeout_seconds: int = Field(default=30, ge=1, le=900)
+    serving_job_lease_seconds: int = Field(default=120, ge=30, le=900)
+    serving_payload_ttl_seconds: int = Field(default=120, ge=30, le=3600)
+    scheduler_safety_reserve_bytes: int = Field(default=1073741824, ge=0)
+    scheduler_safety_reserve_percentage: float = Field(default=0.05, ge=0.0, le=0.5)
+    scheduler_runtime_margin_bytes: int = Field(default=268435456, ge=0)
+    scheduler_deployment_margin_bytes: int = Field(default=134217728, ge=0)
+    scheduler_deployment_margin_percentage: float = Field(default=0.10, ge=0.0, le=1.0)
+    scheduler_global_queue_limit: int = Field(default=128, ge=1, le=10000)
+    scheduler_telemetry_stale_seconds: int = Field(default=90, ge=10, le=3600)
+    scheduler_pressure_stable_seconds: int = Field(default=30, ge=1, le=600)
+    scheduler_eviction_cooldown_seconds: int = Field(default=60, ge=1, le=3600)
+    scheduler_placement_history_limit: int = Field(default=500, ge=10, le=10000)
+    serving_reconciliation_enabled: bool = False
+    serving_reconciliation_interval_seconds: int = Field(default=5, ge=1, le=300)
+    lifecycle_reconciliation_enabled: bool = False
+    migration_reconciliation_enabled: bool = False
+    observability_monitor_enabled: bool = False
+    observability_poll_interval_seconds: int = Field(default=60, ge=30, le=3600)
+    backup_root: Path = Path("/data/backups")
+    backup_restore_root: Path = Path("/data/restore")
+    backup_encryption_key: SecretStr | None = None
+    backup_encryption_key_id: str = Field(default="modelforge-backup-key-1", max_length=128)
+    alembic_directory: Path | None = None
+    backup_pg_dump_path: str = "pg_dump"
+    backup_pg_restore_path: str = "pg_restore"
+    backup_psql_path: str = "psql"
+    backup_command_timeout_seconds: int = Field(default=1800, ge=30, le=86_400)
+    backup_stale_after_seconds: int = Field(default=93_600, ge=3600, le=2_592_000)
+    backup_minimum_free_bytes: int = Field(default=1_073_741_824, ge=0)
+    backup_capacity_headroom_ratio: float = Field(default=3.0, ge=1.0, le=100.0)
+    recovery_reconciliation_enabled: bool = False
+    restore_allow_production_target: bool = False
+
+    # Build identity. These are stamped into the image at build time and are never inferred at
+    # runtime: a build identity a running process can talk itself into is worth nothing during an
+    # incident. Absent values are reported as null rather than guessed.
+    build_commit: str | None = None
+    build_timestamp: str | None = None
+    build_image_digest: str | None = None
+
+    @property
+    def cors_origin_list(self) -> list[str]:
+        return [item.strip() for item in self.cors_origins.split(",") if item.strip()]
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()