from __future__ import annotations

import hashlib
import hmac
import itertools
import json
import math
import posixpath
import secrets
import time
import uuid
from dataclasses import asdict, dataclass, replace
from datetime import UTC, datetime, timedelta
from pathlib import PurePosixPath
from typing import Any, cast

from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session

from modelforge_api.domain.serving import (
    AgentServingJobComplete,
    AgentServingJobFailure,
    AgentServingJobLease,
    AgentServingStateAck,
    AgentServingStateReport,
    CapabilityDeploymentResponse,
    CapabilityExperimentCreate,
    CapabilityExperimentResponse,
    CapabilityPromotionCreate,
    CoResidencyEvidenceCreate,
    CoResidencyEvidenceResponse,
    EmbeddingInvokeRequest,
    EmbeddingInvokeResponse,
    EmbeddingSpaceResponse,
    GatewayExecution,
    GatewayRequestResponse,
    GatewayTiming,
    GatewayUsage,
    PlacementEvictionResponse,
    PlacementPlanRequest,
    PlacementPlanResponse,
    ProductionApprovalCreate,
    ProductionApprovalResponse,
    ProjectFitEvidenceCreate,
    ProjectFitEvidenceResponse,
    ProjectIntegrationResponse,
    ProjectIntegrationUsage,
    RerankDocument,
    RerankingInvokeRequest,
    RerankingInvokeResponse,
    RerankingResult,
    ResidencyPolicyUpdate,
    ResidencyResponse,
    ResourceEnvelopeResponse,
    SchedulerBudgetResponse,
    SchedulerMetricsResponse,
    SchedulerPolicyResponse,
    SchedulerPolicyUpdate,
    ServiceClientCreate,
    ServiceClientCreated,
    ServiceClientResponse,
    SupplyChainReview,
)
from modelforge_api.persistence.models import (
    Capability,
    CapabilityContract,
    CapabilityDeployment,
    CapabilityExperimentRoute,
    CapabilityResourceEnvelope,
    ComputeNode,
    CoResidencyEvidence,
    EmbeddingSpace,
    GatewayRequest,
    PlacementPlanRecord,
    ProductionExecutionApproval,
    Project,
    ProjectBinding,
    ProjectFitEvidence,
    ResidencyAllocation,
    SchedulerAcceleratorState,
    SchedulerEviction,
    SchedulerPolicyRevision,
    ServiceClient,
    ServiceCredential,
    ServingGpuLease,
    ServingJob,
)
from modelforge_api.persistence.serving_repository import ServingRepository
from modelforge_api.services.audit import AuditWriter
from modelforge_api.services.gpu_scheduler import (
    AccountingSnapshot,
    AttributionConfidence,
    LeaseAllocation,
    PlacementInput,
    PlacementVerdict,
    PressureState,
    ResidentCandidate,
    SchedulerPolicy,
    calculate_accounting,
    plan_placement,
    required_envelope,
)
from modelforge_api.services.manifest_registry import ManifestRegistry
from modelforge_api.services.runtime import RuntimeService, _selected_manifest_rows
from modelforge_api.settings import Settings

REQUIRED_SERVING_CAPABILITIES = {
    "deployment.load.v1",
    "deployment.invoke.v1",
    "deployment.health.v1",
    "deployment.drain.v1",
    "deployment.unload.v1",
}


def _now() -> datetime:
    return datetime.now(UTC)


def _aware(value: datetime) -> datetime:
    return value if value.tzinfo else value.replace(tzinfo=UTC)


def _canonical_hash(payload: Any) -> str:
    raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
    return hashlib.sha256(raw.encode()).hexdigest()


def _percentile(values: list[float], fraction: float) -> float | None:
    if not values:
        return None
    index = max(0, min(len(values) - 1, math.ceil(len(values) * fraction) - 1))
    return float(values[index])


def _shape_bucket(encoded_bytes: int) -> str:
    if encoded_bytes <= 64 * 1024:
        return "SMALL"
    if encoded_bytes <= 1024 * 1024:
        return "MEDIUM"
    return "LARGE"


def _safe_parent(paths: list[str]) -> str:
    normalized = [path.replace("\\", "/") for path in paths]
    parent = posixpath.commonpath(normalized)
    if parent in normalized:
        parent = PurePosixPath(parent).parent.as_posix()
    if parent in {"", "."} or parent.startswith("/") or ".." in PurePosixPath(parent).parts:
        raise ServingError(409, "ARTIFACT_PATH_INVALID", "artifact location is not confined")
    return parent


class ServingError(RuntimeError):
    def __init__(
        self,
        status_code: int,
        code: str,
        message: str,
        details: dict[str, Any] | None = None,
    ) -> None:
        super().__init__(message)
        self.status_code = status_code
        self.code = code
        self.details = details or {}


@dataclass(frozen=True, slots=True)
class CapabilityAuthenticationEvidence:
    """Secret-free proof of one exact pre-body capability-client authentication."""

    credential_id: uuid.UUID
    client_id: uuid.UUID
    capability: str
    credential_sha256: str


class ServingService:
    def __init__(
        self,
        session: Session,
        settings: Settings,
        manifests: ManifestRegistry,
        payload_store: Any,
        *,
        actor_type: str = "operator",
        actor_id: str = "local-operator",
    ) -> None:
        self.session = session
        self.settings = settings
        self.manifests = manifests
        self.payload_store = payload_store
        self.repo = ServingRepository(session)
        self.audit = AuditWriter(session, actor_type, actor_id)

    def _commit(self) -> None:
        try:
            self.session.commit()
        except IntegrityError as exc:
            self.session.rollback()
            raise ServingError(
                409, "STATE_CONFLICT", "serving state conflicts with existing data"
            ) from exc

    def _commit_authentication(self) -> None:
        """Persist credential usage without misclassifying auth-store failures as conflicts."""

        try:
            self.session.commit()
        except IntegrityError as exc:
            self.session.rollback()
            raise ServingError(
                503,
                "AUTH_BACKEND_UNAVAILABLE",
                "capability authentication backend is unavailable",
            ) from exc

    def ensure_contract(self, key: str = "rag.embedding", version: int = 1) -> CapabilityContract:
        existing = self.repo.contract(key, version)
        if existing:
            return existing
        manifest = next(
            (
                item
                for item in self.manifests.capabilities()
                if item.capability == key and item.version == version
            ),
            None,
        )
        if not manifest:
            raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability contract is not registered")
        capability = self.repo.capability(key)
        if not capability:
            capability = self.repo.add(Capability(key=key, description=manifest.description))
        contract = self.repo.add(
            CapabilityContract(
                capability_id=capability.id,
                version=version,
                input_schema=manifest.input_schema,
                output_schema=manifest.output_schema,
                contract=manifest.model_dump(mode="json"),
                upgrade_class=manifest.upgrade_class.value,
                immutable_at=_now(),
            )
        )
        self._commit()
        return contract

    def _approval_facts(
        self,
        candidate_id: uuid.UUID,
        contract: CapabilityContract,
        request: ProductionApprovalCreate,
    ) -> tuple[dict[str, Any], Any, Any, Any, ComputeNode]:
        candidate = self.repo.candidate(candidate_id)
        if not candidate or candidate.status != "lab_ready" or candidate.production:
            raise ServingError(
                409, "CANDIDATE_NOT_LAB_READY", "a LAB_READY M4 candidate is required"
            )
        probe = self.repo.probe(candidate.runtime_probe_id)
        profile = self.repo.profile(candidate.runtime_profile_id)
        artifact_set = self.repo.artifact_set(candidate.artifact_set_id)
        node = self.repo.node(candidate.compute_node_id)
        if not probe or probe.status != "completed":
            raise ServingError(
                409, "RUNTIME_PROBE_REQUIRED", "successful M4 runtime probe is required"
            )
        runtime_assessment = RuntimeService(self.session, self.settings).assessment(
            candidate.compatibility_assessment_id
        )
        if runtime_assessment.stale or runtime_assessment.status != "compatible":
            raise ServingError(
                409, "COMPATIBILITY_STALE", "current compatible evidence is required"
            )
        if not profile or not artifact_set or artifact_set.status != "verified":
            raise ServingError(
                409, "ARTIFACT_NOT_VERIFIED", "verified exact ArtifactSet is required"
            )
        if (
            not node
            or node.decommissioned_at is not None
            or not node.enabled
            or not node.production_eligible
            or node.liveness_state != "online"
        ):
            raise ServingError(
                409, "NO_ELIGIBLE_NODE", "candidate node is not production eligible and online"
            )
        missing_capabilities = REQUIRED_SERVING_CAPABILITIES - set(node.agent_capabilities)
        if missing_capabilities:
            raise ServingError(
                409,
                "NODE_SERVING_CAPABILITY_MISSING",
                "target node does not advertise the typed serving protocol",
                {"missing": sorted(missing_capabilities)},
            )
        artifacts = self.repo.set_artifacts(artifact_set.id)
        risk_flags = sorted(
            {
                flag
                for artifact in artifacts
                for flag in artifact.verification_details.get("risk_flags", [])
            }
        )
        if any(artifact.status != "verified" or artifact.quarantined for artifact in artifacts):
            raise ServingError(
                409, "ARTIFACT_NOT_VERIFIED", "all ArtifactSet members must be verified"
            )
        if any(
            artifact.serialization_format in {"pickle", "pytorch_pickle"} for artifact in artifacts
        ):
            raise ServingError(
                409, "PICKLE_PRESENT", "pickle artifacts cannot be production approved"
            )
        if any("remote_code" in flag.lower() for flag in risk_flags):
            raise ServingError(409, "REMOTE_CODE_REQUIRED", "remote-code artifacts are blocked")
        health = probe.health_result
        inference = probe.inference_result
        unload = probe.unload_result
        if not all(health.get(layer) == "healthy" for layer in ("process", "runtime", "model")):
            raise ServingError(
                409, "HEALTHCHECK_FAILED", "M4 layered health evidence is incomplete"
            )
        if inference.get("dimension") != 1024 or inference.get("finite") is not True:
            raise ServingError(409, "INVALID_OUTPUT", "M4 embedding output does not match contract")
        if (
            unload.get("reclaimed") is not True
            or probe.runtime_facts.get("offline_local_only") is not True
        ):
            raise ServingError(
                409, "RUNTIME_SECURITY_GATE_FAILED", "offline unload evidence is required"
            )
        facts = {
            "candidate_id": str(candidate.id),
            "artifact_set_id": str(candidate.artifact_set_id),
            "runtime_profile_id": str(candidate.runtime_profile_id),
            "compute_node_id": str(candidate.compute_node_id),
            "runtime_probe_id": str(probe.id),
            "probe_environment_fingerprint": probe.environment_fingerprint,
            "compatibility_assessment_id": str(candidate.compatibility_assessment_id),
            "contract_id": str(contract.id),
            "contract_version": contract.version,
            "artifact_status": artifact_set.status,
            "artifact_files": [
                {
                    "filename": item.filename,
                    "sha256": item.sha256,
                    "format": item.serialization_format,
                }
                for item in artifacts
            ],
            "risk_flags": risk_flags,
            "runtime_profile_fingerprint": profile.fingerprint,
            "node_hardware_fingerprint": node.hardware_fingerprint,
            "deployment_config": request.deployment_config,
            "supply_chain_review": request.supply_chain_review.model_dump(mode="json"),
        }
        return facts, candidate, probe, profile, node

    def approve_production(
        self, candidate_id: uuid.UUID, request: ProductionApprovalCreate
    ) -> ProductionApprovalResponse:
        contract = self.ensure_contract(request.capability, request.contract_version)
        facts, candidate, _probe, _profile, _node = self._approval_facts(
            candidate_id, contract, request
        )
        fingerprint = _canonical_hash(facts)
        existing = self.repo.approval_by_fingerprint(fingerprint)
        if existing:
            return self.approval_response(existing)
        approval = self.repo.add(
            ProductionExecutionApproval(
                deployment_candidate_id=candidate.id,
                capability_contract_id=contract.id,
                artifact_set_id=candidate.artifact_set_id,
                runtime_profile_id=candidate.runtime_profile_id,
                compute_node_id=candidate.compute_node_id,
                deployment_config=request.deployment_config,
                supply_chain_evidence=request.supply_chain_review.model_dump(mode="json"),
                evidence_fingerprint=fingerprint,
                status="approved",
                approved_by=request.approved_by,
                reason=request.reason,
                approved_at=_now(),
                immutable_at=_now(),
            )
        )
        self.audit.write(
            "PRODUCTION_EXECUTION_APPROVED",
            "production_execution_approval",
            str(approval.id),
            {
                "artifact_set_id": str(candidate.artifact_set_id),
                "runtime_profile_id": str(candidate.runtime_profile_id),
                "capability": "rag.embedding@1",
            },
        )
        self._commit()
        return self.approval_response(approval)

    def approval_response(self, item: ProductionExecutionApproval) -> ProductionApprovalResponse:
        stale = item.revoked_at is not None
        return ProductionApprovalResponse(
            id=item.id,
            deployment_candidate_id=item.deployment_candidate_id,
            capability_contract_id=item.capability_contract_id,
            artifact_set_id=item.artifact_set_id,
            runtime_profile_id=item.runtime_profile_id,
            compute_node_id=item.compute_node_id,
            deployment_config=item.deployment_config,
            supply_chain_evidence=item.supply_chain_evidence,
            evidence_fingerprint=item.evidence_fingerprint,
            status="stale" if stale else item.status,
            approved_by=item.approved_by,
            reason=item.reason,
            approved_at=item.approved_at,
            revoked_at=item.revoked_at,
            stale=stale,
        )

    def approvals(self) -> list[ProductionApprovalResponse]:
        return [self.approval_response(item) for item in self.repo.approvals()]

    def promote(
        self, candidate_id: uuid.UUID, request: CapabilityPromotionCreate
    ) -> CapabilityDeploymentResponse:
        approval = self.repo.approval(request.production_approval_id)
        candidate = self.repo.candidate(candidate_id)
        if not approval or not candidate or approval.deployment_candidate_id != candidate.id:
            raise ServingError(
                404, "PRODUCTION_APPROVAL_NOT_FOUND", "exact production approval is required"
            )
        if approval.revoked_at or approval.status != "approved":
            raise ServingError(
                409, "PRODUCTION_APPROVAL_STALE", "production approval is not current"
            )
        contract = self.session.get(CapabilityContract, approval.capability_contract_id)
        if not contract:
            raise ServingError(
                409, "CAPABILITY_NOT_FOUND", "approved capability contract disappeared"
            )
        verification_request = ProductionApprovalCreate(
            capability="rag.embedding",
            contract_version=1,
            approved_by=approval.approved_by,
            reason=approval.reason,
            supply_chain_review=SupplyChainReview.model_validate(approval.supply_chain_evidence),
            deployment_config=approval.deployment_config,
        )
        current_facts, _, probe, profile, node = self._approval_facts(
            candidate_id, contract, verification_request
        )
        if _canonical_hash(current_facts) != approval.evidence_fingerprint:
            raise ServingError(
                409, "PRODUCTION_APPROVAL_STALE", "reviewed production evidence changed"
            )
        accelerators = self.repo.node_accelerators(node.id)
        probe_gpu_uuid = str(probe.runtime_facts.get("gpu_uuid", ""))
        accelerator = next(
            (item for item in accelerators if item.device_uuid == probe_gpu_uuid), None
        )
        if not accelerator:
            raise ServingError(409, "NO_ELIGIBLE_NODE", "probed accelerator is no longer available")
        embedding_facts = {
            "artifact_set_id": str(candidate.artifact_set_id),
            "runtime_profile_fingerprint": profile.fingerprint,
            "capability_contract_id": str(contract.id),
            "dimension": 1024,
            "normalized": True,
            "migration_class": "requires_reindex",
        }
        embedding_digest = _canonical_hash(embedding_facts)
        space = self.repo.embedding_space_by_digest(embedding_digest)
        if not space:
            space = self.repo.add(
                EmbeddingSpace(
                    capability_contract_id=contract.id,
                    artifact_set_id=candidate.artifact_set_id,
                    runtime_profile_id=candidate.runtime_profile_id,
                    identity_digest=embedding_digest,
                    dimension=1024,
                    normalized=True,
                    migration_class="requires_reindex",
                    identity_facts=embedding_facts,
                    created_at=_now(),
                    immutable_at=_now(),
                )
            )
        config = {
            "candidate_id": str(candidate.id),
            "production_approval_id": str(approval.id),
            "capability_contract_id": str(contract.id),
            "embedding_space_id": str(space.id),
            "artifact_set_id": str(candidate.artifact_set_id),
            "runtime_profile_id": str(candidate.runtime_profile_id),
            "compute_node_id": str(candidate.compute_node_id),
            "accelerator_id": str(accelerator.id),
            "residency_policy": request.residency_policy,
            "keep_warm_seconds": request.keep_warm_seconds,
            "max_concurrency": request.max_concurrency,
            "max_queue_depth": request.max_queue_depth,
            "routing_weight": request.routing_weight,
            "rollback_policy": request.rollback_policy,
        }
        fingerprint = _canonical_hash(config)
        deployment = self.repo.deployment_by_fingerprint(fingerprint)
        if deployment:
            return self.deployment_response(deployment)
        if self.repo.stable_deployment(contract.id):
            raise ServingError(
                409, "STABLE_DEPLOYMENT_EXISTS", "drain the current stable deployment first"
            )
        deployment = self.repo.add(
            CapabilityDeployment(
                capability_contract_id=contract.id,
                deployment_candidate_id=candidate.id,
                production_approval_id=approval.id,
                embedding_space_id=space.id,
                artifact_set_id=candidate.artifact_set_id,
                runtime_profile_id=candidate.runtime_profile_id,
                compute_node_id=candidate.compute_node_id,
                accelerator_id=accelerator.id,
                channel="stable",
                status="stable",
                production=True,
                health_status="ready_on_demand",
                routing_weight=request.routing_weight,
                fallback_policy={"allowed": False, "mode": "hard_fail", "deployments": []},
                residency_policy=request.residency_policy,
                keep_warm_seconds=request.keep_warm_seconds,
                max_concurrency=request.max_concurrency,
                max_queue_depth=request.max_queue_depth,
                config_fingerprint=fingerprint,
                provenance=current_facts,
                rollback_policy=request.rollback_policy,
                promoted_at=_now(),
            )
        )
        measured = probe.measured_resources
        samples = measured.get("samples", {})
        baseline = int(samples.get("before_load", {}).get("process_used_vram_bytes", 0))
        loaded = int(samples.get("loaded", {}).get("process_used_vram_bytes", 0))
        peak = int(samples.get("probe_peak", {}).get("process_used_vram_bytes", 0))
        if peak <= baseline or loaded <= baseline:
            baseline = int(measured["baseline_vram_bytes"])
            loaded = int(measured["loaded_vram_bytes"])
            peak = int(measured["peak_vram_bytes"])
        required = max(loaded, peak) - baseline
        self.repo.add(
            CapabilityResourceEnvelope(
                capability_deployment_id=deployment.id,
                runtime_probe_id=probe.id,
                accelerator_kind=accelerator.name,
                accelerator_uuid=accelerator.device_uuid,
                environment_fingerprint=probe.environment_fingerprint,
                concurrency=1,
                batch_size=int(measured.get("batch_size", 1)),
                max_sequence_length=int(measured.get("max_sequence_length", 128)),
                baseline_vram_bytes=baseline,
                resident_vram_bytes=max(0, loaded - baseline),
                peak_vram_bytes=peak,
                required_vram_bytes=required,
                cold_load_time_ms=float(probe.load_result["load_time_ms"]),
                inference_latency_ms=float(probe.inference_result["latency_ms"]),
                evidence={"kind": "measured", "runtime_probe_id": str(probe.id), "raw": measured},
                stale=False,
                immutable_at=_now(),
            )
        )
        self.repo.add(
            ResidencyAllocation(
                capability_deployment_id=deployment.id,
                compute_node_id=node.id,
                accelerator_id=accelerator.id,
                state="cold",
                load_count=0,
                active_requests=0,
                measured_resident_vram_bytes=0,
                external_baseline_vram_bytes=0,
                health={"process": "unavailable", "runtime": "unavailable", "model": "unavailable"},
            )
        )
        self.audit.write(
            "CAPABILITY_DEPLOYMENT_CREATED",
            "capability_deployment",
            str(deployment.id),
            {"capability": "rag.embedding@1", "production": True},
        )
        self.audit.write(
            "CAPABILITY_PROMOTED_STABLE",
            "capability_deployment",
            str(deployment.id),
            {
                "candidate_id": str(candidate.id),
                "approval_id": str(approval.id),
                "embedding_space_id": str(space.id),
            },
        )
        self._commit()
        return self.deployment_response(deployment)

    def embedding_response(self, item: EmbeddingSpace) -> EmbeddingSpaceResponse:
        return EmbeddingSpaceResponse.model_validate(item)

    def envelope_response(self, item: CapabilityResourceEnvelope) -> ResourceEnvelopeResponse:
        return ResourceEnvelopeResponse.model_validate(item)

    def residency_response(self, item: ResidencyAllocation) -> ResidencyResponse:
        return ResidencyResponse.model_validate(item)

    def deployment_response(self, item: CapabilityDeployment) -> CapabilityDeploymentResponse:
        contract = self.session.get(CapabilityContract, item.capability_contract_id)
        capability = self.session.get(Capability, contract.capability_id) if contract else None
        space = self.repo.embedding_space(item.embedding_space_id)
        envelope = self.repo.envelope(item.id)
        residency = self.repo.residency(item.id)
        if not contract or not capability or not envelope or not residency:
            raise ServingError(
                409, "DEPLOYMENT_PROVENANCE_INCOMPLETE", "deployment evidence is incomplete"
            )
        if capability.key == "rag.embedding" and not space:
            raise ServingError(
                409, "DEPLOYMENT_PROVENANCE_INCOMPLETE", "embedding identity is missing"
            )
        return CapabilityDeploymentResponse(
            id=item.id,
            capability=capability.key,
            contract_version=contract.version,
            deployment_candidate_id=item.deployment_candidate_id,
            production_approval_id=item.production_approval_id,
            execution_approval_id=item.execution_approval_id,
            artifact_set_id=item.artifact_set_id,
            runtime_profile_id=item.runtime_profile_id,
            compute_node_id=item.compute_node_id,
            accelerator_id=item.accelerator_id,
            embedding_space=self.embedding_response(space) if space else None,
            resource_envelope=self.envelope_response(envelope),
            residency=self.residency_response(residency),
            channel=item.channel,
            status=item.status,
            production=item.production,
            health_status=self._health_status(item, residency),
            routing_weight=item.routing_weight,
            fallback_policy=item.fallback_policy,
            residency_policy=item.residency_policy,
            keep_warm_seconds=item.keep_warm_seconds,
            max_concurrency=item.max_concurrency,
            max_queue_depth=item.max_queue_depth,
            config_fingerprint=item.config_fingerprint,
            provenance=item.provenance,
            rollback_policy=item.rollback_policy,
            promoted_at=item.promoted_at,
            created_at=item.created_at,
        )

    def deployments(self) -> list[CapabilityDeploymentResponse]:
        return [self.deployment_response(item) for item in self.repo.deployments()]

    def create_experiment(
        self,
        candidate_id: uuid.UUID,
        request: CapabilityExperimentCreate,
    ) -> CapabilityExperimentResponse:
        existing_route = self.repo.experiment_route(request.route_key)
        if existing_route:
            return self.experiment_response(existing_route)
        candidate = self.repo.candidate(candidate_id)
        approval = self.repo.execution_approval(request.execution_approval_id)
        if not candidate or candidate.status != "lab_ready" or candidate.production:
            raise ServingError(409, "CANDIDATE_NOT_LAB_READY", "LAB_READY candidate is required")
        if (
            not approval
            or approval.id != request.execution_approval_id
            or approval.artifact_set_id != candidate.artifact_set_id
            or approval.scope != "lab_execution"
            or approval.status != "approved"
            or approval.revoked_at is not None
            or (approval.expires_at and _aware(approval.expires_at) <= _now())
        ):
            raise ServingError(
                409, "LAB_APPROVAL_STALE", "current exact LAB execution approval is required"
            )
        probe = self.repo.probe(candidate.runtime_probe_id)
        profile = self.repo.profile(candidate.runtime_profile_id)
        node = self.repo.node(candidate.compute_node_id)
        if (
            not probe
            or probe.status != "completed"
            or probe.execution_approval_id != approval.id
            or not profile
            or not node
            or not node.enabled
            or node.liveness_state != "online"
        ):
            raise ServingError(409, "CANDIDATE_EVIDENCE_STALE", "candidate evidence is incomplete")
        dimension = int(probe.inference_result.get("dimension", 0))
        if request.capability in {"rag.embedding", "vision.embedding"}:
            if (
                dimension != request.expected_dimension
                or probe.inference_result.get("finite") is not True
                or probe.inference_result.get("output_type") not in {None, "float_vector"}
            ):
                raise ServingError(
                    409, "INVALID_OUTPUT", "probe dimension does not match experiment"
                )
        elif request.capability == "rag.reranking" and (
            probe.inference_result.get("output_type") != "ranked_scores"
            or probe.inference_result.get("finite") is not True
            or probe.runtime_facts.get("adapter") != "qwen3_reranker"
        ):
            raise ServingError(409, "INVALID_OUTPUT", "reranking probe evidence is invalid")
        elif request.capability == "document.ocr" and (
            probe.inference_result.get("output_type") != "ocr_text"
            or probe.inference_result.get("output_valid") is not True
        ):
            raise ServingError(409, "INVALID_OUTPUT", "OCR probe evidence is invalid")
        elif request.capability == "speech.transcription" and (
            probe.inference_result.get("output_type") != "transcription"
            or probe.inference_result.get("output_valid") is not True
        ):
            raise ServingError(409, "INVALID_OUTPUT", "ASR probe evidence is invalid")
        accelerators = self.repo.node_accelerators(node.id)
        gpu_uuid = str(probe.runtime_facts.get("gpu_uuid", ""))
        accelerator = next((item for item in accelerators if item.device_uuid == gpu_uuid), None)
        if not accelerator:
            raise ServingError(409, "NO_ELIGIBLE_NODE", "probed accelerator is unavailable")
        contract = self.ensure_contract(request.capability, 1)
        space: EmbeddingSpace | None = None
        if request.capability in {"rag.embedding", "vision.embedding"}:
            semantics_key = (
                "embedding_input_semantics"
                if request.capability == "rag.embedding"
                else "visual_preprocessing"
            )
            semantics = profile.launch_arguments.get(semantics_key, {})
            if not isinstance(semantics, dict):
                raise ServingError(
                    409, "PROFILE_SEMANTICS_INVALID", "input semantics must be typed"
                )
            embedding_facts = {
                "artifact_set_id": str(candidate.artifact_set_id),
                "runtime_profile_fingerprint": profile.fingerprint,
                "capability_contract_id": str(contract.id),
                "dimension": dimension,
                "normalized": True,
                "input_semantics": semantics,
                "modality": "text" if request.capability == "rag.embedding" else "image_text",
                "migration_class": "requires_reindex",
            }
            digest = _canonical_hash(embedding_facts)
            space = self.repo.embedding_space_by_digest(digest)
            if not space:
                space = self.repo.add(
                    EmbeddingSpace(
                        capability_contract_id=contract.id,
                        artifact_set_id=candidate.artifact_set_id,
                        runtime_profile_id=candidate.runtime_profile_id,
                        identity_digest=digest,
                        dimension=dimension,
                        normalized=True,
                        migration_class="requires_reindex",
                        identity_facts=embedding_facts,
                        created_at=_now(),
                        immutable_at=_now(),
                    )
                )
        config = {
            "candidate_id": str(candidate.id),
            "execution_approval_id": str(approval.id),
            "capability_contract_id": str(contract.id),
            "embedding_space_id": str(space.id) if space else None,
            "capability": request.capability,
            "accelerator_id": str(accelerator.id),
            "route_key": request.route_key,
            "residency_policy": request.residency_policy,
            "keep_warm_seconds": request.keep_warm_seconds,
            "max_queue_depth": request.max_queue_depth,
        }
        fingerprint = _canonical_hash(config)
        deployment = self.repo.deployment_by_fingerprint(fingerprint)
        if not deployment:
            deployment = self.repo.add(
                CapabilityDeployment(
                    capability_contract_id=contract.id,
                    deployment_candidate_id=candidate.id,
                    production_approval_id=None,
                    execution_approval_id=approval.id,
                    embedding_space_id=space.id if space else None,
                    artifact_set_id=candidate.artifact_set_id,
                    runtime_profile_id=candidate.runtime_profile_id,
                    compute_node_id=node.id,
                    accelerator_id=accelerator.id,
                    channel="experiment",
                    status="candidate",
                    production=False,
                    health_status="ready_on_demand",
                    routing_weight=0,
                    fallback_policy={"allowed": False, "mode": "hard_fail", "deployments": []},
                    residency_policy=request.residency_policy,
                    keep_warm_seconds=request.keep_warm_seconds,
                    max_concurrency=1,
                    max_queue_depth=request.max_queue_depth,
                    config_fingerprint=fingerprint,
                    provenance={
                        "runtime_probe_id": str(probe.id),
                        "execution_approval_id": str(approval.id),
                        "runtime_profile_fingerprint": profile.fingerprint,
                        "production": False,
                        "capability": request.capability,
                    },
                    rollback_policy={"mode": "unload_experiment", "stable_unchanged": True},
                )
            )
            measured = probe.measured_resources
            samples = measured.get("samples", {})
            baseline = int(samples.get("before_load", {}).get("process_used_vram_bytes", 0))
            loaded = int(samples.get("loaded", {}).get("process_used_vram_bytes", 0))
            peak = int(samples.get("probe_peak", {}).get("process_used_vram_bytes", 0))
            if peak <= baseline or loaded <= baseline:
                baseline = int(measured["baseline_vram_bytes"])
                loaded = int(measured["loaded_vram_bytes"])
                peak = int(measured["peak_vram_bytes"])
            self.repo.add(
                CapabilityResourceEnvelope(
                    capability_deployment_id=deployment.id,
                    runtime_probe_id=probe.id,
                    accelerator_kind=accelerator.name,
                    accelerator_uuid=accelerator.device_uuid,
                    environment_fingerprint=probe.environment_fingerprint,
                    concurrency=1,
                    batch_size=int(measured.get("batch_size", 1)),
                    max_sequence_length=int(measured.get("max_sequence_length", 128)),
                    baseline_vram_bytes=baseline,
                    resident_vram_bytes=max(0, loaded - baseline),
                    peak_vram_bytes=peak,
                    required_vram_bytes=max(loaded, peak) - baseline,
                    cold_load_time_ms=float(probe.load_result["load_time_ms"]),
                    inference_latency_ms=float(probe.inference_result["latency_ms"]),
                    evidence={"kind": "measured", "runtime_probe_id": str(probe.id)},
                    stale=False,
                    immutable_at=_now(),
                )
            )
            self.repo.add(
                ResidencyAllocation(
                    capability_deployment_id=deployment.id,
                    compute_node_id=node.id,
                    accelerator_id=accelerator.id,
                    state="cold",
                    load_count=0,
                    active_requests=0,
                    measured_resident_vram_bytes=0,
                    external_baseline_vram_bytes=0,
                    health={
                        "process": "unavailable",
                        "runtime": "unavailable",
                        "model": "unavailable",
                    },
                )
            )
        route = self.repo.add(
            CapabilityExperimentRoute(
                route_key=request.route_key,
                capability_contract_id=contract.id,
                capability_deployment_id=deployment.id,
                status="active",
                purpose=request.purpose,
                evidence={
                    "candidate_id": str(candidate.id),
                    "embedding_space_id": str(space.id) if space else None,
                    "capability": request.capability,
                    "production": False,
                },
            )
        )
        self.audit.write(
            "CAPABILITY_EXPERIMENT_CREATED",
            "capability_experiment_route",
            str(route.id),
            {"route_key": route.route_key, "production": False},
        )
        self._commit()
        return self.experiment_response(route)

    def experiment_response(self, route: CapabilityExperimentRoute) -> CapabilityExperimentResponse:
        deployment = self.repo.deployment(route.capability_deployment_id)
        contract = self.session.get(CapabilityContract, route.capability_contract_id)
        capability = self.session.get(Capability, contract.capability_id) if contract else None
        if not deployment or not contract or not capability:
            raise ServingError(409, "DEPLOYMENT_PROVENANCE_INCOMPLETE", "route is incomplete")
        return CapabilityExperimentResponse(
            id=route.id,
            route_key=route.route_key,
            capability=capability.key,
            contract_version=contract.version,
            capability_deployment_id=deployment.id,
            status=route.status,
            purpose=route.purpose,
            evidence=route.evidence,
            deployment=self.deployment_response(deployment),
            created_at=route.created_at,
        )

    def experiments(self) -> list[CapabilityExperimentResponse]:
        return [self.experiment_response(item) for item in self.repo.experiment_routes()]

    def deactivate_experiment(self, route_id: uuid.UUID) -> CapabilityExperimentResponse:
        route = self.repo.experiment_route_by_id(route_id)
        if not route:
            raise ServingError(404, "EXPERIMENT_NOT_FOUND", "capability experiment is unavailable")
        if route.status == "active":
            route.status = "inactive"
            self.audit.write(
                "CAPABILITY_EXPERIMENT_DEACTIVATED",
                "capability_experiment_route",
                str(route.id),
                {"route_key": route.route_key, "production": False},
            )
            self._commit()
        return self.experiment_response(route)

    def _health_status(
        self, deployment: CapabilityDeployment, residency: ResidencyAllocation
    ) -> str:
        node = self.repo.node(deployment.compute_node_id)
        if not node or not node.enabled or node.liveness_state != "online":
            return "unavailable"
        if deployment.status == "draining":
            return "degraded"
        if residency.state in {"warm", "busy", "idle"}:
            health = residency.health
            return (
                "healthy"
                if all(health.get(layer) == "healthy" for layer in ("process", "runtime", "model"))
                else "degraded"
            )
        if residency.state in {"cold", "loading"}:
            return "ready_on_demand"
        return "unavailable"

    def create_client(self, request: ServiceClientCreate) -> ServiceClientCreated:
        if self.repo.client_by_name(request.name):
            raise ServingError(409, "CLIENT_EXISTS", "service client name already exists")
        project: Project | None = None
        project_binding: ProjectBinding | None = None
        for scope in request.allowed_capabilities:
            key, raw_version = scope.rsplit("@", 1)
            if not self.repo.contract(key, int(raw_version)):
                self.ensure_contract(key, int(raw_version))
        if request.project_key is not None:
            project = self.session.scalar(select(Project).where(Project.key == request.project_key))
            if project is None or not project.active:
                raise ServingError(404, "PROJECT_NOT_FOUND", "active project was not found")
            key, raw_version = request.allowed_capabilities[0].rsplit("@", 1)
            contract = self.repo.contract(key, int(raw_version))
            if contract is None:
                raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability was not found")
            bindings = list(
                self.session.scalars(
                    select(ProjectBinding)
                    .where(
                        ProjectBinding.project_id == project.id,
                        ProjectBinding.capability_contract_id == contract.id,
                        ProjectBinding.deprecated_at.is_(None),
                    )
                    .order_by(ProjectBinding.created_at.desc())
                )
            )
            preferred_channels = (
                {"stable"}
                if request.integration_environment == "production"
                else {"experimental", "candidate"}
            )
            project_binding = next(
                (item for item in bindings if item.channel in preferred_channels),
                bindings[0] if bindings else None,
            )
            if project_binding is None:
                raise ServingError(
                    409,
                    "PROJECT_CAPABILITY_NOT_BOUND",
                    "project does not have an active binding for this capability",
                )
        secret = "mf_" + secrets.token_urlsafe(36)
        client = self.repo.add(
            ServiceClient(
                name=request.name,
                status="active",
                allowed_capabilities=request.allowed_capabilities,
                requests_per_minute=request.requests_per_minute,
                max_concurrent_requests=request.max_concurrent_requests,
                workload_priority=request.workload_priority,
                project_binding_id=project_binding.id if project_binding else None,
                integration_environment=request.integration_environment,
                purpose=request.purpose,
            )
        )
        credential = self.repo.add(
            ServiceCredential(
                service_client_id=client.id,
                secret_hash=hashlib.sha256(secret.encode()).hexdigest(),
                secret_prefix=secret[:12],
                created_at=_now(),
                expires_at=request.credential_expires_at,
            )
        )
        self.audit.write(
            "CLIENT_CREATED",
            "service_client",
            str(client.id),
            {
                "name": client.name,
                "allowed_capabilities": client.allowed_capabilities,
                "project_key": project.key if project else None,
                "integration_environment": client.integration_environment,
            },
        )
        self._commit()
        return ServiceClientCreated(
            **self.client_response(client, credential).model_dump(), credential=secret
        )

    def client_response(
        self, client: ServiceClient, credential: ServiceCredential | None = None
    ) -> ServiceClientResponse:
        credential = credential or self.repo.active_credential(client.id)
        project_key = None
        if client.project_binding_id is not None:
            binding = self.session.get(ProjectBinding, client.project_binding_id)
            project = self.session.get(Project, binding.project_id) if binding else None
            project_key = project.key if project else None
        return ServiceClientResponse(
            id=client.id,
            name=client.name,
            status=client.status,
            allowed_capabilities=client.allowed_capabilities,
            requests_per_minute=client.requests_per_minute,
            max_concurrent_requests=client.max_concurrent_requests,
            workload_priority=client.workload_priority,
            project_key=project_key,
            project_binding_id=client.project_binding_id,
            integration_environment=client.integration_environment,
            purpose=client.purpose,
            credential_prefix=credential.secret_prefix if credential else None,
            credential_created_at=credential.created_at if credential else None,
            credential_expires_at=credential.expires_at if credential else None,
            credential_revoked_at=credential.revoked_at if credential else None,
            last_used_at=client.last_used_at,
            created_at=client.created_at,
        )

    def clients(self) -> list[ServiceClientResponse]:
        return [self.client_response(item) for item in self.repo.clients()]

    def revoke_client_credential(self, client_id: uuid.UUID) -> ServiceClientResponse:
        client = self.repo.client(client_id)
        credential = self.repo.active_credential(client_id) if client else None
        if not client:
            raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
        if credential and not credential.revoked_at:
            credential.revoked_at = _now()
            self.audit.write(
                "CLIENT_CREDENTIAL_REVOKED", "service_client", str(client.id), {"name": client.name}
            )
            self._commit()
        return self.client_response(client, credential)

    def rotate_client_credential(self, client_id: uuid.UUID) -> ServiceClientCreated:
        client = self.repo.client(client_id)
        if client is None:
            raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
        current = self.repo.active_credential(client_id)
        if current is not None:
            current.revoked_at = _now()
            self.session.flush()
        secret = "mf_" + secrets.token_urlsafe(36)
        credential = self.repo.add(
            ServiceCredential(
                service_client_id=client.id,
                secret_hash=hashlib.sha256(secret.encode()).hexdigest(),
                secret_prefix=secret[:12],
                created_at=_now(),
            )
        )
        self.audit.write(
            "CLIENT_CREDENTIAL_ROTATED", "service_client", str(client.id), {"name": client.name}
        )
        self._commit()
        return ServiceClientCreated(
            **self.client_response(client, credential).model_dump(), credential=secret
        )

    def record_project_fit(self, request: ProjectFitEvidenceCreate) -> ProjectFitEvidenceResponse:
        project = self.session.scalar(select(Project).where(Project.key == request.project_key))
        if project is None:
            raise ServingError(404, "PROJECT_NOT_FOUND", "project was not found")
        key, raw_version = request.capability.rsplit("@", 1)
        contract = self.repo.contract(key, int(raw_version))
        if contract is None:
            raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability contract was not found")
        bindings = list(
            self.session.scalars(
                select(ProjectBinding)
                .where(
                    ProjectBinding.project_id == project.id,
                    ProjectBinding.capability_contract_id == contract.id,
                    ProjectBinding.deprecated_at.is_(None),
                )
                .order_by(ProjectBinding.created_at.desc())
            )
        )
        preferred_channels = (
            {"stable"} if request.environment == "production" else {"experimental", "candidate"}
        )
        binding = next(
            (item for item in bindings if item.channel in preferred_channels),
            bindings[0] if bindings else None,
        )
        if binding is None:
            raise ServingError(409, "PROJECT_CAPABILITY_NOT_BOUND", "project binding was not found")
        if request.capability_deployment_id is not None:
            deployment = self.repo.deployment(request.capability_deployment_id)
            if deployment is None:
                raise ServingError(
                    404, "DEPLOYMENT_NOT_FOUND", "capability deployment was not found"
                )
            if deployment.capability_contract_id != contract.id:
                raise ServingError(
                    422,
                    "PROJECT_FIT_DEPLOYMENT_MISMATCH",
                    "project-fit deployment does not implement the requested capability",
                )
        if request.recommendation == "PROMOTION_ELIGIBLE" and (
            request.evidence_class != "OWNER_PHOTO"
            or request.production_validation != "SATISFIED"
            or request.case_count == 0
            or request.critical_errors > 0
            or request.blockers
        ):
            raise ServingError(
                422,
                "PROJECT_FIT_GATES_FAILED",
                "project production eligibility requires reviewed owner-photo cases, a satisfied "
                "production gate, zero critical errors and no blockers",
            )
        if request.production_validation == "SATISFIED" and (
            request.evidence_class != "OWNER_PHOTO"
            or request.recommendation != "PROMOTION_ELIGIBLE"
            or request.case_count == 0
            or request.critical_errors > 0
            or request.blockers
        ):
            raise ServingError(
                422,
                "PROJECT_PRODUCTION_VALIDATION_FAILED",
                "production validation can only be satisfied by eligible owner-photo evidence",
            )
        payload = request.model_dump(mode="json", exclude={"capability_deployment_id"})
        digest = _canonical_hash(payload)
        existing = self.session.scalar(
            select(ProjectFitEvidence).where(ProjectFitEvidence.evidence_digest == digest)
        )
        if existing is not None:
            return self._project_fit_response(existing, project, request.capability)
        item = self.repo.add(
            ProjectFitEvidence(
                project_binding_id=binding.id,
                capability_deployment_id=request.capability_deployment_id,
                environment=request.environment,
                recommendation=request.recommendation,
                evidence_class=request.evidence_class,
                engineering_integration=request.engineering_integration,
                production_validation=request.production_validation,
                production_action=request.production_action,
                case_count=request.case_count,
                metric_values=request.metric_values,
                critical_errors=request.critical_errors,
                blockers=request.blockers,
                evidence_digest=digest,
                evidence=request.evidence,
            )
        )
        self.audit.write(
            "PROJECT_FIT_RECORDED",
            "project_fit_evidence",
            str(item.id),
            {
                "project_key": project.key,
                "capability": request.capability,
                "recommendation": item.recommendation,
                "evidence_class": item.evidence_class,
                "engineering_integration": item.engineering_integration,
                "production_validation": item.production_validation,
                "production_action": item.production_action,
            },
        )
        self._commit()
        return self._project_fit_response(item, project, request.capability)

    def _project_fit_response(
        self, item: ProjectFitEvidence, project: Project, capability: str
    ) -> ProjectFitEvidenceResponse:
        return ProjectFitEvidenceResponse(
            id=item.id,
            project_key=project.key,
            capability=capability,
            environment=item.environment,
            recommendation=item.recommendation,
            evidence_class=item.evidence_class,
            engineering_integration=item.engineering_integration,
            production_validation=item.production_validation,
            production_action=item.production_action,
            case_count=item.case_count,
            metric_values=item.metric_values,
            critical_errors=item.critical_errors,
            blockers=item.blockers,
            evidence_digest=item.evidence_digest,
            evidence=item.evidence,
            capability_deployment_id=item.capability_deployment_id,
            created_at=item.created_at,
        )

    def project_integrations(self) -> list[ProjectIntegrationResponse]:
        responses: list[ProjectIntegrationResponse] = []
        for client in self.repo.clients():
            if client.project_binding_id is None:
                continue
            binding = self.session.get(ProjectBinding, client.project_binding_id)
            project = self.session.get(Project, binding.project_id) if binding else None
            contract = (
                self.session.get(CapabilityContract, binding.capability_contract_id)
                if binding
                else None
            )
            capability = self.session.get(Capability, contract.capability_id) if contract else None
            if binding is None or project is None or contract is None or capability is None:
                continue
            capability_key = f"{capability.key}@{contract.version}"
            requests = list(
                self.session.scalars(
                    select(GatewayRequest)
                    .where(GatewayRequest.service_client_id == client.id)
                    .order_by(GatewayRequest.created_at)
                )
            )
            successful = [item for item in requests if item.status == "completed"]
            errors = [item for item in requests if item.status == "failed"]
            latencies = sorted(
                item.total_latency_ms for item in successful if item.total_latency_ms is not None
            )
            latest_request = requests[-1] if requests else None
            fit = self.session.scalar(
                select(ProjectFitEvidence)
                .where(ProjectFitEvidence.project_binding_id == binding.id)
                .order_by(ProjectFitEvidence.created_at.desc())
            )
            deployment_id = (
                fit.capability_deployment_id
                if fit and fit.capability_deployment_id
                else (latest_request.capability_deployment_id if latest_request else None)
            )
            envelope = (
                self.session.scalar(
                    select(CapabilityResourceEnvelope)
                    .where(CapabilityResourceEnvelope.capability_deployment_id == deployment_id)
                    .order_by(CapabilityResourceEnvelope.created_at.desc())
                )
                if deployment_id
                else None
            )
            responses.append(
                ProjectIntegrationResponse(
                    project_key=project.key,
                    project_name=project.name,
                    capability=capability_key,
                    environment=client.integration_environment or binding.channel,
                    state="ACTIVE" if client.status == "active" else client.status.upper(),
                    purpose=client.purpose or "unspecified",
                    client_id=client.id,
                    client_name=client.name,
                    deployment_id=deployment_id,
                    resource_impact_bytes=envelope.required_vram_bytes if envelope else None,
                    usage=ProjectIntegrationUsage(
                        request_volume=len(requests),
                        successful_requests=len(successful),
                        error_count=len(errors),
                        last_used_at=client.last_used_at,
                        latency_p50_ms=_percentile(latencies, 0.50),
                        latency_p95_ms=_percentile(latencies, 0.95),
                    ),
                    project_fit=(
                        self._project_fit_response(fit, project, capability_key) if fit else None
                    ),
                )
            )
        return responses

    @staticmethod
    def _credential_secret(authorization: str | None) -> str:
        if not authorization or not authorization.startswith("Bearer "):
            raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is required")
        secret = authorization.removeprefix("Bearer ").strip()
        if not secret:
            raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is required")
        return secret

    def _validate_capability_access(
        self,
        credential: ServiceCredential,
        client: ServiceClient,
        capability: str,
        *,
        enforce_request_limits: bool,
    ) -> None:
        now = _now()
        if credential.revoked_at:
            raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is revoked")
        if credential.expires_at and _aware(credential.expires_at) <= now:
            raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is expired")
        if client.status != "active" or client.disabled_at:
            raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "service client is disabled")
        if capability not in client.allowed_capabilities:
            raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability scope is not granted")
        if client.project_binding_id is not None:
            binding = self.session.get(ProjectBinding, client.project_binding_id)
            contract = (
                self.session.get(CapabilityContract, binding.capability_contract_id)
                if binding
                else None
            )
            bound_capability = (
                self.session.get(Capability, contract.capability_id) if contract else None
            )
            bound_key = (
                f"{bound_capability.key}@{contract.version}"
                if bound_capability and contract
                else None
            )
            if binding is None or binding.deprecated_at is not None or bound_key != capability:
                raise ServingError(
                    403, "PROJECT_CAPABILITY_NOT_BOUND", "project binding is inactive"
                )
        if not enforce_request_limits:
            return
        since = now - timedelta(minutes=1)
        if self.repo.recent_request_count(client.id, since) >= client.requests_per_minute:
            raise ServingError(429, "RATE_LIMITED", "service client rate limit exceeded")
        if self.repo.active_client_requests(client.id) >= client.max_concurrent_requests:
            raise ServingError(429, "QUEUE_FULL", "service client concurrency limit reached")

    def authenticate_with_evidence(
        self, authorization: str | None, capability: str
    ) -> tuple[ServiceClient, CapabilityAuthenticationEvidence]:
        secret = self._credential_secret(authorization)
        credential_sha256 = hashlib.sha256(secret.encode()).hexdigest()
        match = self.repo.credential_by_hash(credential_sha256)
        if not match:
            raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is invalid")
        credential, client = match
        self._validate_capability_access(
            credential,
            client,
            capability,
            enforce_request_limits=True,
        )
        now = _now()
        credential.last_used_at = now
        client.last_used_at = now
        self._commit_authentication()
        return client, CapabilityAuthenticationEvidence(
            credential_id=credential.id,
            client_id=client.id,
            capability=capability,
            credential_sha256=credential_sha256,
        )

    def authenticate(self, authorization: str | None, capability: str) -> ServiceClient:
        client, _evidence = self.authenticate_with_evidence(authorization, capability)
        return client

    def reuse_authentication(
        self,
        evidence: CapabilityAuthenticationEvidence,
        authorization: str | None,
        capability: str,
    ) -> ServiceClient:
        """Revalidate current scope/client state while reusing the pre-body credential proof."""

        secret = self._credential_secret(authorization)
        credential_sha256 = hashlib.sha256(secret.encode()).hexdigest()
        if capability != evidence.capability or not hmac.compare_digest(
            credential_sha256, evidence.credential_sha256
        ):
            raise ServingError(
                403,
                "CAPABILITY_NOT_AUTHORIZED",
                "pre-authenticated capability scope does not match this route",
            )
        credential = self.session.get(ServiceCredential, evidence.credential_id)
        client = self.session.get(ServiceClient, evidence.client_id)
        if (
            credential is None
            or client is None
            or credential.service_client_id != client.id
            or not hmac.compare_digest(credential.secret_hash, credential_sha256)
        ):
            raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is invalid")
        self._validate_capability_access(
            credential,
            client,
            capability,
            enforce_request_limits=False,
        )
        return client

    def _budget(self, deployment: CapabilityDeployment) -> SchedulerBudgetResponse:
        node = self.repo.node(deployment.compute_node_id)
        accelerator = self.repo.accelerator(deployment.accelerator_id)
        telemetry = self.repo.telemetry(deployment.accelerator_id)
        residency = self.repo.residency(deployment.id)
        if not node or not accelerator or not residency or accelerator.total_vram_bytes is None:
            raise ServingError(
                503, "NO_ELIGIBLE_NODE", "deployment hardware evidence is unavailable"
            )
        observed_used = (
            int(telemetry.used_vram_bytes)
            if telemetry and telemetry.used_vram_bytes is not None
            else None
        )
        resident = sum(
            item.measured_resident_vram_bytes
            for candidate in self.repo.deployments()
            if candidate.accelerator_id == accelerator.id
            and candidate.status in {"stable", "draining", "candidate"}
            and (item := self.repo.residency(candidate.id)) is not None
            and item.state != "cold"
        )
        leases = self.repo.active_leases(accelerator.id, _now())
        policy = self._scheduler_policy()
        pressure_row = self.repo.scheduler_accelerator_state(accelerator.id)
        previous_pressure = (
            PressureState(pressure_row.pressure_state) if pressure_row else PressureState.NORMAL
        )
        snapshot = calculate_accounting(
            total_bytes=accelerator.total_vram_bytes,
            observed_used_bytes=observed_used,
            resident_bytes=resident,
            leases=[
                LeaseAllocation(item.reserved_vram_bytes, item.materialized_vram_bytes)
                for item in leases
            ],
            policy=policy,
            telemetry_fresh=(
                telemetry is not None
                and _aware(telemetry.observed_at)
                >= _now() - timedelta(seconds=self.settings.scheduler_telemetry_stale_seconds)
            ),
            previous_pressure=previous_pressure,
        )
        snapshot = replace(
            snapshot,
            pressure=self._stabilize_pressure(accelerator.id, snapshot.pressure, pressure_row),
        )
        return SchedulerBudgetResponse(
            compute_node_id=node.id,
            node_name=node.display_name or node.hostname,
            accelerator_id=accelerator.id,
            accelerator_uuid=accelerator.device_uuid,
            accelerator_name=accelerator.name,
            total_vram_bytes=accelerator.total_vram_bytes,
            observed_used_vram_bytes=observed_used or 0,
            external_vram_bytes=snapshot.external_bytes,
            resident_vram_bytes=resident,
            leased_vram_bytes=snapshot.future_lease_bytes,
            safety_reserve_bytes=snapshot.reserve_bytes,
            schedulable_free_vram_bytes=snapshot.schedulable_bytes,
            pressure=snapshot.pressure.value != "NORMAL",
            pressure_state=snapshot.pressure.value,
            attribution_confidence=snapshot.attribution.value,
            invariant_delta_bytes=snapshot.invariant_delta_bytes,
            policy_revision=policy.revision,
            observed_at=telemetry.observed_at if telemetry else None,
        )

    def _stabilize_pressure(
        self,
        accelerator_id: uuid.UUID,
        candidate: PressureState,
        row: SchedulerAcceleratorState | None,
    ) -> PressureState:
        now = _now()
        if row is None:
            self.repo.add(
                SchedulerAcceleratorState(
                    accelerator_id=accelerator_id,
                    pressure_state=candidate.value,
                    pressure_changed_at=now,
                    last_observed_at=now,
                )
            )
            return candidate
        current = PressureState(row.pressure_state)
        severity = {
            PressureState.NORMAL: 0,
            PressureState.ELEVATED: 1,
            PressureState.HIGH: 2,
            PressureState.CRITICAL: 3,
        }
        row.last_observed_at = now
        if severity[candidate] > severity[current]:
            row.pressure_state = candidate.value
            row.pressure_changed_at = now
            row.recovery_candidate = None
            row.recovery_candidate_since = None
            return candidate
        if candidate == current:
            row.recovery_candidate = None
            row.recovery_candidate_since = None
            return current
        if row.recovery_candidate != candidate.value:
            row.recovery_candidate = candidate.value
            row.recovery_candidate_since = now
            return current
        if (
            row.recovery_candidate_since
            and _aware(row.recovery_candidate_since)
            + timedelta(seconds=self.settings.scheduler_pressure_stable_seconds)
            <= now
        ):
            row.pressure_state = candidate.value
            row.pressure_changed_at = now
            row.recovery_candidate = None
            row.recovery_candidate_since = None
            return candidate
        return current

    def _scheduler_policy(self) -> SchedulerPolicy:
        configured = SchedulerPolicy(
            reserve_minimum_bytes=self.settings.scheduler_safety_reserve_bytes,
            reserve_percentage=self.settings.scheduler_safety_reserve_percentage,
            runtime_margin_bytes=self.settings.scheduler_runtime_margin_bytes,
            deployment_margin_minimum_bytes=self.settings.scheduler_deployment_margin_bytes,
            deployment_margin_percentage=self.settings.scheduler_deployment_margin_percentage,
            global_queue_limit=self.settings.scheduler_global_queue_limit,
            pressure_stable_seconds=self.settings.scheduler_pressure_stable_seconds,
            eviction_cooldown_seconds=self.settings.scheduler_eviction_cooldown_seconds,
            placement_history_limit=self.settings.scheduler_placement_history_limit,
        )
        active = self.repo.active_scheduler_policy()
        if not active:
            return configured
        allowed = set(SchedulerPolicy.__dataclass_fields__)
        values = {key: value for key, value in active.configuration.items() if key in allowed}
        values["revision"] = active.revision
        return SchedulerPolicy(**values)

    def scheduler_policy(self) -> SchedulerPolicyResponse:
        policy = self._scheduler_policy()
        active = self.repo.active_scheduler_policy()
        return SchedulerPolicyResponse(
            revision=policy.revision,
            active=True,
            configuration=asdict(policy),
            created_at=active.created_at if active else _now(),
        )

    def scheduler_metrics(self) -> SchedulerMetricsResponse:
        plans = self.repo.placement_plans(self.settings.scheduler_placement_history_limit)
        requests = self.repo.gateway_requests(self.settings.scheduler_placement_history_limit)
        budgets = self.scheduler_overview()
        counters = {
            "scheduler_admissions_total": sum(
                item.verdict in {"ADMIT", "ADMIT_AFTER_EVICTION"} for item in plans
            ),
            "scheduler_rejections_total": sum(item.verdict.startswith("REJECT_") for item in plans),
            "scheduler_evictions_total": sum(
                item.verdict == "ADMIT_AFTER_EVICTION" for item in plans
            ),
            "cold_loads": sum(item.cold is True for item in requests),
            "warm_hits": sum(item.cold is False for item in requests),
            "reclaim_failures": sum(
                item.failure_code == "GPU_MEMORY_NOT_RECLAIMED" for item in requests
            ),
        }
        queue_values = [item.queue_time_ms / 1000 for item in requests if item.queue_time_ms]
        return SchedulerMetricsResponse(
            counters=counters,
            gauges={
                "resident_bytes": sum(item.resident_vram_bytes for item in budgets),
                "leased_bytes": sum(item.leased_vram_bytes for item in budgets),
                "external_bytes": sum(item.external_vram_bytes for item in budgets),
                "schedulable_bytes": sum(item.schedulable_free_vram_bytes for item in budgets),
                "pressure_state": max((item.pressure_state for item in budgets), default="UNKNOWN"),
            },
            queue_seconds={
                "count": float(len(queue_values)),
                "maximum": max(queue_values, default=0.0),
                "average": sum(queue_values) / len(queue_values) if queue_values else 0.0,
            },
        )

    def update_scheduler_policy(self, request: SchedulerPolicyUpdate) -> SchedulerPolicyResponse:
        current = self._scheduler_policy()
        active = self.repo.active_scheduler_policy()
        if active:
            active.active = False
        revision = f"m10-v{int(active.revision.removeprefix('m10-v') if active else '1') + 1}"
        configuration = {**asdict(current), "revision": revision, "lab_paused": request.lab_paused}
        record = self.repo.add(
            SchedulerPolicyRevision(
                revision=revision, configuration=configuration, active=True, created_at=_now()
            )
        )
        self.audit.write(
            "SCHEDULER_POLICY_CHANGED",
            "scheduler_policy",
            str(record.id),
            {"revision": revision, "lab_paused": request.lab_paused},
        )
        self._commit()
        return self.scheduler_policy()

    def _capability_key(self, deployment: CapabilityDeployment) -> str:
        contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
        capability = self.session.get(Capability, contract.capability_id) if contract else None
        return capability.key if capability else "unknown.capability"

    def _placement_plan(
        self,
        deployment: CapabilityDeployment,
        *,
        priority: str,
        deadline_remaining_ms: float | None = None,
        request_id: uuid.UUID | None = None,
        dry_run: bool,
        persist: bool = True,
        commit: bool = True,
    ) -> PlacementPlanResponse:
        residency = self.repo.residency(deployment.id)
        envelope = self.repo.envelope(deployment.id)
        node = self.repo.node(deployment.compute_node_id)
        if not residency or not envelope or not node:
            raise ServingError(503, "SCHEDULER_STATE_STALE", "scheduler state is incomplete")
        budget = self._budget(deployment)
        policy = self._scheduler_policy()
        residents: list[ResidentCandidate] = []
        for item in self.repo.accelerator_residencies(deployment.accelerator_id):
            if item.state in {"cold", "failed", "load_failed"}:
                continue
            other = self.repo.deployment(item.capability_deployment_id)
            other_envelope = self.repo.envelope(item.capability_deployment_id)
            if not other:
                continue
            residents.append(
                ResidentCandidate(
                    deployment_id=str(other.id),
                    capability=self._capability_key(other),
                    resident_bytes=max(0, item.measured_resident_vram_bytes),
                    active_requests=item.active_requests,
                    priority="production" if other.production else "lab",
                    policy=other.residency_policy,
                    idle_since=item.last_used_at,
                    cold_load_ms=other_envelope.cold_load_time_ms if other_envelope else 0.0,
                    last_used_at=item.last_used_at,
                    pinned=other.residency_policy == "always_warm",
                )
            )
        is_resident = residency.state not in {"cold", "failed", "load_failed"}
        required = (
            policy.request_execution_floor_bytes
            if is_resident
            else max(
                envelope.required_vram_bytes, required_envelope(envelope.peak_vram_bytes, policy)
            )
        )
        snapshot = AccountingSnapshot(
            total_bytes=budget.total_vram_bytes,
            observed_used_bytes=budget.observed_used_vram_bytes,
            resident_bytes=budget.resident_vram_bytes,
            future_lease_bytes=budget.leased_vram_bytes,
            reserve_bytes=budget.safety_reserve_bytes,
            external_bytes=budget.external_vram_bytes,
            schedulable_bytes=budget.schedulable_free_vram_bytes,
            attribution=AttributionConfidence(budget.attribution_confidence),
            invariant_delta_bytes=budget.invariant_delta_bytes,
            pressure=PressureState(budget.pressure_state),
        )
        decision = plan_placement(
            PlacementInput(
                deployment_id=str(deployment.id),
                capability=self._capability_key(deployment),
                node_id=str(node.id),
                accelerator_id=str(deployment.accelerator_id),
                priority=priority,
                required_bytes=required,
                cold_load_ms=envelope.cold_load_time_ms,
                is_resident=is_resident,
                envelope_stale=envelope.stale,
                runtime_healthy=deployment.health_status not in {"failed", "unavailable"},
                node_eligible=node.enabled and node.liveness_state == "online",
                deadline_remaining_ms=deadline_remaining_ms,
            ),
            snapshot,
            residents,
            policy,
            global_queue_depth=sum(
                self.repo.queued_job_count(item.id) for item in self.repo.deployments()
            ),
        )
        created = _now()
        record: PlacementPlanRecord | None = None
        evidence = {
            **decision.evidence(),
            "capability": self._capability_key(deployment),
            "node_id": str(node.id),
            "accelerator_id": str(deployment.accelerator_id),
            "budget": budget.model_dump(mode="json"),
            "priority": priority,
        }
        if persist:
            record = self.repo.add(
                PlacementPlanRecord(
                    capability_deployment_id=deployment.id,
                    request_id=request_id,
                    policy_revision=policy.revision,
                    verdict=decision.verdict.value,
                    reason_codes=list(decision.reason_codes),
                    evidence=evidence,
                    evidence_fingerprint=decision.fingerprint,
                    dry_run=dry_run,
                    created_at=created,
                )
            )
            self.repo.trim_placement_plans(policy.placement_history_limit)
            if commit:
                self._commit()
        return PlacementPlanResponse(
            id=record.id if record else None,
            deployment_id=deployment.id,
            capability=self._capability_key(deployment),
            node_id=node.id,
            accelerator_id=deployment.accelerator_id,
            policy_revision=policy.revision,
            verdict=decision.verdict.value,
            reason_codes=list(decision.reason_codes),
            required_vram_bytes=decision.required_bytes,
            headroom_before_bytes=decision.headroom_before_bytes,
            headroom_after_bytes=decision.headroom_after_bytes,
            expected_cold_load_ms=envelope.cold_load_time_ms,
            evictions=[
                PlacementEvictionResponse(
                    deployment_id=uuid.UUID(item.deployment_id),
                    capability=item.capability,
                    expected_reclaimed_bytes=item.expected_reclaimed_bytes,
                    reason=item.reason,
                )
                for item in decision.evictions
            ],
            decision_fingerprint=decision.fingerprint,
            dry_run=dry_run,
            created_at=created,
        )

    def dry_run_placement(
        self, deployment_id: uuid.UUID, request: PlacementPlanRequest
    ) -> PlacementPlanResponse:
        deployment = self.repo.deployment(deployment_id)
        if not deployment:
            raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
        return self._placement_plan(
            deployment,
            priority=request.priority,
            deadline_remaining_ms=request.deadline_remaining_ms,
            dry_run=True,
        )

    def placement_history(self, limit: int = 100) -> list[PlacementPlanResponse]:
        responses: list[PlacementPlanResponse] = []
        for record in self.repo.placement_plans(limit):
            deployment = self.repo.deployment(record.capability_deployment_id)
            if not deployment:
                continue
            evidence = record.evidence
            responses.append(
                PlacementPlanResponse(
                    id=record.id,
                    deployment_id=deployment.id,
                    capability=str(evidence.get("capability", self._capability_key(deployment))),
                    node_id=uuid.UUID(str(evidence["node_id"])),
                    accelerator_id=uuid.UUID(str(evidence["accelerator_id"])),
                    policy_revision=record.policy_revision,
                    verdict=cast(Any, record.verdict),
                    reason_codes=record.reason_codes,
                    required_vram_bytes=int(evidence["required_bytes"]),
                    headroom_before_bytes=int(evidence["headroom_before_bytes"]),
                    headroom_after_bytes=int(evidence["headroom_after_bytes"]),
                    expected_cold_load_ms=float(evidence.get("cold_load_ms", 0.0)),
                    evictions=[
                        PlacementEvictionResponse.model_validate(item)
                        for item in evidence.get("evictions", [])
                    ],
                    decision_fingerprint=record.evidence_fingerprint,
                    dry_run=record.dry_run,
                    created_at=record.created_at,
                )
            )
        return responses

    def scheduler_overview(self) -> list[SchedulerBudgetResponse]:
        representatives: dict[uuid.UUID, CapabilityDeployment] = {}
        for deployment in self.repo.deployments():
            if deployment.status not in {"stable", "draining"}:
                continue
            current = representatives.get(deployment.accelerator_id)
            if current is None or (current.status != "stable" and deployment.status == "stable"):
                representatives[deployment.accelerator_id] = deployment
        budgets = [self._budget(item) for item in representatives.values()]
        self._commit()
        return budgets

    def co_residency_matrix(self) -> list[CoResidencyEvidenceResponse]:
        eligible = [
            item
            for item in self.repo.deployments()
            if item.status in {"stable", "candidate", "draining"}
            and self.repo.envelope(item.id) is not None
        ]
        result: list[CoResidencyEvidenceResponse] = []
        for left, right in itertools.combinations(eligible, 2):
            if left.accelerator_id != right.accelerator_id:
                continue
            left_envelope = self.repo.envelope(left.id)
            right_envelope = self.repo.envelope(right.id)
            if not left_envelope or not right_envelope:
                continue
            evidence = self.repo.co_residency_evidence(left.id, right.id)
            budget = self._budget(left)
            policy = self._scheduler_policy()
            left_required = max(
                left_envelope.required_vram_bytes,
                required_envelope(left_envelope.peak_vram_bytes, policy),
            )
            right_required = max(
                right_envelope.required_vram_bytes,
                required_envelope(right_envelope.peak_vram_bytes, policy),
            )
            combined = left_required + right_required
            status = (
                evidence.status
                if evidence
                else (
                    "UNKNOWN"
                    if left_envelope.stale
                    or right_envelope.stale
                    or budget.attribution_confidence == "UNKNOWN"
                    else (
                        "EXPECTED_SAFE"
                        if combined
                        <= budget.total_vram_bytes
                        - budget.external_vram_bytes
                        - budget.safety_reserve_bytes
                        else "NOT_SAFE"
                    )
                )
            )
            result.append(
                CoResidencyEvidenceResponse(
                    left_deployment_id=left.id,
                    left_capability=self._capability_key(left),
                    right_deployment_id=right.id,
                    right_capability=self._capability_key(right),
                    left_alone_bytes=left_required,
                    right_alone_bytes=right_required,
                    expected_combined_bytes=combined,
                    measured_combined_bytes=(
                        evidence.measured_combined_bytes if evidence else None
                    ),
                    status=cast(Any, status),
                    evidence=evidence.evidence
                    if evidence
                    else {"source": "authoritative_envelopes"},
                    measured_at=evidence.measured_at if evidence else None,
                )
            )
        self._commit()
        return result

    def record_co_residency_evidence(
        self, request: CoResidencyEvidenceCreate
    ) -> CoResidencyEvidenceResponse:
        left_id, right_id = sorted(
            (request.left_deployment_id, request.right_deployment_id), key=str
        )
        left = self.repo.deployment(left_id)
        right = self.repo.deployment(right_id)
        if not left or not right:
            raise ServingError(404, "CAPABILITY_NOT_FOUND", "deployment was not found")
        if (
            left.compute_node_id != right.compute_node_id
            or left.accelerator_id != right.accelerator_id
        ):
            raise ServingError(
                409,
                "RESIDENCY_CONFLICT",
                "co-residency evidence requires one node and accelerator",
            )
        left_residency = self.repo.residency(left.id, lock=True)
        right_residency = self.repo.residency(right.id, lock=True)
        left_envelope = self.repo.envelope(left.id)
        right_envelope = self.repo.envelope(right.id)
        if not left_residency or not right_residency or not left_envelope or not right_envelope:
            raise ServingError(503, "SCHEDULER_STATE_STALE", "scheduler state is incomplete")
        resident_states = {"warm", "busy", "idle"}
        exact_worker = (
            left_residency.worker_instance_id
            and left_residency.worker_instance_id == right_residency.worker_instance_id
            and left_residency.generation == right_residency.generation
        )
        if (
            left_residency.state not in resident_states
            or right_residency.state not in resident_states
            or left_residency.measured_resident_vram_bytes <= 0
            or right_residency.measured_resident_vram_bytes <= 0
            or not exact_worker
        ):
            raise ServingError(
                409,
                "RESIDENCY_CONFLICT",
                "both exact deployments must be resident in the same worker generation",
            )
        budget = self._budget(left)
        if budget.attribution_confidence != "KNOWN" or budget.observed_at is None:
            raise ServingError(
                503,
                "SCHEDULER_STATE_STALE",
                "known current GPU attribution is required for live evidence",
            )
        policy = self._scheduler_policy()
        left_required = max(
            left_envelope.required_vram_bytes,
            required_envelope(left_envelope.peak_vram_bytes, policy),
        )
        right_required = max(
            right_envelope.required_vram_bytes,
            required_envelope(right_envelope.peak_vram_bytes, policy),
        )
        measured = (
            left_residency.measured_resident_vram_bytes
            + right_residency.measured_resident_vram_bytes
        )
        row = self.repo.co_residency_evidence(left.id, right.id)
        if row is None:
            row = self.repo.add(
                CoResidencyEvidence(
                    left_deployment_id=left.id,
                    right_deployment_id=right.id,
                    status="PROVEN_SAFE",
                    expected_combined_bytes=left_required + right_required,
                    measured_combined_bytes=measured,
                    evidence={},
                )
            )
        row.status = "PROVEN_SAFE"
        row.expected_combined_bytes = left_required + right_required
        row.measured_combined_bytes = measured
        row.measured_at = budget.observed_at
        row.evidence = {
            "source": "live_worker_residency_and_nvml",
            "worker_generation": left_residency.generation,
            "policy_revision": policy.revision,
            "observed_used_vram_bytes": budget.observed_used_vram_bytes,
            "external_vram_bytes": budget.external_vram_bytes,
            "safety_reserve_bytes": budget.safety_reserve_bytes,
            "schedulable_free_vram_bytes": budget.schedulable_free_vram_bytes,
            "attribution_confidence": budget.attribution_confidence,
            "invariant_delta_bytes": budget.invariant_delta_bytes,
            "content_persisted": False,
        }
        self._commit()
        return CoResidencyEvidenceResponse(
            left_deployment_id=left.id,
            left_capability=self._capability_key(left),
            right_deployment_id=right.id,
            right_capability=self._capability_key(right),
            left_alone_bytes=left_required,
            right_alone_bytes=right_required,
            expected_combined_bytes=row.expected_combined_bytes,
            measured_combined_bytes=row.measured_combined_bytes,
            status="PROVEN_SAFE",
            evidence=row.evidence,
            measured_at=row.measured_at,
        )

    def request_history(self, limit: int = 100) -> list[GatewayRequestResponse]:
        client_names = {item.id: item.name for item in self.repo.clients()}
        return [
            GatewayRequestResponse(
                request_id=item.request_id,
                capability=f"{item.capability_key}@{item.capability_version}",
                client=client_names.get(item.service_client_id) if item.service_client_id else None,
                status=item.status,
                cold=item.cold,
                queue_time_ms=item.queue_time_ms,
                load_time_ms=item.load_time_ms,
                inference_time_ms=item.inference_time_ms,
                total_latency_ms=item.total_latency_ms,
                failure_code=item.failure_code,
                latency_breakdown={
                    str(key): float(value)
                    for key, value in item.decision_evidence.get("latency_ms", {}).items()
                    if isinstance(value, int | float)
                },
                created_at=item.created_at,
            )
            for item in self.repo.gateway_requests(limit)
        ]

    def _stable_embedding_deployment(self) -> CapabilityDeployment:
        contract = self.ensure_contract("rag.embedding", 1)
        deployment = self.repo.stable_deployment(contract.id)
        if not deployment:
            raise ServingError(
                503, "CAPABILITY_UNAVAILABLE", "rag.embedding@1 has no stable deployment"
            )
        if deployment.status != "stable" or not deployment.production:
            raise ServingError(503, "CAPABILITY_UNAVAILABLE", "stable deployment is not routable")
        node = self.repo.node(deployment.compute_node_id)
        if (
            not node
            or node.decommissioned_at is not None
            or not node.enabled
            or not node.production_eligible
            or node.liveness_state != "online"
        ):
            raise ServingError(
                503, "NO_ELIGIBLE_NODE", "no production-eligible online node is available"
            )
        return deployment

    def _reranking_deployment(self) -> tuple[CapabilityDeployment, str]:
        contract = self.ensure_contract("rag.reranking", 1)
        routes = [
            route
            for route in self.repo.experiment_routes()
            if route.capability_contract_id == contract.id and route.status == "active"
        ]
        deployments = [
            (self.repo.deployment(route.capability_deployment_id), route.route_key)
            for route in routes
        ]
        routable = [
            (deployment, route_key)
            for deployment, route_key in deployments
            if deployment
            and not deployment.production
            and deployment.channel == "experiment"
            and deployment.status == "candidate"
        ]
        if len(routable) != 1:
            raise ServingError(
                503,
                "RERANKER_NOT_AVAILABLE",
                "rag.reranking@1 requires exactly one active evaluated experiment deployment",
            )
        deployment, route_key = routable[0]
        node = self.repo.node(deployment.compute_node_id)
        if (
            not node
            or node.decommissioned_at is not None
            or not node.enabled
            or node.liveness_state != "online"
        ):
            raise ServingError(503, "NO_ELIGIBLE_NODE", "reranker node is not online")
        return deployment, route_key

    def _experimental_deployment(self, capability_key: str) -> tuple[CapabilityDeployment, str]:
        contract = self.ensure_contract(capability_key, 1)
        routable: list[tuple[CapabilityDeployment, str]] = []
        for route in self.repo.experiment_routes():
            if route.capability_contract_id != contract.id or route.status != "active":
                continue
            deployment = self.repo.deployment(route.capability_deployment_id)
            if (
                deployment
                and not deployment.production
                and deployment.channel == "experiment"
                and deployment.status == "candidate"
            ):
                routable.append((deployment, route.route_key))
        if len(routable) != 1:
            raise ServingError(
                503,
                "CAPABILITY_UNAVAILABLE",
                f"{capability_key}@1 requires exactly one active LAB deployment",
            )
        deployment, route_key = routable[0]
        node = self.repo.node(deployment.compute_node_id)
        if not node or not node.enabled or node.liveness_state != "online":
            raise ServingError(503, "NO_ELIGIBLE_NODE", "capability node is not online")
        return deployment, route_key

    def _validate_inputs(self, request: EmbeddingInvokeRequest) -> list[str]:
        inputs = request.inputs()
        if len(inputs) > self.settings.gateway_max_batch_size:
            raise ServingError(
                422, "INVALID_INPUT", "embedding batch exceeds the configured maximum"
            )
        if any(len(item) > self.settings.gateway_max_input_characters for item in inputs):
            raise ServingError(422, "INVALID_INPUT", "embedding input exceeds the configured limit")
        size = len(json.dumps({"input": inputs}, ensure_ascii=False).encode())
        if size > self.settings.gateway_max_payload_bytes:
            raise ServingError(413, "INVALID_INPUT", "embedding request payload is too large")
        return inputs

    def _validate_reranking_input(self, request: RerankingInvokeRequest) -> list[dict[str, str]]:
        documents = [document.model_dump() for document in request.documents]
        size = len(
            json.dumps(
                {"query": request.query, "documents": documents, "top_n": request.top_n},
                ensure_ascii=False,
            ).encode()
        )
        if size > min(self.settings.gateway_max_payload_bytes, 262_144):
            raise ServingError(413, "RERANK_REQUEST_TOO_LARGE", "reranking payload is too large")
        return documents

    def _placement_evidence(self, deployment: CapabilityDeployment) -> dict[str, Any]:
        candidates = []
        for node in self.session.scalars(
            select(ComputeNode)
            .where(ComputeNode.decommissioned_at.is_(None))
            .order_by(ComputeNode.display_name)
        ):
            reasons: list[str] = []
            if not node.enabled:
                reasons.append("NODE_DISABLED")
            if not node.production_eligible:
                reasons.append("NODE_NOT_PRODUCTION_ELIGIBLE")
            if node.liveness_state != "online":
                reasons.append("NODE_NOT_ONLINE")
            if node.id != deployment.compute_node_id:
                reasons.append("DEPLOYMENT_NODE_CONSTRAINT")
            candidates.append(
                {
                    "node_id": str(node.id),
                    "node": node.display_name or node.hostname,
                    "eligible": not reasons,
                    "reasons": reasons,
                }
            )
        return {"candidate_nodes": candidates, "selected_node_id": str(deployment.compute_node_id)}

    def _expire_leases(self) -> None:
        now = _now()
        for lease in self.repo.expired_leases(now):
            lease.state = "expired"
            lease.released_at = now
            lease.failure_code = "LEASE_TIMEOUT"
            if lease.request_id:
                request = self.session.scalar(
                    select(GatewayRequest).where(GatewayRequest.request_id == lease.request_id)
                )
                if request and request.status in {"queued", "running"}:
                    request.status = "failed"
                    request.failure_code = "LEASE_TIMEOUT"
                    request.finished_at = now

    def _acquire_request_lease(
        self,
        deployment: CapabilityDeployment,
        gateway_request: GatewayRequest,
        client: ServiceClient,
    ) -> ServingGpuLease:
        deadline = time.monotonic() + self.settings.gateway_queue_timeout_seconds
        while time.monotonic() < deadline:
            self.session.expire_all()
            self._expire_leases()
            self.repo.lock_accelerator(deployment.accelerator_id)
            current = self.session.get(GatewayRequest, gateway_request.id)
            residency = self.repo.residency(deployment.id, lock=True)
            envelope = self.repo.envelope(deployment.id)
            if not current or not residency or not envelope:
                raise ServingError(503, "CAPABILITY_UNAVAILABLE", "scheduler state is incomplete")
            if envelope.stale:
                raise ServingError(
                    503,
                    "SCHEDULER_STATE_STALE",
                    "measured resource envelope is stale and must be revalidated",
                    {"stale_reason": envelope.stale_reason},
                )
            active = self.repo.active_leases(deployment.accelerator_id, _now())
            active_for_deployment = [
                item
                for item in active
                if item.capability_deployment_id == deployment.id
                and item.state in {"granted", "active"}
            ]
            if len(active_for_deployment) >= deployment.max_concurrency:
                self.session.commit()
                time.sleep(0.05)
                continue
            resident = residency.measured_resident_vram_bytes if residency.state != "cold" else 0
            plan = self._placement_plan(
                deployment,
                priority=client.workload_priority,
                deadline_remaining_ms=float(self.settings.gateway_request_timeout_seconds * 1000),
                request_id=current.request_id,
                dry_run=False,
                commit=False,
            )
            if plan.verdict == PlacementVerdict.ADMIT_AFTER_EVICTION.value:
                for eviction in plan.evictions:
                    candidate = self.repo.residency(eviction.deployment_id, lock=True)
                    other = self.repo.deployment(eviction.deployment_id)
                    if not candidate or not other or candidate.active_requests > 0:
                        raise ServingError(
                            409, "EVICTION_NOT_ALLOWED", "planned residency is no longer idle"
                        )
                    candidate.state = "unloading"
                    candidate.transition_reason = eviction.reason
                    self.repo.add(
                        SchedulerEviction(
                            capability_deployment_id=other.id,
                            requested_deployment_id=deployment.id,
                            reason_code=eviction.reason,
                            expected_reclaimed_bytes=eviction.expected_reclaimed_bytes,
                            state="queued",
                        )
                    )
                    self._commit()
                    unload = self._enqueue_job(
                        other,
                        "unload",
                        key_suffix=(
                            f"scheduler-{plan.decision_fingerprint[:16]}-{candidate.load_count}"
                        ),
                        priority=client.workload_priority,
                    )
                    self._wait_job(unload.id, deadline)
                continue
            if plan.verdict == PlacementVerdict.QUEUE.value:
                self.session.commit()
                time.sleep(0.05)
                continue
            if plan.verdict != PlacementVerdict.ADMIT.value:
                code = (
                    plan.reason_codes[0] if plan.reason_codes else "INSUFFICIENT_SCHEDULABLE_VRAM"
                )
                self._commit()
                raise ServingError(
                    503,
                    code,
                    "scheduler rejected placement using measured capacity and policy",
                    {
                        "verdict": plan.verdict,
                        "reason_codes": plan.reason_codes,
                        "required_vram_bytes": plan.required_vram_bytes,
                        "schedulable_free_vram_bytes": plan.headroom_before_bytes,
                        "decision_fingerprint": plan.decision_fingerprint,
                    },
                )
            required = plan.required_vram_bytes
            now = _now()
            lease = self.repo.add(
                ServingGpuLease(
                    accelerator_id=deployment.accelerator_id,
                    capability_deployment_id=deployment.id,
                    request_id=current.request_id,
                    reserved_vram_bytes=required,
                    priority=client.workload_priority,
                    state="active",
                    owner=client.name,
                    created_at=now,
                    acquired_at=now,
                    heartbeat_at=now,
                    expires_at=now
                    + timedelta(seconds=self.settings.gateway_request_timeout_seconds),
                    lease_type=("request_execution" if resident else "residency_reservation"),
                    materialized_vram_bytes=0,
                    generation=residency.generation,
                )
            )
            current.status = "running"
            current.decision_evidence = {
                **current.decision_evidence,
                **self._placement_evidence(deployment),
                "placement_plan_id": str(plan.id),
                "decision": plan.verdict,
                "reason_codes": plan.reason_codes,
                "decision_fingerprint": plan.decision_fingerprint,
                "required_vram_bytes": required,
            }
            residency.active_requests += 1
            if residency.state in {"warm", "idle"}:
                residency.state = "busy"
            self._commit()
            return lease
        raise ServingError(504, "REQUEST_TIMEOUT", "request timed out in the bounded queue")

    def _enqueue_job(
        self,
        deployment: CapabilityDeployment,
        operation: str,
        *,
        request: GatewayRequest | None = None,
        payload_reference: str | None = None,
        key_suffix: str,
        priority: str = "production",
    ) -> ServingJob:
        key = _canonical_hash(
            {
                "deployment": str(deployment.id),
                "operation": operation,
                "suffix": key_suffix,
            }
        )
        existing = self.repo.job_by_key(key)
        if existing:
            return existing
        job = self.repo.add(
            ServingJob(
                capability_deployment_id=deployment.id,
                compute_node_id=deployment.compute_node_id,
                gateway_request_id=request.id if request else None,
                operation=operation,
                status="queued",
                priority=priority,
                idempotency_key=key,
                payload_reference=payload_reference,
                attempt_count=0,
                result_summary={},
            )
        )
        self._commit()
        return job

    def _wait_job(self, job_id: uuid.UUID, deadline: float) -> tuple[ServingJob, dict[str, Any]]:
        while time.monotonic() < deadline:
            self.session.expire_all()
            job = self.repo.job(job_id)
            if not job:
                raise ServingError(503, "CAPABILITY_UNAVAILABLE", "serving job disappeared")
            if job.status == "completed":
                result = (
                    self.payload_store.get(job.result_reference) if job.result_reference else {}
                )
                return job, result or {}
            if job.status == "failed":
                raise ServingError(
                    503,
                    job.failure_code or "INFERENCE_FAILED",
                    job.failure_message or "serving worker failed",
                )
            time.sleep(0.05)
        raise ServingError(504, "REQUEST_TIMEOUT", "serving worker response timed out")

    def _ensure_loaded(
        self, deployment: CapabilityDeployment, deadline: float, *, priority: str
    ) -> tuple[bool, float]:
        self._switch_resident_deployment(deployment, deadline, priority=priority)
        self.session.expire_all()
        residency = self.repo.residency(deployment.id, lock=True)
        if not residency:
            raise ServingError(503, "CAPABILITY_UNAVAILABLE", "residency state is missing")
        if residency.state in {"warm", "busy", "idle"}:
            self.session.commit()
            return False, 0.0
        job: ServingJob | None = None
        if residency.state == "loading":
            job = self.session.scalar(
                select(ServingJob)
                .where(
                    ServingJob.capability_deployment_id == deployment.id,
                    ServingJob.operation == "load",
                    ServingJob.status.in_(("queued", "running")),
                )
                .order_by(ServingJob.created_at.desc())
                .limit(1)
            )
            self.session.commit()
        else:
            residency.state = "loading"
            residency.transition_started_at = _now()
            residency.failure_code = None
            residency.failure_message = None
            residency.generation += 1
            residency.transition_reason = "LOAD_COMPLETED"
            for active_lease in self.repo.active_leases(deployment.accelerator_id, _now()):
                if active_lease.capability_deployment_id == deployment.id:
                    active_lease.materialized_vram_bytes = min(
                        active_lease.reserved_vram_bytes,
                        residency.measured_resident_vram_bytes,
                    )
            load_number = residency.load_count + 1
            self._commit()
            job = self._enqueue_job(
                deployment,
                "load",
                key_suffix=f"load-{load_number}",
                priority=priority,
            )
        if not job:
            raise ServingError(503, "RESIDENCY_LOAD_FAILED", "loading state has no worker job")
        completed, _ = self._wait_job(job.id, deadline)
        return True, float(completed.result_summary.get("load_time_ms", 0.0))

    def _switch_resident_deployment(
        self, deployment: CapabilityDeployment, deadline: float, *, priority: str
    ) -> None:
        # Admission and any typed idle evictions are decided under the
        # accelerator row lock. Other proven residents intentionally remain.
        del deployment, deadline, priority

    @staticmethod
    def _validate_vectors(
        vectors: Any, expected_count: int, expected_dimension: int = 1024
    ) -> list[list[float]]:
        if not isinstance(vectors, list) or len(vectors) != expected_count:
            raise ServingError(502, "INVALID_OUTPUT", "runtime returned an invalid embedding batch")
        validated: list[list[float]] = []
        for vector in vectors:
            if not isinstance(vector, list) or len(vector) != expected_dimension:
                raise ServingError(
                    502, "INVALID_OUTPUT", "runtime returned the wrong vector dimension"
                )
            numeric = [float(value) for value in vector]
            if not all(math.isfinite(value) for value in numeric):
                raise ServingError(502, "INVALID_OUTPUT", "runtime returned non-finite values")
            validated.append(numeric)
        return validated

    @staticmethod
    def _validate_reranking_results(
        results: Any, *, expected_ids: set[str] | None = None, expected_top_n: int | None = None
    ) -> list[dict[str, Any]]:
        if not isinstance(results, list) or not results or len(results) > 10:
            raise ServingError(502, "INVALID_OUTPUT", "runtime returned an invalid ranking")
        if expected_top_n is not None and len(results) != expected_top_n:
            raise ServingError(502, "INVALID_OUTPUT", "runtime returned the wrong ranking length")
        validated: list[dict[str, Any]] = []
        seen: set[str] = set()
        previous_score = math.inf
        for expected_rank, item in enumerate(results, 1):
            if not isinstance(item, dict):
                raise ServingError(502, "INVALID_OUTPUT", "reranker result is malformed")
            identifier = str(item.get("id", ""))
            score = float(item.get("score", math.nan))
            rank = int(item.get("rank", 0))
            if (
                not identifier
                or identifier in seen
                or not math.isfinite(score)
                or rank != expected_rank
                or score > previous_score
                or (expected_ids is not None and identifier not in expected_ids)
            ):
                raise ServingError(502, "INVALID_OUTPUT", "reranker output contract was violated")
            seen.add(identifier)
            previous_score = score
            validated.append({"id": identifier, "score": score, "rank": rank})
        return validated

    def invoke(
        self,
        request: EmbeddingInvokeRequest,
        client: ServiceClient,
        *,
        deployment: CapabilityDeployment | None = None,
        experiment_route: str | None = None,
    ) -> EmbeddingInvokeResponse:
        started = time.perf_counter()
        validation_started = time.perf_counter()
        inputs = self._validate_inputs(request)
        validation_ms = (time.perf_counter() - validation_started) * 1000
        resolution_started = time.perf_counter()
        deployment = deployment or self._stable_embedding_deployment()
        if experiment_route and (
            deployment.production
            or deployment.channel != "experiment"
            or deployment.status != "candidate"
        ):
            raise ServingError(409, "EXPERIMENT_NOT_ROUTABLE", "experiment route is not active")
        resolution_ms = (time.perf_counter() - resolution_started) * 1000
        if self.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
            raise ServingError(429, "QUEUE_FULL", "capability queue is full")
        if self.repo.active_gateway_requests() >= self.settings.scheduler_global_queue_limit:
            raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
        request_id = uuid.uuid4()
        input_hash = _canonical_hash(inputs)
        record = self.repo.add(
            GatewayRequest(
                request_id=request_id,
                service_client_id=client.id,
                capability_key="rag.embedding",
                capability_version=1,
                capability_deployment_id=deployment.id,
                compute_node_id=deployment.compute_node_id,
                status="queued",
                priority=client.workload_priority,
                input_sha256=input_hash,
                input_count=len(inputs),
                decision_evidence={
                    "input_persisted": False,
                    "input_type": request.input_type,
                    "experiment_route": experiment_route,
                    "shape": {
                        "bucket": _shape_bucket(sum(len(item.encode()) for item in inputs)),
                        "item_count": len(inputs),
                        "maximum_characters": max(map(len, inputs), default=0),
                    },
                },
            )
        )
        self._commit()
        deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
        lease: ServingGpuLease | None = None
        result_reference: str | None = None
        try:
            scheduling_started = time.perf_counter()
            lease = self._acquire_request_lease(deployment, record, client)
            scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
            cold, load_ms = self._ensure_loaded(
                deployment, deadline, priority=client.workload_priority
            )
            payload_started = time.perf_counter()
            payload_reference = self.payload_store.put(
                {"input": inputs, "input_type": request.input_type},
                self.settings.serving_payload_ttl_seconds,
            )
            job = self._enqueue_job(
                deployment,
                "invoke",
                request=record,
                payload_reference=payload_reference,
                key_suffix=f"request-{request_id}",
                priority=client.workload_priority,
            )
            payload_ms = (time.perf_counter() - payload_started) * 1000
            wait_started = time.perf_counter()
            completed, result = self._wait_job(job.id, deadline)
            wait_ms = (time.perf_counter() - wait_started) * 1000
            result_reference = completed.result_reference
            result_started = time.perf_counter()
            space = self.repo.embedding_space(deployment.embedding_space_id)
            if not space:
                raise ServingError(503, "CAPABILITY_UNAVAILABLE", "embedding space disappeared")
            vectors = self._validate_vectors(result.get("vectors"), len(inputs), space.dimension)
            inference_ms = float(completed.result_summary.get("inference_time_ms", 0.0))
            worker_preprocess_ms = float(completed.result_summary.get("preprocess_time_ms", 0.0))
            worker_serialize_ms = float(completed.result_summary.get("serialization_time_ms", 0.0))
            worker_total_ms = float(
                completed.result_summary.get("worker_total_time_ms", inference_ms)
            )
            dispatch_ms = 0.0
            if completed.started_at:
                dispatch_ms = max(
                    0.0,
                    (_aware(completed.started_at) - _aware(completed.created_at)).total_seconds()
                    * 1000,
                )
            completion_transport_ms = max(0.0, wait_ms - dispatch_ms - worker_total_ms)
            total_ms = (time.perf_counter() - started) * 1000
            self.session.expire_all()
            current = self.session.get(GatewayRequest, record.id)
            residency = self.repo.residency(deployment.id, lock=True)
            if not current or not residency:
                raise ServingError(
                    503, "CAPABILITY_UNAVAILABLE", "request completion state vanished"
                )
            current.status = "completed"
            current.cold = cold
            current.queue_time_ms = scheduling_ms
            current.load_time_ms = load_ms
            current.inference_time_ms = inference_ms
            current.total_latency_ms = total_ms
            current.finished_at = _now()
            residency.state = "warm"
            residency.active_requests = max(0, residency.active_requests - 1)
            residency.last_used_at = _now()
            lease.state = "released"
            lease.released_at = _now()
            result_ms = (time.perf_counter() - result_started) * 1000
            accounted_ms = (
                validation_ms
                + resolution_ms
                + scheduling_ms
                + load_ms
                + payload_ms
                + dispatch_ms
                + worker_total_ms
                + completion_transport_ms
                + result_ms
            )
            gateway_ms = max(
                0.0,
                validation_ms
                + resolution_ms
                + payload_ms
                + completion_transport_ms
                + result_ms
                + max(0.0, total_ms - accounted_ms),
            )
            latency_ms = {
                "validation": validation_ms,
                "resolution": resolution_ms,
                "scheduling": scheduling_ms,
                "load": load_ms,
                "payload": payload_ms,
                "dispatch": dispatch_ms,
                "worker_preprocess": worker_preprocess_ms,
                "inference": inference_ms,
                "worker_serialize": worker_serialize_ms,
                "worker_total": worker_total_ms,
                "completion_transport": completion_transport_ms,
                "result": result_ms,
                "gateway": gateway_ms,
                "total": total_ms,
            }
            current.decision_evidence = {
                **current.decision_evidence,
                "latency_clock": "process-local-monotonic; control-plane timestamps for dispatch",
                "latency_ms": latency_ms,
            }
            self._commit()
            selected_node = self.repo.node(deployment.compute_node_id)
            if not selected_node:
                raise ServingError(503, "NO_ELIGIBLE_NODE", "selected node disappeared")
            return EmbeddingInvokeResponse(
                dimension=space.dimension,
                embedding_space_id=space.id,
                data=vectors,
                request_id=request_id,
                execution=GatewayExecution(
                    cold=cold,
                    node=selected_node.display_name or selected_node.hostname,
                    residency="warm",
                    load_count=residency.load_count,
                    timings=GatewayTiming(
                        validation_ms=validation_ms,
                        resolution_ms=resolution_ms,
                        scheduling_ms=scheduling_ms,
                        payload_ms=payload_ms,
                        dispatch_ms=dispatch_ms,
                        worker_preprocess_ms=worker_preprocess_ms,
                        worker_serialize_ms=worker_serialize_ms,
                        worker_total_ms=worker_total_ms,
                        completion_transport_ms=completion_transport_ms,
                        result_ms=result_ms,
                        gateway_ms=gateway_ms,
                        queue_ms=scheduling_ms,
                        load_ms=load_ms,
                        inference_ms=inference_ms,
                        total_ms=total_ms,
                    ),
                ),
                usage=GatewayUsage(
                    input_count=len(inputs),
                    input_tokens=int(result.get("prompt_tokens", 0)),
                ),
            )
        except ServingError as exc:
            self.session.rollback()
            self.session.expire_all()
            current = self.session.get(GatewayRequest, record.id)
            residency = self.repo.residency(deployment.id, lock=True)
            active_lease = self.repo.lease_for_request(request_id)
            if current and current.status != "completed":
                current.status = "failed"
                current.failure_code = exc.code
                current.total_latency_ms = (time.perf_counter() - started) * 1000
                current.finished_at = _now()
            if residency and residency.active_requests:
                residency.active_requests -= 1
                if residency.state == "busy":
                    residency.state = "warm"
            if active_lease and active_lease.state not in {"released", "expired", "failed"}:
                active_lease.state = "failed"
                active_lease.failure_code = exc.code
                active_lease.released_at = _now()
            self._commit()
            raise
        finally:
            if result_reference:
                self.payload_store.delete(result_reference)

    def invoke_reranking(
        self, request: RerankingInvokeRequest, client: ServiceClient
    ) -> RerankingInvokeResponse:
        started = time.perf_counter()
        validation_started = time.perf_counter()
        documents = self._validate_reranking_input(request)
        validation_ms = (time.perf_counter() - validation_started) * 1000
        resolution_started = time.perf_counter()
        deployment, experiment_route = self._reranking_deployment()
        resolution_ms = (time.perf_counter() - resolution_started) * 1000
        if self.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
            raise ServingError(429, "QUEUE_FULL", "capability queue is full")
        if self.repo.active_gateway_requests() >= self.settings.scheduler_global_queue_limit:
            raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
        request_id = uuid.uuid4()
        content_facts = {
            "query_sha256": hashlib.sha256(request.query.encode()).hexdigest(),
            "documents": [
                {
                    "id": document["id"],
                    "content_sha256": hashlib.sha256(document["text"].encode()).hexdigest(),
                }
                for document in documents
            ],
            "top_n": request.top_n,
        }
        record = self.repo.add(
            GatewayRequest(
                request_id=request_id,
                service_client_id=client.id,
                capability_key="rag.reranking",
                capability_version=1,
                capability_deployment_id=deployment.id,
                compute_node_id=deployment.compute_node_id,
                status="queued",
                priority=client.workload_priority,
                input_sha256=_canonical_hash(content_facts),
                input_count=len(documents),
                decision_evidence={
                    "input_persisted": False,
                    "content_logged": False,
                    "document_count": len(documents),
                    "top_n": request.top_n,
                    "experiment_route": experiment_route,
                    "shape": {
                        "bucket": _shape_bucket(
                            len(request.query.encode())
                            + sum(len(document["text"].encode()) for document in documents)
                        ),
                        "document_count": len(documents),
                        "top_n": request.top_n,
                    },
                },
            )
        )
        self._commit()
        deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
        lease: ServingGpuLease | None = None
        input_reference: str | None = None
        result_reference: str | None = None
        try:
            scheduling_started = time.perf_counter()
            lease = self._acquire_request_lease(deployment, record, client)
            scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
            cold, load_ms = self._ensure_loaded(
                deployment, deadline, priority=client.workload_priority
            )
            payload_started = time.perf_counter()
            input_reference = self.payload_store.put(
                {"query": request.query, "documents": documents, "top_n": request.top_n},
                self.settings.serving_payload_ttl_seconds,
            )
            job = self._enqueue_job(
                deployment,
                "invoke",
                request=record,
                payload_reference=input_reference,
                key_suffix=f"request-{request_id}",
                priority=client.workload_priority,
            )
            payload_ms = (time.perf_counter() - payload_started) * 1000
            wait_started = time.perf_counter()
            completed, result = self._wait_job(job.id, deadline)
            wait_ms = (time.perf_counter() - wait_started) * 1000
            result_reference = completed.result_reference
            result_started = time.perf_counter()
            ranking = self._validate_reranking_results(
                result.get("results"),
                expected_ids={document["id"] for document in documents},
                expected_top_n=request.top_n,
            )
            inference_ms = float(completed.result_summary.get("inference_time_ms", 0.0))
            worker_preprocess_ms = float(completed.result_summary.get("preprocess_time_ms", 0.0))
            worker_serialize_ms = float(completed.result_summary.get("serialization_time_ms", 0.0))
            worker_total_ms = float(
                completed.result_summary.get("worker_total_time_ms", inference_ms)
            )
            dispatch_ms = 0.0
            if completed.started_at:
                dispatch_ms = max(
                    0.0,
                    (_aware(completed.started_at) - _aware(completed.created_at)).total_seconds()
                    * 1000,
                )
            completion_transport_ms = max(0.0, wait_ms - dispatch_ms - worker_total_ms)
            total_ms = (time.perf_counter() - started) * 1000
            self.session.expire_all()
            current = self.session.get(GatewayRequest, record.id)
            residency = self.repo.residency(deployment.id, lock=True)
            if not current or not residency:
                raise ServingError(503, "CAPABILITY_UNAVAILABLE", "completion state vanished")
            current.status = "completed"
            current.cold = cold
            current.queue_time_ms = scheduling_ms
            current.load_time_ms = load_ms
            current.inference_time_ms = inference_ms
            current.total_latency_ms = total_ms
            current.finished_at = _now()
            residency.state = "warm"
            residency.active_requests = max(0, residency.active_requests - 1)
            residency.last_used_at = _now()
            lease.state = "released"
            lease.released_at = _now()
            result_ms = (time.perf_counter() - result_started) * 1000
            accounted_ms = (
                validation_ms
                + resolution_ms
                + scheduling_ms
                + load_ms
                + payload_ms
                + dispatch_ms
                + worker_total_ms
                + completion_transport_ms
                + result_ms
            )
            gateway_ms = max(0.0, total_ms - scheduling_ms - load_ms - worker_total_ms)
            latency_ms = {
                "validation": validation_ms,
                "resolution": resolution_ms,
                "scheduling": scheduling_ms,
                "load": load_ms,
                "payload": payload_ms,
                "dispatch": dispatch_ms,
                "worker_preprocess": worker_preprocess_ms,
                "inference": inference_ms,
                "worker_serialize": worker_serialize_ms,
                "worker_total": worker_total_ms,
                "completion_transport": completion_transport_ms,
                "result": result_ms,
                "gateway": gateway_ms + max(0.0, total_ms - accounted_ms),
                "total": total_ms,
            }
            current.decision_evidence = {
                **current.decision_evidence,
                "latency_clock": "process-local-monotonic; control-plane timestamps for dispatch",
                "latency_ms": latency_ms,
            }
            self._commit()
            selected_node = self.repo.node(deployment.compute_node_id)
            if not selected_node:
                raise ServingError(503, "NO_ELIGIBLE_NODE", "selected node disappeared")
            return RerankingInvokeResponse(
                results=[RerankingResult.model_validate(item) for item in ranking],
                request_id=request_id,
                execution=GatewayExecution(
                    cold=cold,
                    node=selected_node.display_name or selected_node.hostname,
                    residency="warm",
                    load_count=residency.load_count,
                    timings=GatewayTiming(
                        validation_ms=validation_ms,
                        resolution_ms=resolution_ms,
                        scheduling_ms=scheduling_ms,
                        payload_ms=payload_ms,
                        dispatch_ms=dispatch_ms,
                        worker_preprocess_ms=worker_preprocess_ms,
                        worker_serialize_ms=worker_serialize_ms,
                        worker_total_ms=worker_total_ms,
                        completion_transport_ms=completion_transport_ms,
                        result_ms=result_ms,
                        gateway_ms=latency_ms["gateway"],
                        queue_ms=scheduling_ms,
                        load_ms=load_ms,
                        inference_ms=inference_ms,
                        total_ms=total_ms,
                    ),
                ),
                usage=GatewayUsage(
                    input_count=len(documents),
                    input_tokens=int(result.get("prompt_tokens", 0)),
                ),
            )
        except ServingError as exc:
            self.session.rollback()
            self.session.expire_all()
            current = self.session.get(GatewayRequest, record.id)
            residency = self.repo.residency(deployment.id, lock=True)
            active_lease = self.repo.lease_for_request(request_id)
            if current and current.status != "completed":
                current.status = "failed"
                current.failure_code = exc.code
                current.total_latency_ms = (time.perf_counter() - started) * 1000
                current.finished_at = _now()
            if residency and residency.active_requests:
                residency.active_requests -= 1
                if residency.state == "busy":
                    residency.state = "warm"
            if active_lease and active_lease.state not in {"released", "expired", "failed"}:
                active_lease.state = "failed"
                active_lease.failure_code = exc.code
                active_lease.released_at = _now()
            self._commit()
            raise
        finally:
            if input_reference:
                self.payload_store.delete(input_reference)
            if result_reference:
                self.payload_store.delete(result_reference)

    def _validate_modality_output(
        self,
        capability_key: str,
        result: dict[str, Any],
        deployment: CapabilityDeployment,
        expected_count: int,
    ) -> dict[str, Any]:
        if capability_key == "document.ocr":
            text = result.get("text")
            pages = result.get("pages")
            if not isinstance(text, str) or len(text) > 1_000_000:
                raise ServingError(502, "INVALID_OUTPUT", "OCR text output is invalid")
            if not isinstance(pages, list) or len(pages) != 1 or not isinstance(pages[0], dict):
                raise ServingError(502, "INVALID_OUTPUT", "OCR page output is invalid")
            width = int(pages[0].get("width", 0))
            height = int(pages[0].get("height", 0))
            if not 1 <= width <= 4096 or not 1 <= height <= 4096:
                raise ServingError(502, "INVALID_OUTPUT", "OCR dimensions violate the contract")
            return {
                "text": text,
                "pages": pages,
                "confidence": result.get("confidence"),
            }
        if capability_key == "vision.embedding":
            space = self.repo.embedding_space(deployment.embedding_space_id)
            if not space:
                raise ServingError(
                    503, "CAPABILITY_UNAVAILABLE", "visual embedding space is missing"
                )
            vectors = self._validate_vectors(result.get("vectors"), expected_count, space.dimension)
            if result.get("normalized") is not True:
                raise ServingError(502, "INVALID_OUTPUT", "visual vectors are not normalized")
            return {
                "vectors": vectors,
                "dimension": space.dimension,
                "normalized": True,
                "embedding_space_id": str(space.id),
            }
        if capability_key == "speech.transcription":
            text = result.get("text")
            language = result.get("language")
            duration = float(result.get("duration_seconds", 0.0))
            segments = result.get("segments", [])
            if (
                not isinstance(text, str)
                or not isinstance(language, str)
                or not 0 < duration <= 120
                or not isinstance(segments, list)
            ):
                raise ServingError(502, "INVALID_OUTPUT", "transcription output is invalid")
            return {
                "text": text,
                "language": language,
                "duration_seconds": duration,
                "segments": segments,
            }
        raise ServingError(409, "CAPABILITY_UNAVAILABLE", "unsupported modality capability")

    def invoke_modality(
        self,
        capability_key: str,
        payload: dict[str, Any],
        client: ServiceClient,
    ) -> tuple[dict[str, Any], uuid.UUID, GatewayExecution]:
        started = time.perf_counter()
        resolution_started = time.perf_counter()
        deployment, experiment_route = self._experimental_deployment(capability_key)
        resolution_ms = (time.perf_counter() - resolution_started) * 1000
        if self.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
            raise ServingError(429, "QUEUE_FULL", "capability queue is full")
        if self.repo.active_gateway_requests() >= self.settings.scheduler_global_queue_limit:
            raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
        request_id = uuid.uuid4()
        input_count = len(payload.get("items", [])) if capability_key == "vision.embedding" else 1
        encoded_lengths = [
            len(value)
            for key, value in payload.items()
            if key.endswith("_base64") and isinstance(value, str)
        ]
        for item in payload.get("items", []):
            if isinstance(item, dict):
                encoded_lengths.extend(
                    len(value)
                    for key, value in item.items()
                    if key.endswith("_base64") and isinstance(value, str)
                )
        approximate_bytes = sum(encoded_lengths) * 3 // 4
        record = self.repo.add(
            GatewayRequest(
                request_id=request_id,
                service_client_id=client.id,
                capability_key=capability_key,
                capability_version=1,
                capability_deployment_id=deployment.id,
                compute_node_id=deployment.compute_node_id,
                status="queued",
                priority=client.workload_priority,
                input_sha256=_canonical_hash(payload),
                input_count=input_count,
                decision_evidence={
                    "input_persisted": False,
                    "content_logged": False,
                    "experiment_route": experiment_route,
                    "modality": capability_key.split(".", 1)[0],
                    "shape": {
                        "bucket": _shape_bucket(approximate_bytes),
                        "encoded_payload_bytes_approximate": approximate_bytes,
                        "item_count": input_count,
                    },
                },
            )
        )
        self._commit()
        deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
        lease: ServingGpuLease | None = None
        input_reference: str | None = None
        result_reference: str | None = None
        try:
            scheduling_started = time.perf_counter()
            lease = self._acquire_request_lease(deployment, record, client)
            scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
            cold, load_ms = self._ensure_loaded(
                deployment, deadline, priority=client.workload_priority
            )
            payload_started = time.perf_counter()
            input_reference = self.payload_store.put(
                payload, self.settings.serving_payload_ttl_seconds
            )
            job = self._enqueue_job(
                deployment,
                "invoke",
                request=record,
                payload_reference=input_reference,
                key_suffix=f"request-{request_id}",
                priority=client.workload_priority,
            )
            payload_ms = (time.perf_counter() - payload_started) * 1000
            wait_started = time.perf_counter()
            completed, raw_result = self._wait_job(job.id, deadline)
            wait_ms = (time.perf_counter() - wait_started) * 1000
            result_reference = completed.result_reference
            result_started = time.perf_counter()
            result = self._validate_modality_output(
                capability_key, raw_result, deployment, input_count
            )
            result_ms = (time.perf_counter() - result_started) * 1000
            inference_ms = float(completed.result_summary.get("inference_time_ms", 0.0))
            preprocess_ms = float(completed.result_summary.get("preprocess_time_ms", 0.0))
            serialization_ms = float(completed.result_summary.get("serialization_time_ms", 0.0))
            worker_total_ms = float(
                completed.result_summary.get("worker_total_time_ms", inference_ms)
            )
            dispatch_ms = 0.0
            if completed.started_at:
                dispatch_ms = max(
                    0.0,
                    (_aware(completed.started_at) - _aware(completed.created_at)).total_seconds()
                    * 1000,
                )
            completion_transport_ms = max(0.0, wait_ms - dispatch_ms - worker_total_ms)
            total_ms = (time.perf_counter() - started) * 1000
            self.session.expire_all()
            current = self.session.get(GatewayRequest, record.id)
            residency = self.repo.residency(deployment.id, lock=True)
            if not current or not residency:
                raise ServingError(503, "CAPABILITY_UNAVAILABLE", "completion state vanished")
            current.status = "completed"
            current.cold = cold
            current.queue_time_ms = scheduling_ms
            current.load_time_ms = load_ms
            current.inference_time_ms = inference_ms
            current.total_latency_ms = total_ms
            current.finished_at = _now()
            current.decision_evidence = {
                **current.decision_evidence,
                "latency_ms": {
                    "resolution": resolution_ms,
                    "scheduling": scheduling_ms,
                    "payload": payload_ms,
                    "dispatch": dispatch_ms,
                    "worker_preprocess": preprocess_ms,
                    "worker_serialize": serialization_ms,
                    "worker_total": worker_total_ms,
                    "completion_transport": completion_transport_ms,
                    "result": result_ms,
                    "total": total_ms,
                },
            }
            residency.state = "warm"
            residency.active_requests = max(0, residency.active_requests - 1)
            residency.last_used_at = _now()
            lease.state = "released"
            lease.released_at = _now()
            self._commit()
            gateway_ms = max(
                0.0,
                resolution_ms + payload_ms + completion_transport_ms + result_ms,
            )
            return (
                result,
                request_id,
                GatewayExecution(
                    cold=cold,
                    node=str(deployment.compute_node_id),
                    residency=residency.state,
                    load_count=residency.load_count,
                    timings=GatewayTiming(
                        validation_ms=0.0,
                        resolution_ms=resolution_ms,
                        scheduling_ms=scheduling_ms,
                        payload_ms=payload_ms,
                        dispatch_ms=dispatch_ms,
                        worker_preprocess_ms=preprocess_ms,
                        worker_serialize_ms=serialization_ms,
                        worker_total_ms=worker_total_ms,
                        completion_transport_ms=completion_transport_ms,
                        result_ms=result_ms,
                        gateway_ms=gateway_ms,
                        queue_ms=scheduling_ms,
                        load_ms=load_ms,
                        inference_ms=inference_ms,
                        total_ms=total_ms,
                    ),
                ),
            )
        except ServingError as exc:
            self.session.rollback()
            self.session.expire_all()
            current = self.session.get(GatewayRequest, record.id)
            residency = self.repo.residency(deployment.id, lock=True)
            active_lease = self.repo.lease_for_request(request_id)
            if current and current.status != "completed":
                current.status = "failed"
                current.failure_code = exc.code
                current.total_latency_ms = (time.perf_counter() - started) * 1000
                current.finished_at = _now()
            if residency and residency.active_requests:
                residency.active_requests -= 1
                if residency.state == "busy":
                    residency.state = "warm"
            if active_lease and active_lease.state not in {"released", "expired", "failed"}:
                active_lease.state = "failed"
                active_lease.failure_code = exc.code
                active_lease.released_at = _now()
            self._commit()
            raise
        finally:
            if input_reference:
                self.payload_store.delete(input_reference)
            if result_reference:
                self.payload_store.delete(result_reference)

    def _job_manifest(self, deployment: CapabilityDeployment) -> dict[str, Any]:
        artifacts = self.repo.set_artifacts(deployment.artifact_set_id)
        locations = self.repo.artifact_locations([item.id for item in artifacts])
        artifact_set = self.repo.artifact_set(deployment.artifact_set_id)
        revision = self.repo.revision(artifact_set.revision_id) if artifact_set else None
        if not artifact_set or not revision:
            raise ServingError(409, "ARTIFACT_INCOMPLETE", "ArtifactSet revision is missing")
        manifest_rows = _selected_manifest_rows(
            artifact_set.selected_paths,
            [(None, artifact) for artifact in artifacts],
            locations,
        )
        if len(manifest_rows) != len(artifact_set.selected_paths):
            raise ServingError(
                409, "ARTIFACT_INCOMPLETE", "one verified location per selected path is required"
            )
        roots = {location.storage_root_id for _, location, _ in manifest_rows}
        if len(roots) != 1:
            raise ServingError(409, "ARTIFACT_PATH_INVALID", "ArtifactSet spans storage roots")
        root = self.repo.storage_root(roots.pop())
        if not root or root.compute_node_id != deployment.compute_node_id:
            raise ServingError(409, "ARTIFACT_PATH_INVALID", "ArtifactSet is on the wrong node")
        return {
            "revision_sha": revision.resolved_commit_sha,
            "relative_parent": _safe_parent(
                [location.relative_path for _, location, _ in manifest_rows]
            ),
            "files": [
                {
                    "path": selected_path,
                    "sha256": artifact.sha256,
                    "size_bytes": artifact.size_bytes,
                }
                for selected_path, _, artifact in manifest_rows
            ],
        }

    def claim_next(
        self, node: ComputeNode, *, wait_seconds: float = 0.0
    ) -> AgentServingJobLease | None:
        deadline = time.monotonic() + min(max(wait_seconds, 0.0), 5.0)
        job = self.repo.claimable_job(node.id, _now())
        while not job and time.monotonic() < deadline:
            # Bounded server-side wait keeps the outbound-only worker protocol
            # while removing the fixed idle-poll latency from gateway requests.
            self.session.expire_all()
            time.sleep(min(0.025, max(0.0, deadline - time.monotonic())))
            job = self.repo.claimable_job(node.id, _now())
        if not job:
            return None
        deployment = self.repo.deployment(job.capability_deployment_id)
        if not deployment or deployment.compute_node_id != node.id:
            raise ServingError(
                409, "NODE_OWNERSHIP_MISMATCH", "serving job belongs to another node"
            )
        profile = self.repo.profile(deployment.runtime_profile_id)
        if not profile or not profile.runtime_environment_id:
            raise ServingError(409, "RUNTIME_DEPENDENCY_MISSING", "runtime profile is incomplete")
        environment = self.repo.environment(profile.runtime_environment_id)
        if not environment:
            raise ServingError(409, "RUNTIME_DEPENDENCY_MISSING", "runtime environment is missing")
        manifest = self._job_manifest(deployment)
        contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
        capability = self.session.get(Capability, contract.capability_id) if contract else None
        supported_capabilities = {
            "rag.embedding",
            "rag.reranking",
            "document.ocr",
            "vision.embedding",
            "speech.transcription",
        }
        if not capability or capability.key not in supported_capabilities:
            raise ServingError(409, "CAPABILITY_UNAVAILABLE", "serving capability is unsupported")
        input_value: list[str] | None = None
        input_type = "raw"
        rerank_query: str | None = None
        rerank_documents: list[dict[str, str]] | None = None
        top_n: int | None = None
        modality_payload: dict[str, Any] | None = None
        if job.operation == "invoke":
            transient = (
                self.payload_store.get(job.payload_reference) if job.payload_reference else None
            )
            valid_payload = bool(
                transient
                and (
                    (capability.key == "rag.embedding" and isinstance(transient.get("input"), list))
                    or (
                        capability.key == "rag.reranking"
                        and isinstance(transient.get("query"), str)
                        and isinstance(transient.get("documents"), list)
                        and isinstance(transient.get("top_n"), int)
                    )
                    or (
                        capability.key
                        in {
                            "document.ocr",
                            "vision.embedding",
                            "speech.transcription",
                        }
                        and isinstance(transient, dict)
                    )
                )
            )
            if not valid_payload or transient is None:
                job.status = "failed"
                job.failure_code = "REQUEST_TIMEOUT"
                job.failure_message = "transient request payload expired before claim"
                job.finished_at = _now()
                self._commit()
                return None
            if capability.key == "rag.embedding":
                input_value = [str(item) for item in transient["input"]]
                input_type = str(transient.get("input_type", "raw"))
            elif capability.key == "rag.reranking":
                rerank_query = str(transient["query"])
                rerank_documents = [
                    {"id": str(item["id"]), "text": str(item["text"])}
                    for item in transient["documents"]
                    if isinstance(item, dict) and "id" in item and "text" in item
                ]
                top_n = int(transient["top_n"])
            else:
                modality_payload = transient
        token = secrets.token_urlsafe(32)
        now = _now()
        job.status = "running"
        job.attempt_count += 1
        job.lease_token_hash = hashlib.sha256(token.encode()).hexdigest()
        job.lease_expires_at = now + timedelta(seconds=self.settings.serving_job_lease_seconds)
        job.started_at = job.started_at or now
        self._commit()
        runtime = RuntimeService(self.session, self.settings)
        gateway_request = (
            self.session.get(GatewayRequest, job.gateway_request_id)
            if job.gateway_request_id
            else None
        )
        return AgentServingJobLease(
            job_id=job.id,
            lease_token=token,
            lease_expires_at=job.lease_expires_at,
            operation=cast(Any, job.operation),
            deployment_id=deployment.id,
            artifact_set_id=deployment.artifact_set_id,
            revision_sha=manifest["revision_sha"],
            artifact_root=self.settings.runtime_artifact_root,
            artifact_relative_path=manifest["relative_parent"],
            expected_manifest={
                "artifact_set_id": str(deployment.artifact_set_id),
                "revision_sha": manifest["revision_sha"],
                "files": manifest["files"],
            },
            runtime_profile=runtime.profile_response(profile).model_dump(mode="json"),
            runtime_environment=runtime.environment_response(environment).model_dump(mode="json"),
            capability=cast(Any, capability.key),
            embedding_space_id=deployment.embedding_space_id,
            expected_dimension=(
                space.dimension
                if (space := self.repo.embedding_space(deployment.embedding_space_id))
                else None
            ),
            normalize=True if capability.key == "rag.embedding" else None,
            input=input_value,
            input_type=cast(Any, input_type),
            rerank_query=rerank_query,
            rerank_documents=(
                [RerankDocument.model_validate(item) for item in rerank_documents]
                if rerank_documents is not None
                else None
            ),
            top_n=top_n,
            modality_payload=modality_payload,
            request_id=gateway_request.request_id if gateway_request else None,
        )

    @staticmethod
    def _assert_job_lease(job: ServingJob, node: ComputeNode, token: str) -> None:
        if job.compute_node_id != node.id:
            raise ServingError(403, "NODE_OWNERSHIP_MISMATCH", "job belongs to another node")
        digest = hashlib.sha256(token.encode()).hexdigest()
        if not job.lease_token_hash or not hmac.compare_digest(digest, job.lease_token_hash):
            raise ServingError(409, "LEASE_TIMEOUT", "serving job lease is invalid")
        if not job.lease_expires_at or _aware(job.lease_expires_at) <= _now():
            raise ServingError(409, "LEASE_TIMEOUT", "serving job lease expired")

    def complete_job(
        self,
        job_id: uuid.UUID,
        node: ComputeNode,
        request: AgentServingJobComplete,
    ) -> ServingJob:
        job = self.repo.job(job_id)
        if not job:
            raise ServingError(404, "SERVING_JOB_NOT_FOUND", "serving job was not found")
        self._assert_job_lease(job, node, request.lease_token)
        deployment = self.repo.deployment(job.capability_deployment_id)
        residency = self.repo.residency(job.capability_deployment_id, lock=True)
        if not deployment or not residency:
            raise ServingError(409, "CAPABILITY_UNAVAILABLE", "deployment residency is missing")
        contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
        capability = self.session.get(Capability, contract.capability_id) if contract else None
        if not capability:
            raise ServingError(409, "CAPABILITY_UNAVAILABLE", "deployment contract is missing")
        result = request.result
        metrics = request.metrics
        if job.operation in {"invoke", "health"}:
            transient_result: dict[str, Any]
            summary: dict[str, Any]
            if capability.key == "rag.embedding":
                vectors = result.get("vectors")
                expected_count = int(result.get("count", 1))
                space = self.repo.embedding_space(deployment.embedding_space_id)
                if not space:
                    raise ServingError(409, "CAPABILITY_UNAVAILABLE", "embedding space is missing")
                validated = self._validate_vectors(vectors, expected_count, space.dimension)
                transient_result = {
                    "vectors": validated,
                    "prompt_tokens": int(result.get("prompt_tokens", 0)),
                }
                summary = {
                    "count": expected_count,
                    "dimension": space.dimension,
                    "finite": True,
                    "normalized": result.get("normalized") is True,
                }
            elif capability.key == "rag.reranking":
                transient_input = (
                    self.payload_store.get(job.payload_reference)
                    if job.operation == "invoke" and job.payload_reference
                    else None
                )
                expected_ids = (
                    {str(item["id"]) for item in transient_input.get("documents", [])}
                    if transient_input
                    else None
                )
                expected_top_n = int(transient_input["top_n"]) if transient_input else None
                validated_ranking = self._validate_reranking_results(
                    result.get("results"),
                    expected_ids=expected_ids,
                    expected_top_n=expected_top_n,
                )
                transient_result = {
                    "results": validated_ranking,
                    "prompt_tokens": int(result.get("prompt_tokens", 0)),
                }
                summary = {
                    "count": int(result.get("count", len(expected_ids or []))),
                    "top_n": len(validated_ranking),
                    "finite": True,
                }
            elif capability.key in {
                "document.ocr",
                "vision.embedding",
                "speech.transcription",
            }:
                if job.operation == "health":
                    if result.get("status") != "passed":
                        raise ServingError(502, "HEALTHCHECK_FAILED", "worker health probe failed")
                    transient_result = {"health": "passed"}
                    summary = {
                        "count": 1,
                        "output_type": result.get("output_type"),
                        "content_logged": False,
                    }
                else:
                    transient_input = (
                        self.payload_store.get(job.payload_reference)
                        if job.payload_reference
                        else {}
                    ) or {}
                    expected_count = (
                        len(transient_input.get("items", []))
                        if capability.key == "vision.embedding"
                        else 1
                    )
                    transient_result = self._validate_modality_output(
                        capability.key, result, deployment, expected_count
                    )
                    summary = {
                        "count": expected_count,
                        "output_type": result.get("output_type"),
                        "content_logged": False,
                    }
            else:
                raise ServingError(409, "CAPABILITY_UNAVAILABLE", "unsupported capability")
            result_reference = self.payload_store.put(
                transient_result, self.settings.serving_payload_ttl_seconds
            )
            job.result_reference = result_reference
            job.result_summary = {
                **summary,
                "inference_time_ms": float(metrics.get("inference_time_ms", 0.0)),
                "preprocess_time_ms": float(metrics.get("preprocess_time_ms", 0.0)),
                "serialization_time_ms": float(metrics.get("serialization_time_ms", 0.0)),
                "worker_total_time_ms": float(metrics.get("worker_total_time_ms", 0.0)),
                "prompt_tokens": int(result.get("prompt_tokens", 0)),
            }
            residency.state = "warm"
            residency.last_used_at = _now()
            residency.health = request.health
        elif job.operation == "load":
            total_baseline = int(metrics.get("baseline_vram_bytes", 0))
            total_loaded = int(metrics.get("loaded_vram_bytes", 0))
            baseline = int(
                metrics.get("baseline_process_vram_bytes", metrics.get("baseline_vram_bytes", 0))
            )
            loaded = int(
                metrics.get("loaded_process_vram_bytes", metrics.get("loaded_vram_bytes", 0))
            )
            if loaded <= baseline:
                raise ServingError(
                    409, "RESIDENCY_LOAD_FAILED", "load reported no resident allocation"
                )
            residency.state = "warm"
            residency.worker_instance_id = request.worker_instance_id
            residency.load_count += 1
            residency.measured_resident_vram_bytes = loaded - baseline
            residency.external_baseline_vram_bytes = total_baseline
            residency.resident_since = _now()
            residency.last_used_at = _now()
            residency.health = request.health
            residency.failure_code = None
            residency.failure_message = None
            job.result_summary = {
                "load_time_ms": float(metrics.get("load_time_ms", 0.0)),
                "baseline_vram_bytes": baseline,
                "loaded_vram_bytes": loaded,
                "observed_baseline_vram_bytes": total_baseline,
                "observed_loaded_vram_bytes": total_loaded,
                "resident_vram_bytes": loaded - baseline,
            }
        elif job.operation == "drain":
            residency.state = "draining"
            job.result_summary = {"drained": True}
        elif job.operation == "unload":
            if result.get("reclaimed") is not True:
                raise ServingError(
                    409, "RESIDENCY_UNLOAD_FAILED", "worker did not reclaim GPU memory"
                )
            residency.state = "cold"
            residency.worker_instance_id = request.worker_instance_id
            residency.active_requests = 0
            residency.measured_resident_vram_bytes = 0
            residency.health = {
                "process": "healthy",
                "runtime": "unavailable",
                "model": "unavailable",
            }
            residency.resident_since = None
            residency.transition_reason = residency.transition_reason or "ADMIN_UNLOAD"
            job.result_summary = {
                "reclaimed": True,
                "after_unload_vram_bytes": metrics.get("after_unload_vram_bytes"),
                "reclaimed_vram_bytes": metrics.get("reclaimed_vram_bytes"),
            }
            deployment.health_status = (
                "ready_on_demand" if deployment.status in {"stable", "candidate"} else "unavailable"
            )
            self.audit.write(
                "DEPLOYMENT_UNLOADED",
                "capability_deployment",
                str(deployment.id),
                {"reclaimed_vram_bytes": metrics.get("reclaimed_vram_bytes")},
            )
            eviction = self.session.scalar(
                select(SchedulerEviction)
                .where(
                    SchedulerEviction.capability_deployment_id == deployment.id,
                    SchedulerEviction.state.in_(("planned", "queued")),
                )
                .order_by(SchedulerEviction.created_at.desc())
                .limit(1)
            )
            if eviction:
                eviction.state = "completed"
                eviction.actual_reclaimed_bytes = int(metrics.get("reclaimed_vram_bytes", 0))
        job.status = "completed"
        job.finished_at = _now()
        job.lease_token_hash = None
        job.lease_expires_at = None
        if job.payload_reference:
            self.payload_store.delete(job.payload_reference)
            job.payload_reference = None
        self._commit()
        return job

    def fail_job(
        self,
        job_id: uuid.UUID,
        node: ComputeNode,
        request: AgentServingJobFailure,
    ) -> ServingJob:
        job = self.repo.job(job_id)
        if not job:
            raise ServingError(404, "SERVING_JOB_NOT_FOUND", "serving job was not found")
        self._assert_job_lease(job, node, request.lease_token)
        residency = self.repo.residency(job.capability_deployment_id, lock=True)
        job.status = "failed"
        job.failure_code = request.failure_code
        job.failure_message = request.failure_message
        job.result_summary = {"details": request.details}
        job.finished_at = _now()
        job.lease_token_hash = None
        job.lease_expires_at = None
        if job.payload_reference:
            self.payload_store.delete(job.payload_reference)
            job.payload_reference = None
        if residency:
            residency.state = "cold" if job.operation in {"load", "invoke"} else "failed"
            residency.failure_code = request.failure_code
            residency.failure_message = request.failure_message
            residency.measured_resident_vram_bytes = (
                0 if residency.state == "cold" else residency.measured_resident_vram_bytes
            )
            residency.active_requests = max(0, residency.active_requests - 1)
        self._commit()
        return job

    def report_state(
        self, node: ComputeNode, request: AgentServingStateReport
    ) -> AgentServingStateAck:
        if request.residencies:
            reported_ids: set[uuid.UUID] = set()
            desired_unload = False
            for report in request.residencies:
                deployment_id = report.deployment_id
                deployment = self.repo.deployment(deployment_id)
                if not deployment or deployment.compute_node_id != node.id:
                    raise ServingError(
                        403, "NODE_OWNERSHIP_MISMATCH", "reported deployment belongs elsewhere"
                    )
                profile = self.repo.profile(deployment.runtime_profile_id)
                environment = (
                    self.repo.environment(profile.runtime_environment_id)
                    if profile and profile.runtime_environment_id
                    else None
                )
                exact_identity = (
                    report.artifact_set_id == deployment.artifact_set_id
                    and report.runtime_profile_id == deployment.runtime_profile_id
                    and environment is not None
                    and report.runtime_environment_fingerprint == environment.fingerprint
                )
                if not exact_identity:
                    raise ServingError(
                        409,
                        "RESIDENCY_CONFLICT",
                        "worker residency cannot be adopted without exact immutable identity",
                    )
                reported_ids.add(deployment.id)
                residency = self.repo.residency(deployment.id, lock=True)
                if not residency:
                    continue
                adopted = residency.state == "cold"
                residency.worker_instance_id = request.worker_instance_id
                residency.generation = request.generation
                residency.state = report.state
                residency.load_count = max(residency.load_count, report.load_count)
                residency.health = report.health
                residency.measured_resident_vram_bytes = max(
                    residency.measured_resident_vram_bytes,
                    report.resident_vram_bytes,
                )
                if adopted:
                    residency.resident_since = request.observed_at
                    residency.transition_reason = "WORKER_RESIDENCY_ADOPTED"
                desired_unload = desired_unload or deployment.status == "draining"
            for deployment in self.repo.deployments():
                if deployment.compute_node_id != node.id or deployment.id in reported_ids:
                    continue
                residency = self.repo.residency(deployment.id, lock=True)
                if residency and residency.state != "cold":
                    residency.state = "cold"
                    residency.worker_instance_id = request.worker_instance_id
                    residency.generation = request.generation
                    residency.measured_resident_vram_bytes = 0
                    residency.active_requests = 0
                    residency.resident_since = None
                    residency.transition_reason = "WORKER_ABSENCE_RECONCILED"
                    residency.failure_code = "WORKER_RESTART_RECONCILED"
            self._commit()
            return AgentServingStateAck(desired_operation="unload" if desired_unload else "none")
        if request.deployment_id:
            deployment = self.repo.deployment(request.deployment_id)
            if not deployment or deployment.compute_node_id != node.id:
                raise ServingError(
                    403, "NODE_OWNERSHIP_MISMATCH", "reported deployment belongs elsewhere"
                )
            residency = self.repo.residency(deployment.id, lock=True)
            if residency:
                residency.worker_instance_id = request.worker_instance_id
                residency.generation = request.generation
                residency.state = request.state
                residency.load_count = max(residency.load_count, request.load_count)
                residency.health = request.health
                if request.state == "cold":
                    residency.measured_resident_vram_bytes = 0
                    residency.resident_since = None
                self._commit()
            return AgentServingStateAck(
                desired_operation="unload" if deployment.status == "draining" else "none"
            )
        for deployment in self.repo.deployments():
            if deployment.compute_node_id != node.id:
                continue
            residency = self.repo.residency(deployment.id, lock=True)
            if (
                residency
                and residency.worker_instance_id != request.worker_instance_id
                and residency.state != "cold"
            ):
                residency.state = "cold"
                residency.worker_instance_id = request.worker_instance_id
                residency.generation = request.generation
                residency.measured_resident_vram_bytes = 0
                residency.active_requests = 0
                residency.resident_since = None
                residency.transition_reason = "WORKER_ABSENCE_RECONCILED"
                residency.health = {
                    "process": "healthy",
                    "runtime": "unavailable",
                    "model": "unavailable",
                }
                residency.failure_code = "WORKER_RESTART_RECONCILED"
                residency.failure_message = None
        self._commit()
        return AgentServingStateAck()

    def request_unload(
        self, deployment_id: uuid.UUID, *, drain: bool = False
    ) -> CapabilityDeploymentResponse:
        deployment = self.repo.deployment(deployment_id)
        residency = self.repo.residency(deployment_id, lock=True) if deployment else None
        if not deployment or not residency:
            raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
        if residency.active_requests:
            raise ServingError(409, "DEPLOYMENT_BUSY", "active requests must finish before unload")
        if drain:
            deployment.status = "draining"
            deployment.draining_at = _now()
            deployment.health_status = "degraded"
            self.audit.write("DEPLOYMENT_DRAINED", "capability_deployment", str(deployment.id), {})
        if residency.state == "cold":
            self._commit()
            return self.deployment_response(deployment)
        residency.state = "draining"
        self._commit()
        self._enqueue_job(
            deployment,
            "drain",
            key_suffix=f"drain-{residency.load_count}",
            priority="maintenance",
        )
        self._enqueue_job(
            deployment,
            "unload",
            key_suffix=f"unload-{residency.load_count}",
            priority="maintenance",
        )
        return self.deployment_response(deployment)

    def update_residency_policy(
        self, deployment_id: uuid.UUID, request: ResidencyPolicyUpdate
    ) -> CapabilityDeploymentResponse:
        deployment = self.repo.deployment(deployment_id)
        if not deployment:
            raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
        previous = deployment.residency_policy
        deployment.residency_policy = request.residency_policy
        deployment.keep_warm_seconds = request.keep_warm_seconds
        self.audit.write(
            "RESIDENCY_POLICY_CHANGED",
            "capability_deployment",
            str(deployment.id),
            {
                "previous": previous,
                "current": request.residency_policy,
                "keep_warm_seconds": request.keep_warm_seconds,
            },
        )
        self._commit()
        return self.deployment_response(deployment)

    def reconcile(self) -> dict[str, int]:
        self._expire_leases()
        now = _now()
        expired = 0
        unloads = 0
        for deployment in self.repo.deployments():
            residency = self.repo.residency(deployment.id, lock=True)
            node = self.repo.node(deployment.compute_node_id)
            if not residency:
                continue
            if not node or node.liveness_state != "online":
                deployment.health_status = "unavailable"
            elif (
                residency.state == "cold"
                and deployment.status in {"stable", "candidate"}
                and deployment.health_status == "unavailable"
            ):
                # Repair the legacy M9 state where a successful LAB unload was
                # persisted as unavailable even though it remained routable.
                deployment.health_status = "ready_on_demand"
            if (
                deployment.status == "stable"
                and deployment.residency_policy == "keep_warm"
                and residency.state in {"warm", "idle"}
                and residency.active_requests == 0
                and residency.last_used_at
                and _aware(residency.last_used_at) + timedelta(seconds=deployment.keep_warm_seconds)
                <= now
            ):
                self.request_unload(deployment.id)
                unloads += 1
        for lease in self.repo.expired_leases(now):
            if lease.state == "expired":
                expired += 1
        self._commit()
        return {"expired_leases": expired, "unloads_queued": unloads}