Jens/ModelForge-Public
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7082ab955a939a10eca9cf72fdcf72a5db7e7804
ModelForge-Public/backend/src/modelforge_api/services/serving.py
T

3779 lines
168 KiB
Python
Raw Blame History

from __future__ import annotations
import hashlib
import hmac
import itertools
import json
import math
import posixpath
import secrets
import time
import uuid
from dataclasses import asdict, dataclass, replace
from datetime import UTC, datetime, timedelta
from pathlib import PurePosixPath
from typing import Any, cast
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from modelforge_api.domain.serving import (
AgentServingJobComplete,
AgentServingJobFailure,
AgentServingJobLease,
AgentServingStateAck,
AgentServingStateReport,
CapabilityDeploymentResponse,
CapabilityExperimentCreate,
CapabilityExperimentResponse,
CapabilityPromotionCreate,
CoResidencyEvidenceCreate,
CoResidencyEvidenceResponse,
EmbeddingInvokeRequest,
EmbeddingInvokeResponse,
EmbeddingSpaceResponse,
GatewayExecution,
GatewayRequestResponse,
GatewayTiming,
GatewayUsage,
PlacementEvictionResponse,
PlacementPlanRequest,
PlacementPlanResponse,
ProductionApprovalCreate,
ProductionApprovalResponse,
ProjectFitEvidenceCreate,
ProjectFitEvidenceResponse,
ProjectIntegrationResponse,
ProjectIntegrationUsage,
RerankDocument,
RerankingInvokeRequest,
RerankingInvokeResponse,
RerankingResult,
ResidencyPolicyUpdate,
ResidencyResponse,
ResourceEnvelopeResponse,
SchedulerBudgetResponse,
SchedulerMetricsResponse,
SchedulerPolicyResponse,
SchedulerPolicyUpdate,
ServiceClientCreate,
ServiceClientCreated,
ServiceClientResponse,
SupplyChainReview,
)
from modelforge_api.persistence.models import (
Capability,
CapabilityContract,
CapabilityDeployment,
CapabilityExperimentRoute,
CapabilityResourceEnvelope,
ComputeNode,
CoResidencyEvidence,
EmbeddingSpace,
GatewayRequest,
PlacementPlanRecord,
ProductionExecutionApproval,
Project,
ProjectBinding,
ProjectFitEvidence,
ResidencyAllocation,
SchedulerAcceleratorState,
SchedulerEviction,
SchedulerPolicyRevision,
ServiceClient,
ServiceCredential,
ServingGpuLease,
ServingJob,
)
from modelforge_api.persistence.serving_repository import ServingRepository
from modelforge_api.services.audit import AuditWriter
from modelforge_api.services.gpu_scheduler import (
AccountingSnapshot,
AttributionConfidence,
LeaseAllocation,
PlacementInput,
PlacementVerdict,
PressureState,
ResidentCandidate,
SchedulerPolicy,
calculate_accounting,
plan_placement,
required_envelope,
)
from modelforge_api.services.manifest_registry import ManifestRegistry
from modelforge_api.services.runtime import RuntimeService, _selected_manifest_rows
from modelforge_api.settings import Settings
REQUIRED_SERVING_CAPABILITIES = {
"deployment.load.v1",
"deployment.invoke.v1",
"deployment.health.v1",
"deployment.drain.v1",
"deployment.unload.v1",
}
def _now() -> datetime:
return datetime.now(UTC)
def _aware(value: datetime) -> datetime:
return value if value.tzinfo else value.replace(tzinfo=UTC)
def _canonical_hash(payload: Any) -> str:
raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
return hashlib.sha256(raw.encode()).hexdigest()
def _percentile(values: list[float], fraction: float) -> float | None:
if not values:
return None
index = max(0, min(len(values) - 1, math.ceil(len(values) * fraction) - 1))
return float(values[index])
def _shape_bucket(encoded_bytes: int) -> str:
if encoded_bytes <= 64 * 1024:
return "SMALL"
if encoded_bytes <= 1024 * 1024:
return "MEDIUM"
return "LARGE"
def _safe_parent(paths: list[str]) -> str:
normalized = [path.replace("\\", "/") for path in paths]
parent = posixpath.commonpath(normalized)
if parent in normalized:
parent = PurePosixPath(parent).parent.as_posix()
if parent in {"", "."} or parent.startswith("/") or ".." in PurePosixPath(parent).parts:
raise ServingError(409, "ARTIFACT_PATH_INVALID", "artifact location is not confined")
return parent
class ServingError(RuntimeError):
def __init__(
self,
status_code: int,
code: str,
message: str,
details: dict[str, Any] | None = None,
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.details = details or {}
@dataclass(frozen=True, slots=True)
class CapabilityAuthenticationEvidence:
"""Secret-free proof of one exact pre-body capability-client authentication."""
credential_id: uuid.UUID
client_id: uuid.UUID
capability: str
credential_sha256: str
class ServingService:
def __init__(
self,
session: Session,
settings: Settings,
manifests: ManifestRegistry,
payload_store: Any,
*,
actor_type: str = "operator",
actor_id: str = "local-operator",
) -> None:
self.session = session
self.settings = settings
self.manifests = manifests
self.payload_store = payload_store
self.repo = ServingRepository(session)
self.audit = AuditWriter(session, actor_type, actor_id)
def _commit(self) -> None:
try:
self.session.commit()
except IntegrityError as exc:
self.session.rollback()
raise ServingError(
409, "STATE_CONFLICT", "serving state conflicts with existing data"
) from exc
def _commit_authentication(self) -> None:
"""Persist credential usage without misclassifying auth-store failures as conflicts."""
try:
self.session.commit()
except IntegrityError as exc:
self.session.rollback()
raise ServingError(
503,
"AUTH_BACKEND_UNAVAILABLE",
"capability authentication backend is unavailable",
) from exc
def ensure_contract(self, key: str = "rag.embedding", version: int = 1) -> CapabilityContract:
existing = self.repo.contract(key, version)
if existing:
return existing
manifest = next(
(
item
for item in self.manifests.capabilities()
if item.capability == key and item.version == version
),
None,
)
if not manifest:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability contract is not registered")
capability = self.repo.capability(key)
if not capability:
capability = self.repo.add(Capability(key=key, description=manifest.description))
contract = self.repo.add(
CapabilityContract(
capability_id=capability.id,
version=version,
input_schema=manifest.input_schema,
output_schema=manifest.output_schema,
contract=manifest.model_dump(mode="json"),
upgrade_class=manifest.upgrade_class.value,
immutable_at=_now(),
)
)
self._commit()
return contract
def _approval_facts(
self,
candidate_id: uuid.UUID,
contract: CapabilityContract,
request: ProductionApprovalCreate,
) -> tuple[dict[str, Any], Any, Any, Any, ComputeNode]:
candidate = self.repo.candidate(candidate_id)
if not candidate or candidate.status != "lab_ready" or candidate.production:
raise ServingError(
409, "CANDIDATE_NOT_LAB_READY", "a LAB_READY M4 candidate is required"
)
probe = self.repo.probe(candidate.runtime_probe_id)
profile = self.repo.profile(candidate.runtime_profile_id)
artifact_set = self.repo.artifact_set(candidate.artifact_set_id)
node = self.repo.node(candidate.compute_node_id)
if not probe or probe.status != "completed":
raise ServingError(
409, "RUNTIME_PROBE_REQUIRED", "successful M4 runtime probe is required"
)
runtime_assessment = RuntimeService(self.session, self.settings).assessment(
candidate.compatibility_assessment_id
)
if runtime_assessment.stale or runtime_assessment.status != "compatible":
raise ServingError(
409, "COMPATIBILITY_STALE", "current compatible evidence is required"
)
if not profile or not artifact_set or artifact_set.status != "verified":
raise ServingError(
409, "ARTIFACT_NOT_VERIFIED", "verified exact ArtifactSet is required"
)
if (
not node
or node.decommissioned_at is not None
or not node.enabled
or not node.production_eligible
or node.liveness_state != "online"
):
raise ServingError(
409, "NO_ELIGIBLE_NODE", "candidate node is not production eligible and online"
)
missing_capabilities = REQUIRED_SERVING_CAPABILITIES - set(node.agent_capabilities)
if missing_capabilities:
raise ServingError(
409,
"NODE_SERVING_CAPABILITY_MISSING",
"target node does not advertise the typed serving protocol",
{"missing": sorted(missing_capabilities)},
)
artifacts = self.repo.set_artifacts(artifact_set.id)
risk_flags = sorted(
{
flag
for artifact in artifacts
for flag in artifact.verification_details.get("risk_flags", [])
}
)
if any(artifact.status != "verified" or artifact.quarantined for artifact in artifacts):
raise ServingError(
409, "ARTIFACT_NOT_VERIFIED", "all ArtifactSet members must be verified"
)
if any(
artifact.serialization_format in {"pickle", "pytorch_pickle"} for artifact in artifacts
):
raise ServingError(
409, "PICKLE_PRESENT", "pickle artifacts cannot be production approved"
)
if any("remote_code" in flag.lower() for flag in risk_flags):
raise ServingError(409, "REMOTE_CODE_REQUIRED", "remote-code artifacts are blocked")
health = probe.health_result
inference = probe.inference_result
unload = probe.unload_result
if not all(health.get(layer) == "healthy" for layer in ("process", "runtime", "model")):
raise ServingError(
409, "HEALTHCHECK_FAILED", "M4 layered health evidence is incomplete"
)
if inference.get("dimension") != 1024 or inference.get("finite") is not True:
raise ServingError(409, "INVALID_OUTPUT", "M4 embedding output does not match contract")
if (
unload.get("reclaimed") is not True
or probe.runtime_facts.get("offline_local_only") is not True
):
raise ServingError(
409, "RUNTIME_SECURITY_GATE_FAILED", "offline unload evidence is required"
)
facts = {
"candidate_id": str(candidate.id),
"artifact_set_id": str(candidate.artifact_set_id),
"runtime_profile_id": str(candidate.runtime_profile_id),
"compute_node_id": str(candidate.compute_node_id),
"runtime_probe_id": str(probe.id),
"probe_environment_fingerprint": probe.environment_fingerprint,
"compatibility_assessment_id": str(candidate.compatibility_assessment_id),
"contract_id": str(contract.id),
"contract_version": contract.version,
"artifact_status": artifact_set.status,
"artifact_files": [
{
"filename": item.filename,
"sha256": item.sha256,
"format": item.serialization_format,
}
for item in artifacts
],
"risk_flags": risk_flags,
"runtime_profile_fingerprint": profile.fingerprint,
"node_hardware_fingerprint": node.hardware_fingerprint,
"deployment_config": request.deployment_config,
"supply_chain_review": request.supply_chain_review.model_dump(mode="json"),
}
return facts, candidate, probe, profile, node
def approve_production(
self, candidate_id: uuid.UUID, request: ProductionApprovalCreate
) -> ProductionApprovalResponse:
contract = self.ensure_contract(request.capability, request.contract_version)
facts, candidate, _probe, _profile, _node = self._approval_facts(
candidate_id, contract, request
)
fingerprint = _canonical_hash(facts)
existing = self.repo.approval_by_fingerprint(fingerprint)
if existing:
return self.approval_response(existing)
approval = self.repo.add(
ProductionExecutionApproval(
deployment_candidate_id=candidate.id,
capability_contract_id=contract.id,
artifact_set_id=candidate.artifact_set_id,
runtime_profile_id=candidate.runtime_profile_id,
compute_node_id=candidate.compute_node_id,
deployment_config=request.deployment_config,
supply_chain_evidence=request.supply_chain_review.model_dump(mode="json"),
evidence_fingerprint=fingerprint,
status="approved",
approved_by=request.approved_by,
reason=request.reason,
approved_at=_now(),
immutable_at=_now(),
)
)
self.audit.write(
"PRODUCTION_EXECUTION_APPROVED",
"production_execution_approval",
str(approval.id),
{
"artifact_set_id": str(candidate.artifact_set_id),
"runtime_profile_id": str(candidate.runtime_profile_id),
"capability": "rag.embedding@1",
},
)
self._commit()
return self.approval_response(approval)
def approval_response(self, item: ProductionExecutionApproval) -> ProductionApprovalResponse:
stale = item.revoked_at is not None
return ProductionApprovalResponse(
id=item.id,
deployment_candidate_id=item.deployment_candidate_id,
capability_contract_id=item.capability_contract_id,
artifact_set_id=item.artifact_set_id,
runtime_profile_id=item.runtime_profile_id,
compute_node_id=item.compute_node_id,
deployment_config=item.deployment_config,
supply_chain_evidence=item.supply_chain_evidence,
evidence_fingerprint=item.evidence_fingerprint,
status="stale" if stale else item.status,
approved_by=item.approved_by,
reason=item.reason,
approved_at=item.approved_at,
revoked_at=item.revoked_at,
stale=stale,
)
def approvals(self) -> list[ProductionApprovalResponse]:
return [self.approval_response(item) for item in self.repo.approvals()]
def promote(
self, candidate_id: uuid.UUID, request: CapabilityPromotionCreate
) -> CapabilityDeploymentResponse:
approval = self.repo.approval(request.production_approval_id)
candidate = self.repo.candidate(candidate_id)
if not approval or not candidate or approval.deployment_candidate_id != candidate.id:
raise ServingError(
404, "PRODUCTION_APPROVAL_NOT_FOUND", "exact production approval is required"
)
if approval.revoked_at or approval.status != "approved":
raise ServingError(
409, "PRODUCTION_APPROVAL_STALE", "production approval is not current"
)
contract = self.session.get(CapabilityContract, approval.capability_contract_id)
if not contract:
raise ServingError(
409, "CAPABILITY_NOT_FOUND", "approved capability contract disappeared"
)
verification_request = ProductionApprovalCreate(
capability="rag.embedding",
contract_version=1,
approved_by=approval.approved_by,
reason=approval.reason,
supply_chain_review=SupplyChainReview.model_validate(approval.supply_chain_evidence),
deployment_config=approval.deployment_config,
)
current_facts, _, probe, profile, node = self._approval_facts(
candidate_id, contract, verification_request
)
if _canonical_hash(current_facts) != approval.evidence_fingerprint:
raise ServingError(
409, "PRODUCTION_APPROVAL_STALE", "reviewed production evidence changed"
)
accelerators = self.repo.node_accelerators(node.id)
probe_gpu_uuid = str(probe.runtime_facts.get("gpu_uuid", ""))
accelerator = next(
(item for item in accelerators if item.device_uuid == probe_gpu_uuid), None
)
if not accelerator:
raise ServingError(409, "NO_ELIGIBLE_NODE", "probed accelerator is no longer available")
embedding_facts = {
"artifact_set_id": str(candidate.artifact_set_id),
"runtime_profile_fingerprint": profile.fingerprint,
"capability_contract_id": str(contract.id),
"dimension": 1024,
"normalized": True,
"migration_class": "requires_reindex",
}
embedding_digest = _canonical_hash(embedding_facts)
space = self.repo.embedding_space_by_digest(embedding_digest)
if not space:
space = self.repo.add(
EmbeddingSpace(
capability_contract_id=contract.id,
artifact_set_id=candidate.artifact_set_id,
runtime_profile_id=candidate.runtime_profile_id,
identity_digest=embedding_digest,
dimension=1024,
normalized=True,
migration_class="requires_reindex",
identity_facts=embedding_facts,
created_at=_now(),
immutable_at=_now(),
)
)
config = {
"candidate_id": str(candidate.id),
"production_approval_id": str(approval.id),
"capability_contract_id": str(contract.id),
"embedding_space_id": str(space.id),
"artifact_set_id": str(candidate.artifact_set_id),
"runtime_profile_id": str(candidate.runtime_profile_id),
"compute_node_id": str(candidate.compute_node_id),
"accelerator_id": str(accelerator.id),
"residency_policy": request.residency_policy,
"keep_warm_seconds": request.keep_warm_seconds,
"max_concurrency": request.max_concurrency,
"max_queue_depth": request.max_queue_depth,
"routing_weight": request.routing_weight,
"rollback_policy": request.rollback_policy,
}
fingerprint = _canonical_hash(config)
deployment = self.repo.deployment_by_fingerprint(fingerprint)
if deployment:
return self.deployment_response(deployment)
if self.repo.stable_deployment(contract.id):
raise ServingError(
409, "STABLE_DEPLOYMENT_EXISTS", "drain the current stable deployment first"
)
deployment = self.repo.add(
CapabilityDeployment(
capability_contract_id=contract.id,
deployment_candidate_id=candidate.id,
production_approval_id=approval.id,
embedding_space_id=space.id,
artifact_set_id=candidate.artifact_set_id,
runtime_profile_id=candidate.runtime_profile_id,
compute_node_id=candidate.compute_node_id,
accelerator_id=accelerator.id,
channel="stable",
status="stable",
production=True,
health_status="ready_on_demand",
routing_weight=request.routing_weight,
fallback_policy={"allowed": False, "mode": "hard_fail", "deployments": []},
residency_policy=request.residency_policy,
keep_warm_seconds=request.keep_warm_seconds,
max_concurrency=request.max_concurrency,
max_queue_depth=request.max_queue_depth,
config_fingerprint=fingerprint,
provenance=current_facts,
rollback_policy=request.rollback_policy,
promoted_at=_now(),
)
)
measured = probe.measured_resources
samples = measured.get("samples", {})
baseline = int(samples.get("before_load", {}).get("process_used_vram_bytes", 0))
loaded = int(samples.get("loaded", {}).get("process_used_vram_bytes", 0))
peak = int(samples.get("probe_peak", {}).get("process_used_vram_bytes", 0))
if peak <= baseline or loaded <= baseline:
baseline = int(measured["baseline_vram_bytes"])
loaded = int(measured["loaded_vram_bytes"])
peak = int(measured["peak_vram_bytes"])
required = max(loaded, peak) - baseline
self.repo.add(
CapabilityResourceEnvelope(
capability_deployment_id=deployment.id,
runtime_probe_id=probe.id,
accelerator_kind=accelerator.name,
accelerator_uuid=accelerator.device_uuid,
environment_fingerprint=probe.environment_fingerprint,
concurrency=1,
batch_size=int(measured.get("batch_size", 1)),
max_sequence_length=int(measured.get("max_sequence_length", 128)),
baseline_vram_bytes=baseline,
resident_vram_bytes=max(0, loaded - baseline),
peak_vram_bytes=peak,
required_vram_bytes=required,
cold_load_time_ms=float(probe.load_result["load_time_ms"]),
inference_latency_ms=float(probe.inference_result["latency_ms"]),
evidence={"kind": "measured", "runtime_probe_id": str(probe.id), "raw": measured},
stale=False,
immutable_at=_now(),
)
)
self.repo.add(
ResidencyAllocation(
capability_deployment_id=deployment.id,
compute_node_id=node.id,
accelerator_id=accelerator.id,
state="cold",
load_count=0,
active_requests=0,
measured_resident_vram_bytes=0,
external_baseline_vram_bytes=0,
health={"process": "unavailable", "runtime": "unavailable", "model": "unavailable"},
)
)
self.audit.write(
"CAPABILITY_DEPLOYMENT_CREATED",
"capability_deployment",
str(deployment.id),
{"capability": "rag.embedding@1", "production": True},
)
self.audit.write(
"CAPABILITY_PROMOTED_STABLE",
"capability_deployment",
str(deployment.id),
{
"candidate_id": str(candidate.id),
"approval_id": str(approval.id),
"embedding_space_id": str(space.id),
},
)
self._commit()
return self.deployment_response(deployment)
def embedding_response(self, item: EmbeddingSpace) -> EmbeddingSpaceResponse:
return EmbeddingSpaceResponse.model_validate(item)
def envelope_response(self, item: CapabilityResourceEnvelope) -> ResourceEnvelopeResponse:
return ResourceEnvelopeResponse.model_validate(item)
def residency_response(self, item: ResidencyAllocation) -> ResidencyResponse:
return ResidencyResponse.model_validate(item)
def deployment_response(self, item: CapabilityDeployment) -> CapabilityDeploymentResponse:
contract = self.session.get(CapabilityContract, item.capability_contract_id)
capability = self.session.get(Capability, contract.capability_id) if contract else None
space = self.repo.embedding_space(item.embedding_space_id)
envelope = self.repo.envelope(item.id)
residency = self.repo.residency(item.id)
if not contract or not capability or not envelope or not residency:
raise ServingError(
409, "DEPLOYMENT_PROVENANCE_INCOMPLETE", "deployment evidence is incomplete"
)
if capability.key == "rag.embedding" and not space:
raise ServingError(
409, "DEPLOYMENT_PROVENANCE_INCOMPLETE", "embedding identity is missing"
)
return CapabilityDeploymentResponse(
id=item.id,
capability=capability.key,
contract_version=contract.version,
deployment_candidate_id=item.deployment_candidate_id,
production_approval_id=item.production_approval_id,
execution_approval_id=item.execution_approval_id,
artifact_set_id=item.artifact_set_id,
runtime_profile_id=item.runtime_profile_id,
compute_node_id=item.compute_node_id,
accelerator_id=item.accelerator_id,
embedding_space=self.embedding_response(space) if space else None,
resource_envelope=self.envelope_response(envelope),
residency=self.residency_response(residency),
channel=item.channel,
status=item.status,
production=item.production,
health_status=self._health_status(item, residency),
routing_weight=item.routing_weight,
fallback_policy=item.fallback_policy,
residency_policy=item.residency_policy,
keep_warm_seconds=item.keep_warm_seconds,
max_concurrency=item.max_concurrency,
max_queue_depth=item.max_queue_depth,
config_fingerprint=item.config_fingerprint,
provenance=item.provenance,
rollback_policy=item.rollback_policy,
promoted_at=item.promoted_at,
created_at=item.created_at,
)
def deployments(self) -> list[CapabilityDeploymentResponse]:
return [self.deployment_response(item) for item in self.repo.deployments()]
def create_experiment(
self,
candidate_id: uuid.UUID,
request: CapabilityExperimentCreate,
) -> CapabilityExperimentResponse:
existing_route = self.repo.experiment_route(request.route_key)
if existing_route:
return self.experiment_response(existing_route)
candidate = self.repo.candidate(candidate_id)
approval = self.repo.execution_approval(request.execution_approval_id)
if not candidate or candidate.status != "lab_ready" or candidate.production:
raise ServingError(409, "CANDIDATE_NOT_LAB_READY", "LAB_READY candidate is required")
if (
not approval
or approval.id != request.execution_approval_id
or approval.artifact_set_id != candidate.artifact_set_id
or approval.scope != "lab_execution"
or approval.status != "approved"
or approval.revoked_at is not None
or (approval.expires_at and _aware(approval.expires_at) <= _now())
):
raise ServingError(
409, "LAB_APPROVAL_STALE", "current exact LAB execution approval is required"
)
probe = self.repo.probe(candidate.runtime_probe_id)
profile = self.repo.profile(candidate.runtime_profile_id)
node = self.repo.node(candidate.compute_node_id)
if (
not probe
or probe.status != "completed"
or probe.execution_approval_id != approval.id
or not profile
or not node
or not node.enabled
or node.liveness_state != "online"
):
raise ServingError(409, "CANDIDATE_EVIDENCE_STALE", "candidate evidence is incomplete")
dimension = int(probe.inference_result.get("dimension", 0))
if request.capability in {"rag.embedding", "vision.embedding"}:
if (
dimension != request.expected_dimension
or probe.inference_result.get("finite") is not True
or probe.inference_result.get("output_type") not in {None, "float_vector"}
):
raise ServingError(
409, "INVALID_OUTPUT", "probe dimension does not match experiment"
)
elif request.capability == "rag.reranking" and (
probe.inference_result.get("output_type") != "ranked_scores"
or probe.inference_result.get("finite") is not True
or probe.runtime_facts.get("adapter") != "qwen3_reranker"
):
raise ServingError(409, "INVALID_OUTPUT", "reranking probe evidence is invalid")
elif request.capability == "document.ocr" and (
probe.inference_result.get("output_type") != "ocr_text"
or probe.inference_result.get("output_valid") is not True
):
raise ServingError(409, "INVALID_OUTPUT", "OCR probe evidence is invalid")
elif request.capability == "speech.transcription" and (
probe.inference_result.get("output_type") != "transcription"
or probe.inference_result.get("output_valid") is not True
):
raise ServingError(409, "INVALID_OUTPUT", "ASR probe evidence is invalid")
accelerators = self.repo.node_accelerators(node.id)
gpu_uuid = str(probe.runtime_facts.get("gpu_uuid", ""))
accelerator = next((item for item in accelerators if item.device_uuid == gpu_uuid), None)
if not accelerator:
raise ServingError(409, "NO_ELIGIBLE_NODE", "probed accelerator is unavailable")
contract = self.ensure_contract(request.capability, 1)
space: EmbeddingSpace | None = None
if request.capability in {"rag.embedding", "vision.embedding"}:
semantics_key = (
"embedding_input_semantics"
if request.capability == "rag.embedding"
else "visual_preprocessing"
)
semantics = profile.launch_arguments.get(semantics_key, {})
if not isinstance(semantics, dict):
raise ServingError(
409, "PROFILE_SEMANTICS_INVALID", "input semantics must be typed"
)
embedding_facts = {
"artifact_set_id": str(candidate.artifact_set_id),
"runtime_profile_fingerprint": profile.fingerprint,
"capability_contract_id": str(contract.id),
"dimension": dimension,
"normalized": True,
"input_semantics": semantics,
"modality": "text" if request.capability == "rag.embedding" else "image_text",
"migration_class": "requires_reindex",
}
digest = _canonical_hash(embedding_facts)
space = self.repo.embedding_space_by_digest(digest)
if not space:
space = self.repo.add(
EmbeddingSpace(
capability_contract_id=contract.id,
artifact_set_id=candidate.artifact_set_id,
runtime_profile_id=candidate.runtime_profile_id,
identity_digest=digest,
dimension=dimension,
normalized=True,
migration_class="requires_reindex",
identity_facts=embedding_facts,
created_at=_now(),
immutable_at=_now(),
)
)
config = {
"candidate_id": str(candidate.id),
"execution_approval_id": str(approval.id),
"capability_contract_id": str(contract.id),
"embedding_space_id": str(space.id) if space else None,
"capability": request.capability,
"accelerator_id": str(accelerator.id),
"route_key": request.route_key,
"residency_policy": request.residency_policy,
"keep_warm_seconds": request.keep_warm_seconds,
"max_queue_depth": request.max_queue_depth,
}
fingerprint = _canonical_hash(config)
deployment = self.repo.deployment_by_fingerprint(fingerprint)
if not deployment:
deployment = self.repo.add(
CapabilityDeployment(
capability_contract_id=contract.id,
deployment_candidate_id=candidate.id,
production_approval_id=None,
execution_approval_id=approval.id,
embedding_space_id=space.id if space else None,
artifact_set_id=candidate.artifact_set_id,
runtime_profile_id=candidate.runtime_profile_id,
compute_node_id=node.id,
accelerator_id=accelerator.id,
channel="experiment",
status="candidate",
production=False,
health_status="ready_on_demand",
routing_weight=0,
fallback_policy={"allowed": False, "mode": "hard_fail", "deployments": []},
residency_policy=request.residency_policy,
keep_warm_seconds=request.keep_warm_seconds,
max_concurrency=1,
max_queue_depth=request.max_queue_depth,
config_fingerprint=fingerprint,
provenance={
"runtime_probe_id": str(probe.id),
"execution_approval_id": str(approval.id),
"runtime_profile_fingerprint": profile.fingerprint,
"production": False,
"capability": request.capability,
},
rollback_policy={"mode": "unload_experiment", "stable_unchanged": True},
)
)
measured = probe.measured_resources
samples = measured.get("samples", {})
baseline = int(samples.get("before_load", {}).get("process_used_vram_bytes", 0))
loaded = int(samples.get("loaded", {}).get("process_used_vram_bytes", 0))
peak = int(samples.get("probe_peak", {}).get("process_used_vram_bytes", 0))
if peak <= baseline or loaded <= baseline:
baseline = int(measured["baseline_vram_bytes"])
loaded = int(measured["loaded_vram_bytes"])
peak = int(measured["peak_vram_bytes"])
self.repo.add(
CapabilityResourceEnvelope(
capability_deployment_id=deployment.id,
runtime_probe_id=probe.id,
accelerator_kind=accelerator.name,
accelerator_uuid=accelerator.device_uuid,
environment_fingerprint=probe.environment_fingerprint,
concurrency=1,
batch_size=int(measured.get("batch_size", 1)),
max_sequence_length=int(measured.get("max_sequence_length", 128)),
baseline_vram_bytes=baseline,
resident_vram_bytes=max(0, loaded - baseline),
peak_vram_bytes=peak,
required_vram_bytes=max(loaded, peak) - baseline,
cold_load_time_ms=float(probe.load_result["load_time_ms"]),
inference_latency_ms=float(probe.inference_result["latency_ms"]),
evidence={"kind": "measured", "runtime_probe_id": str(probe.id)},
stale=False,
immutable_at=_now(),
)
)
self.repo.add(
ResidencyAllocation(
capability_deployment_id=deployment.id,
compute_node_id=node.id,
accelerator_id=accelerator.id,
state="cold",
load_count=0,
active_requests=0,
measured_resident_vram_bytes=0,
external_baseline_vram_bytes=0,
health={
"process": "unavailable",
"runtime": "unavailable",
"model": "unavailable",
},
)
)
route = self.repo.add(
CapabilityExperimentRoute(
route_key=request.route_key,
capability_contract_id=contract.id,
capability_deployment_id=deployment.id,
status="active",
purpose=request.purpose,
evidence={
"candidate_id": str(candidate.id),
"embedding_space_id": str(space.id) if space else None,
"capability": request.capability,
"production": False,
},
)
)
self.audit.write(
"CAPABILITY_EXPERIMENT_CREATED",
"capability_experiment_route",
str(route.id),
{"route_key": route.route_key, "production": False},
)
self._commit()
return self.experiment_response(route)
def experiment_response(self, route: CapabilityExperimentRoute) -> CapabilityExperimentResponse:
deployment = self.repo.deployment(route.capability_deployment_id)
contract = self.session.get(CapabilityContract, route.capability_contract_id)
capability = self.session.get(Capability, contract.capability_id) if contract else None
if not deployment or not contract or not capability:
raise ServingError(409, "DEPLOYMENT_PROVENANCE_INCOMPLETE", "route is incomplete")
return CapabilityExperimentResponse(
id=route.id,
route_key=route.route_key,
capability=capability.key,
contract_version=contract.version,
capability_deployment_id=deployment.id,
status=route.status,
purpose=route.purpose,
evidence=route.evidence,
deployment=self.deployment_response(deployment),
created_at=route.created_at,
)
def experiments(self) -> list[CapabilityExperimentResponse]:
return [self.experiment_response(item) for item in self.repo.experiment_routes()]
def deactivate_experiment(self, route_id: uuid.UUID) -> CapabilityExperimentResponse:
route = self.repo.experiment_route_by_id(route_id)
if not route:
raise ServingError(404, "EXPERIMENT_NOT_FOUND", "capability experiment is unavailable")
if route.status == "active":
route.status = "inactive"
self.audit.write(
"CAPABILITY_EXPERIMENT_DEACTIVATED",
"capability_experiment_route",
str(route.id),
{"route_key": route.route_key, "production": False},
)
self._commit()
return self.experiment_response(route)
def _health_status(
self, deployment: CapabilityDeployment, residency: ResidencyAllocation
) -> str:
node = self.repo.node(deployment.compute_node_id)
if not node or not node.enabled or node.liveness_state != "online":
return "unavailable"
if deployment.status == "draining":
return "degraded"
if residency.state in {"warm", "busy", "idle"}:
health = residency.health
return (
"healthy"
if all(health.get(layer) == "healthy" for layer in ("process", "runtime", "model"))
else "degraded"
)
if residency.state in {"cold", "loading"}:
return "ready_on_demand"
return "unavailable"
def create_client(self, request: ServiceClientCreate) -> ServiceClientCreated:
if self.repo.client_by_name(request.name):
raise ServingError(409, "CLIENT_EXISTS", "service client name already exists")
project: Project | None = None
project_binding: ProjectBinding | None = None
for scope in request.allowed_capabilities:
key, raw_version = scope.rsplit("@", 1)
if not self.repo.contract(key, int(raw_version)):
self.ensure_contract(key, int(raw_version))
if request.project_key is not None:
project = self.session.scalar(select(Project).where(Project.key == request.project_key))
if project is None or not project.active:
raise ServingError(404, "PROJECT_NOT_FOUND", "active project was not found")
key, raw_version = request.allowed_capabilities[0].rsplit("@", 1)
contract = self.repo.contract(key, int(raw_version))
if contract is None:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability was not found")
bindings = list(
self.session.scalars(
select(ProjectBinding)
.where(
ProjectBinding.project_id == project.id,
ProjectBinding.capability_contract_id == contract.id,
ProjectBinding.deprecated_at.is_(None),
)
.order_by(ProjectBinding.created_at.desc())
)
)
preferred_channels = (
{"stable"}
if request.integration_environment == "production"
else {"experimental", "candidate"}
)
project_binding = next(
(item for item in bindings if item.channel in preferred_channels),
bindings[0] if bindings else None,
)
if project_binding is None:
raise ServingError(
409,
"PROJECT_CAPABILITY_NOT_BOUND",
"project does not have an active binding for this capability",
)
secret = "mf_" + secrets.token_urlsafe(36)
client = self.repo.add(
ServiceClient(
name=request.name,
status="active",
allowed_capabilities=request.allowed_capabilities,
requests_per_minute=request.requests_per_minute,
max_concurrent_requests=request.max_concurrent_requests,
workload_priority=request.workload_priority,
project_binding_id=project_binding.id if project_binding else None,
integration_environment=request.integration_environment,
purpose=request.purpose,
)
)
credential = self.repo.add(
ServiceCredential(
service_client_id=client.id,
secret_hash=hashlib.sha256(secret.encode()).hexdigest(),
secret_prefix=secret[:12],
created_at=_now(),
expires_at=request.credential_expires_at,
)
)
self.audit.write(
"CLIENT_CREATED",
"service_client",
str(client.id),
{
"name": client.name,
"allowed_capabilities": client.allowed_capabilities,
"project_key": project.key if project else None,
"integration_environment": client.integration_environment,
},
)
self._commit()
return ServiceClientCreated(
**self.client_response(client, credential).model_dump(), credential=secret
)
def client_response(
self, client: ServiceClient, credential: ServiceCredential | None = None
) -> ServiceClientResponse:
credential = credential or self.repo.active_credential(client.id)
project_key = None
if client.project_binding_id is not None:
binding = self.session.get(ProjectBinding, client.project_binding_id)
project = self.session.get(Project, binding.project_id) if binding else None
project_key = project.key if project else None
return ServiceClientResponse(
id=client.id,
name=client.name,
status=client.status,
allowed_capabilities=client.allowed_capabilities,
requests_per_minute=client.requests_per_minute,
max_concurrent_requests=client.max_concurrent_requests,
workload_priority=client.workload_priority,
project_key=project_key,
project_binding_id=client.project_binding_id,
integration_environment=client.integration_environment,
purpose=client.purpose,
credential_prefix=credential.secret_prefix if credential else None,
credential_created_at=credential.created_at if credential else None,
credential_expires_at=credential.expires_at if credential else None,
credential_revoked_at=credential.revoked_at if credential else None,
last_used_at=client.last_used_at,
created_at=client.created_at,
)
def clients(self) -> list[ServiceClientResponse]:
return [self.client_response(item) for item in self.repo.clients()]
def revoke_client_credential(self, client_id: uuid.UUID) -> ServiceClientResponse:
client = self.repo.client(client_id)
credential = self.repo.active_credential(client_id) if client else None
if not client:
raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
if credential and not credential.revoked_at:
credential.revoked_at = _now()
self.audit.write(
"CLIENT_CREDENTIAL_REVOKED", "service_client", str(client.id), {"name": client.name}
)
self._commit()
return self.client_response(client, credential)
def rotate_client_credential(self, client_id: uuid.UUID) -> ServiceClientCreated:
client = self.repo.client(client_id)
if client is None:
raise ServingError(404, "CLIENT_NOT_FOUND", "service client was not found")
current = self.repo.active_credential(client_id)
if current is not None:
current.revoked_at = _now()
self.session.flush()
secret = "mf_" + secrets.token_urlsafe(36)
credential = self.repo.add(
ServiceCredential(
service_client_id=client.id,
secret_hash=hashlib.sha256(secret.encode()).hexdigest(),
secret_prefix=secret[:12],
created_at=_now(),
)
)
self.audit.write(
"CLIENT_CREDENTIAL_ROTATED", "service_client", str(client.id), {"name": client.name}
)
self._commit()
return ServiceClientCreated(
**self.client_response(client, credential).model_dump(), credential=secret
)
def record_project_fit(self, request: ProjectFitEvidenceCreate) -> ProjectFitEvidenceResponse:
project = self.session.scalar(select(Project).where(Project.key == request.project_key))
if project is None:
raise ServingError(404, "PROJECT_NOT_FOUND", "project was not found")
key, raw_version = request.capability.rsplit("@", 1)
contract = self.repo.contract(key, int(raw_version))
if contract is None:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability contract was not found")
bindings = list(
self.session.scalars(
select(ProjectBinding)
.where(
ProjectBinding.project_id == project.id,
ProjectBinding.capability_contract_id == contract.id,
ProjectBinding.deprecated_at.is_(None),
)
.order_by(ProjectBinding.created_at.desc())
)
)
preferred_channels = (
{"stable"} if request.environment == "production" else {"experimental", "candidate"}
)
binding = next(
(item for item in bindings if item.channel in preferred_channels),
bindings[0] if bindings else None,
)
if binding is None:
raise ServingError(409, "PROJECT_CAPABILITY_NOT_BOUND", "project binding was not found")
if request.capability_deployment_id is not None:
deployment = self.repo.deployment(request.capability_deployment_id)
if deployment is None:
raise ServingError(
404, "DEPLOYMENT_NOT_FOUND", "capability deployment was not found"
)
if deployment.capability_contract_id != contract.id:
raise ServingError(
422,
"PROJECT_FIT_DEPLOYMENT_MISMATCH",
"project-fit deployment does not implement the requested capability",
)
if request.recommendation == "PROMOTION_ELIGIBLE" and (
request.evidence_class != "OWNER_PHOTO"
or request.production_validation != "SATISFIED"
or request.case_count == 0
or request.critical_errors > 0
or request.blockers
):
raise ServingError(
422,
"PROJECT_FIT_GATES_FAILED",
"project production eligibility requires reviewed owner-photo cases, a satisfied "
"production gate, zero critical errors and no blockers",
)
if request.production_validation == "SATISFIED" and (
request.evidence_class != "OWNER_PHOTO"
or request.recommendation != "PROMOTION_ELIGIBLE"
or request.case_count == 0
or request.critical_errors > 0
or request.blockers
):
raise ServingError(
422,
"PROJECT_PRODUCTION_VALIDATION_FAILED",
"production validation can only be satisfied by eligible owner-photo evidence",
)
payload = request.model_dump(mode="json", exclude={"capability_deployment_id"})
digest = _canonical_hash(payload)
existing = self.session.scalar(
select(ProjectFitEvidence).where(ProjectFitEvidence.evidence_digest == digest)
)
if existing is not None:
return self._project_fit_response(existing, project, request.capability)
item = self.repo.add(
ProjectFitEvidence(
project_binding_id=binding.id,
capability_deployment_id=request.capability_deployment_id,
environment=request.environment,
recommendation=request.recommendation,
evidence_class=request.evidence_class,
engineering_integration=request.engineering_integration,
production_validation=request.production_validation,
production_action=request.production_action,
case_count=request.case_count,
metric_values=request.metric_values,
critical_errors=request.critical_errors,
blockers=request.blockers,
evidence_digest=digest,
evidence=request.evidence,
)
)
self.audit.write(
"PROJECT_FIT_RECORDED",
"project_fit_evidence",
str(item.id),
{
"project_key": project.key,
"capability": request.capability,
"recommendation": item.recommendation,
"evidence_class": item.evidence_class,
"engineering_integration": item.engineering_integration,
"production_validation": item.production_validation,
"production_action": item.production_action,
},
)
self._commit()
return self._project_fit_response(item, project, request.capability)
def _project_fit_response(
self, item: ProjectFitEvidence, project: Project, capability: str
) -> ProjectFitEvidenceResponse:
return ProjectFitEvidenceResponse(
id=item.id,
project_key=project.key,
capability=capability,
environment=item.environment,
recommendation=item.recommendation,
evidence_class=item.evidence_class,
engineering_integration=item.engineering_integration,
production_validation=item.production_validation,
production_action=item.production_action,
case_count=item.case_count,
metric_values=item.metric_values,
critical_errors=item.critical_errors,
blockers=item.blockers,
evidence_digest=item.evidence_digest,
evidence=item.evidence,
capability_deployment_id=item.capability_deployment_id,
created_at=item.created_at,
)
def project_integrations(self) -> list[ProjectIntegrationResponse]:
responses: list[ProjectIntegrationResponse] = []
for client in self.repo.clients():
if client.project_binding_id is None:
continue
binding = self.session.get(ProjectBinding, client.project_binding_id)
project = self.session.get(Project, binding.project_id) if binding else None
contract = (
self.session.get(CapabilityContract, binding.capability_contract_id)
if binding
else None
)
capability = self.session.get(Capability, contract.capability_id) if contract else None
if binding is None or project is None or contract is None or capability is None:
continue
capability_key = f"{capability.key}@{contract.version}"
requests = list(
self.session.scalars(
select(GatewayRequest)
.where(GatewayRequest.service_client_id == client.id)
.order_by(GatewayRequest.created_at)
)
)
successful = [item for item in requests if item.status == "completed"]
errors = [item for item in requests if item.status == "failed"]
latencies = sorted(
item.total_latency_ms for item in successful if item.total_latency_ms is not None
)
latest_request = requests[-1] if requests else None
fit = self.session.scalar(
select(ProjectFitEvidence)
.where(ProjectFitEvidence.project_binding_id == binding.id)
.order_by(ProjectFitEvidence.created_at.desc())
)
deployment_id = (
fit.capability_deployment_id
if fit and fit.capability_deployment_id
else (latest_request.capability_deployment_id if latest_request else None)
)
envelope = (
self.session.scalar(
select(CapabilityResourceEnvelope)
.where(CapabilityResourceEnvelope.capability_deployment_id == deployment_id)
.order_by(CapabilityResourceEnvelope.created_at.desc())
)
if deployment_id
else None
)
responses.append(
ProjectIntegrationResponse(
project_key=project.key,
project_name=project.name,
capability=capability_key,
environment=client.integration_environment or binding.channel,
state="ACTIVE" if client.status == "active" else client.status.upper(),
purpose=client.purpose or "unspecified",
client_id=client.id,
client_name=client.name,
deployment_id=deployment_id,
resource_impact_bytes=envelope.required_vram_bytes if envelope else None,
usage=ProjectIntegrationUsage(
request_volume=len(requests),
successful_requests=len(successful),
error_count=len(errors),
last_used_at=client.last_used_at,
latency_p50_ms=_percentile(latencies, 0.50),
latency_p95_ms=_percentile(latencies, 0.95),
),
project_fit=(
self._project_fit_response(fit, project, capability_key) if fit else None
),
)
)
return responses
@staticmethod
def _credential_secret(authorization: str | None) -> str:
if not authorization or not authorization.startswith("Bearer "):
raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is required")
secret = authorization.removeprefix("Bearer ").strip()
if not secret:
raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is required")
return secret
def _validate_capability_access(
self,
credential: ServiceCredential,
client: ServiceClient,
capability: str,
*,
enforce_request_limits: bool,
) -> None:
now = _now()
if credential.revoked_at:
raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is revoked")
if credential.expires_at and _aware(credential.expires_at) <= now:
raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is expired")
if client.status != "active" or client.disabled_at:
raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "service client is disabled")
if capability not in client.allowed_capabilities:
raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability scope is not granted")
if client.project_binding_id is not None:
binding = self.session.get(ProjectBinding, client.project_binding_id)
contract = (
self.session.get(CapabilityContract, binding.capability_contract_id)
if binding
else None
)
bound_capability = (
self.session.get(Capability, contract.capability_id) if contract else None
)
bound_key = (
f"{bound_capability.key}@{contract.version}"
if bound_capability and contract
else None
)
if binding is None or binding.deprecated_at is not None or bound_key != capability:
raise ServingError(
403, "PROJECT_CAPABILITY_NOT_BOUND", "project binding is inactive"
)
if not enforce_request_limits:
return
since = now - timedelta(minutes=1)
if self.repo.recent_request_count(client.id, since) >= client.requests_per_minute:
raise ServingError(429, "RATE_LIMITED", "service client rate limit exceeded")
if self.repo.active_client_requests(client.id) >= client.max_concurrent_requests:
raise ServingError(429, "QUEUE_FULL", "service client concurrency limit reached")
def authenticate_with_evidence(
self, authorization: str | None, capability: str
) -> tuple[ServiceClient, CapabilityAuthenticationEvidence]:
secret = self._credential_secret(authorization)
credential_sha256 = hashlib.sha256(secret.encode()).hexdigest()
match = self.repo.credential_by_hash(credential_sha256)
if not match:
raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is invalid")
credential, client = match
self._validate_capability_access(
credential,
client,
capability,
enforce_request_limits=True,
)
now = _now()
credential.last_used_at = now
client.last_used_at = now
self._commit_authentication()
return client, CapabilityAuthenticationEvidence(
credential_id=credential.id,
client_id=client.id,
capability=capability,
credential_sha256=credential_sha256,
)
def authenticate(self, authorization: str | None, capability: str) -> ServiceClient:
client, _evidence = self.authenticate_with_evidence(authorization, capability)
return client
def reuse_authentication(
self,
evidence: CapabilityAuthenticationEvidence,
authorization: str | None,
capability: str,
) -> ServiceClient:
"""Revalidate current scope/client state while reusing the pre-body credential proof."""
secret = self._credential_secret(authorization)
credential_sha256 = hashlib.sha256(secret.encode()).hexdigest()
if capability != evidence.capability or not hmac.compare_digest(
credential_sha256, evidence.credential_sha256
):
raise ServingError(
403,
"CAPABILITY_NOT_AUTHORIZED",
"pre-authenticated capability scope does not match this route",
)
credential = self.session.get(ServiceCredential, evidence.credential_id)
client = self.session.get(ServiceClient, evidence.client_id)
if (
credential is None
or client is None
or credential.service_client_id != client.id
or not hmac.compare_digest(credential.secret_hash, credential_sha256)
):
raise ServingError(401, "CAPABILITY_NOT_AUTHORIZED", "service credential is invalid")
self._validate_capability_access(
credential,
client,
capability,
enforce_request_limits=False,
)
return client
def _budget(self, deployment: CapabilityDeployment) -> SchedulerBudgetResponse:
node = self.repo.node(deployment.compute_node_id)
accelerator = self.repo.accelerator(deployment.accelerator_id)
telemetry = self.repo.telemetry(deployment.accelerator_id)
residency = self.repo.residency(deployment.id)
if not node or not accelerator or not residency or accelerator.total_vram_bytes is None:
raise ServingError(
503, "NO_ELIGIBLE_NODE", "deployment hardware evidence is unavailable"
)
observed_used = (
int(telemetry.used_vram_bytes)
if telemetry and telemetry.used_vram_bytes is not None
else None
)
resident = sum(
item.measured_resident_vram_bytes
for candidate in self.repo.deployments()
if candidate.accelerator_id == accelerator.id
and candidate.status in {"stable", "draining", "candidate"}
and (item := self.repo.residency(candidate.id)) is not None
and item.state != "cold"
)
leases = self.repo.active_leases(accelerator.id, _now())
policy = self._scheduler_policy()
pressure_row = self.repo.scheduler_accelerator_state(accelerator.id)
previous_pressure = (
PressureState(pressure_row.pressure_state) if pressure_row else PressureState.NORMAL
)
snapshot = calculate_accounting(
total_bytes=accelerator.total_vram_bytes,
observed_used_bytes=observed_used,
resident_bytes=resident,
leases=[
LeaseAllocation(item.reserved_vram_bytes, item.materialized_vram_bytes)
for item in leases
],
policy=policy,
telemetry_fresh=(
telemetry is not None
and _aware(telemetry.observed_at)
>= _now() - timedelta(seconds=self.settings.scheduler_telemetry_stale_seconds)
),
previous_pressure=previous_pressure,
)
snapshot = replace(
snapshot,
pressure=self._stabilize_pressure(accelerator.id, snapshot.pressure, pressure_row),
)
return SchedulerBudgetResponse(
compute_node_id=node.id,
node_name=node.display_name or node.hostname,
accelerator_id=accelerator.id,
accelerator_uuid=accelerator.device_uuid,
accelerator_name=accelerator.name,
total_vram_bytes=accelerator.total_vram_bytes,
observed_used_vram_bytes=observed_used or 0,
external_vram_bytes=snapshot.external_bytes,
resident_vram_bytes=resident,
leased_vram_bytes=snapshot.future_lease_bytes,
safety_reserve_bytes=snapshot.reserve_bytes,
schedulable_free_vram_bytes=snapshot.schedulable_bytes,
pressure=snapshot.pressure.value != "NORMAL",
pressure_state=snapshot.pressure.value,
attribution_confidence=snapshot.attribution.value,
invariant_delta_bytes=snapshot.invariant_delta_bytes,
policy_revision=policy.revision,
observed_at=telemetry.observed_at if telemetry else None,
)
def _stabilize_pressure(
self,
accelerator_id: uuid.UUID,
candidate: PressureState,
row: SchedulerAcceleratorState | None,
) -> PressureState:
now = _now()
if row is None:
self.repo.add(
SchedulerAcceleratorState(
accelerator_id=accelerator_id,
pressure_state=candidate.value,
pressure_changed_at=now,
last_observed_at=now,
)
)
return candidate
current = PressureState(row.pressure_state)
severity = {
PressureState.NORMAL: 0,
PressureState.ELEVATED: 1,
PressureState.HIGH: 2,
PressureState.CRITICAL: 3,
}
row.last_observed_at = now
if severity[candidate] > severity[current]:
row.pressure_state = candidate.value
row.pressure_changed_at = now
row.recovery_candidate = None
row.recovery_candidate_since = None
return candidate
if candidate == current:
row.recovery_candidate = None
row.recovery_candidate_since = None
return current
if row.recovery_candidate != candidate.value:
row.recovery_candidate = candidate.value
row.recovery_candidate_since = now
return current
if (
row.recovery_candidate_since
and _aware(row.recovery_candidate_since)
+ timedelta(seconds=self.settings.scheduler_pressure_stable_seconds)
<= now
):
row.pressure_state = candidate.value
row.pressure_changed_at = now
row.recovery_candidate = None
row.recovery_candidate_since = None
return candidate
return current
def _scheduler_policy(self) -> SchedulerPolicy:
configured = SchedulerPolicy(
reserve_minimum_bytes=self.settings.scheduler_safety_reserve_bytes,
reserve_percentage=self.settings.scheduler_safety_reserve_percentage,
runtime_margin_bytes=self.settings.scheduler_runtime_margin_bytes,
deployment_margin_minimum_bytes=self.settings.scheduler_deployment_margin_bytes,
deployment_margin_percentage=self.settings.scheduler_deployment_margin_percentage,
global_queue_limit=self.settings.scheduler_global_queue_limit,
pressure_stable_seconds=self.settings.scheduler_pressure_stable_seconds,
eviction_cooldown_seconds=self.settings.scheduler_eviction_cooldown_seconds,
placement_history_limit=self.settings.scheduler_placement_history_limit,
)
active = self.repo.active_scheduler_policy()
if not active:
return configured
allowed = set(SchedulerPolicy.__dataclass_fields__)
values = {key: value for key, value in active.configuration.items() if key in allowed}
values["revision"] = active.revision
return SchedulerPolicy(**values)
def scheduler_policy(self) -> SchedulerPolicyResponse:
policy = self._scheduler_policy()
active = self.repo.active_scheduler_policy()
return SchedulerPolicyResponse(
revision=policy.revision,
active=True,
configuration=asdict(policy),
created_at=active.created_at if active else _now(),
)
def scheduler_metrics(self) -> SchedulerMetricsResponse:
plans = self.repo.placement_plans(self.settings.scheduler_placement_history_limit)
requests = self.repo.gateway_requests(self.settings.scheduler_placement_history_limit)
budgets = self.scheduler_overview()
counters = {
"scheduler_admissions_total": sum(
item.verdict in {"ADMIT", "ADMIT_AFTER_EVICTION"} for item in plans
),
"scheduler_rejections_total": sum(item.verdict.startswith("REJECT_") for item in plans),
"scheduler_evictions_total": sum(
item.verdict == "ADMIT_AFTER_EVICTION" for item in plans
),
"cold_loads": sum(item.cold is True for item in requests),
"warm_hits": sum(item.cold is False for item in requests),
"reclaim_failures": sum(
item.failure_code == "GPU_MEMORY_NOT_RECLAIMED" for item in requests
),
}
queue_values = [item.queue_time_ms / 1000 for item in requests if item.queue_time_ms]
return SchedulerMetricsResponse(
counters=counters,
gauges={
"resident_bytes": sum(item.resident_vram_bytes for item in budgets),
"leased_bytes": sum(item.leased_vram_bytes for item in budgets),
"external_bytes": sum(item.external_vram_bytes for item in budgets),
"schedulable_bytes": sum(item.schedulable_free_vram_bytes for item in budgets),
"pressure_state": max((item.pressure_state for item in budgets), default="UNKNOWN"),
},
queue_seconds={
"count": float(len(queue_values)),
"maximum": max(queue_values, default=0.0),
"average": sum(queue_values) / len(queue_values) if queue_values else 0.0,
},
)
def update_scheduler_policy(self, request: SchedulerPolicyUpdate) -> SchedulerPolicyResponse:
current = self._scheduler_policy()
active = self.repo.active_scheduler_policy()
if active:
active.active = False
revision = f"m10-v{int(active.revision.removeprefix('m10-v') if active else '1') + 1}"
configuration = {**asdict(current), "revision": revision, "lab_paused": request.lab_paused}
record = self.repo.add(
SchedulerPolicyRevision(
revision=revision, configuration=configuration, active=True, created_at=_now()
)
)
self.audit.write(
"SCHEDULER_POLICY_CHANGED",
"scheduler_policy",
str(record.id),
{"revision": revision, "lab_paused": request.lab_paused},
)
self._commit()
return self.scheduler_policy()
def _capability_key(self, deployment: CapabilityDeployment) -> str:
contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
capability = self.session.get(Capability, contract.capability_id) if contract else None
return capability.key if capability else "unknown.capability"
def _placement_plan(
self,
deployment: CapabilityDeployment,
*,
priority: str,
deadline_remaining_ms: float | None = None,
request_id: uuid.UUID | None = None,
dry_run: bool,
persist: bool = True,
commit: bool = True,
) -> PlacementPlanResponse:
residency = self.repo.residency(deployment.id)
envelope = self.repo.envelope(deployment.id)
node = self.repo.node(deployment.compute_node_id)
if not residency or not envelope or not node:
raise ServingError(503, "SCHEDULER_STATE_STALE", "scheduler state is incomplete")
budget = self._budget(deployment)
policy = self._scheduler_policy()
residents: list[ResidentCandidate] = []
for item in self.repo.accelerator_residencies(deployment.accelerator_id):
if item.state in {"cold", "failed", "load_failed"}:
continue
other = self.repo.deployment(item.capability_deployment_id)
other_envelope = self.repo.envelope(item.capability_deployment_id)
if not other:
continue
residents.append(
ResidentCandidate(
deployment_id=str(other.id),
capability=self._capability_key(other),
resident_bytes=max(0, item.measured_resident_vram_bytes),
active_requests=item.active_requests,
priority="production" if other.production else "lab",
policy=other.residency_policy,
idle_since=item.last_used_at,
cold_load_ms=other_envelope.cold_load_time_ms if other_envelope else 0.0,
last_used_at=item.last_used_at,
pinned=other.residency_policy == "always_warm",
)
)
is_resident = residency.state not in {"cold", "failed", "load_failed"}
required = (
policy.request_execution_floor_bytes
if is_resident
else max(
envelope.required_vram_bytes, required_envelope(envelope.peak_vram_bytes, policy)
)
)
snapshot = AccountingSnapshot(
total_bytes=budget.total_vram_bytes,
observed_used_bytes=budget.observed_used_vram_bytes,
resident_bytes=budget.resident_vram_bytes,
future_lease_bytes=budget.leased_vram_bytes,
reserve_bytes=budget.safety_reserve_bytes,
external_bytes=budget.external_vram_bytes,
schedulable_bytes=budget.schedulable_free_vram_bytes,
attribution=AttributionConfidence(budget.attribution_confidence),
invariant_delta_bytes=budget.invariant_delta_bytes,
pressure=PressureState(budget.pressure_state),
)
decision = plan_placement(
PlacementInput(
deployment_id=str(deployment.id),
capability=self._capability_key(deployment),
node_id=str(node.id),
accelerator_id=str(deployment.accelerator_id),
priority=priority,
required_bytes=required,
cold_load_ms=envelope.cold_load_time_ms,
is_resident=is_resident,
envelope_stale=envelope.stale,
runtime_healthy=deployment.health_status not in {"failed", "unavailable"},
node_eligible=node.enabled and node.liveness_state == "online",
deadline_remaining_ms=deadline_remaining_ms,
),
snapshot,
residents,
policy,
global_queue_depth=sum(
self.repo.queued_job_count(item.id) for item in self.repo.deployments()
),
)
created = _now()
record: PlacementPlanRecord | None = None
evidence = {
**decision.evidence(),
"capability": self._capability_key(deployment),
"node_id": str(node.id),
"accelerator_id": str(deployment.accelerator_id),
"budget": budget.model_dump(mode="json"),
"priority": priority,
}
if persist:
record = self.repo.add(
PlacementPlanRecord(
capability_deployment_id=deployment.id,
request_id=request_id,
policy_revision=policy.revision,
verdict=decision.verdict.value,
reason_codes=list(decision.reason_codes),
evidence=evidence,
evidence_fingerprint=decision.fingerprint,
dry_run=dry_run,
created_at=created,
)
)
self.repo.trim_placement_plans(policy.placement_history_limit)
if commit:
self._commit()
return PlacementPlanResponse(
id=record.id if record else None,
deployment_id=deployment.id,
capability=self._capability_key(deployment),
node_id=node.id,
accelerator_id=deployment.accelerator_id,
policy_revision=policy.revision,
verdict=decision.verdict.value,
reason_codes=list(decision.reason_codes),
required_vram_bytes=decision.required_bytes,
headroom_before_bytes=decision.headroom_before_bytes,
headroom_after_bytes=decision.headroom_after_bytes,
expected_cold_load_ms=envelope.cold_load_time_ms,
evictions=[
PlacementEvictionResponse(
deployment_id=uuid.UUID(item.deployment_id),
capability=item.capability,
expected_reclaimed_bytes=item.expected_reclaimed_bytes,
reason=item.reason,
)
for item in decision.evictions
],
decision_fingerprint=decision.fingerprint,
dry_run=dry_run,
created_at=created,
)
def dry_run_placement(
self, deployment_id: uuid.UUID, request: PlacementPlanRequest
) -> PlacementPlanResponse:
deployment = self.repo.deployment(deployment_id)
if not deployment:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
return self._placement_plan(
deployment,
priority=request.priority,
deadline_remaining_ms=request.deadline_remaining_ms,
dry_run=True,
)
def placement_history(self, limit: int = 100) -> list[PlacementPlanResponse]:
responses: list[PlacementPlanResponse] = []
for record in self.repo.placement_plans(limit):
deployment = self.repo.deployment(record.capability_deployment_id)
if not deployment:
continue
evidence = record.evidence
responses.append(
PlacementPlanResponse(
id=record.id,
deployment_id=deployment.id,
capability=str(evidence.get("capability", self._capability_key(deployment))),
node_id=uuid.UUID(str(evidence["node_id"])),
accelerator_id=uuid.UUID(str(evidence["accelerator_id"])),
policy_revision=record.policy_revision,
verdict=cast(Any, record.verdict),
reason_codes=record.reason_codes,
required_vram_bytes=int(evidence["required_bytes"]),
headroom_before_bytes=int(evidence["headroom_before_bytes"]),
headroom_after_bytes=int(evidence["headroom_after_bytes"]),
expected_cold_load_ms=float(evidence.get("cold_load_ms", 0.0)),
evictions=[
PlacementEvictionResponse.model_validate(item)
for item in evidence.get("evictions", [])
],
decision_fingerprint=record.evidence_fingerprint,
dry_run=record.dry_run,
created_at=record.created_at,
)
)
return responses
def scheduler_overview(self) -> list[SchedulerBudgetResponse]:
representatives: dict[uuid.UUID, CapabilityDeployment] = {}
for deployment in self.repo.deployments():
if deployment.status not in {"stable", "draining"}:
continue
current = representatives.get(deployment.accelerator_id)
if current is None or (current.status != "stable" and deployment.status == "stable"):
representatives[deployment.accelerator_id] = deployment
budgets = [self._budget(item) for item in representatives.values()]
self._commit()
return budgets
def co_residency_matrix(self) -> list[CoResidencyEvidenceResponse]:
eligible = [
item
for item in self.repo.deployments()
if item.status in {"stable", "candidate", "draining"}
and self.repo.envelope(item.id) is not None
]
result: list[CoResidencyEvidenceResponse] = []
for left, right in itertools.combinations(eligible, 2):
if left.accelerator_id != right.accelerator_id:
continue
left_envelope = self.repo.envelope(left.id)
right_envelope = self.repo.envelope(right.id)
if not left_envelope or not right_envelope:
continue
evidence = self.repo.co_residency_evidence(left.id, right.id)
budget = self._budget(left)
policy = self._scheduler_policy()
left_required = max(
left_envelope.required_vram_bytes,
required_envelope(left_envelope.peak_vram_bytes, policy),
)
right_required = max(
right_envelope.required_vram_bytes,
required_envelope(right_envelope.peak_vram_bytes, policy),
)
combined = left_required + right_required
status = (
evidence.status
if evidence
else (
"UNKNOWN"
if left_envelope.stale
or right_envelope.stale
or budget.attribution_confidence == "UNKNOWN"
else (
"EXPECTED_SAFE"
if combined
<= budget.total_vram_bytes
- budget.external_vram_bytes
- budget.safety_reserve_bytes
else "NOT_SAFE"
)
)
)
result.append(
CoResidencyEvidenceResponse(
left_deployment_id=left.id,
left_capability=self._capability_key(left),
right_deployment_id=right.id,
right_capability=self._capability_key(right),
left_alone_bytes=left_required,
right_alone_bytes=right_required,
expected_combined_bytes=combined,
measured_combined_bytes=(
evidence.measured_combined_bytes if evidence else None
),
status=cast(Any, status),
evidence=evidence.evidence
if evidence
else {"source": "authoritative_envelopes"},
measured_at=evidence.measured_at if evidence else None,
)
)
self._commit()
return result
def record_co_residency_evidence(
self, request: CoResidencyEvidenceCreate
) -> CoResidencyEvidenceResponse:
left_id, right_id = sorted(
(request.left_deployment_id, request.right_deployment_id), key=str
)
left = self.repo.deployment(left_id)
right = self.repo.deployment(right_id)
if not left or not right:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "deployment was not found")
if (
left.compute_node_id != right.compute_node_id
or left.accelerator_id != right.accelerator_id
):
raise ServingError(
409,
"RESIDENCY_CONFLICT",
"co-residency evidence requires one node and accelerator",
)
left_residency = self.repo.residency(left.id, lock=True)
right_residency = self.repo.residency(right.id, lock=True)
left_envelope = self.repo.envelope(left.id)
right_envelope = self.repo.envelope(right.id)
if not left_residency or not right_residency or not left_envelope or not right_envelope:
raise ServingError(503, "SCHEDULER_STATE_STALE", "scheduler state is incomplete")
resident_states = {"warm", "busy", "idle"}
exact_worker = (
left_residency.worker_instance_id
and left_residency.worker_instance_id == right_residency.worker_instance_id
and left_residency.generation == right_residency.generation
)
if (
left_residency.state not in resident_states
or right_residency.state not in resident_states
or left_residency.measured_resident_vram_bytes <= 0
or right_residency.measured_resident_vram_bytes <= 0
or not exact_worker
):
raise ServingError(
409,
"RESIDENCY_CONFLICT",
"both exact deployments must be resident in the same worker generation",
)
budget = self._budget(left)
if budget.attribution_confidence != "KNOWN" or budget.observed_at is None:
raise ServingError(
503,
"SCHEDULER_STATE_STALE",
"known current GPU attribution is required for live evidence",
)
policy = self._scheduler_policy()
left_required = max(
left_envelope.required_vram_bytes,
required_envelope(left_envelope.peak_vram_bytes, policy),
)
right_required = max(
right_envelope.required_vram_bytes,
required_envelope(right_envelope.peak_vram_bytes, policy),
)
measured = (
left_residency.measured_resident_vram_bytes
+ right_residency.measured_resident_vram_bytes
)
row = self.repo.co_residency_evidence(left.id, right.id)
if row is None:
row = self.repo.add(
CoResidencyEvidence(
left_deployment_id=left.id,
right_deployment_id=right.id,
status="PROVEN_SAFE",
expected_combined_bytes=left_required + right_required,
measured_combined_bytes=measured,
evidence={},
)
)
row.status = "PROVEN_SAFE"
row.expected_combined_bytes = left_required + right_required
row.measured_combined_bytes = measured
row.measured_at = budget.observed_at
row.evidence = {
"source": "live_worker_residency_and_nvml",
"worker_generation": left_residency.generation,
"policy_revision": policy.revision,
"observed_used_vram_bytes": budget.observed_used_vram_bytes,
"external_vram_bytes": budget.external_vram_bytes,
"safety_reserve_bytes": budget.safety_reserve_bytes,
"schedulable_free_vram_bytes": budget.schedulable_free_vram_bytes,
"attribution_confidence": budget.attribution_confidence,
"invariant_delta_bytes": budget.invariant_delta_bytes,
"content_persisted": False,
}
self._commit()
return CoResidencyEvidenceResponse(
left_deployment_id=left.id,
left_capability=self._capability_key(left),
right_deployment_id=right.id,
right_capability=self._capability_key(right),
left_alone_bytes=left_required,
right_alone_bytes=right_required,
expected_combined_bytes=row.expected_combined_bytes,
measured_combined_bytes=row.measured_combined_bytes,
status="PROVEN_SAFE",
evidence=row.evidence,
measured_at=row.measured_at,
)
def request_history(self, limit: int = 100) -> list[GatewayRequestResponse]:
client_names = {item.id: item.name for item in self.repo.clients()}
return [
GatewayRequestResponse(
request_id=item.request_id,
capability=f"{item.capability_key}@{item.capability_version}",
client=client_names.get(item.service_client_id) if item.service_client_id else None,
status=item.status,
cold=item.cold,
queue_time_ms=item.queue_time_ms,
load_time_ms=item.load_time_ms,
inference_time_ms=item.inference_time_ms,
total_latency_ms=item.total_latency_ms,
failure_code=item.failure_code,
latency_breakdown={
str(key): float(value)
for key, value in item.decision_evidence.get("latency_ms", {}).items()
if isinstance(value, int | float)
},
created_at=item.created_at,
)
for item in self.repo.gateway_requests(limit)
]
def _stable_embedding_deployment(self) -> CapabilityDeployment:
contract = self.ensure_contract("rag.embedding", 1)
deployment = self.repo.stable_deployment(contract.id)
if not deployment:
raise ServingError(
503, "CAPABILITY_UNAVAILABLE", "rag.embedding@1 has no stable deployment"
)
if deployment.status != "stable" or not deployment.production:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "stable deployment is not routable")
node = self.repo.node(deployment.compute_node_id)
if (
not node
or node.decommissioned_at is not None
or not node.enabled
or not node.production_eligible
or node.liveness_state != "online"
):
raise ServingError(
503, "NO_ELIGIBLE_NODE", "no production-eligible online node is available"
)
return deployment
def _reranking_deployment(self) -> tuple[CapabilityDeployment, str]:
contract = self.ensure_contract("rag.reranking", 1)
routes = [
route
for route in self.repo.experiment_routes()
if route.capability_contract_id == contract.id and route.status == "active"
]
deployments = [
(self.repo.deployment(route.capability_deployment_id), route.route_key)
for route in routes
]
routable = [
(deployment, route_key)
for deployment, route_key in deployments
if deployment
and not deployment.production
and deployment.channel == "experiment"
and deployment.status == "candidate"
]
if len(routable) != 1:
raise ServingError(
503,
"RERANKER_NOT_AVAILABLE",
"rag.reranking@1 requires exactly one active evaluated experiment deployment",
)
deployment, route_key = routable[0]
node = self.repo.node(deployment.compute_node_id)
if (
not node
or node.decommissioned_at is not None
or not node.enabled
or node.liveness_state != "online"
):
raise ServingError(503, "NO_ELIGIBLE_NODE", "reranker node is not online")
return deployment, route_key
def _experimental_deployment(self, capability_key: str) -> tuple[CapabilityDeployment, str]:
contract = self.ensure_contract(capability_key, 1)
routable: list[tuple[CapabilityDeployment, str]] = []
for route in self.repo.experiment_routes():
if route.capability_contract_id != contract.id or route.status != "active":
continue
deployment = self.repo.deployment(route.capability_deployment_id)
if (
deployment
and not deployment.production
and deployment.channel == "experiment"
and deployment.status == "candidate"
):
routable.append((deployment, route.route_key))
if len(routable) != 1:
raise ServingError(
503,
"CAPABILITY_UNAVAILABLE",
f"{capability_key}@1 requires exactly one active LAB deployment",
)
deployment, route_key = routable[0]
node = self.repo.node(deployment.compute_node_id)
if not node or not node.enabled or node.liveness_state != "online":
raise ServingError(503, "NO_ELIGIBLE_NODE", "capability node is not online")
return deployment, route_key
def _validate_inputs(self, request: EmbeddingInvokeRequest) -> list[str]:
inputs = request.inputs()
if len(inputs) > self.settings.gateway_max_batch_size:
raise ServingError(
422, "INVALID_INPUT", "embedding batch exceeds the configured maximum"
)
if any(len(item) > self.settings.gateway_max_input_characters for item in inputs):
raise ServingError(422, "INVALID_INPUT", "embedding input exceeds the configured limit")
size = len(json.dumps({"input": inputs}, ensure_ascii=False).encode())
if size > self.settings.gateway_max_payload_bytes:
raise ServingError(413, "INVALID_INPUT", "embedding request payload is too large")
return inputs
def _validate_reranking_input(self, request: RerankingInvokeRequest) -> list[dict[str, str]]:
documents = [document.model_dump() for document in request.documents]
size = len(
json.dumps(
{"query": request.query, "documents": documents, "top_n": request.top_n},
ensure_ascii=False,
).encode()
)
if size > min(self.settings.gateway_max_payload_bytes, 262_144):
raise ServingError(413, "RERANK_REQUEST_TOO_LARGE", "reranking payload is too large")
return documents
def _placement_evidence(self, deployment: CapabilityDeployment) -> dict[str, Any]:
candidates = []
for node in self.session.scalars(
select(ComputeNode)
.where(ComputeNode.decommissioned_at.is_(None))
.order_by(ComputeNode.display_name)
):
reasons: list[str] = []
if not node.enabled:
reasons.append("NODE_DISABLED")
if not node.production_eligible:
reasons.append("NODE_NOT_PRODUCTION_ELIGIBLE")
if node.liveness_state != "online":
reasons.append("NODE_NOT_ONLINE")
if node.id != deployment.compute_node_id:
reasons.append("DEPLOYMENT_NODE_CONSTRAINT")
candidates.append(
{
"node_id": str(node.id),
"node": node.display_name or node.hostname,
"eligible": not reasons,
"reasons": reasons,
}
)
return {"candidate_nodes": candidates, "selected_node_id": str(deployment.compute_node_id)}
def _expire_leases(self) -> None:
now = _now()
for lease in self.repo.expired_leases(now):
lease.state = "expired"
lease.released_at = now
lease.failure_code = "LEASE_TIMEOUT"
if lease.request_id:
request = self.session.scalar(
select(GatewayRequest).where(GatewayRequest.request_id == lease.request_id)
)
if request and request.status in {"queued", "running"}:
request.status = "failed"
request.failure_code = "LEASE_TIMEOUT"
request.finished_at = now
def _acquire_request_lease(
self,
deployment: CapabilityDeployment,
gateway_request: GatewayRequest,
client: ServiceClient,
) -> ServingGpuLease:
deadline = time.monotonic() + self.settings.gateway_queue_timeout_seconds
while time.monotonic() < deadline:
self.session.expire_all()
self._expire_leases()
self.repo.lock_accelerator(deployment.accelerator_id)
current = self.session.get(GatewayRequest, gateway_request.id)
residency = self.repo.residency(deployment.id, lock=True)
envelope = self.repo.envelope(deployment.id)
if not current or not residency or not envelope:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "scheduler state is incomplete")
if envelope.stale:
raise ServingError(
503,
"SCHEDULER_STATE_STALE",
"measured resource envelope is stale and must be revalidated",
{"stale_reason": envelope.stale_reason},
)
active = self.repo.active_leases(deployment.accelerator_id, _now())
active_for_deployment = [
item
for item in active
if item.capability_deployment_id == deployment.id
and item.state in {"granted", "active"}
]
if len(active_for_deployment) >= deployment.max_concurrency:
self.session.commit()
time.sleep(0.05)
continue
resident = residency.measured_resident_vram_bytes if residency.state != "cold" else 0
plan = self._placement_plan(
deployment,
priority=client.workload_priority,
deadline_remaining_ms=float(self.settings.gateway_request_timeout_seconds * 1000),
request_id=current.request_id,
dry_run=False,
commit=False,
)
if plan.verdict == PlacementVerdict.ADMIT_AFTER_EVICTION.value:
for eviction in plan.evictions:
candidate = self.repo.residency(eviction.deployment_id, lock=True)
other = self.repo.deployment(eviction.deployment_id)
if not candidate or not other or candidate.active_requests > 0:
raise ServingError(
409, "EVICTION_NOT_ALLOWED", "planned residency is no longer idle"
)
candidate.state = "unloading"
candidate.transition_reason = eviction.reason
self.repo.add(
SchedulerEviction(
capability_deployment_id=other.id,
requested_deployment_id=deployment.id,
reason_code=eviction.reason,
expected_reclaimed_bytes=eviction.expected_reclaimed_bytes,
state="queued",
)
)
self._commit()
unload = self._enqueue_job(
other,
"unload",
key_suffix=(
f"scheduler-{plan.decision_fingerprint[:16]}-{candidate.load_count}"
),
priority=client.workload_priority,
)
self._wait_job(unload.id, deadline)
continue
if plan.verdict == PlacementVerdict.QUEUE.value:
self.session.commit()
time.sleep(0.05)
continue
if plan.verdict != PlacementVerdict.ADMIT.value:
code = (
plan.reason_codes[0] if plan.reason_codes else "INSUFFICIENT_SCHEDULABLE_VRAM"
)
self._commit()
raise ServingError(
503,
code,
"scheduler rejected placement using measured capacity and policy",
{
"verdict": plan.verdict,
"reason_codes": plan.reason_codes,
"required_vram_bytes": plan.required_vram_bytes,
"schedulable_free_vram_bytes": plan.headroom_before_bytes,
"decision_fingerprint": plan.decision_fingerprint,
},
)
required = plan.required_vram_bytes
now = _now()
lease = self.repo.add(
ServingGpuLease(
accelerator_id=deployment.accelerator_id,
capability_deployment_id=deployment.id,
request_id=current.request_id,
reserved_vram_bytes=required,
priority=client.workload_priority,
state="active",
owner=client.name,
created_at=now,
acquired_at=now,
heartbeat_at=now,
expires_at=now
+ timedelta(seconds=self.settings.gateway_request_timeout_seconds),
lease_type=("request_execution" if resident else "residency_reservation"),
materialized_vram_bytes=0,
generation=residency.generation,
)
)
current.status = "running"
current.decision_evidence = {
**current.decision_evidence,
**self._placement_evidence(deployment),
"placement_plan_id": str(plan.id),
"decision": plan.verdict,
"reason_codes": plan.reason_codes,
"decision_fingerprint": plan.decision_fingerprint,
"required_vram_bytes": required,
}
residency.active_requests += 1
if residency.state in {"warm", "idle"}:
residency.state = "busy"
self._commit()
return lease
raise ServingError(504, "REQUEST_TIMEOUT", "request timed out in the bounded queue")
def _enqueue_job(
self,
deployment: CapabilityDeployment,
operation: str,
*,
request: GatewayRequest | None = None,
payload_reference: str | None = None,
key_suffix: str,
priority: str = "production",
) -> ServingJob:
key = _canonical_hash(
{
"deployment": str(deployment.id),
"operation": operation,
"suffix": key_suffix,
}
)
existing = self.repo.job_by_key(key)
if existing:
return existing
job = self.repo.add(
ServingJob(
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
gateway_request_id=request.id if request else None,
operation=operation,
status="queued",
priority=priority,
idempotency_key=key,
payload_reference=payload_reference,
attempt_count=0,
result_summary={},
)
)
self._commit()
return job
def _wait_job(self, job_id: uuid.UUID, deadline: float) -> tuple[ServingJob, dict[str, Any]]:
while time.monotonic() < deadline:
self.session.expire_all()
job = self.repo.job(job_id)
if not job:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "serving job disappeared")
if job.status == "completed":
result = (
self.payload_store.get(job.result_reference) if job.result_reference else {}
)
return job, result or {}
if job.status == "failed":
raise ServingError(
503,
job.failure_code or "INFERENCE_FAILED",
job.failure_message or "serving worker failed",
)
time.sleep(0.05)
raise ServingError(504, "REQUEST_TIMEOUT", "serving worker response timed out")
def _ensure_loaded(
self, deployment: CapabilityDeployment, deadline: float, *, priority: str
) -> tuple[bool, float]:
self._switch_resident_deployment(deployment, deadline, priority=priority)
self.session.expire_all()
residency = self.repo.residency(deployment.id, lock=True)
if not residency:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "residency state is missing")
if residency.state in {"warm", "busy", "idle"}:
self.session.commit()
return False, 0.0
job: ServingJob | None = None
if residency.state == "loading":
job = self.session.scalar(
select(ServingJob)
.where(
ServingJob.capability_deployment_id == deployment.id,
ServingJob.operation == "load",
ServingJob.status.in_(("queued", "running")),
)
.order_by(ServingJob.created_at.desc())
.limit(1)
)
self.session.commit()
else:
residency.state = "loading"
residency.transition_started_at = _now()
residency.failure_code = None
residency.failure_message = None
residency.generation += 1
residency.transition_reason = "LOAD_COMPLETED"
for active_lease in self.repo.active_leases(deployment.accelerator_id, _now()):
if active_lease.capability_deployment_id == deployment.id:
active_lease.materialized_vram_bytes = min(
active_lease.reserved_vram_bytes,
residency.measured_resident_vram_bytes,
)
load_number = residency.load_count + 1
self._commit()
job = self._enqueue_job(
deployment,
"load",
key_suffix=f"load-{load_number}",
priority=priority,
)
if not job:
raise ServingError(503, "RESIDENCY_LOAD_FAILED", "loading state has no worker job")
completed, _ = self._wait_job(job.id, deadline)
return True, float(completed.result_summary.get("load_time_ms", 0.0))
def _switch_resident_deployment(
self, deployment: CapabilityDeployment, deadline: float, *, priority: str
) -> None:
# Admission and any typed idle evictions are decided under the
# accelerator row lock. Other proven residents intentionally remain.
del deployment, deadline, priority
@staticmethod
def _validate_vectors(
vectors: Any, expected_count: int, expected_dimension: int = 1024
) -> list[list[float]]:
if not isinstance(vectors, list) or len(vectors) != expected_count:
raise ServingError(502, "INVALID_OUTPUT", "runtime returned an invalid embedding batch")
validated: list[list[float]] = []
for vector in vectors:
if not isinstance(vector, list) or len(vector) != expected_dimension:
raise ServingError(
502, "INVALID_OUTPUT", "runtime returned the wrong vector dimension"
)
numeric = [float(value) for value in vector]
if not all(math.isfinite(value) for value in numeric):
raise ServingError(502, "INVALID_OUTPUT", "runtime returned non-finite values")
validated.append(numeric)
return validated
@staticmethod
def _validate_reranking_results(
results: Any, *, expected_ids: set[str] | None = None, expected_top_n: int | None = None
) -> list[dict[str, Any]]:
if not isinstance(results, list) or not results or len(results) > 10:
raise ServingError(502, "INVALID_OUTPUT", "runtime returned an invalid ranking")
if expected_top_n is not None and len(results) != expected_top_n:
raise ServingError(502, "INVALID_OUTPUT", "runtime returned the wrong ranking length")
validated: list[dict[str, Any]] = []
seen: set[str] = set()
previous_score = math.inf
for expected_rank, item in enumerate(results, 1):
if not isinstance(item, dict):
raise ServingError(502, "INVALID_OUTPUT", "reranker result is malformed")
identifier = str(item.get("id", ""))
score = float(item.get("score", math.nan))
rank = int(item.get("rank", 0))
if (
not identifier
or identifier in seen
or not math.isfinite(score)
or rank != expected_rank
or score > previous_score
or (expected_ids is not None and identifier not in expected_ids)
):
raise ServingError(502, "INVALID_OUTPUT", "reranker output contract was violated")
seen.add(identifier)
previous_score = score
validated.append({"id": identifier, "score": score, "rank": rank})
return validated
def invoke(
self,
request: EmbeddingInvokeRequest,
client: ServiceClient,
*,
deployment: CapabilityDeployment | None = None,
experiment_route: str | None = None,
) -> EmbeddingInvokeResponse:
started = time.perf_counter()
validation_started = time.perf_counter()
inputs = self._validate_inputs(request)
validation_ms = (time.perf_counter() - validation_started) * 1000
resolution_started = time.perf_counter()
deployment = deployment or self._stable_embedding_deployment()
if experiment_route and (
deployment.production
or deployment.channel != "experiment"
or deployment.status != "candidate"
):
raise ServingError(409, "EXPERIMENT_NOT_ROUTABLE", "experiment route is not active")
resolution_ms = (time.perf_counter() - resolution_started) * 1000
if self.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
raise ServingError(429, "QUEUE_FULL", "capability queue is full")
if self.repo.active_gateway_requests() >= self.settings.scheduler_global_queue_limit:
raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
request_id = uuid.uuid4()
input_hash = _canonical_hash(inputs)
record = self.repo.add(
GatewayRequest(
request_id=request_id,
service_client_id=client.id,
capability_key="rag.embedding",
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
priority=client.workload_priority,
input_sha256=input_hash,
input_count=len(inputs),
decision_evidence={
"input_persisted": False,
"input_type": request.input_type,
"experiment_route": experiment_route,
"shape": {
"bucket": _shape_bucket(sum(len(item.encode()) for item in inputs)),
"item_count": len(inputs),
"maximum_characters": max(map(len, inputs), default=0),
},
},
)
)
self._commit()
deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
lease: ServingGpuLease | None = None
result_reference: str | None = None
try:
scheduling_started = time.perf_counter()
lease = self._acquire_request_lease(deployment, record, client)
scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
cold, load_ms = self._ensure_loaded(
deployment, deadline, priority=client.workload_priority
)
payload_started = time.perf_counter()
payload_reference = self.payload_store.put(
{"input": inputs, "input_type": request.input_type},
self.settings.serving_payload_ttl_seconds,
)
job = self._enqueue_job(
deployment,
"invoke",
request=record,
payload_reference=payload_reference,
key_suffix=f"request-{request_id}",
priority=client.workload_priority,
)
payload_ms = (time.perf_counter() - payload_started) * 1000
wait_started = time.perf_counter()
completed, result = self._wait_job(job.id, deadline)
wait_ms = (time.perf_counter() - wait_started) * 1000
result_reference = completed.result_reference
result_started = time.perf_counter()
space = self.repo.embedding_space(deployment.embedding_space_id)
if not space:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "embedding space disappeared")
vectors = self._validate_vectors(result.get("vectors"), len(inputs), space.dimension)
inference_ms = float(completed.result_summary.get("inference_time_ms", 0.0))
worker_preprocess_ms = float(completed.result_summary.get("preprocess_time_ms", 0.0))
worker_serialize_ms = float(completed.result_summary.get("serialization_time_ms", 0.0))
worker_total_ms = float(
completed.result_summary.get("worker_total_time_ms", inference_ms)
)
dispatch_ms = 0.0
if completed.started_at:
dispatch_ms = max(
0.0,
(_aware(completed.started_at) - _aware(completed.created_at)).total_seconds()
* 1000,
)
completion_transport_ms = max(0.0, wait_ms - dispatch_ms - worker_total_ms)
total_ms = (time.perf_counter() - started) * 1000
self.session.expire_all()
current = self.session.get(GatewayRequest, record.id)
residency = self.repo.residency(deployment.id, lock=True)
if not current or not residency:
raise ServingError(
503, "CAPABILITY_UNAVAILABLE", "request completion state vanished"
)
current.status = "completed"
current.cold = cold
current.queue_time_ms = scheduling_ms
current.load_time_ms = load_ms
current.inference_time_ms = inference_ms
current.total_latency_ms = total_ms
current.finished_at = _now()
residency.state = "warm"
residency.active_requests = max(0, residency.active_requests - 1)
residency.last_used_at = _now()
lease.state = "released"
lease.released_at = _now()
result_ms = (time.perf_counter() - result_started) * 1000
accounted_ms = (
validation_ms
+ resolution_ms
+ scheduling_ms
+ load_ms
+ payload_ms
+ dispatch_ms
+ worker_total_ms
+ completion_transport_ms
+ result_ms
)
gateway_ms = max(
0.0,
validation_ms
+ resolution_ms
+ payload_ms
+ completion_transport_ms
+ result_ms
+ max(0.0, total_ms - accounted_ms),
)
latency_ms = {
"validation": validation_ms,
"resolution": resolution_ms,
"scheduling": scheduling_ms,
"load": load_ms,
"payload": payload_ms,
"dispatch": dispatch_ms,
"worker_preprocess": worker_preprocess_ms,
"inference": inference_ms,
"worker_serialize": worker_serialize_ms,
"worker_total": worker_total_ms,
"completion_transport": completion_transport_ms,
"result": result_ms,
"gateway": gateway_ms,
"total": total_ms,
}
current.decision_evidence = {
**current.decision_evidence,
"latency_clock": "process-local-monotonic; control-plane timestamps for dispatch",
"latency_ms": latency_ms,
}
self._commit()
selected_node = self.repo.node(deployment.compute_node_id)
if not selected_node:
raise ServingError(503, "NO_ELIGIBLE_NODE", "selected node disappeared")
return EmbeddingInvokeResponse(
dimension=space.dimension,
embedding_space_id=space.id,
data=vectors,
request_id=request_id,
execution=GatewayExecution(
cold=cold,
node=selected_node.display_name or selected_node.hostname,
residency="warm",
load_count=residency.load_count,
timings=GatewayTiming(
validation_ms=validation_ms,
resolution_ms=resolution_ms,
scheduling_ms=scheduling_ms,
payload_ms=payload_ms,
dispatch_ms=dispatch_ms,
worker_preprocess_ms=worker_preprocess_ms,
worker_serialize_ms=worker_serialize_ms,
worker_total_ms=worker_total_ms,
completion_transport_ms=completion_transport_ms,
result_ms=result_ms,
gateway_ms=gateway_ms,
queue_ms=scheduling_ms,
load_ms=load_ms,
inference_ms=inference_ms,
total_ms=total_ms,
),
),
usage=GatewayUsage(
input_count=len(inputs),
input_tokens=int(result.get("prompt_tokens", 0)),
),
)
except ServingError as exc:
self.session.rollback()
self.session.expire_all()
current = self.session.get(GatewayRequest, record.id)
residency = self.repo.residency(deployment.id, lock=True)
active_lease = self.repo.lease_for_request(request_id)
if current and current.status != "completed":
current.status = "failed"
current.failure_code = exc.code
current.total_latency_ms = (time.perf_counter() - started) * 1000
current.finished_at = _now()
if residency and residency.active_requests:
residency.active_requests -= 1
if residency.state == "busy":
residency.state = "warm"
if active_lease and active_lease.state not in {"released", "expired", "failed"}:
active_lease.state = "failed"
active_lease.failure_code = exc.code
active_lease.released_at = _now()
self._commit()
raise
finally:
if result_reference:
self.payload_store.delete(result_reference)
def invoke_reranking(
self, request: RerankingInvokeRequest, client: ServiceClient
) -> RerankingInvokeResponse:
started = time.perf_counter()
validation_started = time.perf_counter()
documents = self._validate_reranking_input(request)
validation_ms = (time.perf_counter() - validation_started) * 1000
resolution_started = time.perf_counter()
deployment, experiment_route = self._reranking_deployment()
resolution_ms = (time.perf_counter() - resolution_started) * 1000
if self.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
raise ServingError(429, "QUEUE_FULL", "capability queue is full")
if self.repo.active_gateway_requests() >= self.settings.scheduler_global_queue_limit:
raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
request_id = uuid.uuid4()
content_facts = {
"query_sha256": hashlib.sha256(request.query.encode()).hexdigest(),
"documents": [
{
"id": document["id"],
"content_sha256": hashlib.sha256(document["text"].encode()).hexdigest(),
}
for document in documents
],
"top_n": request.top_n,
}
record = self.repo.add(
GatewayRequest(
request_id=request_id,
service_client_id=client.id,
capability_key="rag.reranking",
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
priority=client.workload_priority,
input_sha256=_canonical_hash(content_facts),
input_count=len(documents),
decision_evidence={
"input_persisted": False,
"content_logged": False,
"document_count": len(documents),
"top_n": request.top_n,
"experiment_route": experiment_route,
"shape": {
"bucket": _shape_bucket(
len(request.query.encode())
+ sum(len(document["text"].encode()) for document in documents)
),
"document_count": len(documents),
"top_n": request.top_n,
},
},
)
)
self._commit()
deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
lease: ServingGpuLease | None = None
input_reference: str | None = None
result_reference: str | None = None
try:
scheduling_started = time.perf_counter()
lease = self._acquire_request_lease(deployment, record, client)
scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
cold, load_ms = self._ensure_loaded(
deployment, deadline, priority=client.workload_priority
)
payload_started = time.perf_counter()
input_reference = self.payload_store.put(
{"query": request.query, "documents": documents, "top_n": request.top_n},
self.settings.serving_payload_ttl_seconds,
)
job = self._enqueue_job(
deployment,
"invoke",
request=record,
payload_reference=input_reference,
key_suffix=f"request-{request_id}",
priority=client.workload_priority,
)
payload_ms = (time.perf_counter() - payload_started) * 1000
wait_started = time.perf_counter()
completed, result = self._wait_job(job.id, deadline)
wait_ms = (time.perf_counter() - wait_started) * 1000
result_reference = completed.result_reference
result_started = time.perf_counter()
ranking = self._validate_reranking_results(
result.get("results"),
expected_ids={document["id"] for document in documents},
expected_top_n=request.top_n,
)
inference_ms = float(completed.result_summary.get("inference_time_ms", 0.0))
worker_preprocess_ms = float(completed.result_summary.get("preprocess_time_ms", 0.0))
worker_serialize_ms = float(completed.result_summary.get("serialization_time_ms", 0.0))
worker_total_ms = float(
completed.result_summary.get("worker_total_time_ms", inference_ms)
)
dispatch_ms = 0.0
if completed.started_at:
dispatch_ms = max(
0.0,
(_aware(completed.started_at) - _aware(completed.created_at)).total_seconds()
* 1000,
)
completion_transport_ms = max(0.0, wait_ms - dispatch_ms - worker_total_ms)
total_ms = (time.perf_counter() - started) * 1000
self.session.expire_all()
current = self.session.get(GatewayRequest, record.id)
residency = self.repo.residency(deployment.id, lock=True)
if not current or not residency:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "completion state vanished")
current.status = "completed"
current.cold = cold
current.queue_time_ms = scheduling_ms
current.load_time_ms = load_ms
current.inference_time_ms = inference_ms
current.total_latency_ms = total_ms
current.finished_at = _now()
residency.state = "warm"
residency.active_requests = max(0, residency.active_requests - 1)
residency.last_used_at = _now()
lease.state = "released"
lease.released_at = _now()
result_ms = (time.perf_counter() - result_started) * 1000
accounted_ms = (
validation_ms
+ resolution_ms
+ scheduling_ms
+ load_ms
+ payload_ms
+ dispatch_ms
+ worker_total_ms
+ completion_transport_ms
+ result_ms
)
gateway_ms = max(0.0, total_ms - scheduling_ms - load_ms - worker_total_ms)
latency_ms = {
"validation": validation_ms,
"resolution": resolution_ms,
"scheduling": scheduling_ms,
"load": load_ms,
"payload": payload_ms,
"dispatch": dispatch_ms,
"worker_preprocess": worker_preprocess_ms,
"inference": inference_ms,
"worker_serialize": worker_serialize_ms,
"worker_total": worker_total_ms,
"completion_transport": completion_transport_ms,
"result": result_ms,
"gateway": gateway_ms + max(0.0, total_ms - accounted_ms),
"total": total_ms,
}
current.decision_evidence = {
**current.decision_evidence,
"latency_clock": "process-local-monotonic; control-plane timestamps for dispatch",
"latency_ms": latency_ms,
}
self._commit()
selected_node = self.repo.node(deployment.compute_node_id)
if not selected_node:
raise ServingError(503, "NO_ELIGIBLE_NODE", "selected node disappeared")
return RerankingInvokeResponse(
results=[RerankingResult.model_validate(item) for item in ranking],
request_id=request_id,
execution=GatewayExecution(
cold=cold,
node=selected_node.display_name or selected_node.hostname,
residency="warm",
load_count=residency.load_count,
timings=GatewayTiming(
validation_ms=validation_ms,
resolution_ms=resolution_ms,
scheduling_ms=scheduling_ms,
payload_ms=payload_ms,
dispatch_ms=dispatch_ms,
worker_preprocess_ms=worker_preprocess_ms,
worker_serialize_ms=worker_serialize_ms,
worker_total_ms=worker_total_ms,
completion_transport_ms=completion_transport_ms,
result_ms=result_ms,
gateway_ms=latency_ms["gateway"],
queue_ms=scheduling_ms,
load_ms=load_ms,
inference_ms=inference_ms,
total_ms=total_ms,
),
),
usage=GatewayUsage(
input_count=len(documents),
input_tokens=int(result.get("prompt_tokens", 0)),
),
)
except ServingError as exc:
self.session.rollback()
self.session.expire_all()
current = self.session.get(GatewayRequest, record.id)
residency = self.repo.residency(deployment.id, lock=True)
active_lease = self.repo.lease_for_request(request_id)
if current and current.status != "completed":
current.status = "failed"
current.failure_code = exc.code
current.total_latency_ms = (time.perf_counter() - started) * 1000
current.finished_at = _now()
if residency and residency.active_requests:
residency.active_requests -= 1
if residency.state == "busy":
residency.state = "warm"
if active_lease and active_lease.state not in {"released", "expired", "failed"}:
active_lease.state = "failed"
active_lease.failure_code = exc.code
active_lease.released_at = _now()
self._commit()
raise
finally:
if input_reference:
self.payload_store.delete(input_reference)
if result_reference:
self.payload_store.delete(result_reference)
def _validate_modality_output(
self,
capability_key: str,
result: dict[str, Any],
deployment: CapabilityDeployment,
expected_count: int,
) -> dict[str, Any]:
if capability_key == "document.ocr":
text = result.get("text")
pages = result.get("pages")
if not isinstance(text, str) or len(text) > 1_000_000:
raise ServingError(502, "INVALID_OUTPUT", "OCR text output is invalid")
if not isinstance(pages, list) or len(pages) != 1 or not isinstance(pages[0], dict):
raise ServingError(502, "INVALID_OUTPUT", "OCR page output is invalid")
width = int(pages[0].get("width", 0))
height = int(pages[0].get("height", 0))
if not 1 <= width <= 4096 or not 1 <= height <= 4096:
raise ServingError(502, "INVALID_OUTPUT", "OCR dimensions violate the contract")
return {
"text": text,
"pages": pages,
"confidence": result.get("confidence"),
}
if capability_key == "vision.embedding":
space = self.repo.embedding_space(deployment.embedding_space_id)
if not space:
raise ServingError(
503, "CAPABILITY_UNAVAILABLE", "visual embedding space is missing"
)
vectors = self._validate_vectors(result.get("vectors"), expected_count, space.dimension)
if result.get("normalized") is not True:
raise ServingError(502, "INVALID_OUTPUT", "visual vectors are not normalized")
return {
"vectors": vectors,
"dimension": space.dimension,
"normalized": True,
"embedding_space_id": str(space.id),
}
if capability_key == "speech.transcription":
text = result.get("text")
language = result.get("language")
duration = float(result.get("duration_seconds", 0.0))
segments = result.get("segments", [])
if (
not isinstance(text, str)
or not isinstance(language, str)
or not 0 < duration <= 120
or not isinstance(segments, list)
):
raise ServingError(502, "INVALID_OUTPUT", "transcription output is invalid")
return {
"text": text,
"language": language,
"duration_seconds": duration,
"segments": segments,
}
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "unsupported modality capability")
def invoke_modality(
self,
capability_key: str,
payload: dict[str, Any],
client: ServiceClient,
) -> tuple[dict[str, Any], uuid.UUID, GatewayExecution]:
started = time.perf_counter()
resolution_started = time.perf_counter()
deployment, experiment_route = self._experimental_deployment(capability_key)
resolution_ms = (time.perf_counter() - resolution_started) * 1000
if self.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
raise ServingError(429, "QUEUE_FULL", "capability queue is full")
if self.repo.active_gateway_requests() >= self.settings.scheduler_global_queue_limit:
raise ServingError(429, "QUEUE_FULL", "global capability queue is full")
request_id = uuid.uuid4()
input_count = len(payload.get("items", [])) if capability_key == "vision.embedding" else 1
encoded_lengths = [
len(value)
for key, value in payload.items()
if key.endswith("_base64") and isinstance(value, str)
]
for item in payload.get("items", []):
if isinstance(item, dict):
encoded_lengths.extend(
len(value)
for key, value in item.items()
if key.endswith("_base64") and isinstance(value, str)
)
approximate_bytes = sum(encoded_lengths) * 3 // 4
record = self.repo.add(
GatewayRequest(
request_id=request_id,
service_client_id=client.id,
capability_key=capability_key,
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
priority=client.workload_priority,
input_sha256=_canonical_hash(payload),
input_count=input_count,
decision_evidence={
"input_persisted": False,
"content_logged": False,
"experiment_route": experiment_route,
"modality": capability_key.split(".", 1)[0],
"shape": {
"bucket": _shape_bucket(approximate_bytes),
"encoded_payload_bytes_approximate": approximate_bytes,
"item_count": input_count,
},
},
)
)
self._commit()
deadline = time.monotonic() + self.settings.gateway_request_timeout_seconds
lease: ServingGpuLease | None = None
input_reference: str | None = None
result_reference: str | None = None
try:
scheduling_started = time.perf_counter()
lease = self._acquire_request_lease(deployment, record, client)
scheduling_ms = (time.perf_counter() - scheduling_started) * 1000
cold, load_ms = self._ensure_loaded(
deployment, deadline, priority=client.workload_priority
)
payload_started = time.perf_counter()
input_reference = self.payload_store.put(
payload, self.settings.serving_payload_ttl_seconds
)
job = self._enqueue_job(
deployment,
"invoke",
request=record,
payload_reference=input_reference,
key_suffix=f"request-{request_id}",
priority=client.workload_priority,
)
payload_ms = (time.perf_counter() - payload_started) * 1000
wait_started = time.perf_counter()
completed, raw_result = self._wait_job(job.id, deadline)
wait_ms = (time.perf_counter() - wait_started) * 1000
result_reference = completed.result_reference
result_started = time.perf_counter()
result = self._validate_modality_output(
capability_key, raw_result, deployment, input_count
)
result_ms = (time.perf_counter() - result_started) * 1000
inference_ms = float(completed.result_summary.get("inference_time_ms", 0.0))
preprocess_ms = float(completed.result_summary.get("preprocess_time_ms", 0.0))
serialization_ms = float(completed.result_summary.get("serialization_time_ms", 0.0))
worker_total_ms = float(
completed.result_summary.get("worker_total_time_ms", inference_ms)
)
dispatch_ms = 0.0
if completed.started_at:
dispatch_ms = max(
0.0,
(_aware(completed.started_at) - _aware(completed.created_at)).total_seconds()
* 1000,
)
completion_transport_ms = max(0.0, wait_ms - dispatch_ms - worker_total_ms)
total_ms = (time.perf_counter() - started) * 1000
self.session.expire_all()
current = self.session.get(GatewayRequest, record.id)
residency = self.repo.residency(deployment.id, lock=True)
if not current or not residency:
raise ServingError(503, "CAPABILITY_UNAVAILABLE", "completion state vanished")
current.status = "completed"
current.cold = cold
current.queue_time_ms = scheduling_ms
current.load_time_ms = load_ms
current.inference_time_ms = inference_ms
current.total_latency_ms = total_ms
current.finished_at = _now()
current.decision_evidence = {
**current.decision_evidence,
"latency_ms": {
"resolution": resolution_ms,
"scheduling": scheduling_ms,
"payload": payload_ms,
"dispatch": dispatch_ms,
"worker_preprocess": preprocess_ms,
"worker_serialize": serialization_ms,
"worker_total": worker_total_ms,
"completion_transport": completion_transport_ms,
"result": result_ms,
"total": total_ms,
},
}
residency.state = "warm"
residency.active_requests = max(0, residency.active_requests - 1)
residency.last_used_at = _now()
lease.state = "released"
lease.released_at = _now()
self._commit()
gateway_ms = max(
0.0,
resolution_ms + payload_ms + completion_transport_ms + result_ms,
)
return (
result,
request_id,
GatewayExecution(
cold=cold,
node=str(deployment.compute_node_id),
residency=residency.state,
load_count=residency.load_count,
timings=GatewayTiming(
validation_ms=0.0,
resolution_ms=resolution_ms,
scheduling_ms=scheduling_ms,
payload_ms=payload_ms,
dispatch_ms=dispatch_ms,
worker_preprocess_ms=preprocess_ms,
worker_serialize_ms=serialization_ms,
worker_total_ms=worker_total_ms,
completion_transport_ms=completion_transport_ms,
result_ms=result_ms,
gateway_ms=gateway_ms,
queue_ms=scheduling_ms,
load_ms=load_ms,
inference_ms=inference_ms,
total_ms=total_ms,
),
),
)
except ServingError as exc:
self.session.rollback()
self.session.expire_all()
current = self.session.get(GatewayRequest, record.id)
residency = self.repo.residency(deployment.id, lock=True)
active_lease = self.repo.lease_for_request(request_id)
if current and current.status != "completed":
current.status = "failed"
current.failure_code = exc.code
current.total_latency_ms = (time.perf_counter() - started) * 1000
current.finished_at = _now()
if residency and residency.active_requests:
residency.active_requests -= 1
if residency.state == "busy":
residency.state = "warm"
if active_lease and active_lease.state not in {"released", "expired", "failed"}:
active_lease.state = "failed"
active_lease.failure_code = exc.code
active_lease.released_at = _now()
self._commit()
raise
finally:
if input_reference:
self.payload_store.delete(input_reference)
if result_reference:
self.payload_store.delete(result_reference)
def _job_manifest(self, deployment: CapabilityDeployment) -> dict[str, Any]:
artifacts = self.repo.set_artifacts(deployment.artifact_set_id)
locations = self.repo.artifact_locations([item.id for item in artifacts])
artifact_set = self.repo.artifact_set(deployment.artifact_set_id)
revision = self.repo.revision(artifact_set.revision_id) if artifact_set else None
if not artifact_set or not revision:
raise ServingError(409, "ARTIFACT_INCOMPLETE", "ArtifactSet revision is missing")
manifest_rows = _selected_manifest_rows(
artifact_set.selected_paths,
[(None, artifact) for artifact in artifacts],
locations,
)
if len(manifest_rows) != len(artifact_set.selected_paths):
raise ServingError(
409, "ARTIFACT_INCOMPLETE", "one verified location per selected path is required"
)
roots = {location.storage_root_id for _, location, _ in manifest_rows}
if len(roots) != 1:
raise ServingError(409, "ARTIFACT_PATH_INVALID", "ArtifactSet spans storage roots")
root = self.repo.storage_root(roots.pop())
if not root or root.compute_node_id != deployment.compute_node_id:
raise ServingError(409, "ARTIFACT_PATH_INVALID", "ArtifactSet is on the wrong node")
return {
"revision_sha": revision.resolved_commit_sha,
"relative_parent": _safe_parent(
[location.relative_path for _, location, _ in manifest_rows]
),
"files": [
{
"path": selected_path,
"sha256": artifact.sha256,
"size_bytes": artifact.size_bytes,
}
for selected_path, _, artifact in manifest_rows
],
}
def claim_next(
self, node: ComputeNode, *, wait_seconds: float = 0.0
) -> AgentServingJobLease | None:
deadline = time.monotonic() + min(max(wait_seconds, 0.0), 5.0)
job = self.repo.claimable_job(node.id, _now())
while not job and time.monotonic() < deadline:
# Bounded server-side wait keeps the outbound-only worker protocol
# while removing the fixed idle-poll latency from gateway requests.
self.session.expire_all()
time.sleep(min(0.025, max(0.0, deadline - time.monotonic())))
job = self.repo.claimable_job(node.id, _now())
if not job:
return None
deployment = self.repo.deployment(job.capability_deployment_id)
if not deployment or deployment.compute_node_id != node.id:
raise ServingError(
409, "NODE_OWNERSHIP_MISMATCH", "serving job belongs to another node"
)
profile = self.repo.profile(deployment.runtime_profile_id)
if not profile or not profile.runtime_environment_id:
raise ServingError(409, "RUNTIME_DEPENDENCY_MISSING", "runtime profile is incomplete")
environment = self.repo.environment(profile.runtime_environment_id)
if not environment:
raise ServingError(409, "RUNTIME_DEPENDENCY_MISSING", "runtime environment is missing")
manifest = self._job_manifest(deployment)
contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
capability = self.session.get(Capability, contract.capability_id) if contract else None
supported_capabilities = {
"rag.embedding",
"rag.reranking",
"document.ocr",
"vision.embedding",
"speech.transcription",
}
if not capability or capability.key not in supported_capabilities:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "serving capability is unsupported")
input_value: list[str] | None = None
input_type = "raw"
rerank_query: str | None = None
rerank_documents: list[dict[str, str]] | None = None
top_n: int | None = None
modality_payload: dict[str, Any] | None = None
if job.operation == "invoke":
transient = (
self.payload_store.get(job.payload_reference) if job.payload_reference else None
)
valid_payload = bool(
transient
and (
(capability.key == "rag.embedding" and isinstance(transient.get("input"), list))
or (
capability.key == "rag.reranking"
and isinstance(transient.get("query"), str)
and isinstance(transient.get("documents"), list)
and isinstance(transient.get("top_n"), int)
)
or (
capability.key
in {
"document.ocr",
"vision.embedding",
"speech.transcription",
}
and isinstance(transient, dict)
)
)
)
if not valid_payload or transient is None:
job.status = "failed"
job.failure_code = "REQUEST_TIMEOUT"
job.failure_message = "transient request payload expired before claim"
job.finished_at = _now()
self._commit()
return None
if capability.key == "rag.embedding":
input_value = [str(item) for item in transient["input"]]
input_type = str(transient.get("input_type", "raw"))
elif capability.key == "rag.reranking":
rerank_query = str(transient["query"])
rerank_documents = [
{"id": str(item["id"]), "text": str(item["text"])}
for item in transient["documents"]
if isinstance(item, dict) and "id" in item and "text" in item
]
top_n = int(transient["top_n"])
else:
modality_payload = transient
token = secrets.token_urlsafe(32)
now = _now()
job.status = "running"
job.attempt_count += 1
job.lease_token_hash = hashlib.sha256(token.encode()).hexdigest()
job.lease_expires_at = now + timedelta(seconds=self.settings.serving_job_lease_seconds)
job.started_at = job.started_at or now
self._commit()
runtime = RuntimeService(self.session, self.settings)
gateway_request = (
self.session.get(GatewayRequest, job.gateway_request_id)
if job.gateway_request_id
else None
)
return AgentServingJobLease(
job_id=job.id,
lease_token=token,
lease_expires_at=job.lease_expires_at,
operation=cast(Any, job.operation),
deployment_id=deployment.id,
artifact_set_id=deployment.artifact_set_id,
revision_sha=manifest["revision_sha"],
artifact_root=self.settings.runtime_artifact_root,
artifact_relative_path=manifest["relative_parent"],
expected_manifest={
"artifact_set_id": str(deployment.artifact_set_id),
"revision_sha": manifest["revision_sha"],
"files": manifest["files"],
},
runtime_profile=runtime.profile_response(profile).model_dump(mode="json"),
runtime_environment=runtime.environment_response(environment).model_dump(mode="json"),
capability=cast(Any, capability.key),
embedding_space_id=deployment.embedding_space_id,
expected_dimension=(
space.dimension
if (space := self.repo.embedding_space(deployment.embedding_space_id))
else None
),
normalize=True if capability.key == "rag.embedding" else None,
input=input_value,
input_type=cast(Any, input_type),
rerank_query=rerank_query,
rerank_documents=(
[RerankDocument.model_validate(item) for item in rerank_documents]
if rerank_documents is not None
else None
),
top_n=top_n,
modality_payload=modality_payload,
request_id=gateway_request.request_id if gateway_request else None,
)
@staticmethod
def _assert_job_lease(job: ServingJob, node: ComputeNode, token: str) -> None:
if job.compute_node_id != node.id:
raise ServingError(403, "NODE_OWNERSHIP_MISMATCH", "job belongs to another node")
digest = hashlib.sha256(token.encode()).hexdigest()
if not job.lease_token_hash or not hmac.compare_digest(digest, job.lease_token_hash):
raise ServingError(409, "LEASE_TIMEOUT", "serving job lease is invalid")
if not job.lease_expires_at or _aware(job.lease_expires_at) <= _now():
raise ServingError(409, "LEASE_TIMEOUT", "serving job lease expired")
def complete_job(
self,
job_id: uuid.UUID,
node: ComputeNode,
request: AgentServingJobComplete,
) -> ServingJob:
job = self.repo.job(job_id)
if not job:
raise ServingError(404, "SERVING_JOB_NOT_FOUND", "serving job was not found")
self._assert_job_lease(job, node, request.lease_token)
deployment = self.repo.deployment(job.capability_deployment_id)
residency = self.repo.residency(job.capability_deployment_id, lock=True)
if not deployment or not residency:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "deployment residency is missing")
contract = self.session.get(CapabilityContract, deployment.capability_contract_id)
capability = self.session.get(Capability, contract.capability_id) if contract else None
if not capability:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "deployment contract is missing")
result = request.result
metrics = request.metrics
if job.operation in {"invoke", "health"}:
transient_result: dict[str, Any]
summary: dict[str, Any]
if capability.key == "rag.embedding":
vectors = result.get("vectors")
expected_count = int(result.get("count", 1))
space = self.repo.embedding_space(deployment.embedding_space_id)
if not space:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "embedding space is missing")
validated = self._validate_vectors(vectors, expected_count, space.dimension)
transient_result = {
"vectors": validated,
"prompt_tokens": int(result.get("prompt_tokens", 0)),
}
summary = {
"count": expected_count,
"dimension": space.dimension,
"finite": True,
"normalized": result.get("normalized") is True,
}
elif capability.key == "rag.reranking":
transient_input = (
self.payload_store.get(job.payload_reference)
if job.operation == "invoke" and job.payload_reference
else None
)
expected_ids = (
{str(item["id"]) for item in transient_input.get("documents", [])}
if transient_input
else None
)
expected_top_n = int(transient_input["top_n"]) if transient_input else None
validated_ranking = self._validate_reranking_results(
result.get("results"),
expected_ids=expected_ids,
expected_top_n=expected_top_n,
)
transient_result = {
"results": validated_ranking,
"prompt_tokens": int(result.get("prompt_tokens", 0)),
}
summary = {
"count": int(result.get("count", len(expected_ids or []))),
"top_n": len(validated_ranking),
"finite": True,
}
elif capability.key in {
"document.ocr",
"vision.embedding",
"speech.transcription",
}:
if job.operation == "health":
if result.get("status") != "passed":
raise ServingError(502, "HEALTHCHECK_FAILED", "worker health probe failed")
transient_result = {"health": "passed"}
summary = {
"count": 1,
"output_type": result.get("output_type"),
"content_logged": False,
}
else:
transient_input = (
self.payload_store.get(job.payload_reference)
if job.payload_reference
else {}
) or {}
expected_count = (
len(transient_input.get("items", []))
if capability.key == "vision.embedding"
else 1
)
transient_result = self._validate_modality_output(
capability.key, result, deployment, expected_count
)
summary = {
"count": expected_count,
"output_type": result.get("output_type"),
"content_logged": False,
}
else:
raise ServingError(409, "CAPABILITY_UNAVAILABLE", "unsupported capability")
result_reference = self.payload_store.put(
transient_result, self.settings.serving_payload_ttl_seconds
)
job.result_reference = result_reference
job.result_summary = {
**summary,
"inference_time_ms": float(metrics.get("inference_time_ms", 0.0)),
"preprocess_time_ms": float(metrics.get("preprocess_time_ms", 0.0)),
"serialization_time_ms": float(metrics.get("serialization_time_ms", 0.0)),
"worker_total_time_ms": float(metrics.get("worker_total_time_ms", 0.0)),
"prompt_tokens": int(result.get("prompt_tokens", 0)),
}
residency.state = "warm"
residency.last_used_at = _now()
residency.health = request.health
elif job.operation == "load":
total_baseline = int(metrics.get("baseline_vram_bytes", 0))
total_loaded = int(metrics.get("loaded_vram_bytes", 0))
baseline = int(
metrics.get("baseline_process_vram_bytes", metrics.get("baseline_vram_bytes", 0))
)
loaded = int(
metrics.get("loaded_process_vram_bytes", metrics.get("loaded_vram_bytes", 0))
)
if loaded <= baseline:
raise ServingError(
409, "RESIDENCY_LOAD_FAILED", "load reported no resident allocation"
)
residency.state = "warm"
residency.worker_instance_id = request.worker_instance_id
residency.load_count += 1
residency.measured_resident_vram_bytes = loaded - baseline
residency.external_baseline_vram_bytes = total_baseline
residency.resident_since = _now()
residency.last_used_at = _now()
residency.health = request.health
residency.failure_code = None
residency.failure_message = None
job.result_summary = {
"load_time_ms": float(metrics.get("load_time_ms", 0.0)),
"baseline_vram_bytes": baseline,
"loaded_vram_bytes": loaded,
"observed_baseline_vram_bytes": total_baseline,
"observed_loaded_vram_bytes": total_loaded,
"resident_vram_bytes": loaded - baseline,
}
elif job.operation == "drain":
residency.state = "draining"
job.result_summary = {"drained": True}
elif job.operation == "unload":
if result.get("reclaimed") is not True:
raise ServingError(
409, "RESIDENCY_UNLOAD_FAILED", "worker did not reclaim GPU memory"
)
residency.state = "cold"
residency.worker_instance_id = request.worker_instance_id
residency.active_requests = 0
residency.measured_resident_vram_bytes = 0
residency.health = {
"process": "healthy",
"runtime": "unavailable",
"model": "unavailable",
}
residency.resident_since = None
residency.transition_reason = residency.transition_reason or "ADMIN_UNLOAD"
job.result_summary = {
"reclaimed": True,
"after_unload_vram_bytes": metrics.get("after_unload_vram_bytes"),
"reclaimed_vram_bytes": metrics.get("reclaimed_vram_bytes"),
}
deployment.health_status = (
"ready_on_demand" if deployment.status in {"stable", "candidate"} else "unavailable"
)
self.audit.write(
"DEPLOYMENT_UNLOADED",
"capability_deployment",
str(deployment.id),
{"reclaimed_vram_bytes": metrics.get("reclaimed_vram_bytes")},
)
eviction = self.session.scalar(
select(SchedulerEviction)
.where(
SchedulerEviction.capability_deployment_id == deployment.id,
SchedulerEviction.state.in_(("planned", "queued")),
)
.order_by(SchedulerEviction.created_at.desc())
.limit(1)
)
if eviction:
eviction.state = "completed"
eviction.actual_reclaimed_bytes = int(metrics.get("reclaimed_vram_bytes", 0))
job.status = "completed"
job.finished_at = _now()
job.lease_token_hash = None
job.lease_expires_at = None
if job.payload_reference:
self.payload_store.delete(job.payload_reference)
job.payload_reference = None
self._commit()
return job
def fail_job(
self,
job_id: uuid.UUID,
node: ComputeNode,
request: AgentServingJobFailure,
) -> ServingJob:
job = self.repo.job(job_id)
if not job:
raise ServingError(404, "SERVING_JOB_NOT_FOUND", "serving job was not found")
self._assert_job_lease(job, node, request.lease_token)
residency = self.repo.residency(job.capability_deployment_id, lock=True)
job.status = "failed"
job.failure_code = request.failure_code
job.failure_message = request.failure_message
job.result_summary = {"details": request.details}
job.finished_at = _now()
job.lease_token_hash = None
job.lease_expires_at = None
if job.payload_reference:
self.payload_store.delete(job.payload_reference)
job.payload_reference = None
if residency:
residency.state = "cold" if job.operation in {"load", "invoke"} else "failed"
residency.failure_code = request.failure_code
residency.failure_message = request.failure_message
residency.measured_resident_vram_bytes = (
0 if residency.state == "cold" else residency.measured_resident_vram_bytes
)
residency.active_requests = max(0, residency.active_requests - 1)
self._commit()
return job
def report_state(
self, node: ComputeNode, request: AgentServingStateReport
) -> AgentServingStateAck:
if request.residencies:
reported_ids: set[uuid.UUID] = set()
desired_unload = False
for report in request.residencies:
deployment_id = report.deployment_id
deployment = self.repo.deployment(deployment_id)
if not deployment or deployment.compute_node_id != node.id:
raise ServingError(
403, "NODE_OWNERSHIP_MISMATCH", "reported deployment belongs elsewhere"
)
profile = self.repo.profile(deployment.runtime_profile_id)
environment = (
self.repo.environment(profile.runtime_environment_id)
if profile and profile.runtime_environment_id
else None
)
exact_identity = (
report.artifact_set_id == deployment.artifact_set_id
and report.runtime_profile_id == deployment.runtime_profile_id
and environment is not None
and report.runtime_environment_fingerprint == environment.fingerprint
)
if not exact_identity:
raise ServingError(
409,
"RESIDENCY_CONFLICT",
"worker residency cannot be adopted without exact immutable identity",
)
reported_ids.add(deployment.id)
residency = self.repo.residency(deployment.id, lock=True)
if not residency:
continue
adopted = residency.state == "cold"
residency.worker_instance_id = request.worker_instance_id
residency.generation = request.generation
residency.state = report.state
residency.load_count = max(residency.load_count, report.load_count)
residency.health = report.health
residency.measured_resident_vram_bytes = max(
residency.measured_resident_vram_bytes,
report.resident_vram_bytes,
)
if adopted:
residency.resident_since = request.observed_at
residency.transition_reason = "WORKER_RESIDENCY_ADOPTED"
desired_unload = desired_unload or deployment.status == "draining"
for deployment in self.repo.deployments():
if deployment.compute_node_id != node.id or deployment.id in reported_ids:
continue
residency = self.repo.residency(deployment.id, lock=True)
if residency and residency.state != "cold":
residency.state = "cold"
residency.worker_instance_id = request.worker_instance_id
residency.generation = request.generation
residency.measured_resident_vram_bytes = 0
residency.active_requests = 0
residency.resident_since = None
residency.transition_reason = "WORKER_ABSENCE_RECONCILED"
residency.failure_code = "WORKER_RESTART_RECONCILED"
self._commit()
return AgentServingStateAck(desired_operation="unload" if desired_unload else "none")
if request.deployment_id:
deployment = self.repo.deployment(request.deployment_id)
if not deployment or deployment.compute_node_id != node.id:
raise ServingError(
403, "NODE_OWNERSHIP_MISMATCH", "reported deployment belongs elsewhere"
)
residency = self.repo.residency(deployment.id, lock=True)
if residency:
residency.worker_instance_id = request.worker_instance_id
residency.generation = request.generation
residency.state = request.state
residency.load_count = max(residency.load_count, request.load_count)
residency.health = request.health
if request.state == "cold":
residency.measured_resident_vram_bytes = 0
residency.resident_since = None
self._commit()
return AgentServingStateAck(
desired_operation="unload" if deployment.status == "draining" else "none"
)
for deployment in self.repo.deployments():
if deployment.compute_node_id != node.id:
continue
residency = self.repo.residency(deployment.id, lock=True)
if (
residency
and residency.worker_instance_id != request.worker_instance_id
and residency.state != "cold"
):
residency.state = "cold"
residency.worker_instance_id = request.worker_instance_id
residency.generation = request.generation
residency.measured_resident_vram_bytes = 0
residency.active_requests = 0
residency.resident_since = None
residency.transition_reason = "WORKER_ABSENCE_RECONCILED"
residency.health = {
"process": "healthy",
"runtime": "unavailable",
"model": "unavailable",
}
residency.failure_code = "WORKER_RESTART_RECONCILED"
residency.failure_message = None
self._commit()
return AgentServingStateAck()
def request_unload(
self, deployment_id: uuid.UUID, *, drain: bool = False
) -> CapabilityDeploymentResponse:
deployment = self.repo.deployment(deployment_id)
residency = self.repo.residency(deployment_id, lock=True) if deployment else None
if not deployment or not residency:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
if residency.active_requests:
raise ServingError(409, "DEPLOYMENT_BUSY", "active requests must finish before unload")
if drain:
deployment.status = "draining"
deployment.draining_at = _now()
deployment.health_status = "degraded"
self.audit.write("DEPLOYMENT_DRAINED", "capability_deployment", str(deployment.id), {})
if residency.state == "cold":
self._commit()
return self.deployment_response(deployment)
residency.state = "draining"
self._commit()
self._enqueue_job(
deployment,
"drain",
key_suffix=f"drain-{residency.load_count}",
priority="maintenance",
)
self._enqueue_job(
deployment,
"unload",
key_suffix=f"unload-{residency.load_count}",
priority="maintenance",
)
return self.deployment_response(deployment)
def update_residency_policy(
self, deployment_id: uuid.UUID, request: ResidencyPolicyUpdate
) -> CapabilityDeploymentResponse:
deployment = self.repo.deployment(deployment_id)
if not deployment:
raise ServingError(404, "CAPABILITY_NOT_FOUND", "capability deployment was not found")
previous = deployment.residency_policy
deployment.residency_policy = request.residency_policy
deployment.keep_warm_seconds = request.keep_warm_seconds
self.audit.write(
"RESIDENCY_POLICY_CHANGED",
"capability_deployment",
str(deployment.id),
{
"previous": previous,
"current": request.residency_policy,
"keep_warm_seconds": request.keep_warm_seconds,
},
)
self._commit()
return self.deployment_response(deployment)
def reconcile(self) -> dict[str, int]:
self._expire_leases()
now = _now()
expired = 0
unloads = 0
for deployment in self.repo.deployments():
residency = self.repo.residency(deployment.id, lock=True)
node = self.repo.node(deployment.compute_node_id)
if not residency:
continue
if not node or node.liveness_state != "online":
deployment.health_status = "unavailable"
elif (
residency.state == "cold"
and deployment.status in {"stable", "candidate"}
and deployment.health_status == "unavailable"
):
# Repair the legacy M9 state where a successful LAB unload was
# persisted as unavailable even though it remained routable.
deployment.health_status = "ready_on_demand"
if (
deployment.status == "stable"
and deployment.residency_policy == "keep_warm"
and residency.state in {"warm", "idle"}
and residency.active_requests == 0
and residency.last_used_at
and _aware(residency.last_used_at) + timedelta(seconds=deployment.keep_warm_seconds)
<= now
):
self.request_unload(deployment.id)
unloads += 1
for lease in self.repo.expired_leases(now):
if lease.state == "expired":
expired += 1
self._commit()
return {"expired_leases": expired, "unloads_queued": unloads}
Reference in new issue View Git Blame Copy Permalink