from __future__ import annotations import hashlib import json import time import uuid from datetime import UTC, datetime, timedelta from typing import Any, cast from sqlalchemy import func, select, update from sqlalchemy.orm import Session from modelforge_api.db import rows_affected from modelforge_api.domain.lifecycle_contracts import ( ApprovalDecision, ApprovalPolicyCreate, ApprovalPolicyResponse, ApprovalRequestCreate, ApprovalRequestResponse, ApprovalStatus, CanaryObservation, CanaryStatus, CleanupExecutionCreate, CleanupPlanCreate, CleanupPlanResponse, CleanupStatus, DeploymentLifecycleState, LifecycleEventResponse, LifecycleOperationResponse, LifecycleSubjectCreate, LifecycleSubjectResponse, OperationStage, PlanExecutionCreate, PromotionPlanCreate, PromotionPlanResponse, PromotionPlanStatus, RetentionPolicyCreate, RetentionPolicyResponse, RetentionState, assert_deployment_transition, ) from modelforge_api.persistence.models import ( ArtifactLocation, ArtifactLocationRemovalRecord, ArtifactSet, ArtifactSetMember, CapabilityDeployment, CapabilityExperimentRoute, CapabilityResourceEnvelope, DeploymentCandidate, DerivedArtifact, DerivedArtifactSource, EmbeddingMigration, EmbeddingSpace, EvaluationRun, ExecutionApproval, LifecycleApprovalEvidence, LifecycleApprovalRequest, LifecycleCanaryRun, LifecycleCleanupPlan, LifecycleEvent, LifecycleOperation, LifecyclePolicyRevision, LifecyclePromotionPlan, LifecycleRetentionRecord, LifecycleRollbackSnapshot, LifecycleSubject, MigrationPlan, MigrationRollbackSnapshot, ProductionExecutionApproval, ProjectBinding, ProjectFitEvidence, RetentionPolicyRevision, RetrievalPipelineIdentity, RuntimeProbe, RuntimeProfile, ) from modelforge_api.services.audit import AuditContext, AuditWriter from modelforge_api.services.lifecycle_serving import LifecycleServing from modelforge_api.services.serving import ServingError class LifecycleError(RuntimeError): def __init__(self, status_code: int, code: str, message: str) -> None: super().__init__(message) self.status_code = status_code self.code = code self.message = message def _hash(payload: Any) -> str: raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) return hashlib.sha256(raw.encode()).hexdigest() def _now() -> datetime: return datetime.now(UTC) def _serving_changed_at(operation: LifecycleOperation) -> str: if operation.finished_at is None: raise LifecycleError(409, "SERVING_JOURNAL_INVALID", "committed serving timestamp is missing") value = operation.finished_at return (value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)).isoformat() DEFAULT_POLICIES: tuple[dict[str, Any], ...] = ( { "key": "lab-promotion", "scope": "LAB_PROMOTION", "requirements": { "identity": "exact", "integrity": "VERIFIED", "security": "not_blocked", "license": "not_blocked", "runtime": "PROVEN", "scheduler_readiness": "READY", "rollback_target": True, }, }, { "key": "capability-production", "scope": "CAPABILITY_PRODUCTION", "requirements": { "identity": "exact", "integrity": "VERIFIED", "security": "APPROVED", "license": "APPROVED", "runtime": "PROVEN", "evaluation": "PROMOTION_ELIGIBLE", "scheduler_readiness": "READY", "rollback_target": True, "critical_regressions": 0, }, }, { "key": "project-production", "scope": "PROJECT_PRODUCTION", "requirements": { "base": "capability-production", "project_fit": "ELIGIBLE", "production_validation": "SATISFIED", "critical_regressions": 0, }, }, ) class LifecycleService: def __init__(self, session: Session) -> None: self.session = session def ensure_defaults(self) -> None: for definition in DEFAULT_POLICIES: if self.session.scalar( select(LifecyclePolicyRevision).where( LifecyclePolicyRevision.key == definition["key"], LifecyclePolicyRevision.active.is_(True), ) ): continue self.create_policy( ApprovalPolicyCreate( key=definition["key"], scope=definition["scope"], requirements=definition["requirements"], created_by="system-m12-bootstrap", ) ) if not self.session.scalar( select(RetentionPolicyRevision).where( RetentionPolicyRevision.key == "standard-rollback", RetentionPolicyRevision.active.is_(True), ) ): self.create_retention_policy( RetentionPolicyCreate( key="standard-rollback", minimum_rollback_days=30, requirements={ "stable": "retain", "previous_stable": "rollback_window", "active_evaluation": "retain", "legal_hold": "retain", }, created_by="system-m12-bootstrap", ) ) def create_policy(self, request: ApprovalPolicyCreate) -> ApprovalPolicyResponse: previous = self.session.scalar( select(LifecyclePolicyRevision) .where(LifecyclePolicyRevision.key == request.key) .order_by(LifecyclePolicyRevision.revision.desc()) ) revision = (previous.revision + 1) if previous else 1 payload = { "key": request.key, "revision": revision, "scope": request.scope, "requirements": request.requirements, } fingerprint = _hash(payload) existing = self.session.scalar( select(LifecyclePolicyRevision).where( LifecyclePolicyRevision.fingerprint == fingerprint ) ) if existing: return self._policy_response(existing) if previous: previous.active = False item = LifecyclePolicyRevision( **payload, fingerprint=fingerprint, active=True, created_by=request.created_by, created_at=_now(), immutable_at=_now(), ) self.session.add(item) self.session.flush() self._event( "APPROVAL_POLICY_CREATED", "APPROVAL_POLICY", str(item.id), None, f"REVISION_{revision}", request.created_by, "POLICY_AUTHOR", item.id, [], "Versioned lifecycle policy created", {"key": request.key, "scope": request.scope, "fingerprint": fingerprint}, ) self.session.commit() return self._policy_response(item) def policies(self) -> list[ApprovalPolicyResponse]: self.ensure_defaults() items = self.session.scalars( select(LifecyclePolicyRevision).order_by( LifecyclePolicyRevision.key, LifecyclePolicyRevision.revision.desc() ) ).all() return [self._policy_response(item) for item in items] def create_subject(self, request: LifecycleSubjectCreate) -> LifecycleSubjectResponse: existing = self.session.scalar( select(LifecycleSubject).where( LifecycleSubject.target_type == request.target_type, LifecycleSubject.target_ref == request.target_ref, LifecycleSubject.environment == request.environment, ) ) if existing: return self._subject_response(existing) self._validate_target(request.target_type, request.target_ref, allow_rehearsal=True) item = LifecycleSubject( target_type=request.target_type, target_ref=request.target_ref, environment=request.environment, state=request.state, version=1, created_at=_now(), updated_at=_now(), ) self.session.add(item) self.session.flush() self._event( "LIFECYCLE_SUBJECT_REGISTERED", request.target_type, request.target_ref, None, request.state, "local-operator", "OPERATOR", None, [], "Lifecycle subject registered without changing the underlying deployment", {"environment": request.environment, "version": 1}, ) self.session.commit() return self._subject_response(item) def subjects(self) -> list[LifecycleSubjectResponse]: return [ self._subject_response(item) for item in self.session.scalars( select(LifecycleSubject).order_by(LifecycleSubject.created_at.desc()) ).all() ] def request_approval(self, request: ApprovalRequestCreate) -> ApprovalRequestResponse: self.ensure_defaults() policy = self.session.scalar( select(LifecyclePolicyRevision).where( LifecyclePolicyRevision.key == request.policy_key, LifecyclePolicyRevision.active.is_(True), ) ) if not policy: raise LifecycleError(404, "POLICY_NOT_FOUND", "active approval policy not found") if request.subject_id: subject = self.session.get(LifecycleSubject, request.subject_id) if not subject: raise LifecycleError(404, "SUBJECT_NOT_FOUND", "lifecycle subject not found") if ( subject.target_type != request.target_type or subject.target_ref != request.target_ref or subject.environment != request.environment ): raise LifecycleError( 422, "SUBJECT_TARGET_MISMATCH", "approval target does not match lifecycle subject", ) if request.evidence.serving_transition is not None: selected = request.evidence.serving_transition if request.evidence.production_validation != "SATISFIED": raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation") if (request.environment != "PRODUCTION" or policy.scope != "CAPABILITY_PRODUCTION" or request.target_type != "CAPABILITY_DEPLOYMENT" or request.target_ref != str(selected.candidate_deployment_id) or request.requested_transition != "STABLE" or request.evidence.capability_deployment_id != selected.candidate_deployment_id or request.evidence.production_execution_approval_id != selected.production_approval_id or request.evidence.rollback_target_ref != str(selected.current_deployment_id)): raise LifecycleError(422, "SERVING_APPROVAL_MISMATCH", "serving transition requires exact production approval scope and targets") resolved = self._resolve_evidence(request.evidence.model_dump(mode="json")) snapshot = { "declared": request.evidence.model_dump(mode="json"), "resolved": resolved, } blockers, warnings = self._evaluate_policy(policy, snapshot) status = ApprovalStatus.BLOCKED if blockers else ApprovalStatus.PENDING item = LifecycleApprovalRequest( policy_revision_id=policy.id, subject_id=request.subject_id, target_type=request.target_type, target_ref=request.target_ref, environment=request.environment, requested_transition=request.requested_transition, evidence_snapshot=snapshot, evidence_fingerprint=_hash(snapshot), status=status, blockers=blockers, warnings=warnings, requested_by=request.requested_by, reason=request.reason, expires_at=request.expires_at, version=1, created_at=_now(), ) self.session.add(item) self.session.flush() for evidence_type, evidence_ref in self._evidence_refs(snapshot["declared"]): self.session.add( LifecycleApprovalEvidence( approval_request_id=item.id, evidence_type=evidence_type, evidence_ref=evidence_ref, evidence_digest=_hash({evidence_type: evidence_ref}), ) ) self._event( "APPROVAL_REQUESTED", request.target_type, request.target_ref, None, status, request.requested_by, "REQUESTER", policy.id, [ref for _, ref in self._evidence_refs(snapshot["declared"])], request.reason, { "blockers": blockers, "warnings": warnings, "transition": request.requested_transition, }, change_id=str(item.id), ) self.session.commit() return self._approval_response(item, policy) def approval_requests(self) -> list[ApprovalRequestResponse]: items = self.session.scalars( select(LifecycleApprovalRequest).order_by(LifecycleApprovalRequest.created_at.desc()) ).all() return [self._approval_response(item) for item in items] def approval(self, approval_id: uuid.UUID) -> ApprovalRequestResponse: item = self._approval(approval_id) self._refresh_approval(item) self.session.commit() return self._approval_response(item) def decide_approval( self, approval_id: uuid.UUID, decision: ApprovalDecision, action: str ) -> ApprovalRequestResponse: item = self._approval(approval_id) policy = self.session.get(LifecyclePolicyRevision, item.policy_revision_id) if policy is None: raise LifecycleError(500, "POLICY_REFERENCE_MISSING", "approval policy is unavailable") self._refresh_approval(item) previous = item.status event_type: str if action == "approve": if item.status != ApprovalStatus.PENDING: raise LifecycleError( 409, "APPROVAL_NOT_PENDING", f"approval cannot be approved from {item.status}" ) item.status = ApprovalStatus.APPROVED item.approved_by = decision.actor event_type = "APPROVAL_APPROVED" elif action == "reject": if item.status not in {ApprovalStatus.PENDING, ApprovalStatus.BLOCKED}: raise LifecycleError( 409, "APPROVAL_ALREADY_DECIDED", "approval is already terminal" ) item.status = ApprovalStatus.REJECTED event_type = "APPROVAL_REJECTED" elif action == "revoke": if item.status != ApprovalStatus.APPROVED and not ( item.status == ApprovalStatus.STALE and item.approved_by is not None ): raise LifecycleError(409, "APPROVAL_NOT_APPROVED", "only approval can be revoked") item.status = ApprovalStatus.REVOKED event_type = "APPROVAL_REVOKED" else: raise LifecycleError(422, "INVALID_DECISION", "unknown approval decision") item.decided_at = _now() item.version += 1 self._event( event_type, item.target_type, item.target_ref, previous, item.status, decision.actor, "APPROVER", policy.id, [ref for _, ref in self._evidence_refs(item.evidence_snapshot["declared"])], decision.reason, {"evidence_fingerprint": item.evidence_fingerprint}, change_id=str(item.id), ) self.session.commit() return self._approval_response(item, policy) def create_plan(self, request: PromotionPlanCreate) -> PromotionPlanResponse: approval = self._approval(request.approval_request_id) self._refresh_approval(approval) if approval.status != ApprovalStatus.APPROVED: raise LifecycleError( 409, "APPROVAL_REQUIRED", f"approved current evidence is required; got {approval.status}", ) subject = self.session.get(LifecycleSubject, request.subject_id) if not subject: raise LifecycleError(404, "SUBJECT_NOT_FOUND", "lifecycle subject not found") if approval.subject_id != subject.id: raise LifecycleError( 422, "APPROVAL_SUBJECT_MISMATCH", "approval is for another subject" ) if approval.requested_transition != request.desired_state: raise LifecycleError( 422, "APPROVAL_TRANSITION_MISMATCH", "promotion plan requests a transition that was not approved", ) declared = approval.evidence_snapshot["declared"] if declared.get("rollback_target_ref") != request.rollback_target_ref: raise LifecycleError( 422, "APPROVAL_ROLLBACK_TARGET_MISMATCH", "promotion plan rollback target differs from approved evidence", ) approved_candidate = declared.get("capability_deployment_id") if approved_candidate and str(request.candidate_deployment_id) != str(approved_candidate): raise LifecycleError( 422, "APPROVAL_CANDIDATE_MISMATCH", "promotion plan candidate differs from approved evidence", ) approved_migration = declared.get("migration_id") if str(request.migration_id or "") != str(approved_migration or ""): raise LifecycleError( 422, "APPROVAL_MIGRATION_MISMATCH", "promotion plan migration differs from approved evidence", ) serving_contract = approval.evidence_snapshot.get("resolved", {}).get("serving_transition") if "serving_transition" in request.affected_identities: raise LifecycleError(422, "SERVING_CONTRACT_SERVER_OWNED", "serving contract is derived from approved evidence") if serving_contract is not None: if (request.canary_strategy or request.migration_class not in {"transparent", "behavioral"} or subject.target_type != "CAPABILITY_DEPLOYMENT" or subject.environment != "PRODUCTION" or subject.target_ref != str(request.candidate_deployment_id) or subject.state not in {"LAB_READY", "PROMOTION_ELIGIBLE"} or request.desired_state != "STABLE"): raise LifecycleError(422, "SERVING_PLAN_UNSUPPORTED", "only reviewed same-space stable replacement is supported") contract = LifecycleServing(self.session).validate(serving_contract) request = request.model_copy(update={ "affected_identities": {**request.affected_identities, "serving_transition": serving_contract}, "project_consumers": sorted({value["project_id"] for value in contract.bindings.values()}), }) try: intermediate = ( DeploymentLifecycleState.CANARY if request.canary_strategy else request.desired_state ) if serving_contract is None: assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate) except ValueError as exc: raise LifecycleError(422, "INVALID_LIFECYCLE_TRANSITION", str(exc)) from exc if request.migration_class == "requires_reindex": migration = ( self.session.get(EmbeddingMigration, request.migration_id) if request.migration_id else None ) if not migration or migration.status != "ready": raise LifecycleError( 422, "READY_MIGRATION_REQUIRED", "requires_reindex plans require a completed isolated migration", ) mode = str(request.canary_strategy.get("mode", "")) if mode not in {"SHADOW", "DUAL_READ", "EVALUATION_ROUTE"}: raise LifecycleError( 422, "INCOMPATIBLE_EMBEDDING_CANARY", "embedding changes require a separate shadow space; percentage mixing is forbidden", ) if request.migration_class == "schema_breaking" and not request.affected_identities.get( "manual_runbook" ): raise LifecycleError( 422, "MANUAL_RUNBOOK_REQUIRED", "schema-breaking plans require an explicit runbook" ) impact = { "projects": sorted(set(request.project_consumers)), "clients": request.affected_identities.get("clients", []), "embedding_indexes": request.affected_identities.get("embedding_indexes", []), "retrieval_pipelines": request.affected_identities.get("retrieval_pipelines", []), "runtime_deployments": request.affected_identities.get("runtime_deployments", []), "storage_artifacts": request.affected_identities.get("storage_artifacts", []), "scheduler_resource": request.affected_identities.get("resource_envelope_id"), "migration_required": request.migration_class == "requires_reindex", } payload = request.model_dump(mode="json") | { "current_state": subject.state, "subject_version": subject.version, "impact_analysis": impact, "approval_fingerprint": approval.evidence_fingerprint, } fingerprint = _hash(payload) existing = self.session.scalar( select(LifecyclePromotionPlan).where( LifecyclePromotionPlan.plan_fingerprint == fingerprint ) ) if existing: return self._plan_response(existing) plan = LifecyclePromotionPlan( approval_request_id=approval.id, subject_id=subject.id, current_state=subject.state, desired_state=request.desired_state, migration_class=request.migration_class, candidate_deployment_id=request.candidate_deployment_id, rollback_target_ref=request.rollback_target_ref, project_consumers=sorted(set(request.project_consumers)), affected_identities=request.affected_identities, impact_analysis=impact, migration_id=request.migration_id, canary_strategy=request.canary_strategy, drain_strategy=request.drain_strategy, health_gates=request.health_gates, automatic_abort_conditions=request.automatic_abort_conditions, plan_fingerprint=fingerprint, status=PromotionPlanStatus.DRAFT, version=1, created_by=request.created_by, created_at=_now(), ) self.session.add(plan) self.session.flush() self._event( "PROMOTION_PLANNED", subject.target_type, subject.target_ref, subject.state, request.desired_state, request.created_by, "REQUESTER", approval.policy_revision_id, [str(approval.id)], "Immutable promotion intent drafted separately from execution", {"plan_id": str(plan.id), "impact": impact, "migration_class": request.migration_class}, change_id=str(plan.id), ) self.session.commit() return self._plan_response(plan) def plans(self) -> list[PromotionPlanResponse]: return [ self._plan_response(item) for item in self.session.scalars( select(LifecyclePromotionPlan).order_by(LifecyclePromotionPlan.created_at.desc()) ).all() ] def plan(self, plan_id: uuid.UUID) -> PromotionPlanResponse: return self._plan_response(self._plan(plan_id)) def approve_plan(self, plan_id: uuid.UUID, decision: ApprovalDecision) -> PromotionPlanResponse: plan = self._plan(plan_id) if plan.status != PromotionPlanStatus.DRAFT: raise LifecycleError(409, "PLAN_NOT_DRAFT", "only a draft plan can be approved") approval = self._approval(plan.approval_request_id) self._refresh_approval(approval) if approval.status != ApprovalStatus.APPROVED: raise LifecycleError(409, "APPROVAL_STALE", "plan evidence approval is not current") plan.status = PromotionPlanStatus.APPROVED plan.approved_by = decision.actor plan.immutable_at = _now() plan.version += 1 self._event( "PROMOTION_PLAN_APPROVED", "PROMOTION_PLAN", str(plan.id), "DRAFT", "APPROVED", decision.actor, "APPROVER", approval.policy_revision_id, [str(approval.id)], decision.reason, {"plan_fingerprint": plan.plan_fingerprint}, change_id=str(plan.id), ) self.session.commit() return self._plan_response(plan) def execute_plan( self, plan_id: uuid.UUID, request: PlanExecutionCreate ) -> LifecycleOperationResponse: existing = self.session.scalar( select(LifecycleOperation).where( LifecycleOperation.idempotency_key == request.idempotency_key ) ) if existing: return self._operation_response(existing) plan = self._plan(plan_id) if plan.status != PromotionPlanStatus.APPROVED: raise LifecycleError(409, "PLAN_NOT_APPROVED", "approved immutable plan required") if plan.migration_class in {"requires_reindex", "schema_breaking"}: raise LifecycleError( 409, "MANUAL_MIGRATION_EXECUTION_REQUIRED", "M12 records but does not automatically execute this migration class", ) approval = self._approval(plan.approval_request_id) self._refresh_approval(approval) if approval.status != ApprovalStatus.APPROVED: raise LifecycleError(409, "APPROVAL_STALE", "approval changed after plan creation") subject = self.session.get(LifecycleSubject, plan.subject_id) if subject is None: raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable") if subject.version != request.expected_subject_version: raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently") serving_contract = plan.affected_identities.get("serving_transition") if serving_contract is not None: LifecycleServing(self.session).validate(serving_contract) elif subject.environment == "PRODUCTION" and subject.target_type == "CAPABILITY_DEPLOYMENT" and plan.desired_state == "STABLE": raise LifecycleError(409, "SERVING_TRANSITION_REQUIRED", "operational stable promotion requires the approved serving adapter") plan_version = plan.version snapshot_payload = { "subject_id": str(subject.id), "target_type": subject.target_type, "target_ref": subject.target_ref, "environment": subject.environment, "state": subject.state, "version": subject.version, "rollback_target_ref": plan.rollback_target_ref, "affected_identities": plan.affected_identities, "superseded_by_ref": subject.superseded_by_ref, } if serving_contract is not None: snapshot_payload["serving_transition"] = serving_contract snapshot = LifecycleRollbackSnapshot( promotion_plan_id=plan.id, snapshot=snapshot_payload, snapshot_digest=_hash(snapshot_payload), created_at=_now(), immutable_at=_now(), ) operation = LifecycleOperation( promotion_plan_id=plan.id, stage=OperationStage.PREPARING, idempotency_key=request.idempotency_key, expected_subject_version=request.expected_subject_version, requester=approval.requested_by, approver=plan.approved_by or approval.approved_by or "unknown", executor=request.executor, failure_details={}, version=1, started_at=_now(), ) # Claim both the immutable plan and its subject with database-side CAS. # This serializes different plans for one subject as well as duplicate # executors that raced after reading the same approved plan. plan_claim = self.session.execute( update(LifecyclePromotionPlan) .where( LifecyclePromotionPlan.id == plan.id, LifecyclePromotionPlan.status == PromotionPlanStatus.APPROVED, LifecyclePromotionPlan.version == plan_version, ) .values(status=PromotionPlanStatus.EXECUTING, version=plan_version + 1) ) if rows_affected(plan_claim) != 1: self.session.rollback() raced = self.session.scalar( select(LifecycleOperation).where( LifecycleOperation.idempotency_key == request.idempotency_key ) ) if raced: return self._operation_response(raced) raise LifecycleError( 409, "PLAN_EXECUTION_CONFLICT", "promotion plan was claimed by another executor", ) claimed_state = ( DeploymentLifecycleState.CANARY if plan.canary_strategy else subject.state ) subject_claim = self.session.execute( update(LifecycleSubject) .where( LifecycleSubject.id == subject.id, LifecycleSubject.version == request.expected_subject_version, LifecycleSubject.state == plan.current_state, ) .values( state=claimed_state, version=LifecycleSubject.version + 1, updated_at=_now(), ) ) if rows_affected(subject_claim) != 1: self.session.rollback() raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently") self.session.add_all([snapshot, operation]) self.session.flush() self.session.refresh(plan) self.session.refresh(subject) self._event( "PROMOTION_STARTED", subject.target_type, subject.target_ref, plan.current_state, "PREPARING", request.executor, "EXECUTOR", approval.policy_revision_id, [str(approval.id), str(snapshot.id)], "Approved lifecycle operation started", {"operation_id": str(operation.id), "snapshot_digest": snapshot.snapshot_digest}, change_id=str(operation.id), ) if request.rehearsal_pause_stage == "PREPARING": self.session.commit() return self._operation_response(operation) operation.stage = OperationStage.ACTIVATING operation.version += 1 if plan.canary_strategy: self._start_canary(plan, operation, subject) else: operation.stage = OperationStage.VERIFYING self.session.commit() return self._operation_response(operation) def observe_canary( self, operation_id: uuid.UUID, observation: CanaryObservation, *, audit_context: AuditContext | None = None, ) -> LifecycleOperationResponse: operation = self._operation(operation_id) if operation.stage != OperationStage.VERIFYING: raise LifecycleError(409, "CANARY_NOT_RUNNING", "operation is not verifying a canary") canary = self.session.scalar( select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id) ) if not canary or canary.status != CanaryStatus.RUNNING: raise LifecycleError(409, "CANARY_NOT_RUNNING", "canary is not running") plan = self._plan(operation.promotion_plan_id) subject = self._require_manual_canary_rehearsal(plan) context = audit_context or AuditContext( actor_type="manual_lab_rehearsal", actor_id="local-lab-rehearsal", ) canary.request_count += observation.request_count canary.error_count += observation.error_count canary.latency_p95_ms = observation.latency_p95_ms canary.result = { **observation.model_dump(mode="json"), "provenance": { "source": "MANUAL_LAB_REHEARSAL", "authoritative": False, "actor": context.actor_id, "correlation_id": context.correlation_id, }, } trigger = self._canary_trigger(canary, observation) if trigger: canary.status = CanaryStatus.ABORTED canary.abort_trigger = trigger canary.finished_at = _now() self._rollback( operation, trigger, context.actor_id, "Automatic abort from manual LAB rehearsal observation", ) event_type = "CANARY_ABORTED" elif canary.request_count >= canary.target_request_count: canary.status = CanaryStatus.READY_FOR_PROMOTION_REVIEW canary.finished_at = _now() event_type = "CANARY_READY_FOR_PROMOTION_REVIEW" else: event_type = "CANARY_OBSERVED" self._event( event_type, subject.target_type, subject.target_ref, subject.state, canary.status, context.actor_id, "MANUAL_LAB_REHEARSAL", self._approval(plan.approval_request_id).policy_revision_id, [str(canary.id)], trigger or "Bounded canary observation recorded without payload content", { "request_count": canary.request_count, "error_count": canary.error_count, "latency_p95_ms": canary.latency_p95_ms, }, change_id=str(operation.id), ) AuditWriter(self.session, context=context).write( action="lifecycle.manual_lab_canary_observed", resource_type="lifecycle_operation", resource_id=str(operation.id), details={ "source": "MANUAL_LAB_REHEARSAL", "canary_id": str(canary.id), "status": canary.status, "authoritative": False, }, ) self.session.commit() return self._operation_response(operation) def _require_manual_canary_rehearsal( self, plan: LifecyclePromotionPlan, ) -> LifecycleSubject: """No manual or legacy canary evidence can qualify an operational promotion.""" subject = self.session.get(LifecycleSubject, plan.subject_id) approval = self.session.get(LifecycleApprovalRequest, plan.approval_request_id) policy = self.session.get(LifecyclePolicyRevision, approval.policy_revision_id) if ( approval is not None ) else None if ( subject is None or approval is None or policy is None or subject.target_type != "LAB_REHEARSAL" or subject.environment != "LAB" or approval.environment != "LAB" or policy.scope != "LAB_PROMOTION" or plan.desired_state != "LAB_STABLE" or approval.subject_id != subject.id or approval.target_type != subject.target_type or approval.target_ref != subject.target_ref or approval.requested_transition != plan.desired_state ): raise LifecycleError( 409, "MANUAL_CANARY_REHEARSAL_ONLY", "Canary observations and commit require an explicit LAB rehearsal; " "authoritative operational canary evidence ingestion is unavailable", ) return subject def commit_operation( self, operation_id: uuid.UUID, decision: ApprovalDecision ) -> LifecycleOperationResponse: operation = self._operation(operation_id) plan = self._plan(operation.promotion_plan_id) if operation.stage == OperationStage.COMMITTED: transition = plan.affected_identities.get("serving_transition") if transition is not None: snapshot = self.session.scalar(select(LifecycleRollbackSnapshot).where( LifecycleRollbackSnapshot.promotion_plan_id == plan.id)) if (snapshot is None or _hash(snapshot.snapshot) != snapshot.snapshot_digest or snapshot.snapshot.get("serving_transition") != transition): raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact installed serving snapshot is required") LifecycleServing(self.session).installed(transition, _serving_changed_at(operation)) self.session.commit() return self._operation_response(operation) if operation.stage != OperationStage.VERIFYING: raise LifecycleError( 409, "OPERATION_NOT_REVIEWABLE", "operation is not ready for review" ) canary = self.session.scalar( select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id) ) if canary or plan.canary_strategy: self._require_manual_canary_rehearsal(plan) if canary and canary.status != CanaryStatus.READY_FOR_PROMOTION_REVIEW: raise LifecycleError( 409, "CANARY_NOT_REVIEWABLE", "canary never auto-promotes; successful evidence required", ) subject = self.session.get(LifecycleSubject, plan.subject_id) if subject is None: raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable") snapshot = self.session.scalar( select(LifecycleRollbackSnapshot).where( LifecycleRollbackSnapshot.promotion_plan_id == plan.id ) ) if snapshot is None: raise LifecycleError( 409, "ROLLBACK_SNAPSHOT_MISSING", "immutable rollback snapshot is unavailable" ) serving_contract = plan.affected_identities.get("serving_transition") serving_changed_at = None if serving_contract is not None: approval = self._approval(plan.approval_request_id) self._refresh_approval(approval) if approval.status != ApprovalStatus.APPROVED or _hash(approval.evidence_snapshot) != approval.evidence_fingerprint: raise LifecycleError(409, "APPROVAL_STALE", "serving approval is no longer current") if snapshot.snapshot.get("serving_transition") != serving_contract or _hash(snapshot.snapshot) != snapshot.snapshot_digest: raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact immutable serving snapshot is required") serving_changed_at = _now() LifecycleServing(self.session).switch(serving_contract, serving_changed_at.isoformat()) expected_state = ( DeploymentLifecycleState.CANARY if plan.canary_strategy else plan.current_state ) try: if serving_contract is None: assert_deployment_transition( DeploymentLifecycleState(expected_state), DeploymentLifecycleState(plan.desired_state), ) except ValueError as exc: raise LifecycleError(409, "STATE_CHANGED", str(exc)) from exc previous = str(expected_state) superseded_by_ref = ( str(plan.affected_identities["superseded_by_ref"]) if plan.affected_identities.get("superseded_by_ref") else subject.superseded_by_ref ) subject_commit = self.session.execute( update(LifecycleSubject) .where( LifecycleSubject.id == subject.id, LifecycleSubject.version == int(snapshot.snapshot["version"]) + 1, LifecycleSubject.state == expected_state, ) .values( state=plan.desired_state, superseded_by_ref=superseded_by_ref, version=LifecycleSubject.version + 1, updated_at=_now(), ) ) if rows_affected(subject_commit) != 1: self.session.rollback() raise LifecycleError( 409, "VERSION_CONFLICT", "lifecycle subject changed before promotion commit", ) self.session.refresh(subject) operation.stage = OperationStage.COMMITTED operation.finished_at = serving_changed_at or _now() operation.version += 1 plan.status = PromotionPlanStatus.COMMITTED plan.version += 1 self._apply_rollback_retention(plan, decision.actor) event_type = { DeploymentLifecycleState.DEPRECATED: "DEPLOYMENT_DEPRECATED", DeploymentLifecycleState.ARCHIVED: "DEPLOYMENT_ARCHIVED", }.get(DeploymentLifecycleState(plan.desired_state), "PROMOTION_COMMITTED") self._event( event_type, subject.target_type, subject.target_ref, previous, subject.state, decision.actor, "EXECUTOR", self._approval(plan.approval_request_id).policy_revision_id, [str(plan.id), str(operation.id)], decision.reason, {"subject_version": subject.version}, change_id=str(operation.id), ) self.session.commit() return self._operation_response(operation) def rollback_operation( self, operation_id: uuid.UUID, decision: ApprovalDecision ) -> LifecycleOperationResponse: operation = self._operation(operation_id) if operation.stage == OperationStage.ROLLED_BACK: return self._operation_response(operation) self._rollback(operation, "OPERATOR_REQUEST", decision.actor, decision.reason) self.session.commit() return self._operation_response(operation) def operations(self) -> list[LifecycleOperationResponse]: return [ self._operation_response(item) for item in self.session.scalars( select(LifecycleOperation).order_by(LifecycleOperation.started_at.desc()) ).all() ] def operation(self, operation_id: uuid.UUID) -> LifecycleOperationResponse: return self._operation_response(self._operation(operation_id)) def reconcile_incomplete(self) -> int: items = self.session.scalars( select(LifecycleOperation).where( LifecycleOperation.stage.in_( [ OperationStage.PREPARING, OperationStage.ACTIVATING, OperationStage.ROLLING_BACK, ] ) ) ).all() for item in items: self._rollback( item, "CONTROL_PLANE_RESTART", "lifecycle-reconciler", "Incomplete activation was conservatively restored from its immutable snapshot", ) if items: self.session.commit() return len(items) def create_retention_policy(self, request: RetentionPolicyCreate) -> RetentionPolicyResponse: previous = self.session.scalar( select(RetentionPolicyRevision) .where(RetentionPolicyRevision.key == request.key) .order_by(RetentionPolicyRevision.revision.desc()) ) revision = (previous.revision + 1) if previous else 1 payload = request.model_dump(mode="json") | {"revision": revision} fingerprint = _hash(payload) existing = self.session.scalar( select(RetentionPolicyRevision).where( RetentionPolicyRevision.fingerprint == fingerprint ) ) if existing: return RetentionPolicyResponse.model_validate(existing) if previous: previous.active = False item = RetentionPolicyRevision( key=request.key, revision=revision, minimum_rollback_days=request.minimum_rollback_days, requirements=request.requirements, fingerprint=fingerprint, active=True, created_by=request.created_by, created_at=_now(), immutable_at=_now(), ) self.session.add(item) self.session.commit() return RetentionPolicyResponse.model_validate(item) def retention_policies(self) -> list[RetentionPolicyResponse]: self.ensure_defaults() return [ RetentionPolicyResponse.model_validate(item) for item in self.session.scalars( select(RetentionPolicyRevision).order_by( RetentionPolicyRevision.key, RetentionPolicyRevision.revision.desc() ) ).all() ] def create_cleanup_plan(self, request: CleanupPlanCreate) -> CleanupPlanResponse: dependencies, reclaimable, retention_state = self._cleanup_dependencies( request.target_type, request.target_ref ) blockers = [f"{item['type']}:{item['ref']}" for item in dependencies if item["blocking"]] status = CleanupStatus.BLOCKED if blockers else CleanupStatus.READY digest = _hash({"dependencies": dependencies, "retention_state": retention_state}) item = LifecycleCleanupPlan( target_type=request.target_type, target_ref=str(request.target_ref), action=request.action, dependencies=dependencies, dependency_digest=digest, reclaimable_bytes=reclaimable, retention_state=retention_state, blockers=blockers, status=status, created_by=request.created_by, created_at=_now(), ) self.session.add(item) self.session.flush() self._event( "CLEANUP_PLANNED", request.target_type, str(request.target_ref), None, status, request.created_by, "REQUESTER", None, [dep["ref"] for dep in dependencies], "Dependency and retention dry-run created; no bytes were removed", {"reclaimable_bytes": reclaimable, "blockers": blockers}, change_id=str(item.id), ) self.session.commit() return self._cleanup_response(item) def cleanup_plans(self) -> list[CleanupPlanResponse]: return [ self._cleanup_response(item) for item in self.session.scalars( select(LifecycleCleanupPlan).order_by(LifecycleCleanupPlan.created_at.desc()) ).all() ] def cleanup_plan(self, plan_id: uuid.UUID) -> CleanupPlanResponse: item = self.session.get(LifecycleCleanupPlan, plan_id) if not item: raise LifecycleError(404, "CLEANUP_PLAN_NOT_FOUND", "cleanup plan not found") return self._cleanup_response(item) def execute_cleanup( self, plan_id: uuid.UUID, request: CleanupExecutionCreate ) -> CleanupPlanResponse: plan = self.session.get(LifecycleCleanupPlan, plan_id) if not plan: raise LifecycleError(404, "CLEANUP_PLAN_NOT_FOUND", "cleanup plan not found") dependencies, _, retention_state = self._cleanup_dependencies( plan.target_type, uuid.UUID(plan.target_ref) ) current_digest = _hash({"dependencies": dependencies, "retention_state": retention_state}) if current_digest != plan.dependency_digest: plan.status = CleanupStatus.STALE self.session.commit() raise LifecycleError( 409, "CLEANUP_PLAN_STALE", "dependencies changed; create a new plan" ) blockers = [item for item in dependencies if item["blocking"]] if blockers or retention_state in { RetentionState.ACTIVE, RetentionState.ROLLBACK_RETAINED, }: raise LifecycleError( 409, "DEPENDENCY_CONFLICT", "active dependencies or retention block cleanup" ) if plan.action == "RECORD_LOCATION_REMOVAL": if not request.physical_removal_confirmed: raise LifecycleError( 422, "PHYSICAL_REMOVAL_CONFIRMATION_REQUIRED", "ModelForge records removal only after the typed storage action confirms it", ) location = self.session.get(ArtifactLocation, uuid.UUID(plan.target_ref)) if location is None: raise LifecycleError(404, "ARTIFACT_LOCATION_NOT_FOUND", "location disappeared") self.session.add( ArtifactLocationRemovalRecord( artifact_location_id=location.id, cleanup_plan_id=plan.id, prior_status=location.status, prior_observed_sha256=location.observed_sha256, prior_size_bytes=location.size_bytes, removed_by=request.executor, removed_at=_now(), ) ) location.status = "removed" location.last_checked_at = _now() if location.artifact_id: remaining = self.session.scalar( select(func.count(ArtifactLocation.id)).where( ArtifactLocation.artifact_id == location.artifact_id, ArtifactLocation.id != location.id, ArtifactLocation.status.in_(["present", "verified", "local"]), ) ) if not remaining: artifact = location.artifact if artifact: artifact.status = "missing" event_type = "ARTIFACT_LOCATION_REMOVED" else: artifact_set = self.session.get(ArtifactSet, uuid.UUID(plan.target_ref)) if artifact_set is None: raise LifecycleError(404, "ARTIFACT_SET_NOT_FOUND", "artifact set disappeared") artifact_set.status = "archived" event_type = "ARTIFACT_SET_ARCHIVED" plan.status = CleanupStatus.EXECUTED plan.executed_at = _now() self._event( event_type, plan.target_type, plan.target_ref, plan.retention_state, "ARCHIVED" if plan.action == "ARCHIVE_METADATA" else "REMOVED", request.executor, "EXECUTOR", None, [str(plan.id)], "Explicit dependency-rechecked cleanup action completed; provenance retained", {"action": plan.action, "reclaimable_bytes": plan.reclaimable_bytes}, change_id=str(plan.id), ) self.session.commit() return self._cleanup_response(plan) def events(self, limit: int = 200) -> list[LifecycleEventResponse]: return [ LifecycleEventResponse.model_validate(item) for item in self.session.scalars( select(LifecycleEvent).order_by(LifecycleEvent.occurred_at.desc()).limit(limit) ).all() ] def _validate_target(self, target_type: str, target_ref: str, *, allow_rehearsal: bool) -> None: if target_type == "LAB_REHEARSAL" and allow_rehearsal: return try: target_id = uuid.UUID(target_ref) except ValueError as exc: raise LifecycleError( 422, "INVALID_TARGET_REF", "target reference must be a UUID" ) from exc mapping: dict[str, type[Any]] = { "CAPABILITY_DEPLOYMENT": CapabilityDeployment, "PROJECT_BINDING": ProjectBinding, "ARTIFACT_SET": ArtifactSet, } model = mapping.get(target_type) if model is None or self.session.get(model, target_id) is None: raise LifecycleError(404, "TARGET_NOT_FOUND", "lifecycle target not found") def _resolve_evidence(self, declared: dict[str, Any]) -> dict[str, Any]: facts: dict[str, Any] = {} if declared.get("serving_transition") is not None: from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection if declared.get("production_validation") != "SATISFIED": raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation") facts["serving_transition"] = LifecycleServing(self.session).prepare( ServingTransitionSelection.model_validate(declared["serving_transition"]) ).model_dump(mode="json") for field, model in ( ("artifact_set_id", ArtifactSet), ("runtime_profile_id", RuntimeProfile), ("runtime_probe_id", RuntimeProbe), ("production_execution_approval_id", ProductionExecutionApproval), ("capability_deployment_id", CapabilityDeployment), ("project_binding_id", ProjectBinding), ("project_fit_evidence_id", ProjectFitEvidence), ("resource_envelope_id", CapabilityResourceEnvelope), ("embedding_space_id", EmbeddingSpace), ("retrieval_pipeline_id", RetrievalPipelineIdentity), ("migration_id", EmbeddingMigration), ): raw = declared.get(field) if not raw: continue item = self.session.get(model, uuid.UUID(str(raw))) facts[field] = self._fact(item) if item else {"missing": True} run_facts: dict[str, dict[str, Any]] = {} for raw in declared.get("evaluation_run_ids", []): item = self.session.get(EvaluationRun, uuid.UUID(str(raw))) run_facts[str(raw)] = self._fact(item) if item else {"missing": True} if run_facts: facts["evaluation_run_ids"] = run_facts fit_raw = declared.get("project_fit_evidence_id") if fit_raw: fit = self.session.get(ProjectFitEvidence, uuid.UUID(str(fit_raw))) if fit: latest = self.session.scalar( select(ProjectFitEvidence) .where(ProjectFitEvidence.project_binding_id == fit.project_binding_id) .order_by(ProjectFitEvidence.created_at.desc()) ) facts["latest_project_fit_evidence_id"] = str(latest.id) if latest else None return facts @staticmethod def _fact(item: Any | None) -> dict[str, Any]: if item is None: return {"missing": True} names = ( "id", "status", "security_status", "license_status", "completeness", "availability", "fingerprint", "environment_fingerprint", "runtime_image_digest", "production", "channel", "health_status", "stale", "stale_reason", "recommendation", "engineering_integration", "production_validation", "critical_errors", "evidence_digest", "identity_digest", "artifact_set_id", "revision_id", "runtime_profile_id", "runtime_probe_id", "deployment_candidate_id", "capability_contract_id", "resource_envelope_id", "revoked_at", "error_count", "completed_cases", "expected_cases", ) return { name: str(value) if isinstance(value, uuid.UUID) else value for name in names if (value := getattr(item, name, None)) is not None } def _evaluate_policy( self, policy: LifecyclePolicyRevision, snapshot: dict[str, Any] ) -> tuple[list[str], list[str]]: scope = policy.scope requirements = dict(policy.requirements) base_key = requirements.pop("base", None) if base_key: base = self.session.scalar( select(LifecyclePolicyRevision).where( LifecyclePolicyRevision.key == base_key, LifecyclePolicyRevision.active.is_(True), ) ) if base: requirements = {**base.requirements, **requirements} data = snapshot["declared"] resolved = snapshot["resolved"] blockers: list[str] = [] warnings: list[str] = [] exact_fields = ( "artifact_set_id", "runtime_profile_id", "runtime_probe_id", "runtime_image_digest", "resource_envelope_id", "rollback_target_ref", ) for field in exact_fields: if not data.get(field): blockers.append(f"MISSING_{field.upper()}") for field, fact in resolved.items(): if isinstance(fact, dict) and fact.get("missing"): blockers.append(f"MISSING_REFERENCED_{field.upper()}") artifact = resolved.get("artifact_set_id", {}) profile = resolved.get("runtime_profile_id", {}) probe = resolved.get("runtime_probe_id", {}) production_approval = resolved.get("production_execution_approval_id", {}) if data["integrity"] != requirements.get("integrity", "VERIFIED"): blockers.append("ARTIFACT_INTEGRITY_NOT_VERIFIED") if artifact and not artifact.get("missing"): if artifact.get("status") != "verified" or artifact.get("completeness") != "complete": blockers.append("ARTIFACT_INTEGRITY_NOT_VERIFIED") if data.get("model_revision_id") != artifact.get("revision_id"): blockers.append("MODEL_REVISION_ARTIFACT_SET_MISMATCH") if profile and not profile.get("missing"): if profile.get("artifact_set_id") != data.get("artifact_set_id"): blockers.append("RUNTIME_PROFILE_ARTIFACT_SET_MISMATCH") if profile.get("runtime_image_digest") != data.get("runtime_image_digest"): blockers.append("RUNTIME_IMAGE_DIGEST_MISMATCH") if probe and not probe.get("missing") and probe.get("status") != "completed": blockers.append("RUNTIME_PROBE_NOT_COMPLETED") if data["security"] == "BLOCKED": blockers.append("SECURITY_BLOCKED") if data["license"] == "BLOCKED": blockers.append("LICENSE_BLOCKED") if data["runtime"] != requirements.get("runtime", "PROVEN"): blockers.append("RUNTIME_NOT_PROVEN") if data["scheduler_readiness"] != requirements.get("scheduler_readiness", "READY"): blockers.append("SCHEDULER_NOT_READY") if scope != "LAB_PROMOTION": if data["security"] != requirements.get("security", "APPROVED"): blockers.append("SECURITY_NOT_APPROVED") if data["license"] != requirements.get("license", "APPROVED"): blockers.append("LICENSE_NOT_APPROVED") if data["evaluation"] != requirements.get("evaluation", "PROMOTION_ELIGIBLE"): blockers.append("EVALUATION_NOT_PROMOTION_ELIGIBLE") if not data.get("evaluation_run_ids"): blockers.append("MISSING_EVALUATION_RUNS") for run in resolved.get("evaluation_run_ids", {}).values(): if run.get("missing") or run.get("status") != "completed": blockers.append("EVALUATION_RUN_NOT_COMPLETED") if run.get("error_count", 0): blockers.append("EVALUATION_RUN_ERRORS") if data["critical_regressions"] > int(requirements.get("critical_regressions", 0)): blockers.append("CRITICAL_PROJECT_REGRESSION") if not data.get("production_execution_approval_id"): blockers.append("MISSING_PRODUCTION_EXECUTION_APPROVAL") elif not production_approval.get("missing"): if production_approval.get("status") != "approved" or production_approval.get( "revoked_at" ): blockers.append("PRODUCTION_EXECUTION_APPROVAL_INVALID") if production_approval.get("artifact_set_id") != data.get("artifact_set_id"): blockers.append("PRODUCTION_APPROVAL_ARTIFACT_SET_MISMATCH") if production_approval.get("runtime_profile_id") != data.get("runtime_profile_id"): blockers.append("PRODUCTION_APPROVAL_RUNTIME_PROFILE_MISMATCH") if not data.get("evidence_revisions", {}).get("scheduler"): blockers.append("MISSING_SCHEDULER_EVIDENCE_REVISION") if scope == "PROJECT_PRODUCTION": if data["production_validation"] == "DEFERRED_EXTERNAL_VALIDATION": blockers.append("REQUIRED_EXTERNAL_VALIDATION_NOT_SATISFIED") elif data["production_validation"] != "SATISFIED": blockers.append("PROJECT_PRODUCTION_VALIDATION_NOT_SATISFIED") if data["project_fit"] != requirements.get("project_fit", "ELIGIBLE"): blockers.append("PROJECT_PRODUCTION_FIT_NOT_ELIGIBLE") if not data.get("project_fit_evidence_id"): blockers.append("MISSING_PROJECT_FIT_EVIDENCE") fit = resolved.get("project_fit_evidence_id", {}) if fit and not fit.get("missing"): if fit.get("production_validation") == "DEFERRED_EXTERNAL_VALIDATION": blockers.append("REQUIRED_EXTERNAL_VALIDATION_NOT_SATISFIED") if fit.get("recommendation") != "PROMOTION_ELIGIBLE": blockers.append("PROJECT_PRODUCTION_FIT_NOT_ELIGIBLE") if fit.get("critical_errors", 0): blockers.append("CRITICAL_PROJECT_REGRESSION") if ( data["production_validation"] == "DEFERRED_EXTERNAL_VALIDATION" and scope != "PROJECT_PRODUCTION" ): warnings.append("DEFERRED_EXTERNAL_VALIDATION_RETAINED") return sorted(set(blockers)), sorted(set(warnings)) def _refresh_approval(self, item: LifecycleApprovalRequest) -> None: if item.status not in {ApprovalStatus.PENDING, ApprovalStatus.APPROVED}: return now = _now() expires = item.expires_at if expires and expires.tzinfo is None: expires = expires.replace(tzinfo=UTC) if expires and expires <= now: previous = item.status item.status = ApprovalStatus.EXPIRED item.decided_at = now self._event( "APPROVAL_EXPIRED", item.target_type, item.target_ref, previous, item.status, "lifecycle-engine", "SYSTEM", item.policy_revision_id, [], "Approval expiry reached", {}, change_id=str(item.id), ) return stale_details: dict[str, Any] = {} current_resolved: dict[str, Any] | None try: current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"]) except (LifecycleError, ServingError) as exc: # A historical serving approval must remain inspectable after its route # changes. Expected qualification failures invalidate authority; internal # errors still propagate rather than being disguised as stale evidence. if (item.evidence_snapshot["declared"].get("serving_transition") is None or exc.status_code not in {404, 409, 422}): raise current_resolved = None stale_details = {"validation_code": exc.code, "validation_reason": str(exc)} item.blockers = sorted(set(item.blockers) | {exc.code}) if current_resolved is None or _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]): previous = item.status item.status = ApprovalStatus.STALE item.stale_at = now self._event( "APPROVAL_STALE", item.target_type, item.target_ref, previous, item.status, "lifecycle-engine", "SYSTEM", item.policy_revision_id, [], "Referenced evidence changed after approval request creation", stale_details, change_id=str(item.id), ) @staticmethod def _evidence_refs(declared: dict[str, Any]) -> list[tuple[str, str]]: refs: list[tuple[str, str]] = [] for key, value in declared.items(): if key.endswith("_id") and value: refs.append((key, str(value))) elif key.endswith("_ids"): refs.extend((key[:-1], str(item)) for item in value) elif key in {"artifact_digests", "rollback_target_ref"}: values = value if isinstance(value, list) else [value] refs.extend((key, str(item)) for item in values if item) return refs def _start_canary( self, plan: LifecyclePromotionPlan, operation: LifecycleOperation, subject: LifecycleSubject, ) -> None: strategy = plan.canary_strategy mode = str(strategy.get("mode", "REQUEST")) if plan.migration_class == "requires_reindex" and mode not in { "SHADOW", "DUAL_READ", "EVALUATION_ROUTE", }: raise LifecycleError(422, "INCOMPATIBLE_EMBEDDING_CANARY", "mixed vector spaces denied") previous = plan.current_state operation.stage = OperationStage.VERIFYING operation.version += 1 canary = LifecycleCanaryRun( operation_id=operation.id, mode=mode, traffic_percent=int(strategy.get("traffic_percent", 0)), eligible_clients=list(strategy.get("eligible_clients", [])), eligible_projects=list(strategy.get("eligible_projects", [])), target_request_count=int(strategy.get("target_request_count", 1)), thresholds={ "max_error_rate": float(strategy.get("max_error_rate", 0.0)), "max_latency_p95_ms": float(strategy.get("max_latency_p95_ms", 10_000.0)), }, status=CanaryStatus.RUNNING, result={}, started_at=_now(), ) self.session.add(canary) self.session.flush() self._event( "CANARY_STARTED", subject.target_type, subject.target_ref, previous, subject.state, operation.executor, "EXECUTOR", self._approval(plan.approval_request_id).policy_revision_id, [str(canary.id)], "Bounded capability-aware canary started", {"mode": mode, "target_request_count": canary.target_request_count}, change_id=str(operation.id), ) @staticmethod def _canary_trigger(canary: LifecycleCanaryRun, observation: CanaryObservation) -> str | None: if observation.capability_health != "HEALTHY" or not observation.worker_healthy: return "HEALTH_FAILURE" if not observation.scheduler_ready or observation.external_pressure: return "CAPACITY_FAILURE" if observation.critical_project_failures: return "CRITICAL_PROJECT_REGRESSION" total = max(canary.request_count, 1) if canary.error_count / total > float(canary.thresholds["max_error_rate"]): return "ERROR_THRESHOLD" if observation.latency_p95_ms > float(canary.thresholds["max_latency_p95_ms"]): return "LATENCY_THRESHOLD" return None def _rollback( self, operation: LifecycleOperation, trigger: str, actor: str, reason: str ) -> None: started = time.perf_counter() plan = self._plan(operation.promotion_plan_id) self._approval(plan.approval_request_id) # Same approval-before-route lock order as commit. snapshot = self.session.scalar( select(LifecycleRollbackSnapshot).where( LifecycleRollbackSnapshot.promotion_plan_id == plan.id ) ) subject = self.session.get(LifecycleSubject, plan.subject_id) if not snapshot or not subject: operation.stage = OperationStage.FAILED operation.failure_code = "ROLLBACK_FAILED" operation.failure_details = {"reason": "immutable rollback snapshot unavailable"} operation.finished_at = _now() if subject: subject.state = DeploymentLifecycleState.MANUAL_INTERVENTION_REQUIRED subject.version += 1 plan.status = PromotionPlanStatus.FAILED self._event( "ROLLBACK_FAILED", "PROMOTION_PLAN", str(plan.id), None, "MANUAL_INTERVENTION_REQUIRED", actor, "EXECUTOR", self._approval(plan.approval_request_id).policy_revision_id, [], reason, {"trigger": trigger}, change_id=str(operation.id), ) return snapshot_version = int(snapshot.snapshot["version"]) serving_contract = plan.affected_identities.get("serving_transition") was_committed = operation.stage == OperationStage.COMMITTED if serving_contract is not None: if (snapshot.snapshot.get("serving_transition") != serving_contract or _hash(snapshot.snapshot) != snapshot.snapshot_digest): raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact serving snapshot is unavailable") # A subject CAS failure below rolls back the route in the same transaction. # A later route is preserved rather than guessed away. try: LifecycleServing(self.session).restore(serving_contract, _serving_changed_at(operation) if was_committed else _now().isoformat(), committed=was_committed) except Exception: self.session.rollback() raise expected_version = snapshot_version + ( 2 if operation.stage == OperationStage.COMMITTED else 1 ) previous = subject.state operation.stage = OperationStage.ROLLING_BACK restored = self.session.execute( update(LifecycleSubject) .where( LifecycleSubject.id == subject.id, LifecycleSubject.version == expected_version, ) .values( state=snapshot.snapshot["state"], superseded_by_ref=snapshot.snapshot.get("superseded_by_ref"), version=LifecycleSubject.version + 1, updated_at=_now(), ) ) if rows_affected(restored) != 1: if serving_contract is not None: self.session.rollback() raise LifecycleError(409, "ROLLBACK_VERSION_CONFLICT", "lifecycle subject changed; serving state was preserved") operation.stage = OperationStage.FAILED operation.failure_code = "ROLLBACK_VERSION_CONFLICT" operation.failure_details = { "reason": "lifecycle subject changed after this operation snapshot", "expected_subject_version": expected_version, "observed_subject_version": subject.version, } operation.finished_at = _now() operation.version += 1 plan.status = PromotionPlanStatus.FAILED plan.version += 1 self._event( "ROLLBACK_FAILED", subject.target_type, subject.target_ref, previous, "CONCURRENT_STATE_PRESERVED", actor, "EXECUTOR" if actor != "SYSTEM" else "SYSTEM", self._approval(plan.approval_request_id).policy_revision_id, [str(snapshot.id)], reason, { "trigger": trigger, "failure_code": "ROLLBACK_VERSION_CONFLICT", "expected_subject_version": expected_version, }, change_id=str(operation.id), ) return self.session.refresh(subject) operation.stage = OperationStage.ROLLED_BACK operation.failure_code = trigger if trigger != "OPERATOR_REQUEST" else None operation.failure_details = {"rollback_target_ref": plan.rollback_target_ref} operation.rollback_duration_ms = (time.perf_counter() - started) * 1000 operation.finished_at = _now() operation.version += 1 plan.status = PromotionPlanStatus.ROLLED_BACK plan.version += 1 self._event( "ROLLBACK_COMPLETED", subject.target_type, subject.target_ref, previous, subject.state, actor, "EXECUTOR" if actor != "SYSTEM" else "SYSTEM", self._approval(plan.approval_request_id).policy_revision_id, [str(snapshot.id)], reason, { "trigger": trigger, "rollback_duration_ms": operation.rollback_duration_ms, "rollback_target_ref": plan.rollback_target_ref, }, change_id=str(operation.id), ) def _apply_rollback_retention(self, plan: LifecyclePromotionPlan, actor: str) -> None: policy = self.session.scalar( select(RetentionPolicyRevision).where( RetentionPolicyRevision.key == "standard-rollback", RetentionPolicyRevision.active.is_(True), ) ) if not policy: raise LifecycleError( 500, "RETENTION_POLICY_MISSING", "rollback retention policy missing" ) retained_until = _now() + timedelta(days=policy.minimum_rollback_days) transition = plan.affected_identities.get("serving_transition") if transition is not None: prior = transition["current"] for kind, ref in (("ARTIFACT_SET", prior["artifact_set_id"]), ("RUNTIME_PROFILE", prior["runtime_profile_id"]), ("CAPABILITY_DEPLOYMENT", prior["id"])): retained = self.session.scalar(select(LifecycleRetentionRecord).where( LifecycleRetentionRecord.target_type == kind, LifecycleRetentionRecord.target_ref == ref)) if retained is None: self.session.add(LifecycleRetentionRecord( policy_revision_id=policy.id, target_type=kind, target_ref=ref, state=RetentionState.ROLLBACK_RETAINED, retained_until=retained_until, legal_hold=False, reason="Exact prior serving identity retained for approved rollback", created_at=_now())) else: previous_until = retained.retained_until if previous_until is None or previous_until.replace(tzinfo=UTC) < retained_until: retained.retained_until = retained_until retained.state = RetentionState.ROLLBACK_RETAINED existing = self.session.scalar( select(LifecycleRetentionRecord).where( LifecycleRetentionRecord.target_type == "ROLLBACK_TARGET", LifecycleRetentionRecord.target_ref == plan.rollback_target_ref, ) ) if existing: existing.policy_revision_id = policy.id existing.state = RetentionState.ROLLBACK_RETAINED existing.retained_until = retained_until else: self.session.add( LifecycleRetentionRecord( policy_revision_id=policy.id, target_type="ROLLBACK_TARGET", target_ref=plan.rollback_target_ref, state=RetentionState.ROLLBACK_RETAINED, retained_until=retained_until, legal_hold=False, reason="Previous exact state retained for rollback after lifecycle commit", created_at=_now(), ) ) self._event( "RETENTION_APPLIED", "ROLLBACK_TARGET", plan.rollback_target_ref, None, RetentionState.ROLLBACK_RETAINED, actor, "EXECUTOR", None, [str(policy.id), str(plan.id)], "Minimum rollback window applied", {"retained_until": retained_until.isoformat()}, change_id=str(plan.id), ) def _cleanup_dependencies( self, target_type: str, target_ref: uuid.UUID ) -> tuple[list[dict[str, Any]], int, str]: dependencies: list[dict[str, Any]] = [] reclaimable = 0 retention = self.session.scalar( select(LifecycleRetentionRecord).where( LifecycleRetentionRecord.target_ref == str(target_ref) ) ) retention_state = retention.state if retention else RetentionState.ARCHIVABLE if retention: until = retention.retained_until if until and until.tzinfo is None: until = until.replace(tzinfo=UTC) blocking = retention.legal_hold or retention.state in { RetentionState.ACTIVE, RetentionState.ROLLBACK_RETAINED, } if until and until <= _now() and not retention.legal_hold: blocking = False retention_state = RetentionState.ARCHIVABLE dependencies.append( { "type": "RETENTION_POLICY", "ref": str(retention.id), "blocking": blocking, "state": retention.state, } ) if target_type == "ARTIFACT_SET": artifact_set = self.session.get(ArtifactSet, target_ref) if not artifact_set: raise LifecycleError(404, "ARTIFACT_SET_NOT_FOUND", "artifact set not found") reclaimable = artifact_set.total_size_bytes active_locations = self.session.scalars( select(ArtifactLocation) .join( ArtifactSetMember, ArtifactSetMember.artifact_id == ArtifactLocation.artifact_id, ) .where( ArtifactSetMember.artifact_set_id == target_ref, ArtifactLocation.status.in_(["present", "verified", "local"]), ) ).all() for active_location in active_locations: dependencies.append( { "type": "ARTIFACT_LOCATION", "ref": str(active_location.id), "blocking": True, } ) artifact_ids = self.session.scalars( select(ArtifactSetMember.artifact_id).where( ArtifactSetMember.artifact_set_id == target_ref ) ).all() if artifact_ids: derived_ids = self.session.scalars( select(DerivedArtifactSource.derived_artifact_id) .where(DerivedArtifactSource.source_artifact_id.in_(artifact_ids)) .distinct() ).all() for derived in self.session.scalars( select(DerivedArtifact).where(DerivedArtifact.id.in_(derived_ids)) ).all(): dependencies.append( {"type": "DERIVED_ARTIFACT", "ref": str(derived.id), "blocking": True} ) deployments = self.session.scalars( select(CapabilityDeployment).where( CapabilityDeployment.artifact_set_id == target_ref ) ).all() deployment_ids = [item.id for item in deployments] contract_ids = {item.capability_contract_id for item in deployments} for deployment in deployments: dependencies.append( { "type": "CAPABILITY_DEPLOYMENT", "ref": str(deployment.id), "blocking": True, } ) if deployment_ids: for route in self.session.scalars( select(CapabilityExperimentRoute).where( CapabilityExperimentRoute.capability_deployment_id.in_(deployment_ids), CapabilityExperimentRoute.status == "active", ) ).all(): dependencies.append( {"type": "EXPERIMENT_ROUTE", "ref": str(route.id), "blocking": True} ) for run in self.session.scalars( select(EvaluationRun).where( EvaluationRun.capability_deployment_id.in_(deployment_ids) ) ).all(): dependencies.append( {"type": "EVALUATION_RUN", "ref": str(run.id), "blocking": True} ) if contract_ids: for binding in self.session.scalars( select(ProjectBinding).where( ProjectBinding.capability_contract_id.in_(contract_ids) ) ).all(): dependencies.append( {"type": "PROJECT_BINDING", "ref": str(binding.id), "blocking": True} ) spaces = self.session.scalars( select(EmbeddingSpace).where(EmbeddingSpace.artifact_set_id == target_ref) ).all() for space in spaces: dependencies.append( {"type": "EMBEDDING_SPACE", "ref": str(space.id), "blocking": True} ) space_ids = [space.id for space in spaces] if space_ids: for migration in self.session.scalars( select(EmbeddingMigration).where( EmbeddingMigration.target_embedding_space_id.in_(space_ids) ) ).all(): dependencies.append( {"type": "MIGRATION", "ref": str(migration.id), "blocking": True} ) terminal_migration_states = {"ROLLED_BACK", "FAILED", "CANCELLED"} for migration_plan in self.session.scalars(select(MigrationPlan)).all(): source_match = str(target_ref) in json.dumps( migration_plan.source_identity, default=str ) target_match = str(target_ref) in json.dumps( migration_plan.target_identity, default=str ) if not source_match and not target_match: continue active = migration_plan.state not in terminal_migration_states rollback = self.session.scalar( select(MigrationRollbackSnapshot).where( MigrationRollbackSnapshot.migration_plan_id == migration_plan.id ) ) retained = False if rollback is not None: retain_until = rollback.retain_until if retain_until.tzinfo is None: retain_until = retain_until.replace(tzinfo=UTC) retained = retain_until > _now() if active or retained: dependencies.append( { "type": ( "MIGRATION_PLAN_SOURCE" if source_match else "MIGRATION_PLAN_TARGET" ), "ref": str(migration_plan.id), "blocking": True, "state": migration_plan.state, "rollback_retained": retained, } ) for model, field, kind, active in ( ( DeploymentCandidate, DeploymentCandidate.artifact_set_id, "DEPLOYMENT_CANDIDATE", True, ), (RuntimeProfile, RuntimeProfile.artifact_set_id, "RUNTIME_PROFILE", True), (ExecutionApproval, ExecutionApproval.artifact_set_id, "EXECUTION_APPROVAL", True), ( ProductionExecutionApproval, ProductionExecutionApproval.artifact_set_id, "PRODUCTION_APPROVAL", True, ), ): for item in self.session.scalars(select(model).where(field == target_ref)).all(): dependencies.append( {"type": kind, "ref": str(cast(Any, item).id), "blocking": active} ) for evidence in self.session.scalars( select(LifecycleApprovalEvidence).where( LifecycleApprovalEvidence.evidence_type == "artifact_set_id", LifecycleApprovalEvidence.evidence_ref == str(target_ref), ) ).all(): dependencies.append( { "type": "LIFECYCLE_APPROVAL_EVIDENCE", "ref": str(evidence.approval_request_id), "blocking": True, } ) else: location = self.session.get(ArtifactLocation, target_ref) if not location: raise LifecycleError( 404, "ARTIFACT_LOCATION_NOT_FOUND", "artifact location not found" ) reclaimable = location.size_bytes or 0 if location.artifact_id: set_ids = self.session.scalars( select(ArtifactSetMember.artifact_set_id).where( ArtifactSetMember.artifact_id == location.artifact_id ) ).all() for artifact_set_id in set_ids: dependencies.append( {"type": "ARTIFACT_SET", "ref": str(artifact_set_id), "blocking": True} ) other_locations = self.session.scalars( select(ArtifactLocation).where( ArtifactLocation.artifact_id == location.artifact_id, ArtifactLocation.id != location.id, ArtifactLocation.status.in_(["present", "verified", "local"]), ) ).all() dependencies.append( { "type": "OTHER_ARTIFACT_LOCATIONS", "ref": str(len(other_locations)), "blocking": False, } ) return ( sorted(dependencies, key=lambda item: (item["type"], item["ref"])), reclaimable, str(retention_state), ) def _approval(self, approval_id: uuid.UUID) -> LifecycleApprovalRequest: item = self.session.get(LifecycleApprovalRequest, approval_id, with_for_update=True, populate_existing=True) if not item: raise LifecycleError(404, "APPROVAL_NOT_FOUND", "approval request not found") return item def _plan(self, plan_id: uuid.UUID) -> LifecyclePromotionPlan: item = self.session.get(LifecyclePromotionPlan, plan_id, with_for_update=True, populate_existing=True) if not item: raise LifecycleError(404, "PROMOTION_PLAN_NOT_FOUND", "promotion plan not found") return item def _operation(self, operation_id: uuid.UUID) -> LifecycleOperation: item = self.session.get(LifecycleOperation, operation_id, with_for_update=True, populate_existing=True) if not item: raise LifecycleError(404, "LIFECYCLE_OPERATION_NOT_FOUND", "operation not found") return item @staticmethod def _policy_response(item: LifecyclePolicyRevision) -> ApprovalPolicyResponse: return ApprovalPolicyResponse.model_validate(item) @staticmethod def _subject_response(item: LifecycleSubject) -> LifecycleSubjectResponse: return LifecycleSubjectResponse.model_validate(item) def _approval_response( self, item: LifecycleApprovalRequest, policy: LifecyclePolicyRevision | None = None ) -> ApprovalRequestResponse: policy = policy or self.session.get(LifecyclePolicyRevision, item.policy_revision_id) if policy is None: raise LifecycleError(500, "POLICY_REFERENCE_MISSING", "approval policy is unavailable") return ApprovalRequestResponse( **{ key: getattr(item, key) for key in ApprovalRequestResponse.model_fields if key not in {"policy_key", "policy_revision"} }, policy_key=policy.key, policy_revision=policy.revision, ) @staticmethod def _plan_response(item: LifecyclePromotionPlan) -> PromotionPlanResponse: return PromotionPlanResponse.model_validate(item) def _operation_response(self, item: LifecycleOperation) -> LifecycleOperationResponse: canary = self.session.scalar( select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == item.id) ) canary_payload = None if canary: canary_payload = { key: value for key, value in canary.__dict__.items() if not key.startswith("_") } canary_payload["id"] = str(canary.id) canary_payload["operation_id"] = str(canary.operation_id) return LifecycleOperationResponse( **{ key: getattr(item, key) for key in LifecycleOperationResponse.model_fields if key != "canary" }, canary=canary_payload, ) @staticmethod def _cleanup_response(item: LifecycleCleanupPlan) -> CleanupPlanResponse: return CleanupPlanResponse.model_validate(item) def _event( self, event_type: str, object_type: str, object_ref: str, from_state: Any | None, to_state: Any | None, actor: str, actor_role: str, policy_revision_id: uuid.UUID | None, evidence_ids: list[str], reason: str, details: dict[str, Any], *, change_id: str | None = None, ) -> None: self.session.add( LifecycleEvent( event_type=event_type, object_type=str(object_type), object_ref=object_ref, from_state=str(from_state) if from_state is not None else None, to_state=str(to_state) if to_state is not None else None, actor=actor, actor_role=actor_role, policy_revision_id=policy_revision_id, evidence_ids=evidence_ids, reason=reason, change_id=change_id or str(uuid.uuid4()), details=details, occurred_at=_now(), ) ) def reconcile_lifecycle_operations(session: Session) -> int: return LifecycleService(session).reconcile_incomplete()