2150 lines
92 KiB
Python
2150 lines
92 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import time
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any, cast
|
|
|
|
from sqlalchemy import func, select, update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.db import rows_affected
|
|
from modelforge_api.domain.lifecycle_contracts import (
|
|
ApprovalDecision,
|
|
ApprovalPolicyCreate,
|
|
ApprovalPolicyResponse,
|
|
ApprovalRequestCreate,
|
|
ApprovalRequestResponse,
|
|
ApprovalStatus,
|
|
CanaryObservation,
|
|
CanaryStatus,
|
|
CleanupExecutionCreate,
|
|
CleanupPlanCreate,
|
|
CleanupPlanResponse,
|
|
CleanupStatus,
|
|
DeploymentLifecycleState,
|
|
LifecycleEventResponse,
|
|
LifecycleOperationResponse,
|
|
LifecycleSubjectCreate,
|
|
LifecycleSubjectResponse,
|
|
OperationStage,
|
|
PlanExecutionCreate,
|
|
PromotionPlanCreate,
|
|
PromotionPlanResponse,
|
|
PromotionPlanStatus,
|
|
RetentionPolicyCreate,
|
|
RetentionPolicyResponse,
|
|
RetentionState,
|
|
assert_deployment_transition,
|
|
)
|
|
from modelforge_api.persistence.models import (
|
|
ArtifactLocation,
|
|
ArtifactLocationRemovalRecord,
|
|
ArtifactSet,
|
|
ArtifactSetMember,
|
|
CapabilityDeployment,
|
|
CapabilityExperimentRoute,
|
|
CapabilityResourceEnvelope,
|
|
DeploymentCandidate,
|
|
DerivedArtifact,
|
|
DerivedArtifactSource,
|
|
EmbeddingMigration,
|
|
EmbeddingSpace,
|
|
EvaluationRun,
|
|
ExecutionApproval,
|
|
LifecycleApprovalEvidence,
|
|
LifecycleApprovalRequest,
|
|
LifecycleCanaryRun,
|
|
LifecycleCleanupPlan,
|
|
LifecycleEvent,
|
|
LifecycleOperation,
|
|
LifecyclePolicyRevision,
|
|
LifecyclePromotionPlan,
|
|
LifecycleRetentionRecord,
|
|
LifecycleRollbackSnapshot,
|
|
LifecycleSubject,
|
|
MigrationPlan,
|
|
MigrationRollbackSnapshot,
|
|
ProductionExecutionApproval,
|
|
ProjectBinding,
|
|
ProjectFitEvidence,
|
|
RetentionPolicyRevision,
|
|
RetrievalPipelineIdentity,
|
|
RuntimeProbe,
|
|
RuntimeProfile,
|
|
)
|
|
from modelforge_api.services.audit import AuditContext, AuditWriter
|
|
from modelforge_api.services.lifecycle_serving import LifecycleServing
|
|
from modelforge_api.services.serving import ServingError
|
|
|
|
|
|
class LifecycleError(RuntimeError):
|
|
def __init__(self, status_code: int, code: str, message: str) -> None:
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.code = code
|
|
self.message = message
|
|
|
|
|
|
def _hash(payload: Any) -> str:
|
|
raw = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
|
|
return hashlib.sha256(raw.encode()).hexdigest()
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def _serving_changed_at(operation: LifecycleOperation) -> str:
|
|
if operation.finished_at is None:
|
|
raise LifecycleError(409, "SERVING_JOURNAL_INVALID", "committed serving timestamp is missing")
|
|
value = operation.finished_at
|
|
return (value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)).isoformat()
|
|
|
|
|
|
DEFAULT_POLICIES: tuple[dict[str, Any], ...] = (
|
|
{
|
|
"key": "lab-promotion",
|
|
"scope": "LAB_PROMOTION",
|
|
"requirements": {
|
|
"identity": "exact",
|
|
"integrity": "VERIFIED",
|
|
"security": "not_blocked",
|
|
"license": "not_blocked",
|
|
"runtime": "PROVEN",
|
|
"scheduler_readiness": "READY",
|
|
"rollback_target": True,
|
|
},
|
|
},
|
|
{
|
|
"key": "capability-production",
|
|
"scope": "CAPABILITY_PRODUCTION",
|
|
"requirements": {
|
|
"identity": "exact",
|
|
"integrity": "VERIFIED",
|
|
"security": "APPROVED",
|
|
"license": "APPROVED",
|
|
"runtime": "PROVEN",
|
|
"evaluation": "PROMOTION_ELIGIBLE",
|
|
"scheduler_readiness": "READY",
|
|
"rollback_target": True,
|
|
"critical_regressions": 0,
|
|
},
|
|
},
|
|
{
|
|
"key": "project-production",
|
|
"scope": "PROJECT_PRODUCTION",
|
|
"requirements": {
|
|
"base": "capability-production",
|
|
"project_fit": "ELIGIBLE",
|
|
"production_validation": "SATISFIED",
|
|
"critical_regressions": 0,
|
|
},
|
|
},
|
|
)
|
|
|
|
|
|
class LifecycleService:
|
|
def __init__(self, session: Session) -> None:
|
|
self.session = session
|
|
|
|
def ensure_defaults(self) -> None:
|
|
for definition in DEFAULT_POLICIES:
|
|
if self.session.scalar(
|
|
select(LifecyclePolicyRevision).where(
|
|
LifecyclePolicyRevision.key == definition["key"],
|
|
LifecyclePolicyRevision.active.is_(True),
|
|
)
|
|
):
|
|
continue
|
|
self.create_policy(
|
|
ApprovalPolicyCreate(
|
|
key=definition["key"],
|
|
scope=definition["scope"],
|
|
requirements=definition["requirements"],
|
|
created_by="system-m12-bootstrap",
|
|
)
|
|
)
|
|
if not self.session.scalar(
|
|
select(RetentionPolicyRevision).where(
|
|
RetentionPolicyRevision.key == "standard-rollback",
|
|
RetentionPolicyRevision.active.is_(True),
|
|
)
|
|
):
|
|
self.create_retention_policy(
|
|
RetentionPolicyCreate(
|
|
key="standard-rollback",
|
|
minimum_rollback_days=30,
|
|
requirements={
|
|
"stable": "retain",
|
|
"previous_stable": "rollback_window",
|
|
"active_evaluation": "retain",
|
|
"legal_hold": "retain",
|
|
},
|
|
created_by="system-m12-bootstrap",
|
|
)
|
|
)
|
|
|
|
def create_policy(self, request: ApprovalPolicyCreate) -> ApprovalPolicyResponse:
|
|
previous = self.session.scalar(
|
|
select(LifecyclePolicyRevision)
|
|
.where(LifecyclePolicyRevision.key == request.key)
|
|
.order_by(LifecyclePolicyRevision.revision.desc())
|
|
)
|
|
revision = (previous.revision + 1) if previous else 1
|
|
payload = {
|
|
"key": request.key,
|
|
"revision": revision,
|
|
"scope": request.scope,
|
|
"requirements": request.requirements,
|
|
}
|
|
fingerprint = _hash(payload)
|
|
existing = self.session.scalar(
|
|
select(LifecyclePolicyRevision).where(
|
|
LifecyclePolicyRevision.fingerprint == fingerprint
|
|
)
|
|
)
|
|
if existing:
|
|
return self._policy_response(existing)
|
|
if previous:
|
|
previous.active = False
|
|
item = LifecyclePolicyRevision(
|
|
**payload,
|
|
fingerprint=fingerprint,
|
|
active=True,
|
|
created_by=request.created_by,
|
|
created_at=_now(),
|
|
immutable_at=_now(),
|
|
)
|
|
self.session.add(item)
|
|
self.session.flush()
|
|
self._event(
|
|
"APPROVAL_POLICY_CREATED",
|
|
"APPROVAL_POLICY",
|
|
str(item.id),
|
|
None,
|
|
f"REVISION_{revision}",
|
|
request.created_by,
|
|
"POLICY_AUTHOR",
|
|
item.id,
|
|
[],
|
|
"Versioned lifecycle policy created",
|
|
{"key": request.key, "scope": request.scope, "fingerprint": fingerprint},
|
|
)
|
|
self.session.commit()
|
|
return self._policy_response(item)
|
|
|
|
def policies(self) -> list[ApprovalPolicyResponse]:
|
|
self.ensure_defaults()
|
|
items = self.session.scalars(
|
|
select(LifecyclePolicyRevision).order_by(
|
|
LifecyclePolicyRevision.key, LifecyclePolicyRevision.revision.desc()
|
|
)
|
|
).all()
|
|
return [self._policy_response(item) for item in items]
|
|
|
|
def create_subject(self, request: LifecycleSubjectCreate) -> LifecycleSubjectResponse:
|
|
existing = self.session.scalar(
|
|
select(LifecycleSubject).where(
|
|
LifecycleSubject.target_type == request.target_type,
|
|
LifecycleSubject.target_ref == request.target_ref,
|
|
LifecycleSubject.environment == request.environment,
|
|
)
|
|
)
|
|
if existing:
|
|
return self._subject_response(existing)
|
|
self._validate_target(request.target_type, request.target_ref, allow_rehearsal=True)
|
|
item = LifecycleSubject(
|
|
target_type=request.target_type,
|
|
target_ref=request.target_ref,
|
|
environment=request.environment,
|
|
state=request.state,
|
|
version=1,
|
|
created_at=_now(),
|
|
updated_at=_now(),
|
|
)
|
|
self.session.add(item)
|
|
self.session.flush()
|
|
self._event(
|
|
"LIFECYCLE_SUBJECT_REGISTERED",
|
|
request.target_type,
|
|
request.target_ref,
|
|
None,
|
|
request.state,
|
|
"local-operator",
|
|
"OPERATOR",
|
|
None,
|
|
[],
|
|
"Lifecycle subject registered without changing the underlying deployment",
|
|
{"environment": request.environment, "version": 1},
|
|
)
|
|
self.session.commit()
|
|
return self._subject_response(item)
|
|
|
|
def subjects(self) -> list[LifecycleSubjectResponse]:
|
|
return [
|
|
self._subject_response(item)
|
|
for item in self.session.scalars(
|
|
select(LifecycleSubject).order_by(LifecycleSubject.created_at.desc())
|
|
).all()
|
|
]
|
|
|
|
def request_approval(self, request: ApprovalRequestCreate) -> ApprovalRequestResponse:
|
|
self.ensure_defaults()
|
|
policy = self.session.scalar(
|
|
select(LifecyclePolicyRevision).where(
|
|
LifecyclePolicyRevision.key == request.policy_key,
|
|
LifecyclePolicyRevision.active.is_(True),
|
|
)
|
|
)
|
|
if not policy:
|
|
raise LifecycleError(404, "POLICY_NOT_FOUND", "active approval policy not found")
|
|
if request.subject_id:
|
|
subject = self.session.get(LifecycleSubject, request.subject_id)
|
|
if not subject:
|
|
raise LifecycleError(404, "SUBJECT_NOT_FOUND", "lifecycle subject not found")
|
|
if (
|
|
subject.target_type != request.target_type
|
|
or subject.target_ref != request.target_ref
|
|
or subject.environment != request.environment
|
|
):
|
|
raise LifecycleError(
|
|
422,
|
|
"SUBJECT_TARGET_MISMATCH",
|
|
"approval target does not match lifecycle subject",
|
|
)
|
|
if request.evidence.serving_transition is not None:
|
|
selected = request.evidence.serving_transition
|
|
if request.evidence.production_validation != "SATISFIED":
|
|
raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
|
|
if (request.environment != "PRODUCTION" or policy.scope != "CAPABILITY_PRODUCTION"
|
|
or request.target_type != "CAPABILITY_DEPLOYMENT"
|
|
or request.target_ref != str(selected.candidate_deployment_id)
|
|
or request.requested_transition != "STABLE"
|
|
or request.evidence.capability_deployment_id != selected.candidate_deployment_id
|
|
or request.evidence.production_execution_approval_id != selected.production_approval_id
|
|
or request.evidence.rollback_target_ref != str(selected.current_deployment_id)):
|
|
raise LifecycleError(422, "SERVING_APPROVAL_MISMATCH", "serving transition requires exact production approval scope and targets")
|
|
resolved = self._resolve_evidence(request.evidence.model_dump(mode="json"))
|
|
snapshot = {
|
|
"declared": request.evidence.model_dump(mode="json"),
|
|
"resolved": resolved,
|
|
}
|
|
blockers, warnings = self._evaluate_policy(policy, snapshot)
|
|
status = ApprovalStatus.BLOCKED if blockers else ApprovalStatus.PENDING
|
|
item = LifecycleApprovalRequest(
|
|
policy_revision_id=policy.id,
|
|
subject_id=request.subject_id,
|
|
target_type=request.target_type,
|
|
target_ref=request.target_ref,
|
|
environment=request.environment,
|
|
requested_transition=request.requested_transition,
|
|
evidence_snapshot=snapshot,
|
|
evidence_fingerprint=_hash(snapshot),
|
|
status=status,
|
|
blockers=blockers,
|
|
warnings=warnings,
|
|
requested_by=request.requested_by,
|
|
reason=request.reason,
|
|
expires_at=request.expires_at,
|
|
version=1,
|
|
created_at=_now(),
|
|
)
|
|
self.session.add(item)
|
|
self.session.flush()
|
|
for evidence_type, evidence_ref in self._evidence_refs(snapshot["declared"]):
|
|
self.session.add(
|
|
LifecycleApprovalEvidence(
|
|
approval_request_id=item.id,
|
|
evidence_type=evidence_type,
|
|
evidence_ref=evidence_ref,
|
|
evidence_digest=_hash({evidence_type: evidence_ref}),
|
|
)
|
|
)
|
|
self._event(
|
|
"APPROVAL_REQUESTED",
|
|
request.target_type,
|
|
request.target_ref,
|
|
None,
|
|
status,
|
|
request.requested_by,
|
|
"REQUESTER",
|
|
policy.id,
|
|
[ref for _, ref in self._evidence_refs(snapshot["declared"])],
|
|
request.reason,
|
|
{
|
|
"blockers": blockers,
|
|
"warnings": warnings,
|
|
"transition": request.requested_transition,
|
|
},
|
|
change_id=str(item.id),
|
|
)
|
|
self.session.commit()
|
|
return self._approval_response(item, policy)
|
|
|
|
def approval_requests(self) -> list[ApprovalRequestResponse]:
|
|
items = self.session.scalars(
|
|
select(LifecycleApprovalRequest).order_by(LifecycleApprovalRequest.created_at.desc())
|
|
).all()
|
|
return [self._approval_response(item) for item in items]
|
|
|
|
def approval(self, approval_id: uuid.UUID) -> ApprovalRequestResponse:
|
|
item = self._approval(approval_id)
|
|
self._refresh_approval(item)
|
|
self.session.commit()
|
|
return self._approval_response(item)
|
|
|
|
def decide_approval(
|
|
self, approval_id: uuid.UUID, decision: ApprovalDecision, action: str
|
|
) -> ApprovalRequestResponse:
|
|
item = self._approval(approval_id)
|
|
policy = self.session.get(LifecyclePolicyRevision, item.policy_revision_id)
|
|
if policy is None:
|
|
raise LifecycleError(500, "POLICY_REFERENCE_MISSING", "approval policy is unavailable")
|
|
self._refresh_approval(item)
|
|
previous = item.status
|
|
event_type: str
|
|
if action == "approve":
|
|
if item.status != ApprovalStatus.PENDING:
|
|
raise LifecycleError(
|
|
409, "APPROVAL_NOT_PENDING", f"approval cannot be approved from {item.status}"
|
|
)
|
|
item.status = ApprovalStatus.APPROVED
|
|
item.approved_by = decision.actor
|
|
event_type = "APPROVAL_APPROVED"
|
|
elif action == "reject":
|
|
if item.status not in {ApprovalStatus.PENDING, ApprovalStatus.BLOCKED}:
|
|
raise LifecycleError(
|
|
409, "APPROVAL_ALREADY_DECIDED", "approval is already terminal"
|
|
)
|
|
item.status = ApprovalStatus.REJECTED
|
|
event_type = "APPROVAL_REJECTED"
|
|
elif action == "revoke":
|
|
if item.status != ApprovalStatus.APPROVED and not (
|
|
item.status == ApprovalStatus.STALE and item.approved_by is not None
|
|
):
|
|
raise LifecycleError(409, "APPROVAL_NOT_APPROVED", "only approval can be revoked")
|
|
item.status = ApprovalStatus.REVOKED
|
|
event_type = "APPROVAL_REVOKED"
|
|
else:
|
|
raise LifecycleError(422, "INVALID_DECISION", "unknown approval decision")
|
|
item.decided_at = _now()
|
|
item.version += 1
|
|
self._event(
|
|
event_type,
|
|
item.target_type,
|
|
item.target_ref,
|
|
previous,
|
|
item.status,
|
|
decision.actor,
|
|
"APPROVER",
|
|
policy.id,
|
|
[ref for _, ref in self._evidence_refs(item.evidence_snapshot["declared"])],
|
|
decision.reason,
|
|
{"evidence_fingerprint": item.evidence_fingerprint},
|
|
change_id=str(item.id),
|
|
)
|
|
self.session.commit()
|
|
return self._approval_response(item, policy)
|
|
|
|
def create_plan(self, request: PromotionPlanCreate) -> PromotionPlanResponse:
|
|
approval = self._approval(request.approval_request_id)
|
|
self._refresh_approval(approval)
|
|
if approval.status != ApprovalStatus.APPROVED:
|
|
raise LifecycleError(
|
|
409,
|
|
"APPROVAL_REQUIRED",
|
|
f"approved current evidence is required; got {approval.status}",
|
|
)
|
|
subject = self.session.get(LifecycleSubject, request.subject_id)
|
|
if not subject:
|
|
raise LifecycleError(404, "SUBJECT_NOT_FOUND", "lifecycle subject not found")
|
|
if approval.subject_id != subject.id:
|
|
raise LifecycleError(
|
|
422, "APPROVAL_SUBJECT_MISMATCH", "approval is for another subject"
|
|
)
|
|
if approval.requested_transition != request.desired_state:
|
|
raise LifecycleError(
|
|
422,
|
|
"APPROVAL_TRANSITION_MISMATCH",
|
|
"promotion plan requests a transition that was not approved",
|
|
)
|
|
declared = approval.evidence_snapshot["declared"]
|
|
if declared.get("rollback_target_ref") != request.rollback_target_ref:
|
|
raise LifecycleError(
|
|
422,
|
|
"APPROVAL_ROLLBACK_TARGET_MISMATCH",
|
|
"promotion plan rollback target differs from approved evidence",
|
|
)
|
|
approved_candidate = declared.get("capability_deployment_id")
|
|
if approved_candidate and str(request.candidate_deployment_id) != str(approved_candidate):
|
|
raise LifecycleError(
|
|
422,
|
|
"APPROVAL_CANDIDATE_MISMATCH",
|
|
"promotion plan candidate differs from approved evidence",
|
|
)
|
|
approved_migration = declared.get("migration_id")
|
|
if str(request.migration_id or "") != str(approved_migration or ""):
|
|
raise LifecycleError(
|
|
422,
|
|
"APPROVAL_MIGRATION_MISMATCH",
|
|
"promotion plan migration differs from approved evidence",
|
|
)
|
|
serving_contract = approval.evidence_snapshot.get("resolved", {}).get("serving_transition")
|
|
if "serving_transition" in request.affected_identities:
|
|
raise LifecycleError(422, "SERVING_CONTRACT_SERVER_OWNED", "serving contract is derived from approved evidence")
|
|
if serving_contract is not None:
|
|
if (request.canary_strategy or request.migration_class not in {"transparent", "behavioral"}
|
|
or subject.target_type != "CAPABILITY_DEPLOYMENT" or subject.environment != "PRODUCTION"
|
|
or subject.target_ref != str(request.candidate_deployment_id)
|
|
or subject.state not in {"LAB_READY", "PROMOTION_ELIGIBLE"}
|
|
or request.desired_state != "STABLE"):
|
|
raise LifecycleError(422, "SERVING_PLAN_UNSUPPORTED", "only reviewed same-space stable replacement is supported")
|
|
contract = LifecycleServing(self.session).validate(serving_contract)
|
|
request = request.model_copy(update={
|
|
"affected_identities": {**request.affected_identities, "serving_transition": serving_contract},
|
|
"project_consumers": sorted({value["project_id"] for value in contract.bindings.values()}),
|
|
})
|
|
try:
|
|
intermediate = (
|
|
DeploymentLifecycleState.CANARY
|
|
if request.canary_strategy
|
|
else request.desired_state
|
|
)
|
|
if serving_contract is None:
|
|
assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
|
|
except ValueError as exc:
|
|
raise LifecycleError(422, "INVALID_LIFECYCLE_TRANSITION", str(exc)) from exc
|
|
if request.migration_class == "requires_reindex":
|
|
migration = (
|
|
self.session.get(EmbeddingMigration, request.migration_id)
|
|
if request.migration_id
|
|
else None
|
|
)
|
|
if not migration or migration.status != "ready":
|
|
raise LifecycleError(
|
|
422,
|
|
"READY_MIGRATION_REQUIRED",
|
|
"requires_reindex plans require a completed isolated migration",
|
|
)
|
|
mode = str(request.canary_strategy.get("mode", ""))
|
|
if mode not in {"SHADOW", "DUAL_READ", "EVALUATION_ROUTE"}:
|
|
raise LifecycleError(
|
|
422,
|
|
"INCOMPATIBLE_EMBEDDING_CANARY",
|
|
"embedding changes require a separate shadow space; percentage mixing is forbidden",
|
|
)
|
|
if request.migration_class == "schema_breaking" and not request.affected_identities.get(
|
|
"manual_runbook"
|
|
):
|
|
raise LifecycleError(
|
|
422, "MANUAL_RUNBOOK_REQUIRED", "schema-breaking plans require an explicit runbook"
|
|
)
|
|
impact = {
|
|
"projects": sorted(set(request.project_consumers)),
|
|
"clients": request.affected_identities.get("clients", []),
|
|
"embedding_indexes": request.affected_identities.get("embedding_indexes", []),
|
|
"retrieval_pipelines": request.affected_identities.get("retrieval_pipelines", []),
|
|
"runtime_deployments": request.affected_identities.get("runtime_deployments", []),
|
|
"storage_artifacts": request.affected_identities.get("storage_artifacts", []),
|
|
"scheduler_resource": request.affected_identities.get("resource_envelope_id"),
|
|
"migration_required": request.migration_class == "requires_reindex",
|
|
}
|
|
payload = request.model_dump(mode="json") | {
|
|
"current_state": subject.state,
|
|
"subject_version": subject.version,
|
|
"impact_analysis": impact,
|
|
"approval_fingerprint": approval.evidence_fingerprint,
|
|
}
|
|
fingerprint = _hash(payload)
|
|
existing = self.session.scalar(
|
|
select(LifecyclePromotionPlan).where(
|
|
LifecyclePromotionPlan.plan_fingerprint == fingerprint
|
|
)
|
|
)
|
|
if existing:
|
|
return self._plan_response(existing)
|
|
plan = LifecyclePromotionPlan(
|
|
approval_request_id=approval.id,
|
|
subject_id=subject.id,
|
|
current_state=subject.state,
|
|
desired_state=request.desired_state,
|
|
migration_class=request.migration_class,
|
|
candidate_deployment_id=request.candidate_deployment_id,
|
|
rollback_target_ref=request.rollback_target_ref,
|
|
project_consumers=sorted(set(request.project_consumers)),
|
|
affected_identities=request.affected_identities,
|
|
impact_analysis=impact,
|
|
migration_id=request.migration_id,
|
|
canary_strategy=request.canary_strategy,
|
|
drain_strategy=request.drain_strategy,
|
|
health_gates=request.health_gates,
|
|
automatic_abort_conditions=request.automatic_abort_conditions,
|
|
plan_fingerprint=fingerprint,
|
|
status=PromotionPlanStatus.DRAFT,
|
|
version=1,
|
|
created_by=request.created_by,
|
|
created_at=_now(),
|
|
)
|
|
self.session.add(plan)
|
|
self.session.flush()
|
|
self._event(
|
|
"PROMOTION_PLANNED",
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
subject.state,
|
|
request.desired_state,
|
|
request.created_by,
|
|
"REQUESTER",
|
|
approval.policy_revision_id,
|
|
[str(approval.id)],
|
|
"Immutable promotion intent drafted separately from execution",
|
|
{"plan_id": str(plan.id), "impact": impact, "migration_class": request.migration_class},
|
|
change_id=str(plan.id),
|
|
)
|
|
self.session.commit()
|
|
return self._plan_response(plan)
|
|
|
|
def plans(self) -> list[PromotionPlanResponse]:
|
|
return [
|
|
self._plan_response(item)
|
|
for item in self.session.scalars(
|
|
select(LifecyclePromotionPlan).order_by(LifecyclePromotionPlan.created_at.desc())
|
|
).all()
|
|
]
|
|
|
|
def plan(self, plan_id: uuid.UUID) -> PromotionPlanResponse:
|
|
return self._plan_response(self._plan(plan_id))
|
|
|
|
def approve_plan(self, plan_id: uuid.UUID, decision: ApprovalDecision) -> PromotionPlanResponse:
|
|
plan = self._plan(plan_id)
|
|
if plan.status != PromotionPlanStatus.DRAFT:
|
|
raise LifecycleError(409, "PLAN_NOT_DRAFT", "only a draft plan can be approved")
|
|
approval = self._approval(plan.approval_request_id)
|
|
self._refresh_approval(approval)
|
|
if approval.status != ApprovalStatus.APPROVED:
|
|
raise LifecycleError(409, "APPROVAL_STALE", "plan evidence approval is not current")
|
|
plan.status = PromotionPlanStatus.APPROVED
|
|
plan.approved_by = decision.actor
|
|
plan.immutable_at = _now()
|
|
plan.version += 1
|
|
self._event(
|
|
"PROMOTION_PLAN_APPROVED",
|
|
"PROMOTION_PLAN",
|
|
str(plan.id),
|
|
"DRAFT",
|
|
"APPROVED",
|
|
decision.actor,
|
|
"APPROVER",
|
|
approval.policy_revision_id,
|
|
[str(approval.id)],
|
|
decision.reason,
|
|
{"plan_fingerprint": plan.plan_fingerprint},
|
|
change_id=str(plan.id),
|
|
)
|
|
self.session.commit()
|
|
return self._plan_response(plan)
|
|
|
|
def execute_plan(
|
|
self, plan_id: uuid.UUID, request: PlanExecutionCreate
|
|
) -> LifecycleOperationResponse:
|
|
existing = self.session.scalar(
|
|
select(LifecycleOperation).where(
|
|
LifecycleOperation.idempotency_key == request.idempotency_key
|
|
)
|
|
)
|
|
if existing:
|
|
return self._operation_response(existing)
|
|
plan = self._plan(plan_id)
|
|
if plan.status != PromotionPlanStatus.APPROVED:
|
|
raise LifecycleError(409, "PLAN_NOT_APPROVED", "approved immutable plan required")
|
|
if plan.migration_class in {"requires_reindex", "schema_breaking"}:
|
|
raise LifecycleError(
|
|
409,
|
|
"MANUAL_MIGRATION_EXECUTION_REQUIRED",
|
|
"M12 records but does not automatically execute this migration class",
|
|
)
|
|
approval = self._approval(plan.approval_request_id)
|
|
self._refresh_approval(approval)
|
|
if approval.status != ApprovalStatus.APPROVED:
|
|
raise LifecycleError(409, "APPROVAL_STALE", "approval changed after plan creation")
|
|
subject = self.session.get(LifecycleSubject, plan.subject_id)
|
|
if subject is None:
|
|
raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
|
|
if subject.version != request.expected_subject_version:
|
|
raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently")
|
|
serving_contract = plan.affected_identities.get("serving_transition")
|
|
if serving_contract is not None:
|
|
LifecycleServing(self.session).validate(serving_contract)
|
|
elif subject.environment == "PRODUCTION" and subject.target_type == "CAPABILITY_DEPLOYMENT" and plan.desired_state == "STABLE":
|
|
raise LifecycleError(409, "SERVING_TRANSITION_REQUIRED", "operational stable promotion requires the approved serving adapter")
|
|
plan_version = plan.version
|
|
snapshot_payload = {
|
|
"subject_id": str(subject.id),
|
|
"target_type": subject.target_type,
|
|
"target_ref": subject.target_ref,
|
|
"environment": subject.environment,
|
|
"state": subject.state,
|
|
"version": subject.version,
|
|
"rollback_target_ref": plan.rollback_target_ref,
|
|
"affected_identities": plan.affected_identities,
|
|
"superseded_by_ref": subject.superseded_by_ref,
|
|
}
|
|
if serving_contract is not None:
|
|
snapshot_payload["serving_transition"] = serving_contract
|
|
snapshot = LifecycleRollbackSnapshot(
|
|
promotion_plan_id=plan.id,
|
|
snapshot=snapshot_payload,
|
|
snapshot_digest=_hash(snapshot_payload),
|
|
created_at=_now(),
|
|
immutable_at=_now(),
|
|
)
|
|
operation = LifecycleOperation(
|
|
promotion_plan_id=plan.id,
|
|
stage=OperationStage.PREPARING,
|
|
idempotency_key=request.idempotency_key,
|
|
expected_subject_version=request.expected_subject_version,
|
|
requester=approval.requested_by,
|
|
approver=plan.approved_by or approval.approved_by or "unknown",
|
|
executor=request.executor,
|
|
failure_details={},
|
|
version=1,
|
|
started_at=_now(),
|
|
)
|
|
# Claim both the immutable plan and its subject with database-side CAS.
|
|
# This serializes different plans for one subject as well as duplicate
|
|
# executors that raced after reading the same approved plan.
|
|
plan_claim = self.session.execute(
|
|
update(LifecyclePromotionPlan)
|
|
.where(
|
|
LifecyclePromotionPlan.id == plan.id,
|
|
LifecyclePromotionPlan.status == PromotionPlanStatus.APPROVED,
|
|
LifecyclePromotionPlan.version == plan_version,
|
|
)
|
|
.values(status=PromotionPlanStatus.EXECUTING, version=plan_version + 1)
|
|
)
|
|
if rows_affected(plan_claim) != 1:
|
|
self.session.rollback()
|
|
raced = self.session.scalar(
|
|
select(LifecycleOperation).where(
|
|
LifecycleOperation.idempotency_key == request.idempotency_key
|
|
)
|
|
)
|
|
if raced:
|
|
return self._operation_response(raced)
|
|
raise LifecycleError(
|
|
409,
|
|
"PLAN_EXECUTION_CONFLICT",
|
|
"promotion plan was claimed by another executor",
|
|
)
|
|
claimed_state = (
|
|
DeploymentLifecycleState.CANARY if plan.canary_strategy else subject.state
|
|
)
|
|
subject_claim = self.session.execute(
|
|
update(LifecycleSubject)
|
|
.where(
|
|
LifecycleSubject.id == subject.id,
|
|
LifecycleSubject.version == request.expected_subject_version,
|
|
LifecycleSubject.state == plan.current_state,
|
|
)
|
|
.values(
|
|
state=claimed_state,
|
|
version=LifecycleSubject.version + 1,
|
|
updated_at=_now(),
|
|
)
|
|
)
|
|
if rows_affected(subject_claim) != 1:
|
|
self.session.rollback()
|
|
raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently")
|
|
self.session.add_all([snapshot, operation])
|
|
self.session.flush()
|
|
self.session.refresh(plan)
|
|
self.session.refresh(subject)
|
|
self._event(
|
|
"PROMOTION_STARTED",
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
plan.current_state,
|
|
"PREPARING",
|
|
request.executor,
|
|
"EXECUTOR",
|
|
approval.policy_revision_id,
|
|
[str(approval.id), str(snapshot.id)],
|
|
"Approved lifecycle operation started",
|
|
{"operation_id": str(operation.id), "snapshot_digest": snapshot.snapshot_digest},
|
|
change_id=str(operation.id),
|
|
)
|
|
if request.rehearsal_pause_stage == "PREPARING":
|
|
self.session.commit()
|
|
return self._operation_response(operation)
|
|
operation.stage = OperationStage.ACTIVATING
|
|
operation.version += 1
|
|
if plan.canary_strategy:
|
|
self._start_canary(plan, operation, subject)
|
|
else:
|
|
operation.stage = OperationStage.VERIFYING
|
|
self.session.commit()
|
|
return self._operation_response(operation)
|
|
|
|
def observe_canary(
|
|
self, operation_id: uuid.UUID, observation: CanaryObservation,
|
|
*, audit_context: AuditContext | None = None,
|
|
) -> LifecycleOperationResponse:
|
|
operation = self._operation(operation_id)
|
|
if operation.stage != OperationStage.VERIFYING:
|
|
raise LifecycleError(409, "CANARY_NOT_RUNNING", "operation is not verifying a canary")
|
|
canary = self.session.scalar(
|
|
select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id)
|
|
)
|
|
if not canary or canary.status != CanaryStatus.RUNNING:
|
|
raise LifecycleError(409, "CANARY_NOT_RUNNING", "canary is not running")
|
|
plan = self._plan(operation.promotion_plan_id)
|
|
subject = self._require_manual_canary_rehearsal(plan)
|
|
context = audit_context or AuditContext(
|
|
actor_type="manual_lab_rehearsal", actor_id="local-lab-rehearsal",
|
|
)
|
|
canary.request_count += observation.request_count
|
|
canary.error_count += observation.error_count
|
|
canary.latency_p95_ms = observation.latency_p95_ms
|
|
canary.result = {
|
|
**observation.model_dump(mode="json"),
|
|
"provenance": {
|
|
"source": "MANUAL_LAB_REHEARSAL",
|
|
"authoritative": False,
|
|
"actor": context.actor_id,
|
|
"correlation_id": context.correlation_id,
|
|
},
|
|
}
|
|
trigger = self._canary_trigger(canary, observation)
|
|
if trigger:
|
|
canary.status = CanaryStatus.ABORTED
|
|
canary.abort_trigger = trigger
|
|
canary.finished_at = _now()
|
|
self._rollback(
|
|
operation, trigger, context.actor_id,
|
|
"Automatic abort from manual LAB rehearsal observation",
|
|
)
|
|
event_type = "CANARY_ABORTED"
|
|
elif canary.request_count >= canary.target_request_count:
|
|
canary.status = CanaryStatus.READY_FOR_PROMOTION_REVIEW
|
|
canary.finished_at = _now()
|
|
event_type = "CANARY_READY_FOR_PROMOTION_REVIEW"
|
|
else:
|
|
event_type = "CANARY_OBSERVED"
|
|
self._event(
|
|
event_type,
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
subject.state,
|
|
canary.status,
|
|
context.actor_id,
|
|
"MANUAL_LAB_REHEARSAL",
|
|
self._approval(plan.approval_request_id).policy_revision_id,
|
|
[str(canary.id)],
|
|
trigger or "Bounded canary observation recorded without payload content",
|
|
{
|
|
"request_count": canary.request_count,
|
|
"error_count": canary.error_count,
|
|
"latency_p95_ms": canary.latency_p95_ms,
|
|
},
|
|
change_id=str(operation.id),
|
|
)
|
|
AuditWriter(self.session, context=context).write(
|
|
action="lifecycle.manual_lab_canary_observed",
|
|
resource_type="lifecycle_operation",
|
|
resource_id=str(operation.id),
|
|
details={
|
|
"source": "MANUAL_LAB_REHEARSAL",
|
|
"canary_id": str(canary.id),
|
|
"status": canary.status,
|
|
"authoritative": False,
|
|
},
|
|
)
|
|
self.session.commit()
|
|
return self._operation_response(operation)
|
|
|
|
def _require_manual_canary_rehearsal(
|
|
self, plan: LifecyclePromotionPlan,
|
|
) -> LifecycleSubject:
|
|
"""No manual or legacy canary evidence can qualify an operational promotion."""
|
|
subject = self.session.get(LifecycleSubject, plan.subject_id)
|
|
approval = self.session.get(LifecycleApprovalRequest, plan.approval_request_id)
|
|
policy = self.session.get(LifecyclePolicyRevision, approval.policy_revision_id) if (
|
|
approval is not None
|
|
) else None
|
|
if (
|
|
subject is None or approval is None or policy is None
|
|
or subject.target_type != "LAB_REHEARSAL"
|
|
or subject.environment != "LAB"
|
|
or approval.environment != "LAB"
|
|
or policy.scope != "LAB_PROMOTION"
|
|
or plan.desired_state != "LAB_STABLE"
|
|
or approval.subject_id != subject.id
|
|
or approval.target_type != subject.target_type
|
|
or approval.target_ref != subject.target_ref
|
|
or approval.requested_transition != plan.desired_state
|
|
):
|
|
raise LifecycleError(
|
|
409, "MANUAL_CANARY_REHEARSAL_ONLY",
|
|
"Canary observations and commit require an explicit LAB rehearsal; "
|
|
"authoritative operational canary evidence ingestion is unavailable",
|
|
)
|
|
return subject
|
|
|
|
def commit_operation(
|
|
self, operation_id: uuid.UUID, decision: ApprovalDecision
|
|
) -> LifecycleOperationResponse:
|
|
operation = self._operation(operation_id)
|
|
plan = self._plan(operation.promotion_plan_id)
|
|
if operation.stage == OperationStage.COMMITTED:
|
|
transition = plan.affected_identities.get("serving_transition")
|
|
if transition is not None:
|
|
snapshot = self.session.scalar(select(LifecycleRollbackSnapshot).where(
|
|
LifecycleRollbackSnapshot.promotion_plan_id == plan.id))
|
|
if (snapshot is None or _hash(snapshot.snapshot) != snapshot.snapshot_digest
|
|
or snapshot.snapshot.get("serving_transition") != transition):
|
|
raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact installed serving snapshot is required")
|
|
LifecycleServing(self.session).installed(transition, _serving_changed_at(operation))
|
|
self.session.commit()
|
|
return self._operation_response(operation)
|
|
if operation.stage != OperationStage.VERIFYING:
|
|
raise LifecycleError(
|
|
409, "OPERATION_NOT_REVIEWABLE", "operation is not ready for review"
|
|
)
|
|
canary = self.session.scalar(
|
|
select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id)
|
|
)
|
|
if canary or plan.canary_strategy:
|
|
self._require_manual_canary_rehearsal(plan)
|
|
if canary and canary.status != CanaryStatus.READY_FOR_PROMOTION_REVIEW:
|
|
raise LifecycleError(
|
|
409,
|
|
"CANARY_NOT_REVIEWABLE",
|
|
"canary never auto-promotes; successful evidence required",
|
|
)
|
|
subject = self.session.get(LifecycleSubject, plan.subject_id)
|
|
if subject is None:
|
|
raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
|
|
snapshot = self.session.scalar(
|
|
select(LifecycleRollbackSnapshot).where(
|
|
LifecycleRollbackSnapshot.promotion_plan_id == plan.id
|
|
)
|
|
)
|
|
if snapshot is None:
|
|
raise LifecycleError(
|
|
409, "ROLLBACK_SNAPSHOT_MISSING", "immutable rollback snapshot is unavailable"
|
|
)
|
|
serving_contract = plan.affected_identities.get("serving_transition")
|
|
serving_changed_at = None
|
|
if serving_contract is not None:
|
|
approval = self._approval(plan.approval_request_id)
|
|
self._refresh_approval(approval)
|
|
if approval.status != ApprovalStatus.APPROVED or _hash(approval.evidence_snapshot) != approval.evidence_fingerprint:
|
|
raise LifecycleError(409, "APPROVAL_STALE", "serving approval is no longer current")
|
|
if snapshot.snapshot.get("serving_transition") != serving_contract or _hash(snapshot.snapshot) != snapshot.snapshot_digest:
|
|
raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact immutable serving snapshot is required")
|
|
serving_changed_at = _now()
|
|
LifecycleServing(self.session).switch(serving_contract, serving_changed_at.isoformat())
|
|
expected_state = (
|
|
DeploymentLifecycleState.CANARY if plan.canary_strategy else plan.current_state
|
|
)
|
|
try:
|
|
if serving_contract is None:
|
|
assert_deployment_transition(
|
|
DeploymentLifecycleState(expected_state),
|
|
DeploymentLifecycleState(plan.desired_state),
|
|
)
|
|
except ValueError as exc:
|
|
raise LifecycleError(409, "STATE_CHANGED", str(exc)) from exc
|
|
previous = str(expected_state)
|
|
superseded_by_ref = (
|
|
str(plan.affected_identities["superseded_by_ref"])
|
|
if plan.affected_identities.get("superseded_by_ref")
|
|
else subject.superseded_by_ref
|
|
)
|
|
subject_commit = self.session.execute(
|
|
update(LifecycleSubject)
|
|
.where(
|
|
LifecycleSubject.id == subject.id,
|
|
LifecycleSubject.version == int(snapshot.snapshot["version"]) + 1,
|
|
LifecycleSubject.state == expected_state,
|
|
)
|
|
.values(
|
|
state=plan.desired_state,
|
|
superseded_by_ref=superseded_by_ref,
|
|
version=LifecycleSubject.version + 1,
|
|
updated_at=_now(),
|
|
)
|
|
)
|
|
if rows_affected(subject_commit) != 1:
|
|
self.session.rollback()
|
|
raise LifecycleError(
|
|
409,
|
|
"VERSION_CONFLICT",
|
|
"lifecycle subject changed before promotion commit",
|
|
)
|
|
self.session.refresh(subject)
|
|
operation.stage = OperationStage.COMMITTED
|
|
operation.finished_at = serving_changed_at or _now()
|
|
operation.version += 1
|
|
plan.status = PromotionPlanStatus.COMMITTED
|
|
plan.version += 1
|
|
self._apply_rollback_retention(plan, decision.actor)
|
|
event_type = {
|
|
DeploymentLifecycleState.DEPRECATED: "DEPLOYMENT_DEPRECATED",
|
|
DeploymentLifecycleState.ARCHIVED: "DEPLOYMENT_ARCHIVED",
|
|
}.get(DeploymentLifecycleState(plan.desired_state), "PROMOTION_COMMITTED")
|
|
self._event(
|
|
event_type,
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
previous,
|
|
subject.state,
|
|
decision.actor,
|
|
"EXECUTOR",
|
|
self._approval(plan.approval_request_id).policy_revision_id,
|
|
[str(plan.id), str(operation.id)],
|
|
decision.reason,
|
|
{"subject_version": subject.version},
|
|
change_id=str(operation.id),
|
|
)
|
|
self.session.commit()
|
|
return self._operation_response(operation)
|
|
|
|
def rollback_operation(
|
|
self, operation_id: uuid.UUID, decision: ApprovalDecision
|
|
) -> LifecycleOperationResponse:
|
|
operation = self._operation(operation_id)
|
|
if operation.stage == OperationStage.ROLLED_BACK:
|
|
return self._operation_response(operation)
|
|
self._rollback(operation, "OPERATOR_REQUEST", decision.actor, decision.reason)
|
|
self.session.commit()
|
|
return self._operation_response(operation)
|
|
|
|
def operations(self) -> list[LifecycleOperationResponse]:
|
|
return [
|
|
self._operation_response(item)
|
|
for item in self.session.scalars(
|
|
select(LifecycleOperation).order_by(LifecycleOperation.started_at.desc())
|
|
).all()
|
|
]
|
|
|
|
def operation(self, operation_id: uuid.UUID) -> LifecycleOperationResponse:
|
|
return self._operation_response(self._operation(operation_id))
|
|
|
|
def reconcile_incomplete(self) -> int:
|
|
items = self.session.scalars(
|
|
select(LifecycleOperation).where(
|
|
LifecycleOperation.stage.in_(
|
|
[
|
|
OperationStage.PREPARING,
|
|
OperationStage.ACTIVATING,
|
|
OperationStage.ROLLING_BACK,
|
|
]
|
|
)
|
|
)
|
|
).all()
|
|
for item in items:
|
|
self._rollback(
|
|
item,
|
|
"CONTROL_PLANE_RESTART",
|
|
"lifecycle-reconciler",
|
|
"Incomplete activation was conservatively restored from its immutable snapshot",
|
|
)
|
|
if items:
|
|
self.session.commit()
|
|
return len(items)
|
|
|
|
def create_retention_policy(self, request: RetentionPolicyCreate) -> RetentionPolicyResponse:
|
|
previous = self.session.scalar(
|
|
select(RetentionPolicyRevision)
|
|
.where(RetentionPolicyRevision.key == request.key)
|
|
.order_by(RetentionPolicyRevision.revision.desc())
|
|
)
|
|
revision = (previous.revision + 1) if previous else 1
|
|
payload = request.model_dump(mode="json") | {"revision": revision}
|
|
fingerprint = _hash(payload)
|
|
existing = self.session.scalar(
|
|
select(RetentionPolicyRevision).where(
|
|
RetentionPolicyRevision.fingerprint == fingerprint
|
|
)
|
|
)
|
|
if existing:
|
|
return RetentionPolicyResponse.model_validate(existing)
|
|
if previous:
|
|
previous.active = False
|
|
item = RetentionPolicyRevision(
|
|
key=request.key,
|
|
revision=revision,
|
|
minimum_rollback_days=request.minimum_rollback_days,
|
|
requirements=request.requirements,
|
|
fingerprint=fingerprint,
|
|
active=True,
|
|
created_by=request.created_by,
|
|
created_at=_now(),
|
|
immutable_at=_now(),
|
|
)
|
|
self.session.add(item)
|
|
self.session.commit()
|
|
return RetentionPolicyResponse.model_validate(item)
|
|
|
|
def retention_policies(self) -> list[RetentionPolicyResponse]:
|
|
self.ensure_defaults()
|
|
return [
|
|
RetentionPolicyResponse.model_validate(item)
|
|
for item in self.session.scalars(
|
|
select(RetentionPolicyRevision).order_by(
|
|
RetentionPolicyRevision.key, RetentionPolicyRevision.revision.desc()
|
|
)
|
|
).all()
|
|
]
|
|
|
|
def create_cleanup_plan(self, request: CleanupPlanCreate) -> CleanupPlanResponse:
|
|
dependencies, reclaimable, retention_state = self._cleanup_dependencies(
|
|
request.target_type, request.target_ref
|
|
)
|
|
blockers = [f"{item['type']}:{item['ref']}" for item in dependencies if item["blocking"]]
|
|
status = CleanupStatus.BLOCKED if blockers else CleanupStatus.READY
|
|
digest = _hash({"dependencies": dependencies, "retention_state": retention_state})
|
|
item = LifecycleCleanupPlan(
|
|
target_type=request.target_type,
|
|
target_ref=str(request.target_ref),
|
|
action=request.action,
|
|
dependencies=dependencies,
|
|
dependency_digest=digest,
|
|
reclaimable_bytes=reclaimable,
|
|
retention_state=retention_state,
|
|
blockers=blockers,
|
|
status=status,
|
|
created_by=request.created_by,
|
|
created_at=_now(),
|
|
)
|
|
self.session.add(item)
|
|
self.session.flush()
|
|
self._event(
|
|
"CLEANUP_PLANNED",
|
|
request.target_type,
|
|
str(request.target_ref),
|
|
None,
|
|
status,
|
|
request.created_by,
|
|
"REQUESTER",
|
|
None,
|
|
[dep["ref"] for dep in dependencies],
|
|
"Dependency and retention dry-run created; no bytes were removed",
|
|
{"reclaimable_bytes": reclaimable, "blockers": blockers},
|
|
change_id=str(item.id),
|
|
)
|
|
self.session.commit()
|
|
return self._cleanup_response(item)
|
|
|
|
def cleanup_plans(self) -> list[CleanupPlanResponse]:
|
|
return [
|
|
self._cleanup_response(item)
|
|
for item in self.session.scalars(
|
|
select(LifecycleCleanupPlan).order_by(LifecycleCleanupPlan.created_at.desc())
|
|
).all()
|
|
]
|
|
|
|
def cleanup_plan(self, plan_id: uuid.UUID) -> CleanupPlanResponse:
|
|
item = self.session.get(LifecycleCleanupPlan, plan_id)
|
|
if not item:
|
|
raise LifecycleError(404, "CLEANUP_PLAN_NOT_FOUND", "cleanup plan not found")
|
|
return self._cleanup_response(item)
|
|
|
|
def execute_cleanup(
|
|
self, plan_id: uuid.UUID, request: CleanupExecutionCreate
|
|
) -> CleanupPlanResponse:
|
|
plan = self.session.get(LifecycleCleanupPlan, plan_id)
|
|
if not plan:
|
|
raise LifecycleError(404, "CLEANUP_PLAN_NOT_FOUND", "cleanup plan not found")
|
|
dependencies, _, retention_state = self._cleanup_dependencies(
|
|
plan.target_type, uuid.UUID(plan.target_ref)
|
|
)
|
|
current_digest = _hash({"dependencies": dependencies, "retention_state": retention_state})
|
|
if current_digest != plan.dependency_digest:
|
|
plan.status = CleanupStatus.STALE
|
|
self.session.commit()
|
|
raise LifecycleError(
|
|
409, "CLEANUP_PLAN_STALE", "dependencies changed; create a new plan"
|
|
)
|
|
blockers = [item for item in dependencies if item["blocking"]]
|
|
if blockers or retention_state in {
|
|
RetentionState.ACTIVE,
|
|
RetentionState.ROLLBACK_RETAINED,
|
|
}:
|
|
raise LifecycleError(
|
|
409, "DEPENDENCY_CONFLICT", "active dependencies or retention block cleanup"
|
|
)
|
|
if plan.action == "RECORD_LOCATION_REMOVAL":
|
|
if not request.physical_removal_confirmed:
|
|
raise LifecycleError(
|
|
422,
|
|
"PHYSICAL_REMOVAL_CONFIRMATION_REQUIRED",
|
|
"ModelForge records removal only after the typed storage action confirms it",
|
|
)
|
|
location = self.session.get(ArtifactLocation, uuid.UUID(plan.target_ref))
|
|
if location is None:
|
|
raise LifecycleError(404, "ARTIFACT_LOCATION_NOT_FOUND", "location disappeared")
|
|
self.session.add(
|
|
ArtifactLocationRemovalRecord(
|
|
artifact_location_id=location.id,
|
|
cleanup_plan_id=plan.id,
|
|
prior_status=location.status,
|
|
prior_observed_sha256=location.observed_sha256,
|
|
prior_size_bytes=location.size_bytes,
|
|
removed_by=request.executor,
|
|
removed_at=_now(),
|
|
)
|
|
)
|
|
location.status = "removed"
|
|
location.last_checked_at = _now()
|
|
if location.artifact_id:
|
|
remaining = self.session.scalar(
|
|
select(func.count(ArtifactLocation.id)).where(
|
|
ArtifactLocation.artifact_id == location.artifact_id,
|
|
ArtifactLocation.id != location.id,
|
|
ArtifactLocation.status.in_(["present", "verified", "local"]),
|
|
)
|
|
)
|
|
if not remaining:
|
|
artifact = location.artifact
|
|
if artifact:
|
|
artifact.status = "missing"
|
|
event_type = "ARTIFACT_LOCATION_REMOVED"
|
|
else:
|
|
artifact_set = self.session.get(ArtifactSet, uuid.UUID(plan.target_ref))
|
|
if artifact_set is None:
|
|
raise LifecycleError(404, "ARTIFACT_SET_NOT_FOUND", "artifact set disappeared")
|
|
artifact_set.status = "archived"
|
|
event_type = "ARTIFACT_SET_ARCHIVED"
|
|
plan.status = CleanupStatus.EXECUTED
|
|
plan.executed_at = _now()
|
|
self._event(
|
|
event_type,
|
|
plan.target_type,
|
|
plan.target_ref,
|
|
plan.retention_state,
|
|
"ARCHIVED" if plan.action == "ARCHIVE_METADATA" else "REMOVED",
|
|
request.executor,
|
|
"EXECUTOR",
|
|
None,
|
|
[str(plan.id)],
|
|
"Explicit dependency-rechecked cleanup action completed; provenance retained",
|
|
{"action": plan.action, "reclaimable_bytes": plan.reclaimable_bytes},
|
|
change_id=str(plan.id),
|
|
)
|
|
self.session.commit()
|
|
return self._cleanup_response(plan)
|
|
|
|
def events(self, limit: int = 200) -> list[LifecycleEventResponse]:
|
|
return [
|
|
LifecycleEventResponse.model_validate(item)
|
|
for item in self.session.scalars(
|
|
select(LifecycleEvent).order_by(LifecycleEvent.occurred_at.desc()).limit(limit)
|
|
).all()
|
|
]
|
|
|
|
def _validate_target(self, target_type: str, target_ref: str, *, allow_rehearsal: bool) -> None:
|
|
if target_type == "LAB_REHEARSAL" and allow_rehearsal:
|
|
return
|
|
try:
|
|
target_id = uuid.UUID(target_ref)
|
|
except ValueError as exc:
|
|
raise LifecycleError(
|
|
422, "INVALID_TARGET_REF", "target reference must be a UUID"
|
|
) from exc
|
|
mapping: dict[str, type[Any]] = {
|
|
"CAPABILITY_DEPLOYMENT": CapabilityDeployment,
|
|
"PROJECT_BINDING": ProjectBinding,
|
|
"ARTIFACT_SET": ArtifactSet,
|
|
}
|
|
model = mapping.get(target_type)
|
|
if model is None or self.session.get(model, target_id) is None:
|
|
raise LifecycleError(404, "TARGET_NOT_FOUND", "lifecycle target not found")
|
|
|
|
def _resolve_evidence(self, declared: dict[str, Any]) -> dict[str, Any]:
|
|
facts: dict[str, Any] = {}
|
|
if declared.get("serving_transition") is not None:
|
|
from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
|
|
|
|
if declared.get("production_validation") != "SATISFIED":
|
|
raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
|
|
facts["serving_transition"] = LifecycleServing(self.session).prepare(
|
|
ServingTransitionSelection.model_validate(declared["serving_transition"])
|
|
).model_dump(mode="json")
|
|
for field, model in (
|
|
("artifact_set_id", ArtifactSet),
|
|
("runtime_profile_id", RuntimeProfile),
|
|
("runtime_probe_id", RuntimeProbe),
|
|
("production_execution_approval_id", ProductionExecutionApproval),
|
|
("capability_deployment_id", CapabilityDeployment),
|
|
("project_binding_id", ProjectBinding),
|
|
("project_fit_evidence_id", ProjectFitEvidence),
|
|
("resource_envelope_id", CapabilityResourceEnvelope),
|
|
("embedding_space_id", EmbeddingSpace),
|
|
("retrieval_pipeline_id", RetrievalPipelineIdentity),
|
|
("migration_id", EmbeddingMigration),
|
|
):
|
|
raw = declared.get(field)
|
|
if not raw:
|
|
continue
|
|
item = self.session.get(model, uuid.UUID(str(raw)))
|
|
facts[field] = self._fact(item) if item else {"missing": True}
|
|
run_facts: dict[str, dict[str, Any]] = {}
|
|
for raw in declared.get("evaluation_run_ids", []):
|
|
item = self.session.get(EvaluationRun, uuid.UUID(str(raw)))
|
|
run_facts[str(raw)] = self._fact(item) if item else {"missing": True}
|
|
if run_facts:
|
|
facts["evaluation_run_ids"] = run_facts
|
|
fit_raw = declared.get("project_fit_evidence_id")
|
|
if fit_raw:
|
|
fit = self.session.get(ProjectFitEvidence, uuid.UUID(str(fit_raw)))
|
|
if fit:
|
|
latest = self.session.scalar(
|
|
select(ProjectFitEvidence)
|
|
.where(ProjectFitEvidence.project_binding_id == fit.project_binding_id)
|
|
.order_by(ProjectFitEvidence.created_at.desc())
|
|
)
|
|
facts["latest_project_fit_evidence_id"] = str(latest.id) if latest else None
|
|
return facts
|
|
|
|
@staticmethod
|
|
def _fact(item: Any | None) -> dict[str, Any]:
|
|
if item is None:
|
|
return {"missing": True}
|
|
names = (
|
|
"id",
|
|
"status",
|
|
"security_status",
|
|
"license_status",
|
|
"completeness",
|
|
"availability",
|
|
"fingerprint",
|
|
"environment_fingerprint",
|
|
"runtime_image_digest",
|
|
"production",
|
|
"channel",
|
|
"health_status",
|
|
"stale",
|
|
"stale_reason",
|
|
"recommendation",
|
|
"engineering_integration",
|
|
"production_validation",
|
|
"critical_errors",
|
|
"evidence_digest",
|
|
"identity_digest",
|
|
"artifact_set_id",
|
|
"revision_id",
|
|
"runtime_profile_id",
|
|
"runtime_probe_id",
|
|
"deployment_candidate_id",
|
|
"capability_contract_id",
|
|
"resource_envelope_id",
|
|
"revoked_at",
|
|
"error_count",
|
|
"completed_cases",
|
|
"expected_cases",
|
|
)
|
|
return {
|
|
name: str(value) if isinstance(value, uuid.UUID) else value
|
|
for name in names
|
|
if (value := getattr(item, name, None)) is not None
|
|
}
|
|
|
|
def _evaluate_policy(
|
|
self, policy: LifecyclePolicyRevision, snapshot: dict[str, Any]
|
|
) -> tuple[list[str], list[str]]:
|
|
scope = policy.scope
|
|
requirements = dict(policy.requirements)
|
|
base_key = requirements.pop("base", None)
|
|
if base_key:
|
|
base = self.session.scalar(
|
|
select(LifecyclePolicyRevision).where(
|
|
LifecyclePolicyRevision.key == base_key,
|
|
LifecyclePolicyRevision.active.is_(True),
|
|
)
|
|
)
|
|
if base:
|
|
requirements = {**base.requirements, **requirements}
|
|
data = snapshot["declared"]
|
|
resolved = snapshot["resolved"]
|
|
blockers: list[str] = []
|
|
warnings: list[str] = []
|
|
exact_fields = (
|
|
"artifact_set_id",
|
|
"runtime_profile_id",
|
|
"runtime_probe_id",
|
|
"runtime_image_digest",
|
|
"resource_envelope_id",
|
|
"rollback_target_ref",
|
|
)
|
|
for field in exact_fields:
|
|
if not data.get(field):
|
|
blockers.append(f"MISSING_{field.upper()}")
|
|
for field, fact in resolved.items():
|
|
if isinstance(fact, dict) and fact.get("missing"):
|
|
blockers.append(f"MISSING_REFERENCED_{field.upper()}")
|
|
artifact = resolved.get("artifact_set_id", {})
|
|
profile = resolved.get("runtime_profile_id", {})
|
|
probe = resolved.get("runtime_probe_id", {})
|
|
production_approval = resolved.get("production_execution_approval_id", {})
|
|
if data["integrity"] != requirements.get("integrity", "VERIFIED"):
|
|
blockers.append("ARTIFACT_INTEGRITY_NOT_VERIFIED")
|
|
if artifact and not artifact.get("missing"):
|
|
if artifact.get("status") != "verified" or artifact.get("completeness") != "complete":
|
|
blockers.append("ARTIFACT_INTEGRITY_NOT_VERIFIED")
|
|
if data.get("model_revision_id") != artifact.get("revision_id"):
|
|
blockers.append("MODEL_REVISION_ARTIFACT_SET_MISMATCH")
|
|
if profile and not profile.get("missing"):
|
|
if profile.get("artifact_set_id") != data.get("artifact_set_id"):
|
|
blockers.append("RUNTIME_PROFILE_ARTIFACT_SET_MISMATCH")
|
|
if profile.get("runtime_image_digest") != data.get("runtime_image_digest"):
|
|
blockers.append("RUNTIME_IMAGE_DIGEST_MISMATCH")
|
|
if probe and not probe.get("missing") and probe.get("status") != "completed":
|
|
blockers.append("RUNTIME_PROBE_NOT_COMPLETED")
|
|
if data["security"] == "BLOCKED":
|
|
blockers.append("SECURITY_BLOCKED")
|
|
if data["license"] == "BLOCKED":
|
|
blockers.append("LICENSE_BLOCKED")
|
|
if data["runtime"] != requirements.get("runtime", "PROVEN"):
|
|
blockers.append("RUNTIME_NOT_PROVEN")
|
|
if data["scheduler_readiness"] != requirements.get("scheduler_readiness", "READY"):
|
|
blockers.append("SCHEDULER_NOT_READY")
|
|
if scope != "LAB_PROMOTION":
|
|
if data["security"] != requirements.get("security", "APPROVED"):
|
|
blockers.append("SECURITY_NOT_APPROVED")
|
|
if data["license"] != requirements.get("license", "APPROVED"):
|
|
blockers.append("LICENSE_NOT_APPROVED")
|
|
if data["evaluation"] != requirements.get("evaluation", "PROMOTION_ELIGIBLE"):
|
|
blockers.append("EVALUATION_NOT_PROMOTION_ELIGIBLE")
|
|
if not data.get("evaluation_run_ids"):
|
|
blockers.append("MISSING_EVALUATION_RUNS")
|
|
for run in resolved.get("evaluation_run_ids", {}).values():
|
|
if run.get("missing") or run.get("status") != "completed":
|
|
blockers.append("EVALUATION_RUN_NOT_COMPLETED")
|
|
if run.get("error_count", 0):
|
|
blockers.append("EVALUATION_RUN_ERRORS")
|
|
if data["critical_regressions"] > int(requirements.get("critical_regressions", 0)):
|
|
blockers.append("CRITICAL_PROJECT_REGRESSION")
|
|
if not data.get("production_execution_approval_id"):
|
|
blockers.append("MISSING_PRODUCTION_EXECUTION_APPROVAL")
|
|
elif not production_approval.get("missing"):
|
|
if production_approval.get("status") != "approved" or production_approval.get(
|
|
"revoked_at"
|
|
):
|
|
blockers.append("PRODUCTION_EXECUTION_APPROVAL_INVALID")
|
|
if production_approval.get("artifact_set_id") != data.get("artifact_set_id"):
|
|
blockers.append("PRODUCTION_APPROVAL_ARTIFACT_SET_MISMATCH")
|
|
if production_approval.get("runtime_profile_id") != data.get("runtime_profile_id"):
|
|
blockers.append("PRODUCTION_APPROVAL_RUNTIME_PROFILE_MISMATCH")
|
|
if not data.get("evidence_revisions", {}).get("scheduler"):
|
|
blockers.append("MISSING_SCHEDULER_EVIDENCE_REVISION")
|
|
if scope == "PROJECT_PRODUCTION":
|
|
if data["production_validation"] == "DEFERRED_EXTERNAL_VALIDATION":
|
|
blockers.append("REQUIRED_EXTERNAL_VALIDATION_NOT_SATISFIED")
|
|
elif data["production_validation"] != "SATISFIED":
|
|
blockers.append("PROJECT_PRODUCTION_VALIDATION_NOT_SATISFIED")
|
|
if data["project_fit"] != requirements.get("project_fit", "ELIGIBLE"):
|
|
blockers.append("PROJECT_PRODUCTION_FIT_NOT_ELIGIBLE")
|
|
if not data.get("project_fit_evidence_id"):
|
|
blockers.append("MISSING_PROJECT_FIT_EVIDENCE")
|
|
fit = resolved.get("project_fit_evidence_id", {})
|
|
if fit and not fit.get("missing"):
|
|
if fit.get("production_validation") == "DEFERRED_EXTERNAL_VALIDATION":
|
|
blockers.append("REQUIRED_EXTERNAL_VALIDATION_NOT_SATISFIED")
|
|
if fit.get("recommendation") != "PROMOTION_ELIGIBLE":
|
|
blockers.append("PROJECT_PRODUCTION_FIT_NOT_ELIGIBLE")
|
|
if fit.get("critical_errors", 0):
|
|
blockers.append("CRITICAL_PROJECT_REGRESSION")
|
|
if (
|
|
data["production_validation"] == "DEFERRED_EXTERNAL_VALIDATION"
|
|
and scope != "PROJECT_PRODUCTION"
|
|
):
|
|
warnings.append("DEFERRED_EXTERNAL_VALIDATION_RETAINED")
|
|
return sorted(set(blockers)), sorted(set(warnings))
|
|
|
|
def _refresh_approval(self, item: LifecycleApprovalRequest) -> None:
|
|
if item.status not in {ApprovalStatus.PENDING, ApprovalStatus.APPROVED}:
|
|
return
|
|
now = _now()
|
|
expires = item.expires_at
|
|
if expires and expires.tzinfo is None:
|
|
expires = expires.replace(tzinfo=UTC)
|
|
if expires and expires <= now:
|
|
previous = item.status
|
|
item.status = ApprovalStatus.EXPIRED
|
|
item.decided_at = now
|
|
self._event(
|
|
"APPROVAL_EXPIRED",
|
|
item.target_type,
|
|
item.target_ref,
|
|
previous,
|
|
item.status,
|
|
"lifecycle-engine",
|
|
"SYSTEM",
|
|
item.policy_revision_id,
|
|
[],
|
|
"Approval expiry reached",
|
|
{},
|
|
change_id=str(item.id),
|
|
)
|
|
return
|
|
stale_details: dict[str, Any] = {}
|
|
current_resolved: dict[str, Any] | None
|
|
try:
|
|
current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
|
|
except (LifecycleError, ServingError) as exc:
|
|
# A historical serving approval must remain inspectable after its route
|
|
# changes. Expected qualification failures invalidate authority; internal
|
|
# errors still propagate rather than being disguised as stale evidence.
|
|
if (item.evidence_snapshot["declared"].get("serving_transition") is None
|
|
or exc.status_code not in {404, 409, 422}):
|
|
raise
|
|
current_resolved = None
|
|
stale_details = {"validation_code": exc.code, "validation_reason": str(exc)}
|
|
item.blockers = sorted(set(item.blockers) | {exc.code})
|
|
if current_resolved is None or _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
|
|
previous = item.status
|
|
item.status = ApprovalStatus.STALE
|
|
item.stale_at = now
|
|
self._event(
|
|
"APPROVAL_STALE",
|
|
item.target_type,
|
|
item.target_ref,
|
|
previous,
|
|
item.status,
|
|
"lifecycle-engine",
|
|
"SYSTEM",
|
|
item.policy_revision_id,
|
|
[],
|
|
"Referenced evidence changed after approval request creation",
|
|
stale_details,
|
|
change_id=str(item.id),
|
|
)
|
|
|
|
@staticmethod
|
|
def _evidence_refs(declared: dict[str, Any]) -> list[tuple[str, str]]:
|
|
refs: list[tuple[str, str]] = []
|
|
for key, value in declared.items():
|
|
if key.endswith("_id") and value:
|
|
refs.append((key, str(value)))
|
|
elif key.endswith("_ids"):
|
|
refs.extend((key[:-1], str(item)) for item in value)
|
|
elif key in {"artifact_digests", "rollback_target_ref"}:
|
|
values = value if isinstance(value, list) else [value]
|
|
refs.extend((key, str(item)) for item in values if item)
|
|
return refs
|
|
|
|
def _start_canary(
|
|
self,
|
|
plan: LifecyclePromotionPlan,
|
|
operation: LifecycleOperation,
|
|
subject: LifecycleSubject,
|
|
) -> None:
|
|
strategy = plan.canary_strategy
|
|
mode = str(strategy.get("mode", "REQUEST"))
|
|
if plan.migration_class == "requires_reindex" and mode not in {
|
|
"SHADOW",
|
|
"DUAL_READ",
|
|
"EVALUATION_ROUTE",
|
|
}:
|
|
raise LifecycleError(422, "INCOMPATIBLE_EMBEDDING_CANARY", "mixed vector spaces denied")
|
|
previous = plan.current_state
|
|
operation.stage = OperationStage.VERIFYING
|
|
operation.version += 1
|
|
canary = LifecycleCanaryRun(
|
|
operation_id=operation.id,
|
|
mode=mode,
|
|
traffic_percent=int(strategy.get("traffic_percent", 0)),
|
|
eligible_clients=list(strategy.get("eligible_clients", [])),
|
|
eligible_projects=list(strategy.get("eligible_projects", [])),
|
|
target_request_count=int(strategy.get("target_request_count", 1)),
|
|
thresholds={
|
|
"max_error_rate": float(strategy.get("max_error_rate", 0.0)),
|
|
"max_latency_p95_ms": float(strategy.get("max_latency_p95_ms", 10_000.0)),
|
|
},
|
|
status=CanaryStatus.RUNNING,
|
|
result={},
|
|
started_at=_now(),
|
|
)
|
|
self.session.add(canary)
|
|
self.session.flush()
|
|
self._event(
|
|
"CANARY_STARTED",
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
previous,
|
|
subject.state,
|
|
operation.executor,
|
|
"EXECUTOR",
|
|
self._approval(plan.approval_request_id).policy_revision_id,
|
|
[str(canary.id)],
|
|
"Bounded capability-aware canary started",
|
|
{"mode": mode, "target_request_count": canary.target_request_count},
|
|
change_id=str(operation.id),
|
|
)
|
|
|
|
@staticmethod
|
|
def _canary_trigger(canary: LifecycleCanaryRun, observation: CanaryObservation) -> str | None:
|
|
if observation.capability_health != "HEALTHY" or not observation.worker_healthy:
|
|
return "HEALTH_FAILURE"
|
|
if not observation.scheduler_ready or observation.external_pressure:
|
|
return "CAPACITY_FAILURE"
|
|
if observation.critical_project_failures:
|
|
return "CRITICAL_PROJECT_REGRESSION"
|
|
total = max(canary.request_count, 1)
|
|
if canary.error_count / total > float(canary.thresholds["max_error_rate"]):
|
|
return "ERROR_THRESHOLD"
|
|
if observation.latency_p95_ms > float(canary.thresholds["max_latency_p95_ms"]):
|
|
return "LATENCY_THRESHOLD"
|
|
return None
|
|
|
|
def _rollback(
|
|
self, operation: LifecycleOperation, trigger: str, actor: str, reason: str
|
|
) -> None:
|
|
started = time.perf_counter()
|
|
plan = self._plan(operation.promotion_plan_id)
|
|
self._approval(plan.approval_request_id) # Same approval-before-route lock order as commit.
|
|
snapshot = self.session.scalar(
|
|
select(LifecycleRollbackSnapshot).where(
|
|
LifecycleRollbackSnapshot.promotion_plan_id == plan.id
|
|
)
|
|
)
|
|
subject = self.session.get(LifecycleSubject, plan.subject_id)
|
|
if not snapshot or not subject:
|
|
operation.stage = OperationStage.FAILED
|
|
operation.failure_code = "ROLLBACK_FAILED"
|
|
operation.failure_details = {"reason": "immutable rollback snapshot unavailable"}
|
|
operation.finished_at = _now()
|
|
if subject:
|
|
subject.state = DeploymentLifecycleState.MANUAL_INTERVENTION_REQUIRED
|
|
subject.version += 1
|
|
plan.status = PromotionPlanStatus.FAILED
|
|
self._event(
|
|
"ROLLBACK_FAILED",
|
|
"PROMOTION_PLAN",
|
|
str(plan.id),
|
|
None,
|
|
"MANUAL_INTERVENTION_REQUIRED",
|
|
actor,
|
|
"EXECUTOR",
|
|
self._approval(plan.approval_request_id).policy_revision_id,
|
|
[],
|
|
reason,
|
|
{"trigger": trigger},
|
|
change_id=str(operation.id),
|
|
)
|
|
return
|
|
snapshot_version = int(snapshot.snapshot["version"])
|
|
serving_contract = plan.affected_identities.get("serving_transition")
|
|
was_committed = operation.stage == OperationStage.COMMITTED
|
|
if serving_contract is not None:
|
|
if (snapshot.snapshot.get("serving_transition") != serving_contract
|
|
or _hash(snapshot.snapshot) != snapshot.snapshot_digest):
|
|
raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact serving snapshot is unavailable")
|
|
# A subject CAS failure below rolls back the route in the same transaction.
|
|
# A later route is preserved rather than guessed away.
|
|
try:
|
|
LifecycleServing(self.session).restore(serving_contract,
|
|
_serving_changed_at(operation) if was_committed else _now().isoformat(),
|
|
committed=was_committed)
|
|
except Exception:
|
|
self.session.rollback()
|
|
raise
|
|
expected_version = snapshot_version + (
|
|
2 if operation.stage == OperationStage.COMMITTED else 1
|
|
)
|
|
previous = subject.state
|
|
operation.stage = OperationStage.ROLLING_BACK
|
|
restored = self.session.execute(
|
|
update(LifecycleSubject)
|
|
.where(
|
|
LifecycleSubject.id == subject.id,
|
|
LifecycleSubject.version == expected_version,
|
|
)
|
|
.values(
|
|
state=snapshot.snapshot["state"],
|
|
superseded_by_ref=snapshot.snapshot.get("superseded_by_ref"),
|
|
version=LifecycleSubject.version + 1,
|
|
updated_at=_now(),
|
|
)
|
|
)
|
|
if rows_affected(restored) != 1:
|
|
if serving_contract is not None:
|
|
self.session.rollback()
|
|
raise LifecycleError(409, "ROLLBACK_VERSION_CONFLICT", "lifecycle subject changed; serving state was preserved")
|
|
operation.stage = OperationStage.FAILED
|
|
operation.failure_code = "ROLLBACK_VERSION_CONFLICT"
|
|
operation.failure_details = {
|
|
"reason": "lifecycle subject changed after this operation snapshot",
|
|
"expected_subject_version": expected_version,
|
|
"observed_subject_version": subject.version,
|
|
}
|
|
operation.finished_at = _now()
|
|
operation.version += 1
|
|
plan.status = PromotionPlanStatus.FAILED
|
|
plan.version += 1
|
|
self._event(
|
|
"ROLLBACK_FAILED",
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
previous,
|
|
"CONCURRENT_STATE_PRESERVED",
|
|
actor,
|
|
"EXECUTOR" if actor != "SYSTEM" else "SYSTEM",
|
|
self._approval(plan.approval_request_id).policy_revision_id,
|
|
[str(snapshot.id)],
|
|
reason,
|
|
{
|
|
"trigger": trigger,
|
|
"failure_code": "ROLLBACK_VERSION_CONFLICT",
|
|
"expected_subject_version": expected_version,
|
|
},
|
|
change_id=str(operation.id),
|
|
)
|
|
return
|
|
self.session.refresh(subject)
|
|
operation.stage = OperationStage.ROLLED_BACK
|
|
operation.failure_code = trigger if trigger != "OPERATOR_REQUEST" else None
|
|
operation.failure_details = {"rollback_target_ref": plan.rollback_target_ref}
|
|
operation.rollback_duration_ms = (time.perf_counter() - started) * 1000
|
|
operation.finished_at = _now()
|
|
operation.version += 1
|
|
plan.status = PromotionPlanStatus.ROLLED_BACK
|
|
plan.version += 1
|
|
self._event(
|
|
"ROLLBACK_COMPLETED",
|
|
subject.target_type,
|
|
subject.target_ref,
|
|
previous,
|
|
subject.state,
|
|
actor,
|
|
"EXECUTOR" if actor != "SYSTEM" else "SYSTEM",
|
|
self._approval(plan.approval_request_id).policy_revision_id,
|
|
[str(snapshot.id)],
|
|
reason,
|
|
{
|
|
"trigger": trigger,
|
|
"rollback_duration_ms": operation.rollback_duration_ms,
|
|
"rollback_target_ref": plan.rollback_target_ref,
|
|
},
|
|
change_id=str(operation.id),
|
|
)
|
|
|
|
def _apply_rollback_retention(self, plan: LifecyclePromotionPlan, actor: str) -> None:
|
|
policy = self.session.scalar(
|
|
select(RetentionPolicyRevision).where(
|
|
RetentionPolicyRevision.key == "standard-rollback",
|
|
RetentionPolicyRevision.active.is_(True),
|
|
)
|
|
)
|
|
if not policy:
|
|
raise LifecycleError(
|
|
500, "RETENTION_POLICY_MISSING", "rollback retention policy missing"
|
|
)
|
|
retained_until = _now() + timedelta(days=policy.minimum_rollback_days)
|
|
transition = plan.affected_identities.get("serving_transition")
|
|
if transition is not None:
|
|
prior = transition["current"]
|
|
for kind, ref in (("ARTIFACT_SET", prior["artifact_set_id"]),
|
|
("RUNTIME_PROFILE", prior["runtime_profile_id"]),
|
|
("CAPABILITY_DEPLOYMENT", prior["id"])):
|
|
retained = self.session.scalar(select(LifecycleRetentionRecord).where(
|
|
LifecycleRetentionRecord.target_type == kind,
|
|
LifecycleRetentionRecord.target_ref == ref))
|
|
if retained is None:
|
|
self.session.add(LifecycleRetentionRecord(
|
|
policy_revision_id=policy.id, target_type=kind, target_ref=ref,
|
|
state=RetentionState.ROLLBACK_RETAINED, retained_until=retained_until,
|
|
legal_hold=False, reason="Exact prior serving identity retained for approved rollback",
|
|
created_at=_now()))
|
|
else:
|
|
previous_until = retained.retained_until
|
|
if previous_until is None or previous_until.replace(tzinfo=UTC) < retained_until:
|
|
retained.retained_until = retained_until
|
|
retained.state = RetentionState.ROLLBACK_RETAINED
|
|
existing = self.session.scalar(
|
|
select(LifecycleRetentionRecord).where(
|
|
LifecycleRetentionRecord.target_type == "ROLLBACK_TARGET",
|
|
LifecycleRetentionRecord.target_ref == plan.rollback_target_ref,
|
|
)
|
|
)
|
|
if existing:
|
|
existing.policy_revision_id = policy.id
|
|
existing.state = RetentionState.ROLLBACK_RETAINED
|
|
existing.retained_until = retained_until
|
|
else:
|
|
self.session.add(
|
|
LifecycleRetentionRecord(
|
|
policy_revision_id=policy.id,
|
|
target_type="ROLLBACK_TARGET",
|
|
target_ref=plan.rollback_target_ref,
|
|
state=RetentionState.ROLLBACK_RETAINED,
|
|
retained_until=retained_until,
|
|
legal_hold=False,
|
|
reason="Previous exact state retained for rollback after lifecycle commit",
|
|
created_at=_now(),
|
|
)
|
|
)
|
|
self._event(
|
|
"RETENTION_APPLIED",
|
|
"ROLLBACK_TARGET",
|
|
plan.rollback_target_ref,
|
|
None,
|
|
RetentionState.ROLLBACK_RETAINED,
|
|
actor,
|
|
"EXECUTOR",
|
|
None,
|
|
[str(policy.id), str(plan.id)],
|
|
"Minimum rollback window applied",
|
|
{"retained_until": retained_until.isoformat()},
|
|
change_id=str(plan.id),
|
|
)
|
|
|
|
def _cleanup_dependencies(
|
|
self, target_type: str, target_ref: uuid.UUID
|
|
) -> tuple[list[dict[str, Any]], int, str]:
|
|
dependencies: list[dict[str, Any]] = []
|
|
reclaimable = 0
|
|
retention = self.session.scalar(
|
|
select(LifecycleRetentionRecord).where(
|
|
LifecycleRetentionRecord.target_ref == str(target_ref)
|
|
)
|
|
)
|
|
retention_state = retention.state if retention else RetentionState.ARCHIVABLE
|
|
if retention:
|
|
until = retention.retained_until
|
|
if until and until.tzinfo is None:
|
|
until = until.replace(tzinfo=UTC)
|
|
blocking = retention.legal_hold or retention.state in {
|
|
RetentionState.ACTIVE,
|
|
RetentionState.ROLLBACK_RETAINED,
|
|
}
|
|
if until and until <= _now() and not retention.legal_hold:
|
|
blocking = False
|
|
retention_state = RetentionState.ARCHIVABLE
|
|
dependencies.append(
|
|
{
|
|
"type": "RETENTION_POLICY",
|
|
"ref": str(retention.id),
|
|
"blocking": blocking,
|
|
"state": retention.state,
|
|
}
|
|
)
|
|
if target_type == "ARTIFACT_SET":
|
|
artifact_set = self.session.get(ArtifactSet, target_ref)
|
|
if not artifact_set:
|
|
raise LifecycleError(404, "ARTIFACT_SET_NOT_FOUND", "artifact set not found")
|
|
reclaimable = artifact_set.total_size_bytes
|
|
active_locations = self.session.scalars(
|
|
select(ArtifactLocation)
|
|
.join(
|
|
ArtifactSetMember,
|
|
ArtifactSetMember.artifact_id == ArtifactLocation.artifact_id,
|
|
)
|
|
.where(
|
|
ArtifactSetMember.artifact_set_id == target_ref,
|
|
ArtifactLocation.status.in_(["present", "verified", "local"]),
|
|
)
|
|
).all()
|
|
for active_location in active_locations:
|
|
dependencies.append(
|
|
{
|
|
"type": "ARTIFACT_LOCATION",
|
|
"ref": str(active_location.id),
|
|
"blocking": True,
|
|
}
|
|
)
|
|
artifact_ids = self.session.scalars(
|
|
select(ArtifactSetMember.artifact_id).where(
|
|
ArtifactSetMember.artifact_set_id == target_ref
|
|
)
|
|
).all()
|
|
if artifact_ids:
|
|
derived_ids = self.session.scalars(
|
|
select(DerivedArtifactSource.derived_artifact_id)
|
|
.where(DerivedArtifactSource.source_artifact_id.in_(artifact_ids))
|
|
.distinct()
|
|
).all()
|
|
for derived in self.session.scalars(
|
|
select(DerivedArtifact).where(DerivedArtifact.id.in_(derived_ids))
|
|
).all():
|
|
dependencies.append(
|
|
{"type": "DERIVED_ARTIFACT", "ref": str(derived.id), "blocking": True}
|
|
)
|
|
deployments = self.session.scalars(
|
|
select(CapabilityDeployment).where(
|
|
CapabilityDeployment.artifact_set_id == target_ref
|
|
)
|
|
).all()
|
|
deployment_ids = [item.id for item in deployments]
|
|
contract_ids = {item.capability_contract_id for item in deployments}
|
|
for deployment in deployments:
|
|
dependencies.append(
|
|
{
|
|
"type": "CAPABILITY_DEPLOYMENT",
|
|
"ref": str(deployment.id),
|
|
"blocking": True,
|
|
}
|
|
)
|
|
if deployment_ids:
|
|
for route in self.session.scalars(
|
|
select(CapabilityExperimentRoute).where(
|
|
CapabilityExperimentRoute.capability_deployment_id.in_(deployment_ids),
|
|
CapabilityExperimentRoute.status == "active",
|
|
)
|
|
).all():
|
|
dependencies.append(
|
|
{"type": "EXPERIMENT_ROUTE", "ref": str(route.id), "blocking": True}
|
|
)
|
|
for run in self.session.scalars(
|
|
select(EvaluationRun).where(
|
|
EvaluationRun.capability_deployment_id.in_(deployment_ids)
|
|
)
|
|
).all():
|
|
dependencies.append(
|
|
{"type": "EVALUATION_RUN", "ref": str(run.id), "blocking": True}
|
|
)
|
|
if contract_ids:
|
|
for binding in self.session.scalars(
|
|
select(ProjectBinding).where(
|
|
ProjectBinding.capability_contract_id.in_(contract_ids)
|
|
)
|
|
).all():
|
|
dependencies.append(
|
|
{"type": "PROJECT_BINDING", "ref": str(binding.id), "blocking": True}
|
|
)
|
|
spaces = self.session.scalars(
|
|
select(EmbeddingSpace).where(EmbeddingSpace.artifact_set_id == target_ref)
|
|
).all()
|
|
for space in spaces:
|
|
dependencies.append(
|
|
{"type": "EMBEDDING_SPACE", "ref": str(space.id), "blocking": True}
|
|
)
|
|
space_ids = [space.id for space in spaces]
|
|
if space_ids:
|
|
for migration in self.session.scalars(
|
|
select(EmbeddingMigration).where(
|
|
EmbeddingMigration.target_embedding_space_id.in_(space_ids)
|
|
)
|
|
).all():
|
|
dependencies.append(
|
|
{"type": "MIGRATION", "ref": str(migration.id), "blocking": True}
|
|
)
|
|
terminal_migration_states = {"ROLLED_BACK", "FAILED", "CANCELLED"}
|
|
for migration_plan in self.session.scalars(select(MigrationPlan)).all():
|
|
source_match = str(target_ref) in json.dumps(
|
|
migration_plan.source_identity, default=str
|
|
)
|
|
target_match = str(target_ref) in json.dumps(
|
|
migration_plan.target_identity, default=str
|
|
)
|
|
if not source_match and not target_match:
|
|
continue
|
|
active = migration_plan.state not in terminal_migration_states
|
|
rollback = self.session.scalar(
|
|
select(MigrationRollbackSnapshot).where(
|
|
MigrationRollbackSnapshot.migration_plan_id == migration_plan.id
|
|
)
|
|
)
|
|
retained = False
|
|
if rollback is not None:
|
|
retain_until = rollback.retain_until
|
|
if retain_until.tzinfo is None:
|
|
retain_until = retain_until.replace(tzinfo=UTC)
|
|
retained = retain_until > _now()
|
|
if active or retained:
|
|
dependencies.append(
|
|
{
|
|
"type": (
|
|
"MIGRATION_PLAN_SOURCE"
|
|
if source_match
|
|
else "MIGRATION_PLAN_TARGET"
|
|
),
|
|
"ref": str(migration_plan.id),
|
|
"blocking": True,
|
|
"state": migration_plan.state,
|
|
"rollback_retained": retained,
|
|
}
|
|
)
|
|
for model, field, kind, active in (
|
|
(
|
|
DeploymentCandidate,
|
|
DeploymentCandidate.artifact_set_id,
|
|
"DEPLOYMENT_CANDIDATE",
|
|
True,
|
|
),
|
|
(RuntimeProfile, RuntimeProfile.artifact_set_id, "RUNTIME_PROFILE", True),
|
|
(ExecutionApproval, ExecutionApproval.artifact_set_id, "EXECUTION_APPROVAL", True),
|
|
(
|
|
ProductionExecutionApproval,
|
|
ProductionExecutionApproval.artifact_set_id,
|
|
"PRODUCTION_APPROVAL",
|
|
True,
|
|
),
|
|
):
|
|
for item in self.session.scalars(select(model).where(field == target_ref)).all():
|
|
dependencies.append(
|
|
{"type": kind, "ref": str(cast(Any, item).id), "blocking": active}
|
|
)
|
|
for evidence in self.session.scalars(
|
|
select(LifecycleApprovalEvidence).where(
|
|
LifecycleApprovalEvidence.evidence_type == "artifact_set_id",
|
|
LifecycleApprovalEvidence.evidence_ref == str(target_ref),
|
|
)
|
|
).all():
|
|
dependencies.append(
|
|
{
|
|
"type": "LIFECYCLE_APPROVAL_EVIDENCE",
|
|
"ref": str(evidence.approval_request_id),
|
|
"blocking": True,
|
|
}
|
|
)
|
|
else:
|
|
location = self.session.get(ArtifactLocation, target_ref)
|
|
if not location:
|
|
raise LifecycleError(
|
|
404, "ARTIFACT_LOCATION_NOT_FOUND", "artifact location not found"
|
|
)
|
|
reclaimable = location.size_bytes or 0
|
|
if location.artifact_id:
|
|
set_ids = self.session.scalars(
|
|
select(ArtifactSetMember.artifact_set_id).where(
|
|
ArtifactSetMember.artifact_id == location.artifact_id
|
|
)
|
|
).all()
|
|
for artifact_set_id in set_ids:
|
|
dependencies.append(
|
|
{"type": "ARTIFACT_SET", "ref": str(artifact_set_id), "blocking": True}
|
|
)
|
|
other_locations = self.session.scalars(
|
|
select(ArtifactLocation).where(
|
|
ArtifactLocation.artifact_id == location.artifact_id,
|
|
ArtifactLocation.id != location.id,
|
|
ArtifactLocation.status.in_(["present", "verified", "local"]),
|
|
)
|
|
).all()
|
|
dependencies.append(
|
|
{
|
|
"type": "OTHER_ARTIFACT_LOCATIONS",
|
|
"ref": str(len(other_locations)),
|
|
"blocking": False,
|
|
}
|
|
)
|
|
return (
|
|
sorted(dependencies, key=lambda item: (item["type"], item["ref"])),
|
|
reclaimable,
|
|
str(retention_state),
|
|
)
|
|
|
|
def _approval(self, approval_id: uuid.UUID) -> LifecycleApprovalRequest:
|
|
item = self.session.get(LifecycleApprovalRequest, approval_id, with_for_update=True, populate_existing=True)
|
|
if not item:
|
|
raise LifecycleError(404, "APPROVAL_NOT_FOUND", "approval request not found")
|
|
return item
|
|
|
|
def _plan(self, plan_id: uuid.UUID) -> LifecyclePromotionPlan:
|
|
item = self.session.get(LifecyclePromotionPlan, plan_id, with_for_update=True, populate_existing=True)
|
|
if not item:
|
|
raise LifecycleError(404, "PROMOTION_PLAN_NOT_FOUND", "promotion plan not found")
|
|
return item
|
|
|
|
def _operation(self, operation_id: uuid.UUID) -> LifecycleOperation:
|
|
item = self.session.get(LifecycleOperation, operation_id, with_for_update=True, populate_existing=True)
|
|
if not item:
|
|
raise LifecycleError(404, "LIFECYCLE_OPERATION_NOT_FOUND", "operation not found")
|
|
return item
|
|
|
|
@staticmethod
|
|
def _policy_response(item: LifecyclePolicyRevision) -> ApprovalPolicyResponse:
|
|
return ApprovalPolicyResponse.model_validate(item)
|
|
|
|
@staticmethod
|
|
def _subject_response(item: LifecycleSubject) -> LifecycleSubjectResponse:
|
|
return LifecycleSubjectResponse.model_validate(item)
|
|
|
|
def _approval_response(
|
|
self, item: LifecycleApprovalRequest, policy: LifecyclePolicyRevision | None = None
|
|
) -> ApprovalRequestResponse:
|
|
policy = policy or self.session.get(LifecyclePolicyRevision, item.policy_revision_id)
|
|
if policy is None:
|
|
raise LifecycleError(500, "POLICY_REFERENCE_MISSING", "approval policy is unavailable")
|
|
return ApprovalRequestResponse(
|
|
**{
|
|
key: getattr(item, key)
|
|
for key in ApprovalRequestResponse.model_fields
|
|
if key not in {"policy_key", "policy_revision"}
|
|
},
|
|
policy_key=policy.key,
|
|
policy_revision=policy.revision,
|
|
)
|
|
|
|
@staticmethod
|
|
def _plan_response(item: LifecyclePromotionPlan) -> PromotionPlanResponse:
|
|
return PromotionPlanResponse.model_validate(item)
|
|
|
|
def _operation_response(self, item: LifecycleOperation) -> LifecycleOperationResponse:
|
|
canary = self.session.scalar(
|
|
select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == item.id)
|
|
)
|
|
canary_payload = None
|
|
if canary:
|
|
canary_payload = {
|
|
key: value for key, value in canary.__dict__.items() if not key.startswith("_")
|
|
}
|
|
canary_payload["id"] = str(canary.id)
|
|
canary_payload["operation_id"] = str(canary.operation_id)
|
|
return LifecycleOperationResponse(
|
|
**{
|
|
key: getattr(item, key)
|
|
for key in LifecycleOperationResponse.model_fields
|
|
if key != "canary"
|
|
},
|
|
canary=canary_payload,
|
|
)
|
|
|
|
@staticmethod
|
|
def _cleanup_response(item: LifecycleCleanupPlan) -> CleanupPlanResponse:
|
|
return CleanupPlanResponse.model_validate(item)
|
|
|
|
def _event(
|
|
self,
|
|
event_type: str,
|
|
object_type: str,
|
|
object_ref: str,
|
|
from_state: Any | None,
|
|
to_state: Any | None,
|
|
actor: str,
|
|
actor_role: str,
|
|
policy_revision_id: uuid.UUID | None,
|
|
evidence_ids: list[str],
|
|
reason: str,
|
|
details: dict[str, Any],
|
|
*,
|
|
change_id: str | None = None,
|
|
) -> None:
|
|
self.session.add(
|
|
LifecycleEvent(
|
|
event_type=event_type,
|
|
object_type=str(object_type),
|
|
object_ref=object_ref,
|
|
from_state=str(from_state) if from_state is not None else None,
|
|
to_state=str(to_state) if to_state is not None else None,
|
|
actor=actor,
|
|
actor_role=actor_role,
|
|
policy_revision_id=policy_revision_id,
|
|
evidence_ids=evidence_ids,
|
|
reason=reason,
|
|
change_id=change_id or str(uuid.uuid4()),
|
|
details=details,
|
|
occurred_at=_now(),
|
|
)
|
|
)
|
|
|
|
|
|
def reconcile_lifecycle_operations(session: Session) -> int:
|
|
return LifecycleService(session).reconcile_incomplete()
|