Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/src/modelforge_api/services/lifecycle.py

@@ -75,6 +75,9 @@ from modelforge_api.persistence.models import (
     RuntimeProbe,
     RuntimeProfile,
 )
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.lifecycle_serving import LifecycleServing
+from modelforge_api.services.serving import ServingError
 
 
 class LifecycleError(RuntimeError):
@@ -94,6 +97,13 @@ def _now() -> datetime:
     return datetime.now(UTC)
 
 
+def _serving_changed_at(operation: LifecycleOperation) -> str:
+    if operation.finished_at is None:
+        raise LifecycleError(409, "SERVING_JOURNAL_INVALID", "committed serving timestamp is missing")
+    value = operation.finished_at
+    return (value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)).isoformat()
+
+
 DEFAULT_POLICIES: tuple[dict[str, Any], ...] = (
     {
         "key": "lab-promotion",
@@ -305,6 +315,18 @@ class LifecycleService:
                     "SUBJECT_TARGET_MISMATCH",
                     "approval target does not match lifecycle subject",
                 )
+        if request.evidence.serving_transition is not None:
+            selected = request.evidence.serving_transition
+            if request.evidence.production_validation != "SATISFIED":
+                raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
+            if (request.environment != "PRODUCTION" or policy.scope != "CAPABILITY_PRODUCTION"
+                    or request.target_type != "CAPABILITY_DEPLOYMENT"
+                    or request.target_ref != str(selected.candidate_deployment_id)
+                    or request.requested_transition != "STABLE"
+                    or request.evidence.capability_deployment_id != selected.candidate_deployment_id
+                    or request.evidence.production_execution_approval_id != selected.production_approval_id
+                    or request.evidence.rollback_target_ref != str(selected.current_deployment_id)):
+                raise LifecycleError(422, "SERVING_APPROVAL_MISMATCH", "serving transition requires exact production approval scope and targets")
         resolved = self._resolve_evidence(request.evidence.model_dump(mode="json"))
         snapshot = {
             "declared": request.evidence.model_dump(mode="json"),
@@ -400,7 +422,9 @@ class LifecycleService:
             item.status = ApprovalStatus.REJECTED
             event_type = "APPROVAL_REJECTED"
         elif action == "revoke":
-            if item.status != ApprovalStatus.APPROVED:
+            if item.status != ApprovalStatus.APPROVED and not (
+                item.status == ApprovalStatus.STALE and item.approved_by is not None
+            ):
                 raise LifecycleError(409, "APPROVAL_NOT_APPROVED", "only approval can be revoked")
             item.status = ApprovalStatus.REVOKED
             event_type = "APPROVAL_REVOKED"
@@ -468,13 +492,29 @@ class LifecycleService:
                 "APPROVAL_MIGRATION_MISMATCH",
                 "promotion plan migration differs from approved evidence",
             )
+        serving_contract = approval.evidence_snapshot.get("resolved", {}).get("serving_transition")
+        if "serving_transition" in request.affected_identities:
+            raise LifecycleError(422, "SERVING_CONTRACT_SERVER_OWNED", "serving contract is derived from approved evidence")
+        if serving_contract is not None:
+            if (request.canary_strategy or request.migration_class not in {"transparent", "behavioral"}
+                    or subject.target_type != "CAPABILITY_DEPLOYMENT" or subject.environment != "PRODUCTION"
+                    or subject.target_ref != str(request.candidate_deployment_id)
+                    or subject.state not in {"LAB_READY", "PROMOTION_ELIGIBLE"}
+                    or request.desired_state != "STABLE"):
+                raise LifecycleError(422, "SERVING_PLAN_UNSUPPORTED", "only reviewed same-space stable replacement is supported")
+            contract = LifecycleServing(self.session).validate(serving_contract)
+            request = request.model_copy(update={
+                "affected_identities": {**request.affected_identities, "serving_transition": serving_contract},
+                "project_consumers": sorted({value["project_id"] for value in contract.bindings.values()}),
+            })
         try:
             intermediate = (
                 DeploymentLifecycleState.CANARY
                 if request.canary_strategy
                 else request.desired_state
             )
-            assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
+            if serving_contract is None:
+                assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
         except ValueError as exc:
             raise LifecycleError(422, "INVALID_LIFECYCLE_TRANSITION", str(exc)) from exc
         if request.migration_class == "requires_reindex":
@@ -635,6 +675,11 @@ class LifecycleService:
             raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
         if subject.version != request.expected_subject_version:
             raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently")
+        serving_contract = plan.affected_identities.get("serving_transition")
+        if serving_contract is not None:
+            LifecycleServing(self.session).validate(serving_contract)
+        elif subject.environment == "PRODUCTION" and subject.target_type == "CAPABILITY_DEPLOYMENT" and plan.desired_state == "STABLE":
+            raise LifecycleError(409, "SERVING_TRANSITION_REQUIRED", "operational stable promotion requires the approved serving adapter")
         plan_version = plan.version
         snapshot_payload = {
             "subject_id": str(subject.id),
@@ -645,7 +690,10 @@ class LifecycleService:
             "version": subject.version,
             "rollback_target_ref": plan.rollback_target_ref,
             "affected_identities": plan.affected_identities,
+            "superseded_by_ref": subject.superseded_by_ref,
         }
+        if serving_contract is not None:
+            snapshot_payload["serving_transition"] = serving_contract
         snapshot = LifecycleRollbackSnapshot(
             promotion_plan_id=plan.id,
             snapshot=snapshot_payload,
@@ -741,7 +789,8 @@ class LifecycleService:
         return self._operation_response(operation)
 
     def observe_canary(
-        self, operation_id: uuid.UUID, observation: CanaryObservation
+        self, operation_id: uuid.UUID, observation: CanaryObservation,
+        *, audit_context: AuditContext | None = None,
     ) -> LifecycleOperationResponse:
         operation = self._operation(operation_id)
         if operation.stage != OperationStage.VERIFYING:
@@ -751,16 +800,32 @@ class LifecycleService:
         )
         if not canary or canary.status != CanaryStatus.RUNNING:
             raise LifecycleError(409, "CANARY_NOT_RUNNING", "canary is not running")
+        plan = self._plan(operation.promotion_plan_id)
+        subject = self._require_manual_canary_rehearsal(plan)
+        context = audit_context or AuditContext(
+            actor_type="manual_lab_rehearsal", actor_id="local-lab-rehearsal",
+        )
         canary.request_count += observation.request_count
         canary.error_count += observation.error_count
         canary.latency_p95_ms = observation.latency_p95_ms
-        canary.result = observation.model_dump(mode="json")
+        canary.result = {
+            **observation.model_dump(mode="json"),
+            "provenance": {
+                "source": "MANUAL_LAB_REHEARSAL",
+                "authoritative": False,
+                "actor": context.actor_id,
+                "correlation_id": context.correlation_id,
+            },
+        }
         trigger = self._canary_trigger(canary, observation)
         if trigger:
             canary.status = CanaryStatus.ABORTED
             canary.abort_trigger = trigger
             canary.finished_at = _now()
-            self._rollback(operation, trigger, "SYSTEM", "Automatic canary abort")
+            self._rollback(
+                operation, trigger, context.actor_id,
+                "Automatic abort from manual LAB rehearsal observation",
+            )
             event_type = "CANARY_ABORTED"
         elif canary.request_count >= canary.target_request_count:
             canary.status = CanaryStatus.READY_FOR_PROMOTION_REVIEW
@@ -768,18 +833,14 @@ class LifecycleService:
             event_type = "CANARY_READY_FOR_PROMOTION_REVIEW"
         else:
             event_type = "CANARY_OBSERVED"
-        plan = self._plan(operation.promotion_plan_id)
-        subject = self.session.get(LifecycleSubject, plan.subject_id)
-        if subject is None:
-            raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
         self._event(
             event_type,
             subject.target_type,
             subject.target_ref,
             subject.state,
             canary.status,
-            "lifecycle-engine",
-            "SYSTEM",
+            context.actor_id,
+            "MANUAL_LAB_REHEARSAL",
             self._approval(plan.approval_request_id).policy_revision_id,
             [str(canary.id)],
             trigger or "Bounded canary observation recorded without payload content",
@@ -790,14 +851,64 @@ class LifecycleService:
             },
             change_id=str(operation.id),
         )
+        AuditWriter(self.session, context=context).write(
+            action="lifecycle.manual_lab_canary_observed",
+            resource_type="lifecycle_operation",
+            resource_id=str(operation.id),
+            details={
+                "source": "MANUAL_LAB_REHEARSAL",
+                "canary_id": str(canary.id),
+                "status": canary.status,
+                "authoritative": False,
+            },
+        )
         self.session.commit()
         return self._operation_response(operation)
 
+    def _require_manual_canary_rehearsal(
+        self, plan: LifecyclePromotionPlan,
+    ) -> LifecycleSubject:
+        """No manual or legacy canary evidence can qualify an operational promotion."""
+        subject = self.session.get(LifecycleSubject, plan.subject_id)
+        approval = self.session.get(LifecycleApprovalRequest, plan.approval_request_id)
+        policy = self.session.get(LifecyclePolicyRevision, approval.policy_revision_id) if (
+            approval is not None
+        ) else None
+        if (
+            subject is None or approval is None or policy is None
+            or subject.target_type != "LAB_REHEARSAL"
+            or subject.environment != "LAB"
+            or approval.environment != "LAB"
+            or policy.scope != "LAB_PROMOTION"
+            or plan.desired_state != "LAB_STABLE"
+            or approval.subject_id != subject.id
+            or approval.target_type != subject.target_type
+            or approval.target_ref != subject.target_ref
+            or approval.requested_transition != plan.desired_state
+        ):
+            raise LifecycleError(
+                409, "MANUAL_CANARY_REHEARSAL_ONLY",
+                "Canary observations and commit require an explicit LAB rehearsal; "
+                "authoritative operational canary evidence ingestion is unavailable",
+            )
+        return subject
+
     def commit_operation(
         self, operation_id: uuid.UUID, decision: ApprovalDecision
     ) -> LifecycleOperationResponse:
         operation = self._operation(operation_id)
         plan = self._plan(operation.promotion_plan_id)
+        if operation.stage == OperationStage.COMMITTED:
+            transition = plan.affected_identities.get("serving_transition")
+            if transition is not None:
+                snapshot = self.session.scalar(select(LifecycleRollbackSnapshot).where(
+                    LifecycleRollbackSnapshot.promotion_plan_id == plan.id))
+                if (snapshot is None or _hash(snapshot.snapshot) != snapshot.snapshot_digest
+                        or snapshot.snapshot.get("serving_transition") != transition):
+                    raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact installed serving snapshot is required")
+                LifecycleServing(self.session).installed(transition, _serving_changed_at(operation))
+            self.session.commit()
+            return self._operation_response(operation)
         if operation.stage != OperationStage.VERIFYING:
             raise LifecycleError(
                 409, "OPERATION_NOT_REVIEWABLE", "operation is not ready for review"
@@ -805,6 +916,8 @@ class LifecycleService:
         canary = self.session.scalar(
             select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id)
         )
+        if canary or plan.canary_strategy:
+            self._require_manual_canary_rehearsal(plan)
         if canary and canary.status != CanaryStatus.READY_FOR_PROMOTION_REVIEW:
             raise LifecycleError(
                 409,
@@ -823,14 +936,26 @@ class LifecycleService:
             raise LifecycleError(
                 409, "ROLLBACK_SNAPSHOT_MISSING", "immutable rollback snapshot is unavailable"
             )
+        serving_contract = plan.affected_identities.get("serving_transition")
+        serving_changed_at = None
+        if serving_contract is not None:
+            approval = self._approval(plan.approval_request_id)
+            self._refresh_approval(approval)
+            if approval.status != ApprovalStatus.APPROVED or _hash(approval.evidence_snapshot) != approval.evidence_fingerprint:
+                raise LifecycleError(409, "APPROVAL_STALE", "serving approval is no longer current")
+            if snapshot.snapshot.get("serving_transition") != serving_contract or _hash(snapshot.snapshot) != snapshot.snapshot_digest:
+                raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact immutable serving snapshot is required")
+            serving_changed_at = _now()
+            LifecycleServing(self.session).switch(serving_contract, serving_changed_at.isoformat())
         expected_state = (
             DeploymentLifecycleState.CANARY if plan.canary_strategy else plan.current_state
         )
         try:
-            assert_deployment_transition(
-                DeploymentLifecycleState(expected_state),
-                DeploymentLifecycleState(plan.desired_state),
-            )
+            if serving_contract is None:
+                assert_deployment_transition(
+                    DeploymentLifecycleState(expected_state),
+                    DeploymentLifecycleState(plan.desired_state),
+                )
         except ValueError as exc:
             raise LifecycleError(409, "STATE_CHANGED", str(exc)) from exc
         previous = str(expected_state)
@@ -862,7 +987,7 @@ class LifecycleService:
             )
         self.session.refresh(subject)
         operation.stage = OperationStage.COMMITTED
-        operation.finished_at = _now()
+        operation.finished_at = serving_changed_at or _now()
         operation.version += 1
         plan.status = PromotionPlanStatus.COMMITTED
         plan.version += 1
@@ -1142,6 +1267,14 @@ class LifecycleService:
 
     def _resolve_evidence(self, declared: dict[str, Any]) -> dict[str, Any]:
         facts: dict[str, Any] = {}
+        if declared.get("serving_transition") is not None:
+            from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
+
+            if declared.get("production_validation") != "SATISFIED":
+                raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
+            facts["serving_transition"] = LifecycleServing(self.session).prepare(
+                ServingTransitionSelection.model_validate(declared["serving_transition"])
+            ).model_dump(mode="json")
         for field, model in (
             ("artifact_set_id", ArtifactSet),
             ("runtime_profile_id", RuntimeProfile),
@@ -1359,8 +1492,21 @@ class LifecycleService:
                 change_id=str(item.id),
             )
             return
-        current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
-        if _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
+        stale_details: dict[str, Any] = {}
+        current_resolved: dict[str, Any] | None
+        try:
+            current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
+        except (LifecycleError, ServingError) as exc:
+            # A historical serving approval must remain inspectable after its route
+            # changes. Expected qualification failures invalidate authority; internal
+            # errors still propagate rather than being disguised as stale evidence.
+            if (item.evidence_snapshot["declared"].get("serving_transition") is None
+                    or exc.status_code not in {404, 409, 422}):
+                raise
+            current_resolved = None
+            stale_details = {"validation_code": exc.code, "validation_reason": str(exc)}
+            item.blockers = sorted(set(item.blockers) | {exc.code})
+        if current_resolved is None or _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
             previous = item.status
             item.status = ApprovalStatus.STALE
             item.stale_at = now
@@ -1375,7 +1521,7 @@ class LifecycleService:
                 item.policy_revision_id,
                 [],
                 "Referenced evidence changed after approval request creation",
-                {},
+                stale_details,
                 change_id=str(item.id),
             )
 
@@ -1461,6 +1607,7 @@ class LifecycleService:
     ) -> None:
         started = time.perf_counter()
         plan = self._plan(operation.promotion_plan_id)
+        self._approval(plan.approval_request_id)  # Same approval-before-route lock order as commit.
         snapshot = self.session.scalar(
             select(LifecycleRollbackSnapshot).where(
                 LifecycleRollbackSnapshot.promotion_plan_id == plan.id
@@ -1492,6 +1639,21 @@ class LifecycleService:
             )
             return
         snapshot_version = int(snapshot.snapshot["version"])
+        serving_contract = plan.affected_identities.get("serving_transition")
+        was_committed = operation.stage == OperationStage.COMMITTED
+        if serving_contract is not None:
+            if (snapshot.snapshot.get("serving_transition") != serving_contract
+                    or _hash(snapshot.snapshot) != snapshot.snapshot_digest):
+                raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact serving snapshot is unavailable")
+            # A subject CAS failure below rolls back the route in the same transaction.
+            # A later route is preserved rather than guessed away.
+            try:
+                LifecycleServing(self.session).restore(serving_contract,
+                    _serving_changed_at(operation) if was_committed else _now().isoformat(),
+                    committed=was_committed)
+            except Exception:
+                self.session.rollback()
+                raise
         expected_version = snapshot_version + (
             2 if operation.stage == OperationStage.COMMITTED else 1
         )
@@ -1511,6 +1673,9 @@ class LifecycleService:
             )
         )
         if rows_affected(restored) != 1:
+            if serving_contract is not None:
+                self.session.rollback()
+                raise LifecycleError(409, "ROLLBACK_VERSION_CONFLICT", "lifecycle subject changed; serving state was preserved")
             operation.stage = OperationStage.FAILED
             operation.failure_code = "ROLLBACK_VERSION_CONFLICT"
             operation.failure_details = {
@@ -1581,6 +1746,26 @@ class LifecycleService:
                 500, "RETENTION_POLICY_MISSING", "rollback retention policy missing"
             )
         retained_until = _now() + timedelta(days=policy.minimum_rollback_days)
+        transition = plan.affected_identities.get("serving_transition")
+        if transition is not None:
+            prior = transition["current"]
+            for kind, ref in (("ARTIFACT_SET", prior["artifact_set_id"]),
+                              ("RUNTIME_PROFILE", prior["runtime_profile_id"]),
+                              ("CAPABILITY_DEPLOYMENT", prior["id"])):
+                retained = self.session.scalar(select(LifecycleRetentionRecord).where(
+                    LifecycleRetentionRecord.target_type == kind,
+                    LifecycleRetentionRecord.target_ref == ref))
+                if retained is None:
+                    self.session.add(LifecycleRetentionRecord(
+                        policy_revision_id=policy.id, target_type=kind, target_ref=ref,
+                        state=RetentionState.ROLLBACK_RETAINED, retained_until=retained_until,
+                        legal_hold=False, reason="Exact prior serving identity retained for approved rollback",
+                        created_at=_now()))
+                else:
+                    previous_until = retained.retained_until
+                    if previous_until is None or previous_until.replace(tzinfo=UTC) < retained_until:
+                        retained.retained_until = retained_until
+                    retained.state = RetentionState.ROLLBACK_RETAINED
         existing = self.session.scalar(
             select(LifecycleRetentionRecord).where(
                 LifecycleRetentionRecord.target_type == "ROLLBACK_TARGET",
@@ -1856,19 +2041,19 @@ class LifecycleService:
         )
 
     def _approval(self, approval_id: uuid.UUID) -> LifecycleApprovalRequest:
-        item = self.session.get(LifecycleApprovalRequest, approval_id)
+        item = self.session.get(LifecycleApprovalRequest, approval_id, with_for_update=True, populate_existing=True)
         if not item:
             raise LifecycleError(404, "APPROVAL_NOT_FOUND", "approval request not found")
         return item
 
     def _plan(self, plan_id: uuid.UUID) -> LifecyclePromotionPlan:
-        item = self.session.get(LifecyclePromotionPlan, plan_id)
+        item = self.session.get(LifecyclePromotionPlan, plan_id, with_for_update=True, populate_existing=True)
         if not item:
             raise LifecycleError(404, "PROMOTION_PLAN_NOT_FOUND", "promotion plan not found")
         return item
 
     def _operation(self, operation_id: uuid.UUID) -> LifecycleOperation:
-        item = self.session.get(LifecycleOperation, operation_id)
+        item = self.session.get(LifecycleOperation, operation_id, with_for_update=True, populate_existing=True)
         if not item:
             raise LifecycleError(404, "LIFECYCLE_OPERATION_NOT_FOUND", "operation not found")
         return item