from __future__ import annotations import uuid from datetime import UTC, datetime, timedelta from types import SimpleNamespace import pytest from fastapi.testclient import TestClient from sqlalchemy import create_engine, delete, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from starlette.requests import Request from modelforge_api.api.authorization import Principal, PrincipalRole from modelforge_api.api.routes.lifecycle import get_lifecycle_service from modelforge_api.api.routes.lifecycle import observe_canary as observe_canary_route from modelforge_api.domain.lifecycle_contracts import ( ApprovalDecision, ApprovalPolicyCreate, ApprovalRequestCreate, ApprovalRequestResponse, CanaryObservation, CleanupExecutionCreate, CleanupPlanCreate, DeploymentLifecycleState, LifecycleEnvironment, LifecycleEvidenceBundle, LifecycleSubjectCreate, PlanExecutionCreate, PromotionPlanCreate, assert_deployment_transition, ) from modelforge_api.main import app from modelforge_api.persistence.models import ( ArtifactLocation, ArtifactLocationRemovalRecord, ArtifactSet, ArtifactSetMember, AuditEvent, Base, CapabilityDeployment, CapabilityResourceEnvelope, ComputeNode, DeploymentCandidate, LifecycleApprovalRequest, LifecycleCanaryRun, LifecycleEvent, LifecyclePolicyRevision, LifecyclePromotionPlan, LifecycleRetentionRecord, LifecycleRollbackSnapshot, LifecycleSubject, ModelArtifact, RuntimeProbe, RuntimeProfile, StorageRoot, ) from modelforge_api.services import audit as audit_service from modelforge_api.services.lifecycle import LifecycleError, LifecycleService from modelforge_api.settings import Settings, get_settings from tests.test_runtime import setup_runtime from tests.test_serving import approve_and_promote, setup_m5 @pytest.fixture def session() -> Session: engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all(engine) with Session(engine) as value: yield value def lab_evidence(session: Session) -> tuple[LifecycleEvidenceBundle, uuid.UUID]: serving, candidate_value, _node = setup_m5(session) _approval, deployment = approve_and_promote(serving, candidate_value.id) candidate = session.get(DeploymentCandidate, candidate_value.id) assert candidate is not None artifact_set = session.get(ArtifactSet, candidate.artifact_set_id) profile = session.get(RuntimeProfile, candidate.runtime_profile_id) probe = session.get(RuntimeProbe, candidate.runtime_probe_id) envelope = session.scalar( select(CapabilityResourceEnvelope).where( CapabilityResourceEnvelope.capability_deployment_id == deployment.id ) ) assert artifact_set is not None and profile is not None and probe is not None assert envelope is not None and profile.runtime_image_digest is not None return ( LifecycleEvidenceBundle( artifact_set_id=artifact_set.id, model_revision_id=artifact_set.revision_id, runtime_profile_id=profile.id, runtime_probe_id=probe.id, runtime_image_digest=profile.runtime_image_digest, capability_deployment_id=deployment.id, resource_envelope_id=envelope.id, rollback_target_ref="lab-baseline-a", integrity="VERIFIED", security="APPROVED", license="APPROVED", runtime="PROVEN", evaluation="EVALUATED", project_fit="KEEP_LAB", production_validation="NOT_REQUIRED", scheduler_readiness="READY", ), deployment.id, ) def approved_lab_plan( session: Session, *, target: str = "m12-safe-lab" ) -> tuple[LifecycleService, object, object]: evidence, deployment_id = lab_evidence(session) service = LifecycleService(session) subject = service.create_subject( LifecycleSubjectCreate( target_type="LAB_REHEARSAL", target_ref=target, environment=LifecycleEnvironment.LAB, state=DeploymentLifecycleState.LAB_READY, ) ) approval = service.request_approval( ApprovalRequestCreate( policy_key="lab-promotion", subject_id=subject.id, target_type="LAB_REHEARSAL", target_ref=target, environment=LifecycleEnvironment.LAB, requested_transition=DeploymentLifecycleState.LAB_STABLE, evidence=evidence, requested_by="requester-a", reason="Safe non-production lifecycle rehearsal with exact evidence", ) ) assert approval.status == "PENDING" approval = service.decide_approval( approval.id, ApprovalDecision(actor="approver-a", reason="Reviewed exact immutable LAB evidence"), "approve", ) plan = service.create_plan( PromotionPlanCreate( approval_request_id=approval.id, subject_id=subject.id, desired_state=DeploymentLifecycleState.LAB_STABLE, migration_class="behavioral", candidate_deployment_id=deployment_id, rollback_target_ref="lab-baseline-a", project_consumers=[], affected_identities={ "runtime_deployments": [str(deployment_id)], "resource_envelope_id": str(evidence.resource_envelope_id), }, canary_strategy={ "mode": "REQUEST", "traffic_percent": 0, "target_request_count": 2, "max_error_rate": 0.0, "max_latency_p95_ms": 1000.0, }, health_gates={"worker": "healthy", "scheduler": "ready"}, automatic_abort_conditions=[ "HEALTH_FAILURE", "ERROR_THRESHOLD", "LATENCY_THRESHOLD", "CAPACITY_FAILURE", ], created_by="requester-a", ) ) plan = service.approve_plan( plan.id, ApprovalDecision(actor="approver-a", reason="Plan and rollback snapshot inputs reviewed"), ) return service, subject, plan def test_independent_lifecycle_states_reject_shortcuts() -> None: assert_deployment_transition( DeploymentLifecycleState.LAB_READY, DeploymentLifecycleState.CANARY ) with pytest.raises(ValueError, match="not allowed"): assert_deployment_transition( DeploymentLifecycleState.CANDIDATE, DeploymentLifecycleState.STABLE ) with pytest.raises(ValueError, match="not allowed"): assert_deployment_transition( DeploymentLifecycleState.DEPRECATED, DeploymentLifecycleState.STABLE ) def test_policy_revisions_are_immutable_and_do_not_reinterpret_approvals( session: Session, ) -> None: service = LifecycleService(session) service.ensure_defaults() original = next(item for item in service.policies() if item.key == "lab-promotion") revised = service.create_policy( ApprovalPolicyCreate( key="lab-promotion", scope="LAB_PROMOTION", requirements={"integrity": "VERIFIED", "runtime": "PROVEN", "review": "stricter"}, created_by="policy-owner", ) ) assert revised.revision == original.revision + 1 assert revised.id != original.id stored = session.get(LifecyclePolicyRevision, original.id) assert stored is not None and stored.active is False def test_deferred_external_validation_blocks_project_production(session: Session) -> None: service = LifecycleService(session) request = service.request_approval( ApprovalRequestCreate( policy_key="project-production", target_type="CAPABILITY", target_ref="vision.embedding@1", environment=LifecycleEnvironment.PRODUCTION, requested_transition=DeploymentLifecycleState.STABLE, evidence=LifecycleEvidenceBundle( project_fit="REQUIRES_MORE_EVIDENCE", production_validation="DEFERRED_EXTERNAL_VALIDATION", ), requested_by="operator", reason="ExampleVision Vision production gate rehearsal", ) ) assert request.status == "BLOCKED" assert "REQUIRED_EXTERNAL_VALIDATION_NOT_SATISFIED" in request.blockers assert "PROJECT_PRODUCTION_FIT_NOT_ELIGIBLE" in request.blockers with pytest.raises(LifecycleError, match="cannot be approved"): service.decide_approval( request.id, ApprovalDecision(actor="operator", reason="Cannot override deferred validation"), "approve", ) def test_approval_becomes_stale_when_referenced_probe_changes(session: Session) -> None: evidence, _deployment_id = lab_evidence(session) service = LifecycleService(session) subject = service.create_subject( LifecycleSubjectCreate( target_type="LAB_REHEARSAL", target_ref="stale-lab", environment=LifecycleEnvironment.LAB, state=DeploymentLifecycleState.LAB_READY, ) ) approval = service.request_approval( ApprovalRequestCreate( policy_key="lab-promotion", subject_id=subject.id, target_type="LAB_REHEARSAL", target_ref="stale-lab", environment=LifecycleEnvironment.LAB, requested_transition=DeploymentLifecycleState.LAB_STABLE, evidence=evidence, requested_by="requester", reason="Approval staleness must bind the exact runtime probe", ) ) approval = service.decide_approval( approval.id, ApprovalDecision(actor="approver", reason="Initial evidence is current and reviewed"), "approve", ) probe = session.get(RuntimeProbe, evidence.runtime_probe_id) assert probe is not None probe.status = "failed" session.commit() assert service.approval(approval.id).status == "STALE" def test_expired_and_revoked_approvals_are_terminal(session: Session) -> None: evidence, _deployment_id = lab_evidence(session) service = LifecycleService(session) expired = service.request_approval( ApprovalRequestCreate( policy_key="lab-promotion", target_type="LAB_REHEARSAL", target_ref="expired-lab", environment=LifecycleEnvironment.LAB, requested_transition=DeploymentLifecycleState.LAB_STABLE, evidence=evidence, requested_by="requester", reason="Expired approval fixture uses exact evidence", expires_at=datetime.now(UTC) - timedelta(seconds=1), ) ) assert service.approval(expired.id).status == "EXPIRED" subject = service.create_subject( LifecycleSubjectCreate( target_type="LAB_REHEARSAL", target_ref="revoke-lab", environment=LifecycleEnvironment.LAB, state=DeploymentLifecycleState.LAB_READY, ) ) current = service.request_approval( ApprovalRequestCreate( policy_key="lab-promotion", subject_id=subject.id, target_type="LAB_REHEARSAL", target_ref="revoke-lab", environment=LifecycleEnvironment.LAB, requested_transition=DeploymentLifecycleState.LAB_STABLE, evidence=evidence, requested_by="requester", reason="Revocation fixture uses exact evidence", ) ) current = service.decide_approval( current.id, ApprovalDecision(actor="approver", reason="Exact LAB evidence reviewed before revocation"), "approve", ) revoked = service.decide_approval( current.id, ApprovalDecision( actor="approver", reason="Approval intentionally revoked before execution" ), "revoke", ) assert revoked.status == "REVOKED" def test_lab_canary_requires_review_then_commits_and_rolls_back(session: Session) -> None: service, subject_value, plan = approved_lab_plan(session) operation = service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="lab-positive-operation", expected_subject_version=subject_value.version, ), ) assert operation.stage == "VERIFYING" assert operation.canary is not None and operation.canary["status"] == "RUNNING" current_subject = next(item for item in service.subjects() if item.id == subject_value.id) assert current_subject.state == "CANARY" operation = service.observe_canary( operation.id, CanaryObservation( request_count=2, error_count=0, latency_p95_ms=125.0, capability_health="HEALTHY", scheduler_ready=True, ), ) assert operation.canary is not None assert operation.canary["status"] == "READY_FOR_PROMOTION_REVIEW" assert ( next(item for item in service.subjects() if item.id == subject_value.id).state == "CANARY" ) operation = service.commit_operation( operation.id, ApprovalDecision(actor="executor-a", reason="Canary evidence reviewed; commit LAB state"), ) assert operation.stage == "COMMITTED" assert ( next(item for item in service.subjects() if item.id == subject_value.id).state == "LAB_STABLE" ) assert session.scalar(select(LifecycleRetentionRecord)) is not None rolled_back = service.rollback_operation( operation.id, ApprovalDecision( actor="executor-a", reason="Intentional non-production rollback rehearsal" ), ) assert rolled_back.stage == "ROLLED_BACK" assert rolled_back.rollback_duration_ms is not None assert ( next(item for item in service.subjects() if item.id == subject_value.id).state == "LAB_READY" ) def test_canary_auto_abort_restores_snapshot(session: Session) -> None: service, subject_value, plan = approved_lab_plan(session, target="abort-lab") operation = service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="lab-abort-operation", expected_subject_version=subject_value.version, ), ) operation = service.observe_canary( operation.id, CanaryObservation( request_count=1, error_count=1, latency_p95_ms=50.0, capability_health="HEALTHY", scheduler_ready=True, ), ) assert operation.stage == "ROLLED_BACK" assert operation.failure_code == "ERROR_THRESHOLD" assert ( next(item for item in service.subjects() if item.id == subject_value.id).state == "LAB_READY" ) @pytest.mark.parametrize("scope_change", [ "subject_environment", "subject_target", "approval_environment", "approval_subject", "approval_target", "approval_ref", "approval_transition", "policy_scope", "plan_desired", ]) @pytest.mark.parametrize("legacy_ready", [False, True]) def test_manual_canary_rejects_persisted_production_scope( session: Session, scope_change: str, legacy_ready: bool, ) -> None: service, subject, plan = approved_lab_plan(session) operation = service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="scope-test", expected_subject_version=subject.version, ), ) stored_plan = session.get(LifecyclePromotionPlan, plan.id) assert stored_plan is not None approval = session.get(LifecycleApprovalRequest, stored_plan.approval_request_id) assert approval is not None changes = { "subject_environment": (LifecycleSubject, subject.id, {"environment": "PRODUCTION"}), "subject_target": (LifecycleSubject, subject.id, {"target_type": "CAPABILITY"}), "approval_environment": ( LifecycleApprovalRequest, approval.id, {"environment": "PRODUCTION"}, ), "approval_subject": (LifecycleApprovalRequest, approval.id, {"subject_id": None}), "approval_target": ( LifecycleApprovalRequest, approval.id, {"target_type": "CAPABILITY"}, ), "approval_ref": (LifecycleApprovalRequest, approval.id, {"target_ref": "another"}), "approval_transition": ( LifecycleApprovalRequest, approval.id, {"requested_transition": "STABLE"}, ), "policy_scope": ( LifecyclePolicyRevision, approval.policy_revision_id, {"scope": "PRODUCTION"}, ), "plan_desired": (LifecyclePromotionPlan, plan.id, {"desired_state": "STABLE"}), } model, identity, values = changes[scope_change] session.execute(update(model).where(model.id == identity).values(**values)) if legacy_ready: session.execute(update(LifecycleCanaryRun).where( LifecycleCanaryRun.operation_id == operation.id, ).values(status="READY_FOR_PROMOTION_REVIEW", request_count=2, result={})) session.commit() before = service.operation(operation.id).model_dump(mode="json") before_events = list(session.scalars(select(LifecycleEvent.id))) before_audit = list(session.scalars(select(AuditEvent.id))) with pytest.raises(LifecycleError) as denied: if legacy_ready: service.commit_operation(operation.id, ApprovalDecision( actor="executor-a", reason="Legacy ready observation is not authoritative", )) else: service.observe_canary(operation.id, CanaryObservation( request_count=2, error_count=0, latency_p95_ms=10, capability_health="HEALTHY", scheduler_ready=True, )) assert denied.value.status_code == 409 assert denied.value.code == "MANUAL_CANARY_REHEARSAL_ONLY" assert service.operation(operation.id).model_dump(mode="json") == before assert list(session.scalars(select(LifecycleEvent.id))) == before_events assert list(session.scalars(select(AuditEvent.id))) == before_audit @pytest.mark.parametrize("abort", [False, True]) @pytest.mark.parametrize("audit_failure", [False, True]) def test_manual_lab_canary_route_attribution_and_atomic_audit( session: Session, monkeypatch: pytest.MonkeyPatch, abort: bool, audit_failure: bool, ) -> None: service, subject, plan = approved_lab_plan(session) operation = service.execute_plan(plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="attribution-test", expected_subject_version=subject.version, )) before = service.operation(operation.id).model_dump(mode="json") before_events = list(session.scalars(select(LifecycleEvent.id))) http_request = Request({"type": "http"}) http_request.state.correlation_id = "manual-lab-request-42" principal = Principal(subject="named-admin", role=PrincipalRole.ADMIN, authentication_method="named_operator_key") def invoke() -> object: return observe_canary_route(operation.id, CanaryObservation( request_count=2, error_count=1 if abort else 0, latency_p95_ms=10, capability_health="HEALTHY", scheduler_ready=True, ), service, principal, http_request) if audit_failure: def fail_append(*args: object, **kwargs: object) -> None: raise RuntimeError("audit unavailable") monkeypatch.setattr(audit_service, "_append_canonical_audit_event", fail_append) with pytest.raises(RuntimeError, match="audit unavailable"): invoke() assert service.operation(operation.id).model_dump(mode="json") == before assert list(session.scalars(select(LifecycleEvent.id))) == before_events assert session.get(LifecycleSubject, subject.id).state == "CANARY" else: invoke() audit = session.scalar(select(AuditEvent).where( AuditEvent.action == "lifecycle.manual_lab_canary_observed", )) assert audit is not None assert audit.actor_id == "named-admin" assert audit.correlation_id == "manual-lab-request-42" canary = session.scalar(select(LifecycleCanaryRun).where( LifecycleCanaryRun.operation_id == operation.id, )) assert canary is not None assert canary.result["provenance"] == { "source": "MANUAL_LAB_REHEARSAL", "authoritative": False, "actor": "named-admin", "correlation_id": "manual-lab-request-42", } event = session.scalar(select(LifecycleEvent).where( LifecycleEvent.event_type == ( "CANARY_ABORTED" if abort else "CANARY_READY_FOR_PROMOTION_REVIEW" ), )) assert event is not None and event.actor == "named-admin" def test_restart_reconciliation_and_cas_are_conservative(session: Session) -> None: service, subject_value, plan = approved_lab_plan(session, target="restart-lab") with pytest.raises(LifecycleError, match="changed concurrently"): service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="wrong-version-operation", expected_subject_version=subject_value.version + 1, ), ) operation = service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="restart-operation", expected_subject_version=subject_value.version, rehearsal_pause_stage="PREPARING", ), ) assert operation.stage == "PREPARING" assert service.reconcile_incomplete() == 1 assert service.operations()[0].stage == "ROLLED_BACK" assert ( next(item for item in service.subjects() if item.id == subject_value.id).state == "LAB_READY" ) def test_approval_transition_and_exact_rollback_target_bind_the_plan( session: Session, ) -> None: service, subject_value, approved = approved_lab_plan(session, target="bound-plan") with pytest.raises(LifecycleError) as transition_error: service.create_plan( PromotionPlanCreate( approval_request_id=approved.approval_request_id, subject_id=subject_value.id, desired_state=DeploymentLifecycleState.DEPRECATED, migration_class="transparent", candidate_deployment_id=approved.candidate_deployment_id, rollback_target_ref=approved.rollback_target_ref, created_by="requester-a", ) ) assert transition_error.value.code == "APPROVAL_TRANSITION_MISMATCH" with pytest.raises(LifecycleError) as rollback_error: service.create_plan( PromotionPlanCreate( approval_request_id=approved.approval_request_id, subject_id=subject_value.id, desired_state=DeploymentLifecycleState.LAB_STABLE, migration_class="behavioral", candidate_deployment_id=approved.candidate_deployment_id, rollback_target_ref="different-unapproved-target", canary_strategy={"mode": "REQUEST", "target_request_count": 1}, created_by="requester-a", ) ) assert rollback_error.value.code == "APPROVAL_ROLLBACK_TARGET_MISMATCH" def test_competing_promotion_plans_are_serialized_by_database_cas(session: Session) -> None: service, subject_value, first_plan = approved_lab_plan(session, target="promotion-race") first_approval = session.get(LifecycleApprovalRequest, first_plan.approval_request_id) assert first_approval is not None evidence = LifecycleEvidenceBundle.model_validate(first_approval.evidence_snapshot["declared"]) second_approval = service.request_approval( ApprovalRequestCreate( policy_key="lab-promotion", subject_id=subject_value.id, target_type="LAB_REHEARSAL", target_ref=subject_value.target_ref, environment=LifecycleEnvironment.LAB, requested_transition=DeploymentLifecycleState.LAB_STABLE, evidence=evidence, requested_by="requester-b", reason="Second operator prepares a competing exact LAB transition", ) ) second_approval = service.decide_approval( second_approval.id, ApprovalDecision(actor="approver-b", reason="Competing plan evidence independently reviewed"), "approve", ) second_plan = service.create_plan( PromotionPlanCreate( approval_request_id=second_approval.id, subject_id=subject_value.id, desired_state=DeploymentLifecycleState.LAB_STABLE, migration_class="behavioral", candidate_deployment_id=first_plan.candidate_deployment_id, rollback_target_ref=first_plan.rollback_target_ref, affected_identities={"race_contender": "b"}, canary_strategy={"mode": "REQUEST", "target_request_count": 1}, created_by="requester-b", ) ) second_plan = service.approve_plan( second_plan.id, ApprovalDecision(actor="approver-b", reason="Second immutable plan approved for CAS test"), ) first = service.execute_plan( first_plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="promotion-race-winner", expected_subject_version=subject_value.version, ), ) retry = service.execute_plan( first_plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="promotion-race-winner", expected_subject_version=subject_value.version, ), ) assert retry.id == first.id assert len([item for item in service.operations() if item.id == first.id]) == 1 with pytest.raises(LifecycleError) as conflict: service.execute_plan( second_plan.id, PlanExecutionCreate( executor="executor-b", idempotency_key="promotion-race-loser", expected_subject_version=subject_value.version, ), ) assert conflict.value.code == "VERSION_CONFLICT" stored_second = session.get(LifecyclePromotionPlan, second_plan.id) assert stored_second is not None and stored_second.status == "APPROVED" def test_rollback_race_preserves_newer_subject_state_and_reports_failure( session: Session, ) -> None: service, subject_value, plan = approved_lab_plan(session, target="rollback-race") operation = service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="rollback-race-operation", expected_subject_version=subject_value.version, ), ) session.execute( update(LifecycleSubject) .where(LifecycleSubject.id == subject_value.id) .values(version=LifecycleSubject.version + 1) ) session.commit() failed = service.rollback_operation( operation.id, ApprovalDecision(actor="executor-a", reason="Exercise rollback versus newer transition"), ) current = next(item for item in service.subjects() if item.id == subject_value.id) assert failed.stage == "FAILED" and failed.failure_code == "ROLLBACK_VERSION_CONFLICT" assert current.state == "CANARY" and current.version == 3 def test_deprecation_supersession_archive_and_retention_expiry_are_independent( session: Session, ) -> None: base_evidence, deployment_id = lab_evidence(session) assert base_evidence.artifact_set_id is not None rollback_ref = str(base_evidence.artifact_set_id) evidence = base_evidence.model_copy(update={"rollback_target_ref": rollback_ref}) service = LifecycleService(session) subject = service.create_subject( LifecycleSubjectCreate( target_type="LAB_REHEARSAL", target_ref="deprecation-retention-lab", environment=LifecycleEnvironment.LAB, state=DeploymentLifecycleState.LAB_READY, ) ) def approved_transition( desired: DeploymentLifecycleState, suffix: str ) -> ApprovalRequestResponse: approval = service.request_approval( ApprovalRequestCreate( policy_key="lab-promotion", subject_id=subject.id, target_type="LAB_REHEARSAL", target_ref=subject.target_ref, environment=LifecycleEnvironment.LAB, requested_transition=desired, evidence=evidence, requested_by=f"requester-{suffix}", reason=f"Exact evidence for logical {desired} transition without binary deletion", ) ) return service.decide_approval( approval.id, ApprovalDecision( actor=f"approver-{suffix}", reason="Lifecycle and retention effects reviewed" ), "approve", ) deprecation_approval = approved_transition(DeploymentLifecycleState.DEPRECATED, "dep") deprecation_plan = service.create_plan( PromotionPlanCreate( approval_request_id=deprecation_approval.id, subject_id=subject.id, desired_state=DeploymentLifecycleState.DEPRECATED, migration_class="transparent", candidate_deployment_id=deployment_id, rollback_target_ref=rollback_ref, affected_identities={"superseded_by_ref": "replacement-lab-revision"}, created_by="requester-dep", ) ) deprecation_plan = service.approve_plan( deprecation_plan.id, ApprovalDecision(actor="approver-dep", reason="Deprecation plan reviewed and frozen"), ) deprecation_operation = service.execute_plan( deprecation_plan.id, PlanExecutionCreate( executor="executor-dep", idempotency_key="deprecation-retention-operation", expected_subject_version=subject.version, ), ) service.commit_operation( deprecation_operation.id, ApprovalDecision(actor="executor-dep", reason="Commit logical deprecation only"), ) deprecated = next(item for item in service.subjects() if item.id == subject.id) artifact_set = session.get(ArtifactSet, base_evidence.artifact_set_id) retention = session.scalar( select(LifecycleRetentionRecord).where( LifecycleRetentionRecord.target_ref == rollback_ref ) ) assert deprecated.state == "DEPRECATED" assert deprecated.superseded_by_ref == "replacement-lab-revision" assert artifact_set is not None and artifact_set.status == "verified" assert retention is not None and retention.state == "ROLLBACK_RETAINED" protected = service.create_cleanup_plan( CleanupPlanCreate( target_type="ARTIFACT_SET", target_ref=base_evidence.artifact_set_id, action="ARCHIVE_METADATA", created_by="operator", ) ) protected_retention = next( item for item in protected.dependencies if item["type"] == "RETENTION_POLICY" ) assert protected.retention_state == "ROLLBACK_RETAINED" assert protected_retention["blocking"] is True retention.retained_until = datetime.now(UTC) - timedelta(seconds=1) session.commit() expired = service.create_cleanup_plan( CleanupPlanCreate( target_type="ARTIFACT_SET", target_ref=base_evidence.artifact_set_id, action="ARCHIVE_METADATA", created_by="operator", ) ) expired_retention = next( item for item in expired.dependencies if item["type"] == "RETENTION_POLICY" ) assert expired.retention_state == "ARCHIVABLE" assert expired_retention["blocking"] is False assert expired.status == "BLOCKED" # independent active dependencies still win archive_approval = approved_transition(DeploymentLifecycleState.ARCHIVED, "archive") archive_plan = service.create_plan( PromotionPlanCreate( approval_request_id=archive_approval.id, subject_id=subject.id, desired_state=DeploymentLifecycleState.ARCHIVED, migration_class="transparent", candidate_deployment_id=deployment_id, rollback_target_ref=rollback_ref, affected_identities={"superseded_by_ref": "replacement-lab-revision"}, created_by="requester-archive", ) ) archive_plan = service.approve_plan( archive_plan.id, ApprovalDecision(actor="approver-archive", reason="Logical archive plan reviewed"), ) archive_operation = service.execute_plan( archive_plan.id, PlanExecutionCreate( executor="executor-archive", idempotency_key="logical-archive-operation", expected_subject_version=deprecated.version, ), ) service.commit_operation( archive_operation.id, ApprovalDecision(actor="executor-archive", reason="Commit logical archive only"), ) archived = next(item for item in service.subjects() if item.id == subject.id) assert archived.state == "ARCHIVED" assert artifact_set.status == "verified" event_types = {item.event_type for item in service.events(1000)} assert {"DEPLOYMENT_DEPRECATED", "DEPLOYMENT_ARCHIVED", "RETENTION_APPLIED"} <= event_types def test_approved_plan_is_immutable(session: Session) -> None: _service, _subject, plan_value = approved_lab_plan(session, target="immutable-plan") plan = session.get(LifecyclePromotionPlan, plan_value.id) assert plan is not None plan.desired_state = "STABLE" with pytest.raises(ValueError, match="immutable approved fields"): session.commit() session.rollback() def test_cleanup_dry_run_blocks_dependencies_and_rechecks_safe_location( session: Session, ) -> None: _runtime, artifact_set, _node, artifacts = setup_runtime(session) service = LifecycleService(session) blocked = service.create_cleanup_plan( CleanupPlanCreate( target_type="ARTIFACT_SET", target_ref=artifact_set.id, action="ARCHIVE_METADATA", created_by="operator", ) ) assert blocked.status == "BLOCKED" assert blocked.reclaimable_bytes == artifact_set.total_size_bytes root = session.scalar(select(StorageRoot)) assert root is not None orphan = ModelArtifact( revision_id=artifacts[0].revision_id, filename="m12-unused-test.safetensors", artifact_type="weights", serialization_format="safetensors", sha256="d" * 64, size_bytes=4096, security_status="verified", license_status="approved", status="verified", quarantined=False, ) session.add(orphan) session.flush() location = ArtifactLocation( artifact_id=orphan.id, storage_root_id=root.id, relative_path="m12/unused-test.safetensors", status="verified", size_bytes=4096, observed_sha256=orphan.sha256, ) session.add(location) session.commit() ready = service.create_cleanup_plan( CleanupPlanCreate( target_type="ARTIFACT_LOCATION", target_ref=location.id, action="RECORD_LOCATION_REMOVAL", created_by="operator", ) ) assert ready.status == "READY" and ready.reclaimable_bytes == 4096 executed = service.execute_cleanup( ready.id, CleanupExecutionCreate(executor="operator", confirm=True, physical_removal_confirmed=True), ) assert executed.status == "EXECUTED" assert location.status == "removed" and orphan.status == "missing" assert session.scalar(select(ArtifactLocationRemovalRecord)) is not None def test_lifecycle_events_are_append_only(session: Session) -> None: service = LifecycleService(session) service.ensure_defaults() event = session.scalar(select(LifecycleEvent)) assert event is not None event.reason = "tampered" with pytest.raises(ValueError, match="append-only"): session.commit() session.rollback() def test_lifecycle_routes_deny_capability_credentials() -> None: app.dependency_overrides[get_settings] = lambda: Settings(operator_api_key="operator-secret") try: client = TestClient(app) assert client.get("/api/v1/admin/lifecycle/approval-policies").status_code == 401 response = client.get( "/api/v1/admin/lifecycle/approval-policies", headers={"Authorization": "Bearer project-capability-secret"}, ) assert response.status_code == 401 finally: app.dependency_overrides.clear() def test_manual_canary_http_denial_is_typed_and_preserves_authentication() -> None: contexts = [] def deny_observation(operation_id: object, observation: object, *, audit_context: object) -> None: contexts.append(audit_context) raise LifecycleError(409, "MANUAL_CANARY_REHEARSAL_ONLY", "LAB rehearsal required") app.dependency_overrides[get_settings] = lambda: Settings(operator_api_key="operator-secret") app.dependency_overrides[get_lifecycle_service] = lambda: SimpleNamespace( observe_canary=deny_observation, ) try: client = TestClient(app) url = f"/api/v1/admin/lifecycle/operations/{uuid.uuid4()}/canary" body = {"request_count": 2, "error_count": 0, "latency_p95_ms": 10, "capability_health": "HEALTHY", "scheduler_ready": True} assert client.post(url, json=body).status_code == 401 assert client.post(url, json=body, headers={ "X-ModelForge-Admin-Token": "mfsvc_project-key", }).status_code == 401 assert contexts == [] result = client.post(url, json=body, headers={ "X-ModelForge-Admin-Token": "operator-secret", "X-Correlation-ID": "canary-denial-42", }) assert result.status_code == 409 assert result.json()["error"]["code"] == "MANUAL_CANARY_REHEARSAL_ONLY" assert result.json()["error"]["correlation_id"] == "canary-denial-42" assert contexts[0].actor_id == "legacy-operator" assert contexts[0].correlation_id == "canary-denial-42" finally: app.dependency_overrides.clear() def test_blocked_approval_is_audited_without_payload_content(session: Session) -> None: service = LifecycleService(session) result = service.request_approval( ApprovalRequestCreate( policy_key="capability-production", target_type="CAPABILITY", target_ref="document.ocr@1", environment=LifecycleEnvironment.PRODUCTION, requested_transition=DeploymentLifecycleState.STABLE, evidence=LifecycleEvidenceBundle(), requested_by="operator", reason="OCR production approval blocker rehearsal", ) ) event = session.scalar(select(LifecycleEvent).where(LifecycleEvent.change_id == str(result.id))) approval = session.get(LifecycleApprovalRequest, result.id) assert event is not None and approval is not None assert event.to_state == "BLOCKED" assert "payload" not in event.details and "content" not in event.details def test_declared_green_states_cannot_replace_exact_production_evidence(session: Session) -> None: evidence, _deployment_id = lab_evidence(session) forged = evidence.model_copy( update={ "model_revision_id": uuid.uuid4(), "runtime_image_digest": "sha256:" + "f" * 64, "security": "APPROVED", "license": "APPROVED", "evaluation": "PROMOTION_ELIGIBLE", "scheduler_readiness": "READY", "critical_regressions": 0, } ) result = LifecycleService(session).request_approval( ApprovalRequestCreate( policy_key="capability-production", target_type="CAPABILITY", target_ref="vision.embedding@1", environment=LifecycleEnvironment.PRODUCTION, requested_transition=DeploymentLifecycleState.STABLE, evidence=forged, requested_by="operator", reason="Attempt to prove that claimed green labels cannot replace database evidence", ) ) assert result.status == "BLOCKED" assert "MODEL_REVISION_ARTIFACT_SET_MISMATCH" in result.blockers assert "RUNTIME_IMAGE_DIGEST_MISMATCH" in result.blockers assert "MISSING_PRODUCTION_EXECUTION_APPROVAL" in result.blockers assert "MISSING_EVALUATION_RUNS" in result.blockers assert "MISSING_SCHEDULER_EVIDENCE_REVISION" in result.blockers def test_database_prevents_two_production_stable_deployments(session: Session) -> None: _evidence, deployment_id = lab_evidence(session) current = session.get(CapabilityDeployment, deployment_id) assert current is not None and current.production and current.status == "stable" duplicate = CapabilityDeployment( capability_contract_id=current.capability_contract_id, deployment_candidate_id=current.deployment_candidate_id, production_approval_id=current.production_approval_id, execution_approval_id=current.execution_approval_id, embedding_space_id=current.embedding_space_id, artifact_set_id=current.artifact_set_id, runtime_profile_id=current.runtime_profile_id, compute_node_id=current.compute_node_id, accelerator_id=current.accelerator_id, channel="stable", status="stable", production=True, health_status="healthy", routing_weight=100, fallback_policy={}, residency_policy="on_demand", keep_warm_seconds=0, max_concurrency=1, max_queue_depth=1, config_fingerprint="f" * 64, provenance={}, rollback_policy={}, ) session.add(duplicate) with pytest.raises(IntegrityError): session.commit() session.rollback() def test_missing_rollback_snapshot_fails_conservatively(session: Session) -> None: service, subject_value, plan = approved_lab_plan(session, target="rollback-failure-lab") operation = service.execute_plan( plan.id, PlanExecutionCreate( executor="executor-a", idempotency_key="rollback-failure-operation", expected_subject_version=subject_value.version, ), ) session.execute( delete(LifecycleRollbackSnapshot).where( LifecycleRollbackSnapshot.promotion_plan_id == plan.id ) ) session.commit() failed = service.rollback_operation( operation.id, ApprovalDecision(actor="executor-a", reason="Exercise unavailable rollback target path"), ) subject = next(item for item in service.subjects() if item.id == subject_value.id) assert failed.stage == "FAILED" and failed.failure_code == "ROLLBACK_FAILED" assert subject.state == "MANUAL_INTERVENTION_REQUIRED" def test_cleanup_plan_becomes_stale_when_dependency_appears(session: Session) -> None: _runtime, artifact_set, _node, artifacts = setup_runtime(session) root = session.scalar(select(StorageRoot)) assert root is not None orphan = ModelArtifact( revision_id=artifacts[0].revision_id, filename="m12-cleanup-race.safetensors", artifact_type="weights", serialization_format="safetensors", sha256="e" * 64, size_bytes=2048, security_status="verified", license_status="approved", status="verified", quarantined=False, ) session.add(orphan) session.flush() location = ArtifactLocation( artifact_id=orphan.id, storage_root_id=root.id, relative_path="m12/cleanup-race.safetensors", status="verified", size_bytes=2048, observed_sha256=orphan.sha256, ) session.add(location) session.commit() service = LifecycleService(session) plan = service.create_cleanup_plan( CleanupPlanCreate( target_type="ARTIFACT_LOCATION", target_ref=location.id, action="RECORD_LOCATION_REMOVAL", created_by="operator", ) ) assert plan.status == "READY" session.add( ArtifactSetMember( artifact_set_id=artifact_set.id, artifact_id=orphan.id, ordinal=artifact_set.file_count, required=False, ) ) session.commit() with pytest.raises(LifecycleError) as error: service.execute_cleanup( plan.id, CleanupExecutionCreate( executor="operator", confirm=True, physical_removal_confirmed=True ), ) assert error.value.code == "CLEANUP_PLAN_STALE" stored = next(item for item in service.cleanup_plans() if item.id == plan.id) assert stored.status == "STALE" and location.status == "verified" def test_node_specific_location_removal_does_not_mark_global_artifact_missing( session: Session, ) -> None: _runtime, _artifact_set, _node, artifacts = setup_runtime(session) first_root = session.scalar(select(StorageRoot)) assert first_root is not None second_node = ComputeNode(key="m12-replica", hostname="m12-replica", liveness_state="online") session.add(second_node) session.flush() second_root = StorageRoot( compute_node_id=second_node.id, name="replica-models", path="/replica/models", agent_path="/data/artifacts/model-registry", status="ready", writable=True, ) orphan = ModelArtifact( revision_id=artifacts[0].revision_id, filename="m12-multi-node.safetensors", artifact_type="weights", serialization_format="safetensors", sha256="9" * 64, size_bytes=1024, security_status="verified", license_status="approved", status="verified", quarantined=False, ) session.add_all([second_root, orphan]) session.flush() first_location = ArtifactLocation( artifact_id=orphan.id, storage_root_id=first_root.id, relative_path="m12/multi-node.safetensors", status="verified", size_bytes=orphan.size_bytes, observed_sha256=orphan.sha256, ) second_location = ArtifactLocation( artifact_id=orphan.id, storage_root_id=second_root.id, relative_path="m12/multi-node.safetensors", status="verified", size_bytes=orphan.size_bytes, observed_sha256=orphan.sha256, ) session.add_all([first_location, second_location]) session.commit() service = LifecycleService(session) plan = service.create_cleanup_plan( CleanupPlanCreate( target_type="ARTIFACT_LOCATION", target_ref=second_location.id, action="RECORD_LOCATION_REMOVAL", created_by="operator", ) ) assert plan.status == "READY" assert ( next(item for item in plan.dependencies if item["type"] == "OTHER_ARTIFACT_LOCATIONS")[ "ref" ] == "1" ) service.execute_cleanup( plan.id, CleanupExecutionCreate(executor="operator", confirm=True, physical_removal_confirmed=True), ) assert second_location.status == "removed" assert first_location.status == "verified" and orphan.status == "verified"