Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_lifecycle_m12.py

@@ -2,13 +2,18 @@ from __future__ import annotations
 
 import uuid
 from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
 
 import pytest
 from fastapi.testclient import TestClient
 from sqlalchemy import create_engine, delete, select, update
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
+from starlette.requests import Request
 
+from modelforge_api.api.authorization import Principal, PrincipalRole
+from modelforge_api.api.routes.lifecycle import get_lifecycle_service
+from modelforge_api.api.routes.lifecycle import observe_canary as observe_canary_route
 from modelforge_api.domain.lifecycle_contracts import (
     ApprovalDecision,
     ApprovalPolicyCreate,
@@ -31,12 +36,14 @@ from modelforge_api.persistence.models import (
     ArtifactLocationRemovalRecord,
     ArtifactSet,
     ArtifactSetMember,
+    AuditEvent,
     Base,
     CapabilityDeployment,
     CapabilityResourceEnvelope,
     ComputeNode,
     DeploymentCandidate,
     LifecycleApprovalRequest,
+    LifecycleCanaryRun,
     LifecycleEvent,
     LifecyclePolicyRevision,
     LifecyclePromotionPlan,
@@ -48,6 +55,7 @@ from modelforge_api.persistence.models import (
     RuntimeProfile,
     StorageRoot,
 )
+from modelforge_api.services import audit as audit_service
 from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
 from modelforge_api.settings import Settings, get_settings
 from tests.test_runtime import setup_runtime
@@ -402,6 +410,128 @@ def test_canary_auto_abort_restores_snapshot(session: Session) -> None:
     )
 
 
+@pytest.mark.parametrize("scope_change", [
+    "subject_environment", "subject_target", "approval_environment", "approval_subject",
+    "approval_target", "approval_ref", "approval_transition", "policy_scope", "plan_desired",
+])
+@pytest.mark.parametrize("legacy_ready", [False, True])
+def test_manual_canary_rejects_persisted_production_scope(
+    session: Session, scope_change: str, legacy_ready: bool,
+) -> None:
+    service, subject, plan = approved_lab_plan(session)
+    operation = service.execute_plan(
+        plan.id,
+        PlanExecutionCreate(
+            executor="executor-a", idempotency_key="scope-test",
+            expected_subject_version=subject.version,
+        ),
+    )
+    stored_plan = session.get(LifecyclePromotionPlan, plan.id)
+    assert stored_plan is not None
+    approval = session.get(LifecycleApprovalRequest, stored_plan.approval_request_id)
+    assert approval is not None
+    changes = {
+        "subject_environment": (LifecycleSubject, subject.id, {"environment": "PRODUCTION"}),
+        "subject_target": (LifecycleSubject, subject.id, {"target_type": "CAPABILITY"}),
+        "approval_environment": (
+            LifecycleApprovalRequest, approval.id, {"environment": "PRODUCTION"},
+        ),
+        "approval_subject": (LifecycleApprovalRequest, approval.id, {"subject_id": None}),
+        "approval_target": (
+            LifecycleApprovalRequest, approval.id, {"target_type": "CAPABILITY"},
+        ),
+        "approval_ref": (LifecycleApprovalRequest, approval.id, {"target_ref": "another"}),
+        "approval_transition": (
+            LifecycleApprovalRequest, approval.id, {"requested_transition": "STABLE"},
+        ),
+        "policy_scope": (
+            LifecyclePolicyRevision, approval.policy_revision_id, {"scope": "PRODUCTION"},
+        ),
+        "plan_desired": (LifecyclePromotionPlan, plan.id, {"desired_state": "STABLE"}),
+    }
+    model, identity, values = changes[scope_change]
+    session.execute(update(model).where(model.id == identity).values(**values))
+    if legacy_ready:
+        session.execute(update(LifecycleCanaryRun).where(
+            LifecycleCanaryRun.operation_id == operation.id,
+        ).values(status="READY_FOR_PROMOTION_REVIEW", request_count=2, result={}))
+    session.commit()
+    before = service.operation(operation.id).model_dump(mode="json")
+    before_events = list(session.scalars(select(LifecycleEvent.id)))
+    before_audit = list(session.scalars(select(AuditEvent.id)))
+    with pytest.raises(LifecycleError) as denied:
+        if legacy_ready:
+            service.commit_operation(operation.id, ApprovalDecision(
+                actor="executor-a", reason="Legacy ready observation is not authoritative",
+            ))
+        else:
+            service.observe_canary(operation.id, CanaryObservation(
+                request_count=2, error_count=0, latency_p95_ms=10,
+                capability_health="HEALTHY", scheduler_ready=True,
+            ))
+    assert denied.value.status_code == 409
+    assert denied.value.code == "MANUAL_CANARY_REHEARSAL_ONLY"
+    assert service.operation(operation.id).model_dump(mode="json") == before
+    assert list(session.scalars(select(LifecycleEvent.id))) == before_events
+    assert list(session.scalars(select(AuditEvent.id))) == before_audit
+
+
+@pytest.mark.parametrize("abort", [False, True])
+@pytest.mark.parametrize("audit_failure", [False, True])
+def test_manual_lab_canary_route_attribution_and_atomic_audit(
+    session: Session, monkeypatch: pytest.MonkeyPatch, abort: bool, audit_failure: bool,
+) -> None:
+    service, subject, plan = approved_lab_plan(session)
+    operation = service.execute_plan(plan.id, PlanExecutionCreate(
+        executor="executor-a", idempotency_key="attribution-test",
+        expected_subject_version=subject.version,
+    ))
+    before = service.operation(operation.id).model_dump(mode="json")
+    before_events = list(session.scalars(select(LifecycleEvent.id)))
+    http_request = Request({"type": "http"})
+    http_request.state.correlation_id = "manual-lab-request-42"
+    principal = Principal(subject="named-admin", role=PrincipalRole.ADMIN,
+                          authentication_method="named_operator_key")
+
+    def invoke() -> object:
+        return observe_canary_route(operation.id, CanaryObservation(
+            request_count=2, error_count=1 if abort else 0, latency_p95_ms=10,
+            capability_health="HEALTHY", scheduler_ready=True,
+        ), service, principal, http_request)
+
+    if audit_failure:
+        def fail_append(*args: object, **kwargs: object) -> None:
+            raise RuntimeError("audit unavailable")
+        monkeypatch.setattr(audit_service, "_append_canonical_audit_event", fail_append)
+        with pytest.raises(RuntimeError, match="audit unavailable"):
+            invoke()
+        assert service.operation(operation.id).model_dump(mode="json") == before
+        assert list(session.scalars(select(LifecycleEvent.id))) == before_events
+        assert session.get(LifecycleSubject, subject.id).state == "CANARY"
+    else:
+        invoke()
+        audit = session.scalar(select(AuditEvent).where(
+            AuditEvent.action == "lifecycle.manual_lab_canary_observed",
+        ))
+        assert audit is not None
+        assert audit.actor_id == "named-admin"
+        assert audit.correlation_id == "manual-lab-request-42"
+        canary = session.scalar(select(LifecycleCanaryRun).where(
+            LifecycleCanaryRun.operation_id == operation.id,
+        ))
+        assert canary is not None
+        assert canary.result["provenance"] == {
+            "source": "MANUAL_LAB_REHEARSAL", "authoritative": False,
+            "actor": "named-admin", "correlation_id": "manual-lab-request-42",
+        }
+        event = session.scalar(select(LifecycleEvent).where(
+            LifecycleEvent.event_type == (
+                "CANARY_ABORTED" if abort else "CANARY_READY_FOR_PROMOTION_REVIEW"
+            ),
+        ))
+        assert event is not None and event.actor == "named-admin"
+
+
 def test_restart_reconciliation_and_cas_are_conservative(session: Session) -> None:
     service, subject_value, plan = approved_lab_plan(session, target="restart-lab")
     with pytest.raises(LifecycleError, match="changed concurrently"):
@@ -806,6 +936,40 @@ def test_lifecycle_routes_deny_capability_credentials() -> None:
         app.dependency_overrides.clear()
 
 
+def test_manual_canary_http_denial_is_typed_and_preserves_authentication() -> None:
+    contexts = []
+
+    def deny_observation(operation_id: object, observation: object, *, audit_context: object) -> None:
+        contexts.append(audit_context)
+        raise LifecycleError(409, "MANUAL_CANARY_REHEARSAL_ONLY", "LAB rehearsal required")
+
+    app.dependency_overrides[get_settings] = lambda: Settings(operator_api_key="operator-secret")
+    app.dependency_overrides[get_lifecycle_service] = lambda: SimpleNamespace(
+        observe_canary=deny_observation,
+    )
+    try:
+        client = TestClient(app)
+        url = f"/api/v1/admin/lifecycle/operations/{uuid.uuid4()}/canary"
+        body = {"request_count": 2, "error_count": 0, "latency_p95_ms": 10,
+                "capability_health": "HEALTHY", "scheduler_ready": True}
+        assert client.post(url, json=body).status_code == 401
+        assert client.post(url, json=body, headers={
+            "X-ModelForge-Admin-Token": "mfsvc_project-key",
+        }).status_code == 401
+        assert contexts == []
+        result = client.post(url, json=body, headers={
+            "X-ModelForge-Admin-Token": "operator-secret",
+            "X-Correlation-ID": "canary-denial-42",
+        })
+        assert result.status_code == 409
+        assert result.json()["error"]["code"] == "MANUAL_CANARY_REHEARSAL_ONLY"
+        assert result.json()["error"]["correlation_id"] == "canary-denial-42"
+        assert contexts[0].actor_id == "legacy-operator"
+        assert contexts[0].correlation_id == "canary-denial-42"
+    finally:
+        app.dependency_overrides.clear()
+
+
 def test_blocked_approval_is_audited_without_payload_content(session: Session) -> None:
     service = LifecycleService(session)
     result = service.request_approval(