"""Approval-bound promotion for the single registered WorkMesh product target.""" from __future__ import annotations import hashlib import json import re import uuid from datetime import timedelta from typing import Any from sqlalchemy import update from sqlmodel import Session, select from ..core.time import utc_now from ..db.models import Approval, WorkMeshPromotion from .approval_service import approval_fingerprint from .audit_logger import log_event from .deployment_broker import DeploymentBrokerClient from .secret_store import get_secret _ACTION = "deployment.target.promote_workmesh" _TARGET = "jens/workmesh" _SECRET_REF = "WORKMESH_DEPLOYMENT_BROKER" _API_VERSION = "itworx.workmesh-promotion/v2" _REVISION = re.compile(r"^[0-9a-f]{40}$") _SHA256 = re.compile(r"^[0-9a-f]{64}$") _IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") _CHECKS = {"exactMain", "pushValidation", "controllerPreflight", "dataBackup"} _SAFE_PREFLIGHT_FIELDS = { "apiVersion", "status", "repository", "revision", "preflightSha256", "backupInputsSha256", "checks", } _TERMINAL = {"succeeded", "failed", "preflight_changed"} class WorkMeshPromotionError(ValueError): """A fail-closed error in the fixed-target promotion contract.""" def _idempotency_digest(value: str) -> str: normalized = value.strip() if not _IDEMPOTENCY.fullmatch(normalized): raise WorkMeshPromotionError("Idempotency-Key must contain 8 to 128 safe characters.") return hashlib.sha256(normalized.encode("utf-8")).hexdigest() def _request_id(revision: str, idempotency_key: str) -> str: value = json.dumps( { "repository": _TARGET, "revision": revision, "operation": "workmesh_promote", "idempotency": idempotency_key, }, sort_keys=True, separators=(",", ":"), ).encode("utf-8") return hashlib.sha256(value).hexdigest() def _safe_preflight(result: dict[str, Any], revision: str) -> dict[str, Any] | None: checks = result.get("checks") digest = str(result.get("preflightSha256") or "") if ( result.get("status") != "preflight_ok" or result.get("apiVersion") != _API_VERSION or result.get("repository") != _TARGET or result.get("revision") != revision or not _SHA256.fullmatch(digest) or not _SHA256.fullmatch(str(result.get("backupInputsSha256") or "")) or not isinstance(checks, dict) or set(checks) != _CHECKS or any(checks.get(key) != "passed" for key in _CHECKS) ): return None return { key: result[key] for key in _SAFE_PREFLIGHT_FIELDS if key in result } def _serialize( row: WorkMeshPromotion, *, idempotent_replay: bool = False, approval_status: str | None = None, ) -> dict[str, Any]: return { "promotionId": row.id, "jobId": row.id, "revision": row.revision, "status": row.status, "stage": row.stage, "preflight": dict(row.preflight_json or {}), "approvalId": row.approval_id, "approvalStatus": approval_status, "request": row.broker_request_id, "evidence": dict(row.evidence_json or {}), "result": row.result_summary, "auditId": row.audit_event_id, "createdAt": row.created_at.isoformat(), "updatedAt": row.updated_at.isoformat(), "completedAt": row.completed_at.isoformat() if row.completed_at else None, "idempotentReplay": idempotent_replay, } def _record_preflight_error(result: dict[str, Any]) -> str: # Never surface arbitrary controller response text or its transport details. status = str(result.get("status") or "unavailable") if status in {"unreachable", "unavailable", "unknown", "missing_config", "missing_secret"}: return "The fixed WorkMesh controller preflight is unavailable." if status in {"preflight_failed", "preflight_changed", "deployment_policy_rejected"}: return "The fixed WorkMesh controller rejected the requested revision preflight." return "The controller response did not satisfy the WorkMesh promotion contract." class WorkMeshPromotionService: def __init__(self, *, broker: DeploymentBrokerClient | None = None): self.broker = broker or DeploymentBrokerClient() def preflight( self, session: Session, *, revision: str, idempotency_key: str, ) -> dict[str, Any]: if not _REVISION.fullmatch(revision): raise WorkMeshPromotionError("revision must be a full lowercase 40-character Git SHA.") key_digest = _idempotency_digest(idempotency_key) existing = session.exec( select(WorkMeshPromotion).where( WorkMeshPromotion.idempotency_sha256 == key_digest ) ).first() if existing: if existing.revision != revision: raise WorkMeshPromotionError( "Idempotency-Key was already used for a different WorkMesh revision." ) if existing.approval_id: approval = session.get(Approval, existing.approval_id) return _serialize( existing, idempotent_replay=True, approval_status=approval.status if approval else None, ) raise WorkMeshPromotionError( "The original preflight request is incomplete and requires operator review." ) try: secret = get_secret(session, _SECRET_REF, touch=False) except Exception: secret = None if not secret: raise WorkMeshPromotionError( "The fixed WorkMesh deployment-controller secret is not configured." ) try: result = self.broker.workmesh_promotion_preflight(revision, secret=secret) except Exception as exc: raise WorkMeshPromotionError( "The fixed WorkMesh controller preflight is unavailable." ) from exc if not isinstance(result, dict): result = {"status": "invalid_response"} safe = _safe_preflight(result, revision) if safe is None: raise WorkMeshPromotionError(_record_preflight_error(result)) promotion_id = f"workmesh-prom-{uuid.uuid4().hex}" context = { "apiVersion": _API_VERSION, "promotionId": promotion_id, "repository": _TARGET, "revision": revision, "preflightSha256": safe["preflightSha256"], "backupInputsSha256": safe["backupInputsSha256"], "preflight": safe, } approval = Approval( id=f"appr-{uuid.uuid4().hex[:12]}", action=_ACTION, target=_TARGET, source="manual", risk="high", status="pending", requester="operator", reason=( f"Promote WorkMesh revision {revision} after preflight digest " f"{safe['preflightSha256']}." ), expires_at=utc_now() + timedelta(minutes=30), request_context_json=context, action_fingerprint=approval_fingerprint( _ACTION, _TARGET, "manual", "high", context ), ) row = WorkMeshPromotion( id=promotion_id, revision=revision, preflight_sha256=safe["preflightSha256"], idempotency_sha256=key_digest, status="approval_required", stage="preflight_accepted", preflight_json=safe, approval_id=approval.id, ) # The approval and its immutable preflight binding become durable together. session.add(row) session.add(approval) session.commit() session.refresh(row) audit = log_event( session, source="deployment", identity="operator", action_id=_ACTION, target=_TARGET, target_type="approval", risk="high", policy_decision="approval_required", reason="WorkMesh promotion preflight passed for the exact revision.", result="approval_requested", execution_plan={ "steps": [ "Verify exact protected-main revision and push validation", "Run fixed WorkMesh controller preflight", "Bind operator approval to the immutable preflight digest", "Verify a PostgreSQL and n8n backup with isolated restore evidence before apply", ], "promotionId": row.id, "revision": revision, "preflightSha256": row.preflight_sha256, "backupInputsSha256": safe["backupInputsSha256"], "approvalId": approval.id, }, redacted_metadata={"promotionId": row.id, "approvalId": approval.id}, ) row.audit_event_id = audit.id row.updated_at = utc_now() session.add(row) session.commit() session.refresh(row) return _serialize(row, approval_status=approval.status) def execute_approved(self, session: Session, approval: Approval) -> dict[str, Any]: context = dict(approval.request_context_json or {}) promotion_id = str(context.get("promotionId") or "") row = session.get(WorkMeshPromotion, promotion_id) if ( approval.action != _ACTION or approval.target != _TARGET or approval.source != "manual" or approval.risk != "high" or approval.status != "approved" or (approval.expires_at is not None and approval.expires_at <= utc_now()) or context.get("apiVersion") != _API_VERSION or not row or row.approval_id != approval.id or row.revision != context.get("revision") or row.preflight_sha256 != context.get("preflightSha256") or dict(row.preflight_json or {}).get("backupInputsSha256") != context.get("backupInputsSha256") ): return { "success": False, "status": "failed", "result": "blocked", "message": "Approval is not bound to a valid fixed WorkMesh promotion preflight.", } if row.status != "approval_required": # The apply boundary is one-shot. This also closes the crash window # after persisting `submitting` but before learning the HTTP outcome. return self._approval_execution_result(row) try: secret = get_secret(session, _SECRET_REF, touch=False) except Exception: secret = None if not secret: return { "success": False, "status": "failed", "result": "blocked", "message": "The fixed WorkMesh deployment-controller secret is unavailable.", } try: current = self.broker.workmesh_promotion_preflight(row.revision, secret=secret) except Exception: current = {"status": "unavailable"} if not isinstance(current, dict): current = {"status": "invalid_response"} safe = _safe_preflight(current, row.revision) if ( safe is None or safe["preflightSha256"] != row.preflight_sha256 or safe["backupInputsSha256"] != dict(row.preflight_json or {}).get("backupInputsSha256") ): now = utc_now() claim = session.exec( update(WorkMeshPromotion) .where( (WorkMeshPromotion.id == row.id) & (WorkMeshPromotion.status == "approval_required") ) .values( status="preflight_changed", stage="approval_invalidated", evidence_json={"preflightChanged": True}, result_summary="The controller preflight changed after approval; no promotion was queued.", completed_at=now, updated_at=now, ) ) if claim.rowcount != 1: session.rollback() session.refresh(row) return self._approval_execution_result(row) approval.status = "expired" approval.decided_at = now session.add(approval) session.commit() session.refresh(row) audit = log_event( session, source="approval", identity="operator", action_id=_ACTION, target=_TARGET, target_type="approval", risk="high", policy_decision="approval_required", reason="WorkMesh preflight changed after operator approval; promotion was not queued.", result="blocked", redacted_metadata={"promotionId": row.id, "revision": row.revision}, ) row.audit_event_id = audit.id row.updated_at = utc_now() session.add(row) session.commit() return { "success": False, "status": row.status, "result": "preflight_changed", "message": row.result_summary, "jobId": row.id, "auditId": audit.id, } idempotency_key = f"workmesh-promotion:{row.id}" request_id = _request_id(row.revision, idempotency_key) # This durable marker is committed before the only mutating controller # request. Re-entry reconciles status and never repeats apply. now = utc_now() claim = session.exec( update(WorkMeshPromotion) .where( (WorkMeshPromotion.id == row.id) & (WorkMeshPromotion.status == "approval_required") ) .values( status="submitting", stage="controller_request_submitted", broker_request_id=request_id, updated_at=now, ) ) session.commit() if claim.rowcount != 1: session.refresh(row) return self._approval_execution_result(row) session.refresh(row) audit = log_event( session, source="approval", identity="operator", action_id=_ACTION, target=_TARGET, target_type="approval", risk="high", policy_decision="approval_required", reason="Submitting the exact approved WorkMesh revision to the fixed controller target.", result="pending", execution_plan={ "promotionId": row.id, "revision": row.revision, "preflightSha256": row.preflight_sha256, "backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"), "controllerRequest": request_id, }, redacted_metadata={"promotionId": row.id, "approvalId": approval.id}, ) row.audit_event_id = audit.id session.add(row) session.commit() try: result = self.broker.workmesh_promotion_apply( row.revision, secret=secret, preflight_sha256=row.preflight_sha256, idempotency_key=idempotency_key, ) except Exception: result = {"status": "unavailable"} if not isinstance(result, dict): result = {"status": "invalid_response"} return self._accept_apply_result(session, row, result, request_id) def _accept_apply_result( self, session: Session, row: WorkMeshPromotion, result: dict[str, Any], request_id: str, ) -> dict[str, Any]: identity_matches = ( result.get("request") == request_id and result.get("repository") == _TARGET and result.get("revision") == row.revision and result.get("operation") == "workmesh_promote" ) status = str(result.get("status") or "unknown") if not identity_matches: self._mark_outcome_unknown(session, row) return self._approval_execution_result(row) if ( status == "preflight_changed" and result.get("reason") == "approved_preflight_no_longer_matches" ): now = utc_now() current_digest = str(result.get("preflightSha256") or "") row.status = "preflight_changed" row.stage = "approval_invalidated" row.evidence_json = { "preflightChanged": True, **( {"currentPreflightSha256": current_digest} if _SHA256.fullmatch(current_digest) else {} ), } row.result_summary = ( "Controller preflight changed before queueing; no deployment was started and approval expired." ) row.completed_at = now row.updated_at = now approval = session.get(Approval, row.approval_id) if row.approval_id else None if approval and approval.status == "approved": approval.status = "expired" approval.decided_at = now session.add(approval) session.add(row) audit = log_event( session, source="approval", identity="operator", action_id=_ACTION, target=_TARGET, target_type="approval", risk="high", policy_decision="approval_required", reason=( "Controller preflight changed before queueing; no deployment was started " "and the approval was expired." ), result="blocked", execution_plan={ "promotionId": row.id, "revision": row.revision, "approvedPreflightSha256": row.preflight_sha256, "controllerRequest": request_id, }, redacted_metadata={"promotionId": row.id, "approvalId": row.approval_id}, ) row.audit_event_id = audit.id row.updated_at = utc_now() session.add(row) session.commit() return self._approval_execution_result(row) if ( status == "deployed" and _SHA256.fullmatch(str(result.get("receiptSha256") or "")) and _SHA256.fullmatch(str(result.get("backupReceiptSha256") or "")) ): row.status = "succeeded" row.stage = "receipt_verified" row.evidence_json = { "repository": _TARGET, "revision": row.revision, "operation": "workmesh_promote", "receiptSha256": result["receiptSha256"], "backupReceiptSha256": result["backupReceiptSha256"], } row.result_summary = "The exact WorkMesh revision has a verified controller receipt." row.completed_at = utc_now() elif status in {"accepted", "already_queued", "pending", "running"}: row.status = "applying" row.stage = "controller_accepted" row.evidence_json = { "repository": _TARGET, "revision": row.revision, "operation": "workmesh_promote", "backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"), } row.result_summary = "The fixed controller accepted the WorkMesh promotion; reconcile its receipt." elif status == "failed" and result.get("reason") in { "controller_rejected_before_mutation", "workmesh_preflight_changed", }: row.status = "preflight_changed" row.stage = "approval_invalidated" row.evidence_json = {"preflightChanged": True, "noMutation": True} row.result_summary = ( "The controller rejected the approved backup preflight before deployment; approval expired." ) approval = session.get(Approval, row.approval_id) if row.approval_id else None if approval and approval.status == "approved": approval.status = "expired" approval.decided_at = utc_now() session.add(approval) row.completed_at = utc_now() audit = log_event( session, source="approval", identity="operator", action_id=_ACTION, target=_TARGET, target_type="approval", risk="high", policy_decision="approval_required", reason="Controller rejected the approved WorkMesh backup preflight before mutation.", result="blocked", execution_plan={ "promotionId": row.id, "revision": row.revision, "preflightSha256": row.preflight_sha256, "backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"), "controllerRequest": request_id, }, redacted_metadata={"promotionId": row.id, "approvalId": row.approval_id}, ) row.audit_event_id = audit.id elif status == "failed": row.status = "failed" row.stage = "controller_failed" row.evidence_json = {"repository": _TARGET, "revision": row.revision} row.result_summary = "The fixed controller recorded a failed WorkMesh promotion." row.completed_at = utc_now() else: self._mark_outcome_unknown(session, row) return self._approval_execution_result(row) row.updated_at = utc_now() session.add(row) session.commit() return self._approval_execution_result(row) @staticmethod def _mark_outcome_unknown(session: Session, row: WorkMeshPromotion) -> None: row.status = "outcome_unknown" row.stage = "reconciliation_required" row.evidence_json = {"reconciliationRequired": True} row.result_summary = ( "Controller acceptance is ambiguous. Reconcile the fixed request status; do not resubmit." ) row.updated_at = utc_now() session.add(row) session.commit() @staticmethod def _approval_execution_result(row: WorkMeshPromotion) -> dict[str, Any]: # Once apply might have crossed the network boundary, success here means # the approval was consumed and cannot be executed a second time. return { "success": row.status != "preflight_changed", "status": row.status, "result": row.status, "message": row.result_summary or "WorkMesh promotion request was recorded.", "jobId": row.id, "auditId": row.audit_event_id or "", } def refresh(self, session: Session, promotion_id: str) -> dict[str, Any] | None: row = session.get(WorkMeshPromotion, promotion_id) if not row: return None if row.status in _TERMINAL or not row.broker_request_id: approval = session.get(Approval, row.approval_id) if row.approval_id else None return _serialize(row, approval_status=approval.status if approval else None) try: secret = get_secret(session, _SECRET_REF, touch=False) except Exception: secret = None if not secret: approval = session.get(Approval, row.approval_id) if row.approval_id else None return _serialize(row, approval_status=approval.status if approval else None) try: result = self.broker.workmesh_promotion_status( row.broker_request_id, secret=secret ) except Exception: result = {"status": "unavailable"} if not isinstance(result, dict): result = {"status": "unavailable"} matches = ( result.get("request") == row.broker_request_id and result.get("repository") == _TARGET and result.get("revision") == row.revision and result.get("operation") == "workmesh_promote" ) observed_status = str(result.get("status") or "unknown") previous = row.status if matches and observed_status == "deployed": receipt_sha256 = str(result.get("receiptSha256") or "") backup_receipt_sha256 = str(result.get("backupReceiptSha256") or "") if _SHA256.fullmatch(receipt_sha256) and _SHA256.fullmatch(backup_receipt_sha256): row.status = "succeeded" row.stage = "receipt_verified" row.evidence_json = { "repository": _TARGET, "revision": row.revision, "operation": "workmesh_promote", "receiptSha256": receipt_sha256, "backupReceiptSha256": backup_receipt_sha256, } row.result_summary = "The exact WorkMesh revision has a verified controller receipt." row.completed_at = utc_now() elif matches and observed_status in {"pending", "running", "accepted", "already_queued"}: row.status = "applying" row.stage = "controller_running" row.evidence_json = { "repository": _TARGET, "revision": row.revision, "operation": "workmesh_promote", "backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"), } elif matches and observed_status == "failed" and result.get("reason") in { "controller_rejected_before_mutation", "workmesh_preflight_changed", }: row.status = "preflight_changed" row.stage = "approval_invalidated" row.evidence_json = {"preflightChanged": True, "noMutation": True} row.result_summary = ( "The controller rejected the approved backup preflight before deployment; approval expired." ) approval = session.get(Approval, row.approval_id) if row.approval_id else None if approval and approval.status == "approved": approval.status = "expired" approval.decided_at = utc_now() session.add(approval) row.completed_at = utc_now() elif matches and observed_status == "failed": row.status = "failed" row.stage = "controller_failed" row.evidence_json = {"repository": _TARGET, "revision": row.revision} row.result_summary = "The fixed controller recorded a failed WorkMesh promotion." row.completed_at = utc_now() else: # Not found, transport failure, missing receipt, or identity drift # cannot prove that it is safe to submit again. row.status = "outcome_unknown" row.stage = "reconciliation_required" row.evidence_json = {"reconciliationRequired": True} row.result_summary = ( "Controller acceptance is not verified. Keep reconciling this request; do not resubmit." ) row.updated_at = utc_now() session.add(row) session.commit() session.refresh(row) if row.status != previous: audit = log_event( session, source="deployment", identity="operator", action_id=_ACTION, target=_TARGET, target_type="deployment", risk="high", policy_decision="approval_required", reason="Reconciled the fixed WorkMesh controller request without resubmission.", result="success" if row.status == "succeeded" else row.status, execution_plan={"promotionId": row.id, "revision": row.revision}, redacted_metadata={"promotionId": row.id, "controllerStatus": observed_status}, ) row.audit_event_id = audit.id row.updated_at = utc_now() session.add(row) session.commit() approval = session.get(Approval, row.approval_id) if row.approval_id else None return _serialize(row, approval_status=approval.status if approval else None) @staticmethod def list_promotions(session: Session, *, limit: int = 20) -> list[dict[str, Any]]: rows = session.exec( select(WorkMeshPromotion) .order_by(WorkMeshPromotion.created_at.desc()) .limit(max(1, min(limit, 50))) ).all() # Listing is a durable local read. Reconciliation is an explicit GET by # promotion id so page loads never create an unbounded controller poll. result = [] for row in rows: approval = session.get(Approval, row.approval_id) if row.approval_id else None result.append(_serialize(row, approval_status=approval.status if approval else None)) return result