Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,722 @@
|
||||
"""Approval-bound promotion for the single registered WorkMesh product target."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import update
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import Approval, WorkMeshPromotion
|
||||
from .approval_service import approval_fingerprint
|
||||
from .audit_logger import log_event
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
from .secret_store import get_secret
|
||||
|
||||
_ACTION = "deployment.target.promote_workmesh"
|
||||
_TARGET = "jens/workmesh"
|
||||
_SECRET_REF = "WORKMESH_DEPLOYMENT_BROKER"
|
||||
_API_VERSION = "itworx.workmesh-promotion/v2"
|
||||
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
_CHECKS = {"exactMain", "pushValidation", "controllerPreflight", "dataBackup"}
|
||||
_SAFE_PREFLIGHT_FIELDS = {
|
||||
"apiVersion",
|
||||
"status",
|
||||
"repository",
|
||||
"revision",
|
||||
"preflightSha256",
|
||||
"backupInputsSha256",
|
||||
"checks",
|
||||
}
|
||||
_TERMINAL = {"succeeded", "failed", "preflight_changed"}
|
||||
|
||||
|
||||
class WorkMeshPromotionError(ValueError):
|
||||
"""A fail-closed error in the fixed-target promotion contract."""
|
||||
|
||||
|
||||
def _idempotency_digest(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not _IDEMPOTENCY.fullmatch(normalized):
|
||||
raise WorkMeshPromotionError("Idempotency-Key must contain 8 to 128 safe characters.")
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _request_id(revision: str, idempotency_key: str) -> str:
|
||||
value = json.dumps(
|
||||
{
|
||||
"repository": _TARGET,
|
||||
"revision": revision,
|
||||
"operation": "workmesh_promote",
|
||||
"idempotency": idempotency_key,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _safe_preflight(result: dict[str, Any], revision: str) -> dict[str, Any] | None:
|
||||
checks = result.get("checks")
|
||||
digest = str(result.get("preflightSha256") or "")
|
||||
if (
|
||||
result.get("status") != "preflight_ok"
|
||||
or result.get("apiVersion") != _API_VERSION
|
||||
or result.get("repository") != _TARGET
|
||||
or result.get("revision") != revision
|
||||
or not _SHA256.fullmatch(digest)
|
||||
or not _SHA256.fullmatch(str(result.get("backupInputsSha256") or ""))
|
||||
or not isinstance(checks, dict)
|
||||
or set(checks) != _CHECKS
|
||||
or any(checks.get(key) != "passed" for key in _CHECKS)
|
||||
):
|
||||
return None
|
||||
return {
|
||||
key: result[key]
|
||||
for key in _SAFE_PREFLIGHT_FIELDS
|
||||
if key in result
|
||||
}
|
||||
|
||||
|
||||
def _serialize(
|
||||
row: WorkMeshPromotion,
|
||||
*,
|
||||
idempotent_replay: bool = False,
|
||||
approval_status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"promotionId": row.id,
|
||||
"jobId": row.id,
|
||||
"revision": row.revision,
|
||||
"status": row.status,
|
||||
"stage": row.stage,
|
||||
"preflight": dict(row.preflight_json or {}),
|
||||
"approvalId": row.approval_id,
|
||||
"approvalStatus": approval_status,
|
||||
"request": row.broker_request_id,
|
||||
"evidence": dict(row.evidence_json or {}),
|
||||
"result": row.result_summary,
|
||||
"auditId": row.audit_event_id,
|
||||
"createdAt": row.created_at.isoformat(),
|
||||
"updatedAt": row.updated_at.isoformat(),
|
||||
"completedAt": row.completed_at.isoformat() if row.completed_at else None,
|
||||
"idempotentReplay": idempotent_replay,
|
||||
}
|
||||
|
||||
|
||||
def _record_preflight_error(result: dict[str, Any]) -> str:
|
||||
# Never surface arbitrary controller response text or its transport details.
|
||||
status = str(result.get("status") or "unavailable")
|
||||
if status in {"unreachable", "unavailable", "unknown", "missing_config", "missing_secret"}:
|
||||
return "The fixed WorkMesh controller preflight is unavailable."
|
||||
if status in {"preflight_failed", "preflight_changed", "deployment_policy_rejected"}:
|
||||
return "The fixed WorkMesh controller rejected the requested revision preflight."
|
||||
return "The controller response did not satisfy the WorkMesh promotion contract."
|
||||
|
||||
|
||||
class WorkMeshPromotionService:
|
||||
def __init__(self, *, broker: DeploymentBrokerClient | None = None):
|
||||
self.broker = broker or DeploymentBrokerClient()
|
||||
|
||||
def preflight(
|
||||
self,
|
||||
session: Session,
|
||||
*,
|
||||
revision: str,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
if not _REVISION.fullmatch(revision):
|
||||
raise WorkMeshPromotionError("revision must be a full lowercase 40-character Git SHA.")
|
||||
key_digest = _idempotency_digest(idempotency_key)
|
||||
existing = session.exec(
|
||||
select(WorkMeshPromotion).where(
|
||||
WorkMeshPromotion.idempotency_sha256 == key_digest
|
||||
)
|
||||
).first()
|
||||
if existing:
|
||||
if existing.revision != revision:
|
||||
raise WorkMeshPromotionError(
|
||||
"Idempotency-Key was already used for a different WorkMesh revision."
|
||||
)
|
||||
if existing.approval_id:
|
||||
approval = session.get(Approval, existing.approval_id)
|
||||
return _serialize(
|
||||
existing,
|
||||
idempotent_replay=True,
|
||||
approval_status=approval.status if approval else None,
|
||||
)
|
||||
raise WorkMeshPromotionError(
|
||||
"The original preflight request is incomplete and requires operator review."
|
||||
)
|
||||
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception:
|
||||
secret = None
|
||||
if not secret:
|
||||
raise WorkMeshPromotionError(
|
||||
"The fixed WorkMesh deployment-controller secret is not configured."
|
||||
)
|
||||
try:
|
||||
result = self.broker.workmesh_promotion_preflight(revision, secret=secret)
|
||||
except Exception as exc:
|
||||
raise WorkMeshPromotionError(
|
||||
"The fixed WorkMesh controller preflight is unavailable."
|
||||
) from exc
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "invalid_response"}
|
||||
safe = _safe_preflight(result, revision)
|
||||
if safe is None:
|
||||
raise WorkMeshPromotionError(_record_preflight_error(result))
|
||||
|
||||
promotion_id = f"workmesh-prom-{uuid.uuid4().hex}"
|
||||
context = {
|
||||
"apiVersion": _API_VERSION,
|
||||
"promotionId": promotion_id,
|
||||
"repository": _TARGET,
|
||||
"revision": revision,
|
||||
"preflightSha256": safe["preflightSha256"],
|
||||
"backupInputsSha256": safe["backupInputsSha256"],
|
||||
"preflight": safe,
|
||||
}
|
||||
approval = Approval(
|
||||
id=f"appr-{uuid.uuid4().hex[:12]}",
|
||||
action=_ACTION,
|
||||
target=_TARGET,
|
||||
source="manual",
|
||||
risk="high",
|
||||
status="pending",
|
||||
requester="operator",
|
||||
reason=(
|
||||
f"Promote WorkMesh revision {revision} after preflight digest "
|
||||
f"{safe['preflightSha256']}."
|
||||
),
|
||||
expires_at=utc_now() + timedelta(minutes=30),
|
||||
request_context_json=context,
|
||||
action_fingerprint=approval_fingerprint(
|
||||
_ACTION, _TARGET, "manual", "high", context
|
||||
),
|
||||
)
|
||||
row = WorkMeshPromotion(
|
||||
id=promotion_id,
|
||||
revision=revision,
|
||||
preflight_sha256=safe["preflightSha256"],
|
||||
idempotency_sha256=key_digest,
|
||||
status="approval_required",
|
||||
stage="preflight_accepted",
|
||||
preflight_json=safe,
|
||||
approval_id=approval.id,
|
||||
)
|
||||
# The approval and its immutable preflight binding become durable together.
|
||||
session.add(row)
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="deployment",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="WorkMesh promotion preflight passed for the exact revision.",
|
||||
result="approval_requested",
|
||||
execution_plan={
|
||||
"steps": [
|
||||
"Verify exact protected-main revision and push validation",
|
||||
"Run fixed WorkMesh controller preflight",
|
||||
"Bind operator approval to the immutable preflight digest",
|
||||
"Verify a PostgreSQL and n8n backup with isolated restore evidence before apply",
|
||||
],
|
||||
"promotionId": row.id,
|
||||
"revision": revision,
|
||||
"preflightSha256": row.preflight_sha256,
|
||||
"backupInputsSha256": safe["backupInputsSha256"],
|
||||
"approvalId": approval.id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": approval.id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
return _serialize(row, approval_status=approval.status)
|
||||
|
||||
def execute_approved(self, session: Session, approval: Approval) -> dict[str, Any]:
|
||||
context = dict(approval.request_context_json or {})
|
||||
promotion_id = str(context.get("promotionId") or "")
|
||||
row = session.get(WorkMeshPromotion, promotion_id)
|
||||
if (
|
||||
approval.action != _ACTION
|
||||
or approval.target != _TARGET
|
||||
or approval.source != "manual"
|
||||
or approval.risk != "high"
|
||||
or approval.status != "approved"
|
||||
or (approval.expires_at is not None and approval.expires_at <= utc_now())
|
||||
or context.get("apiVersion") != _API_VERSION
|
||||
or not row
|
||||
or row.approval_id != approval.id
|
||||
or row.revision != context.get("revision")
|
||||
or row.preflight_sha256 != context.get("preflightSha256")
|
||||
or dict(row.preflight_json or {}).get("backupInputsSha256")
|
||||
!= context.get("backupInputsSha256")
|
||||
):
|
||||
return {
|
||||
"success": False,
|
||||
"status": "failed",
|
||||
"result": "blocked",
|
||||
"message": "Approval is not bound to a valid fixed WorkMesh promotion preflight.",
|
||||
}
|
||||
|
||||
if row.status != "approval_required":
|
||||
# The apply boundary is one-shot. This also closes the crash window
|
||||
# after persisting `submitting` but before learning the HTTP outcome.
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception:
|
||||
secret = None
|
||||
if not secret:
|
||||
return {
|
||||
"success": False,
|
||||
"status": "failed",
|
||||
"result": "blocked",
|
||||
"message": "The fixed WorkMesh deployment-controller secret is unavailable.",
|
||||
}
|
||||
try:
|
||||
current = self.broker.workmesh_promotion_preflight(row.revision, secret=secret)
|
||||
except Exception:
|
||||
current = {"status": "unavailable"}
|
||||
if not isinstance(current, dict):
|
||||
current = {"status": "invalid_response"}
|
||||
safe = _safe_preflight(current, row.revision)
|
||||
if (
|
||||
safe is None
|
||||
or safe["preflightSha256"] != row.preflight_sha256
|
||||
or safe["backupInputsSha256"]
|
||||
!= dict(row.preflight_json or {}).get("backupInputsSha256")
|
||||
):
|
||||
now = utc_now()
|
||||
claim = session.exec(
|
||||
update(WorkMeshPromotion)
|
||||
.where(
|
||||
(WorkMeshPromotion.id == row.id)
|
||||
& (WorkMeshPromotion.status == "approval_required")
|
||||
)
|
||||
.values(
|
||||
status="preflight_changed",
|
||||
stage="approval_invalidated",
|
||||
evidence_json={"preflightChanged": True},
|
||||
result_summary="The controller preflight changed after approval; no promotion was queued.",
|
||||
completed_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if claim.rowcount != 1:
|
||||
session.rollback()
|
||||
session.refresh(row)
|
||||
return self._approval_execution_result(row)
|
||||
approval.status = "expired"
|
||||
approval.decided_at = now
|
||||
session.add(approval)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="WorkMesh preflight changed after operator approval; promotion was not queued.",
|
||||
result="blocked",
|
||||
redacted_metadata={"promotionId": row.id, "revision": row.revision},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return {
|
||||
"success": False,
|
||||
"status": row.status,
|
||||
"result": "preflight_changed",
|
||||
"message": row.result_summary,
|
||||
"jobId": row.id,
|
||||
"auditId": audit.id,
|
||||
}
|
||||
|
||||
idempotency_key = f"workmesh-promotion:{row.id}"
|
||||
request_id = _request_id(row.revision, idempotency_key)
|
||||
# This durable marker is committed before the only mutating controller
|
||||
# request. Re-entry reconciles status and never repeats apply.
|
||||
now = utc_now()
|
||||
claim = session.exec(
|
||||
update(WorkMeshPromotion)
|
||||
.where(
|
||||
(WorkMeshPromotion.id == row.id)
|
||||
& (WorkMeshPromotion.status == "approval_required")
|
||||
)
|
||||
.values(
|
||||
status="submitting",
|
||||
stage="controller_request_submitted",
|
||||
broker_request_id=request_id,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
if claim.rowcount != 1:
|
||||
session.refresh(row)
|
||||
return self._approval_execution_result(row)
|
||||
session.refresh(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="Submitting the exact approved WorkMesh revision to the fixed controller target.",
|
||||
result="pending",
|
||||
execution_plan={
|
||||
"promotionId": row.id,
|
||||
"revision": row.revision,
|
||||
"preflightSha256": row.preflight_sha256,
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
"controllerRequest": request_id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": approval.id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
try:
|
||||
result = self.broker.workmesh_promotion_apply(
|
||||
row.revision,
|
||||
secret=secret,
|
||||
preflight_sha256=row.preflight_sha256,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
except Exception:
|
||||
result = {"status": "unavailable"}
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "invalid_response"}
|
||||
return self._accept_apply_result(session, row, result, request_id)
|
||||
|
||||
def _accept_apply_result(
|
||||
self,
|
||||
session: Session,
|
||||
row: WorkMeshPromotion,
|
||||
result: dict[str, Any],
|
||||
request_id: str,
|
||||
) -> dict[str, Any]:
|
||||
identity_matches = (
|
||||
result.get("request") == request_id
|
||||
and result.get("repository") == _TARGET
|
||||
and result.get("revision") == row.revision
|
||||
and result.get("operation") == "workmesh_promote"
|
||||
)
|
||||
status = str(result.get("status") or "unknown")
|
||||
if not identity_matches:
|
||||
self._mark_outcome_unknown(session, row)
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
if (
|
||||
status == "preflight_changed"
|
||||
and result.get("reason") == "approved_preflight_no_longer_matches"
|
||||
):
|
||||
now = utc_now()
|
||||
current_digest = str(result.get("preflightSha256") or "")
|
||||
row.status = "preflight_changed"
|
||||
row.stage = "approval_invalidated"
|
||||
row.evidence_json = {
|
||||
"preflightChanged": True,
|
||||
**(
|
||||
{"currentPreflightSha256": current_digest}
|
||||
if _SHA256.fullmatch(current_digest)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
row.result_summary = (
|
||||
"Controller preflight changed before queueing; no deployment was started and approval expired."
|
||||
)
|
||||
row.completed_at = now
|
||||
row.updated_at = now
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
if approval and approval.status == "approved":
|
||||
approval.status = "expired"
|
||||
approval.decided_at = now
|
||||
session.add(approval)
|
||||
session.add(row)
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason=(
|
||||
"Controller preflight changed before queueing; no deployment was started "
|
||||
"and the approval was expired."
|
||||
),
|
||||
result="blocked",
|
||||
execution_plan={
|
||||
"promotionId": row.id,
|
||||
"revision": row.revision,
|
||||
"approvedPreflightSha256": row.preflight_sha256,
|
||||
"controllerRequest": request_id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": row.approval_id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
if (
|
||||
status == "deployed"
|
||||
and _SHA256.fullmatch(str(result.get("receiptSha256") or ""))
|
||||
and _SHA256.fullmatch(str(result.get("backupReceiptSha256") or ""))
|
||||
):
|
||||
row.status = "succeeded"
|
||||
row.stage = "receipt_verified"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"receiptSha256": result["receiptSha256"],
|
||||
"backupReceiptSha256": result["backupReceiptSha256"],
|
||||
}
|
||||
row.result_summary = "The exact WorkMesh revision has a verified controller receipt."
|
||||
row.completed_at = utc_now()
|
||||
elif status in {"accepted", "already_queued", "pending", "running"}:
|
||||
row.status = "applying"
|
||||
row.stage = "controller_accepted"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
}
|
||||
row.result_summary = "The fixed controller accepted the WorkMesh promotion; reconcile its receipt."
|
||||
elif status == "failed" and result.get("reason") in {
|
||||
"controller_rejected_before_mutation",
|
||||
"workmesh_preflight_changed",
|
||||
}:
|
||||
row.status = "preflight_changed"
|
||||
row.stage = "approval_invalidated"
|
||||
row.evidence_json = {"preflightChanged": True, "noMutation": True}
|
||||
row.result_summary = (
|
||||
"The controller rejected the approved backup preflight before deployment; approval expired."
|
||||
)
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
if approval and approval.status == "approved":
|
||||
approval.status = "expired"
|
||||
approval.decided_at = utc_now()
|
||||
session.add(approval)
|
||||
row.completed_at = utc_now()
|
||||
audit = log_event(
|
||||
session,
|
||||
source="approval",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="approval",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="Controller rejected the approved WorkMesh backup preflight before mutation.",
|
||||
result="blocked",
|
||||
execution_plan={
|
||||
"promotionId": row.id,
|
||||
"revision": row.revision,
|
||||
"preflightSha256": row.preflight_sha256,
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
"controllerRequest": request_id,
|
||||
},
|
||||
redacted_metadata={"promotionId": row.id, "approvalId": row.approval_id},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
elif status == "failed":
|
||||
row.status = "failed"
|
||||
row.stage = "controller_failed"
|
||||
row.evidence_json = {"repository": _TARGET, "revision": row.revision}
|
||||
row.result_summary = "The fixed controller recorded a failed WorkMesh promotion."
|
||||
row.completed_at = utc_now()
|
||||
else:
|
||||
self._mark_outcome_unknown(session, row)
|
||||
return self._approval_execution_result(row)
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
return self._approval_execution_result(row)
|
||||
|
||||
@staticmethod
|
||||
def _mark_outcome_unknown(session: Session, row: WorkMeshPromotion) -> None:
|
||||
row.status = "outcome_unknown"
|
||||
row.stage = "reconciliation_required"
|
||||
row.evidence_json = {"reconciliationRequired": True}
|
||||
row.result_summary = (
|
||||
"Controller acceptance is ambiguous. Reconcile the fixed request status; do not resubmit."
|
||||
)
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
def _approval_execution_result(row: WorkMeshPromotion) -> dict[str, Any]:
|
||||
# Once apply might have crossed the network boundary, success here means
|
||||
# the approval was consumed and cannot be executed a second time.
|
||||
return {
|
||||
"success": row.status != "preflight_changed",
|
||||
"status": row.status,
|
||||
"result": row.status,
|
||||
"message": row.result_summary or "WorkMesh promotion request was recorded.",
|
||||
"jobId": row.id,
|
||||
"auditId": row.audit_event_id or "",
|
||||
}
|
||||
|
||||
def refresh(self, session: Session, promotion_id: str) -> dict[str, Any] | None:
|
||||
row = session.get(WorkMeshPromotion, promotion_id)
|
||||
if not row:
|
||||
return None
|
||||
if row.status in _TERMINAL or not row.broker_request_id:
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
return _serialize(row, approval_status=approval.status if approval else None)
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception:
|
||||
secret = None
|
||||
if not secret:
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
return _serialize(row, approval_status=approval.status if approval else None)
|
||||
try:
|
||||
result = self.broker.workmesh_promotion_status(
|
||||
row.broker_request_id, secret=secret
|
||||
)
|
||||
except Exception:
|
||||
result = {"status": "unavailable"}
|
||||
if not isinstance(result, dict):
|
||||
result = {"status": "unavailable"}
|
||||
|
||||
matches = (
|
||||
result.get("request") == row.broker_request_id
|
||||
and result.get("repository") == _TARGET
|
||||
and result.get("revision") == row.revision
|
||||
and result.get("operation") == "workmesh_promote"
|
||||
)
|
||||
observed_status = str(result.get("status") or "unknown")
|
||||
previous = row.status
|
||||
if matches and observed_status == "deployed":
|
||||
receipt_sha256 = str(result.get("receiptSha256") or "")
|
||||
backup_receipt_sha256 = str(result.get("backupReceiptSha256") or "")
|
||||
if _SHA256.fullmatch(receipt_sha256) and _SHA256.fullmatch(backup_receipt_sha256):
|
||||
row.status = "succeeded"
|
||||
row.stage = "receipt_verified"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"receiptSha256": receipt_sha256,
|
||||
"backupReceiptSha256": backup_receipt_sha256,
|
||||
}
|
||||
row.result_summary = "The exact WorkMesh revision has a verified controller receipt."
|
||||
row.completed_at = utc_now()
|
||||
elif matches and observed_status in {"pending", "running", "accepted", "already_queued"}:
|
||||
row.status = "applying"
|
||||
row.stage = "controller_running"
|
||||
row.evidence_json = {
|
||||
"repository": _TARGET,
|
||||
"revision": row.revision,
|
||||
"operation": "workmesh_promote",
|
||||
"backupInputsSha256": dict(row.preflight_json or {}).get("backupInputsSha256"),
|
||||
}
|
||||
elif matches and observed_status == "failed" and result.get("reason") in {
|
||||
"controller_rejected_before_mutation",
|
||||
"workmesh_preflight_changed",
|
||||
}:
|
||||
row.status = "preflight_changed"
|
||||
row.stage = "approval_invalidated"
|
||||
row.evidence_json = {"preflightChanged": True, "noMutation": True}
|
||||
row.result_summary = (
|
||||
"The controller rejected the approved backup preflight before deployment; approval expired."
|
||||
)
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
if approval and approval.status == "approved":
|
||||
approval.status = "expired"
|
||||
approval.decided_at = utc_now()
|
||||
session.add(approval)
|
||||
row.completed_at = utc_now()
|
||||
elif matches and observed_status == "failed":
|
||||
row.status = "failed"
|
||||
row.stage = "controller_failed"
|
||||
row.evidence_json = {"repository": _TARGET, "revision": row.revision}
|
||||
row.result_summary = "The fixed controller recorded a failed WorkMesh promotion."
|
||||
row.completed_at = utc_now()
|
||||
else:
|
||||
# Not found, transport failure, missing receipt, or identity drift
|
||||
# cannot prove that it is safe to submit again.
|
||||
row.status = "outcome_unknown"
|
||||
row.stage = "reconciliation_required"
|
||||
row.evidence_json = {"reconciliationRequired": True}
|
||||
row.result_summary = (
|
||||
"Controller acceptance is not verified. Keep reconciling this request; do not resubmit."
|
||||
)
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
session.refresh(row)
|
||||
if row.status != previous:
|
||||
audit = log_event(
|
||||
session,
|
||||
source="deployment",
|
||||
identity="operator",
|
||||
action_id=_ACTION,
|
||||
target=_TARGET,
|
||||
target_type="deployment",
|
||||
risk="high",
|
||||
policy_decision="approval_required",
|
||||
reason="Reconciled the fixed WorkMesh controller request without resubmission.",
|
||||
result="success" if row.status == "succeeded" else row.status,
|
||||
execution_plan={"promotionId": row.id, "revision": row.revision},
|
||||
redacted_metadata={"promotionId": row.id, "controllerStatus": observed_status},
|
||||
)
|
||||
row.audit_event_id = audit.id
|
||||
row.updated_at = utc_now()
|
||||
session.add(row)
|
||||
session.commit()
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
return _serialize(row, approval_status=approval.status if approval else None)
|
||||
|
||||
@staticmethod
|
||||
def list_promotions(session: Session, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
rows = session.exec(
|
||||
select(WorkMeshPromotion)
|
||||
.order_by(WorkMeshPromotion.created_at.desc())
|
||||
.limit(max(1, min(limit, 50)))
|
||||
).all()
|
||||
# Listing is a durable local read. Reconciliation is an explicit GET by
|
||||
# promotion id so page loads never create an unbounded controller poll.
|
||||
result = []
|
||||
for row in rows:
|
||||
approval = session.get(Approval, row.approval_id) if row.approval_id else None
|
||||
result.append(_serialize(row, approval_status=approval.status if approval else None))
|
||||
return result
|
||||
Reference in new issue
Block a user