"""Deployment Control Plane V1 orchestration. The service is intentionally a bounded state machine. It validates a typed plan, records durable idempotent jobs, and talks only to dedicated API clients. There is no free-form command, Compose content, Docker socket or host path input anywhere on the public API. """ from __future__ import annotations import hashlib import json import re import uuid from typing import Any, Literal from sqlmodel import Session, select from ..connectors.authentik import AuthentikConnector from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector from ..connectors.portainer import PortainerConnector from ..connectors.unraid import UnraidConnector from ..core.time import utc_now from ..db.models import DeploymentJob, Policy from ..schemas.deployment import DeploymentPlanV1 from .audit_logger import log_event from .deployment_broker import DeploymentBrokerClient from .policy_engine import evaluate_policy from .secret_store import get_secret, has_secret, secret_storage_status _IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") _TERMINAL = {"preflight_failed", "succeeded", "failed", "rolled_back", "rollback_failed"} _BROKER_EVIDENCE_FIELDS = {"request", "repository", "revision", "status"} _OIDC_EVIDENCE_FIELDS = {"success", "status", "applicationId", "providerId", "clientSecretCreated"} _PROXY_EVIDENCE_FIELDS = {"success", "status", "hostId", "hostname"} _CANONICAL_HUB_REPOSITORY = "jens/itworx_mcp_hub" class DeploymentControlError(ValueError): pass def _canonical_plan(plan: DeploymentPlanV1) -> tuple[dict[str, Any], str]: value = plan.model_dump(mode="json") digest = hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() return value, digest def _idempotency_digest(value: str) -> str: if not _IDEMPOTENCY_KEY.fullmatch(value): raise DeploymentControlError("idempotency key must contain 8 to 128 safe characters") return hashlib.sha256(value.encode("utf-8")).hexdigest() def _check(name: str, status: Literal["passed", "blocked", "warning", "unknown"], detail: str) -> dict[str, str]: return {"name": name, "status": status, "detail": detail} class DeploymentControlPlane: """Durable orchestrator for strict DeploymentPlanV1 requests.""" def __init__( self, *, broker: DeploymentBrokerClient | None = None, portainer: PortainerConnector | None = None, npm: NginxProxyManagerConnector | None = None, authentik: AuthentikConnector | None = None, unraid: UnraidConnector | None = None, ): self.broker = broker or DeploymentBrokerClient() self.portainer = portainer or PortainerConnector() self.npm = npm or NginxProxyManagerConnector() self.authentik = authentik or AuthentikConnector() self.unraid = unraid or UnraidConnector() def preflight( self, session: Session, plan: DeploymentPlanV1, *, operation: Literal["preflight", "apply", "rollback"] = "apply", ) -> dict[str, Any]: checks: list[dict[str, str]] = [] policy_action = "deployment.rollback" if operation == "rollback" else "deployment.apply" policy = evaluate_policy( session, action=policy_action, target=plan.application_id, source="manual", ) exact_policy = session.exec( select(Policy).where( (Policy.action == policy_action) & (Policy.target == plan.application_id) ) ).first() if policy.get("decision") == "allowed" and exact_policy is not None: checks.append(_check("deployment_policy", "passed", "Exact-target deployment policy is present.")) else: checks.append(_check("deployment_policy", "blocked", "Deployment requires an explicit allowed exact-target policy.")) if has_secret(session, plan.broker_secret_ref): checks.append(_check("broker_secret_reference", "passed", "Stored controller secret is available by reference.")) else: checks.append(_check("broker_secret_reference", "blocked", "The referenced deployment-broker secret is not configured.")) containers = self.portainer.list_containers() if containers.get("status") != "ok": checks.append(_check("portainer_inventory", "blocked", "Portainer inventory is unavailable for collision preflight.")) else: names = {str(item.get("name") or "").casefold() for item in containers.get("containers", [])} collisions = sorted(name for name in plan.topology.containers if name.casefold() in names) # Existing expected containers are not unsafe by themselves: an exact # controller receipt decides whether an apply is an upgrade or a # conflicting takeover. detail = "No managed container name is currently present." if not collisions else "Existing containers require controller ownership verification: " + ", ".join(collisions) checks.append(_check("container_ownership", "warning" if collisions else "passed", detail)) if plan.proxy: hosts = self.npm.proxy_hosts() if not hosts.get("success"): checks.append(_check("proxy_hostname", "blocked", "Nginx Proxy Manager inventory is unavailable.")) else: matches = [ item for item in hosts.get("items", []) if plan.proxy.hostname in {str(value).lower() for value in item.get("domainNames", [])} ] if not matches: checks.append(_check("proxy_hostname", "passed", "Requested hostname is free in the NPM inventory.")) elif all( str(item.get("forwardHost") or "") == plan.application_id and int(item.get("forwardPort") or 0) == plan.topology.internal_port for item in matches ): checks.append(_check("proxy_hostname", "passed", "Existing hostname already targets the planned managed container.")) else: checks.append(_check("proxy_hostname", "blocked", "Requested hostname is owned by a different NPM target.")) if plan.oidc: auth_status = self.authentik.test_connection() if auth_status.get("status") != "connected": checks.append(_check("oidc_provider", "blocked", "Authentik is unavailable for the requested OIDC plan.")) else: apps = self.authentik.applications() if not apps.get("success"): checks.append(_check("oidc_provider", "blocked", "Authentik applications cannot be inspected.")) else: found = [item for item in apps.get("items", []) if item.get("slug") == plan.oidc.application_slug] checks.append(_check("oidc_provider", "warning" if found else "passed", "Existing Authentik application will be ownership-checked." if found else "OIDC application can be provisioned from the bounded template.")) storage = secret_storage_status() if not (storage.get("productionReady") or storage.get("status") == "ready"): checks.append(_check("oidc_secret_reference", "blocked", "Encrypted secret storage is not ready for the referenced OIDC client secret.")) elif has_secret(session, plan.oidc.client_secret_ref): checks.append(_check("oidc_secret_reference", "passed", "The write-only OIDC client secret reference is available for the bounded provider setup.")) else: checks.append(_check("oidc_secret_reference", "blocked", "The referenced OIDC client secret is not configured in encrypted secret storage.")) shares = self.unraid.get_share_summary() if shares.get("status") in {"ok", "connected"}: checks.append(_check("trusted_appdata_root", "passed", f"Appdata path is derived as {plan.appdata_path}.")) else: checks.append(_check("trusted_appdata_root", "warning", f"Appdata is derived as {plan.appdata_path}; Unraid share inventory is currently unavailable.")) broker_secret = get_secret(session, plan.broker_secret_ref, touch=False) if broker_secret: result = self.broker.preflight(plan, secret=broker_secret) status = str(result.get("status") or "unknown") if status in {"ok", "preflight_ok", "already_deployed"}: checks.append(_check("deployment_controller", "passed", "Controller accepted immutable repository and topology preflight.")) elif status in {"missing_config", "missing_secret"}: checks.append(_check("deployment_controller", "blocked", "Deployment controller is not configured for the typed API.")) else: checks.append(_check("deployment_controller", "blocked", "Controller rejected typed deployment preflight.")) else: checks.append(_check("deployment_controller", "blocked", "Controller authentication secret is unavailable.")) blockers = [item["name"] for item in checks if item["status"] == "blocked"] warnings = [item["name"] for item in checks if item["status"] == "warning"] return { "status": "blocked" if blockers else "passed", "blockers": blockers, "warnings": warnings, "checks": checks, "appdataPath": plan.appdata_path, "publicUrl": plan.public_url, } def create_job( self, session: Session, *, plan: DeploymentPlanV1, idempotency_key: str, operation: Literal["preflight", "apply", "rollback"], ) -> DeploymentJob: plan_json, plan_digest = _canonical_plan(plan) idempotency_digest = _idempotency_digest(f"{operation}:{idempotency_key}") existing = session.exec( select(DeploymentJob).where(DeploymentJob.idempotency_sha256 == idempotency_digest) ).first() if existing: if existing.plan_sha256 != plan_digest: raise DeploymentControlError("idempotency key was already used for a different deployment plan") return existing job = DeploymentJob( id=f"deploy-{uuid.uuid4().hex[:16]}", application_id=plan.application_id, repository=plan.repository, revision=plan.revision, plan_sha256=plan_digest, idempotency_sha256=idempotency_digest, status="queued", stage="accepted", plan_json=plan_json, evidence_json={"operation": operation}, ) session.add(job) session.commit() session.refresh(job) return job @staticmethod def serialize_job(job: DeploymentJob, *, replay: bool = False) -> dict[str, Any]: return { "id": job.id, "applicationId": job.application_id, "repository": job.repository, "revision": job.revision, "status": job.status, "stage": job.stage, "preflight": job.preflight_json, "evidence": job.evidence_json, "result": job.result_summary, "brokerRequestId": job.broker_request_id, "createdAt": job.created_at.isoformat(), "updatedAt": job.updated_at.isoformat(), "completedAt": job.completed_at.isoformat() if job.completed_at else None, "idempotentReplay": replay, } def _complete_external_integrations( self, session: Session, plan: DeploymentPlanV1, ) -> tuple[bool, dict[str, Any], str]: """Complete only plan-derived edge/OIDC integration after live proof. Application environment secret delivery remains an explicit controller enrollment responsibility. This code resolves the same write-only reference only inside the backend to create/verify the bounded OIDC provider, and never writes it to an API result or audit record. """ evidence: dict[str, Any] = {} if plan.oidc: client_secret = get_secret(session, plan.oidc.client_secret_ref, touch=False) if not client_secret: return False, evidence, "Referenced OIDC client secret is unavailable after deployment." oidc_result = self.authentik.ensure_deployment_oidc_application( application_slug=plan.oidc.application_slug, launch_url=plan.public_url or "", redirect_uri=f"{plan.public_url}{plan.oidc.redirect_path}", client_secret=client_secret, administrator_group=plan.oidc.roles.administrator_group, support_agent_group=plan.oidc.roles.support_agent_group, read_only_group=plan.oidc.roles.read_only_group, ) evidence["oidc"] = { key: oidc_result[key] for key in _OIDC_EVIDENCE_FIELDS if key in oidc_result } if not oidc_result.get("success") or oidc_result.get("status") not in {"created", "already_managed"}: return False, evidence, "Typed OIDC provider/application setup was not verified." if plan.proxy: proxy_result = self.npm.ensure_managed_proxy_host( hostname=plan.proxy.hostname, forward_host=plan.application_id, forward_port=plan.topology.internal_port, certificate_mode=plan.proxy.certificate_mode, force_ssl=plan.proxy.force_ssl, http2=plan.proxy.http2, upload_limit_mb=plan.proxy.upload_limit_mb, read_timeout_seconds=plan.proxy.read_timeout_seconds, ) evidence["proxy"] = { key: proxy_result[key] for key in _PROXY_EVIDENCE_FIELDS if key in proxy_result } if not proxy_result.get("success") or proxy_result.get("status") not in {"created", "already_managed"}: return False, evidence, "Typed reverse-proxy setup was not verified." return True, evidence, "External integration is verified." def _complete_apply_job( self, session: Session, job: DeploymentJob, plan: DeploymentPlanV1, ) -> None: success, integration_evidence, message = self._complete_external_integrations(session, plan) job.evidence_json = { **dict(job.evidence_json or {}), "externalIntegration": integration_evidence, } if success: job.status = "succeeded" job.stage = "controller_and_edge_verified" job.result_summary = "Deployment controller and required edge integration returned verified evidence." job.completed_at = utc_now() return job.evidence_json = { **dict(job.evidence_json or {}), "postApplyVerification": {"status": "failed", "detail": message}, } job.status = "failed" job.stage = "external_integration_failed" if plan.repository.casefold() == _CANONICAL_HUB_REPOSITORY: job.evidence_json = { **dict(job.evidence_json or {}), "recovery": { "status": "manual_required", "reason": "hub_data_aware_recovery_required", }, } job.result_summary = ( f"{message} Automatic image-only rollback is disabled for MCP Hub; " "data-aware recovery is required." ) else: job.result_summary = message job.completed_at = utc_now() def execute( self, session: Session, *, plan: DeploymentPlanV1, idempotency_key: str, operation: Literal["preflight", "apply", "rollback"], ) -> dict[str, Any]: if ( operation in {"apply", "rollback"} and plan.repository.casefold() == _CANONICAL_HUB_REPOSITORY ): raise DeploymentControlError( "MCP Hub apply and rollback require the signed protected push workflow" ) job = self.create_job(session, plan=plan, idempotency_key=idempotency_key, operation=operation) if job.status != "queued": return self.serialize_job(job, replay=True) preflight = self.preflight(session, plan, operation=operation) job.preflight_json = preflight job.updated_at = utc_now() if preflight["status"] != "passed": job.status = "preflight_failed" job.stage = "preflight" job.result_summary = "Deployment preflight was blocked." job.completed_at = utc_now() session.add(job) session.commit() log_event( session, source="deployment-control-plane", identity="operator", action_id=f"deployment.{operation}", target=plan.application_id, target_type="deployment", risk="high", policy_decision="blocked", reason="Typed deployment preflight failed closed.", result="blocked", execution_plan={"planSha256": job.plan_sha256, "operation": operation}, post_checks={"preflight": preflight}, ) return self.serialize_job(job) if operation == "preflight": job.status = "preflight_succeeded" job.stage = "preflight" job.result_summary = "Typed deployment preflight passed." job.completed_at = utc_now() session.add(job) session.commit() return self.serialize_job(job) secret = get_secret(session, plan.broker_secret_ref) if not secret: raise DeploymentControlError("deployment broker secret was unavailable after preflight") broker_result = ( self.broker.apply(plan, secret=secret, idempotency_key=idempotency_key) if operation == "apply" else self.broker.rollback(plan, secret=secret, idempotency_key=idempotency_key) ) broker_status = str(broker_result.get("status") or "failed") job.evidence_json = { **dict(job.evidence_json or {}), "brokerStatus": broker_status, "brokerEvidence": { key: broker_result[key] for key in _BROKER_EVIDENCE_FIELDS if key in broker_result }, } job.broker_request_id = str(broker_result.get("request") or "") or None job.updated_at = utc_now() if broker_status in {"accepted", "already_queued", "running"}: job.status = "applying" if operation == "apply" else "rolling_back" job.stage = "controller_queued" job.result_summary = "Deployment controller accepted the typed request." elif operation == "apply" and broker_status in {"deployed", "already_deployed"}: self._complete_apply_job(session, job, plan) elif operation == "rollback" and broker_status == "rolled_back": job.status = "rolled_back" job.stage = "rollback_verified" job.result_summary = "Deployment controller returned verified rollback evidence." job.completed_at = utc_now() else: job.status = "failed" if operation == "apply" else "rollback_failed" job.stage = "controller_rejected" job.result_summary = "Deployment controller rejected the typed request." job.completed_at = utc_now() session.add(job) session.commit() log_event( session, source="deployment-control-plane", identity="operator", action_id=f"deployment.{operation}", target=plan.application_id, target_type="deployment", risk="high", policy_decision="allowed" if job.status not in {"failed", "rollback_failed"} else "blocked", reason="Typed deployment request sent to the private controller.", result="success" if job.status not in {"failed", "rollback_failed"} else "failed", execution_plan={"planSha256": job.plan_sha256, "operation": operation}, after_state={"status": job.status, "stage": job.stage, "brokerRequestId": job.broker_request_id}, redacted_metadata={"brokerStatus": broker_status}, ) return self.serialize_job(job) def refresh(self, session: Session, job_id: str) -> dict[str, Any]: job = session.get(DeploymentJob, job_id) if not job: raise DeploymentControlError("deployment job was not found") if job.status in _TERMINAL or not job.broker_request_id: return self.serialize_job(job) plan = DeploymentPlanV1.model_validate(job.plan_json) secret = get_secret(session, plan.broker_secret_ref) if not secret: job.status = "failed" job.stage = "controller_auth_lost" job.result_summary = "Deployment controller secret is no longer available." job.completed_at = utc_now() else: result = self.broker.status(job.broker_request_id, secret=secret) status = str(result.get("status") or "unknown") job.evidence_json = { **dict(job.evidence_json or {}), "brokerStatus": status, "brokerEvidence": { key: result[key] for key in _BROKER_EVIDENCE_FIELDS if key in result }, } if status in {"deployed", "already_deployed"}: auto_rollback = dict(job.evidence_json or {}).get("autoRollback") if job.status == "rolling_back": job.status = "failed" if isinstance(auto_rollback, dict) else "rollback_failed" job.stage = ( "auto_rollback_failed" if isinstance(auto_rollback, dict) else "controller_failed" ) job.result_summary = ( "Rollback request returned a deployed state and was not accepted as rollback proof." ) job.completed_at = utc_now() else: self._complete_apply_job(session, job, plan) elif status == "rolled_back": job.status = "rolled_back" auto_rollback = dict(job.evidence_json or {}).get("autoRollback") if isinstance(auto_rollback, dict): job.evidence_json = { **dict(job.evidence_json or {}), "autoRollback": {**auto_rollback, "status": "rolled_back"}, } job.stage = "auto_rollback_verified" job.result_summary = ( "Post-apply verification failed; the controller verified automatic rollback." ) else: job.stage = "rollback_verified" job.result_summary = "Deployment controller verified rollback evidence." job.completed_at = utc_now() elif status in {"failed", "non_forward", "rejected"}: auto_rollback = dict(job.evidence_json or {}).get("autoRollback") if isinstance(auto_rollback, dict) and job.status == "rolling_back": job.status = "failed" job.stage = "auto_rollback_failed" job.evidence_json = { **dict(job.evidence_json or {}), "autoRollback": {**auto_rollback, "status": status}, } job.result_summary = ( "Post-apply verification failed and automatic rollback was not verified." ) else: job.status = "failed" if job.status == "applying" else "rollback_failed" job.stage = "controller_failed" job.result_summary = "Deployment controller reported a typed deployment failure." job.completed_at = utc_now() job.updated_at = utc_now() session.add(job) session.commit() return self.serialize_job(job) def list_jobs(self, session: Session, *, limit: int = 100) -> list[dict[str, Any]]: rows = session.exec( select(DeploymentJob).order_by(DeploymentJob.created_at.desc()).limit(max(1, min(limit, 250))) ).all() return [self.serialize_job(row) for row in rows]