520 lines
24 KiB
Python
520 lines
24 KiB
Python
"""Deployment Control Plane V1 orchestration.
|
|
|
|
The service is intentionally a bounded state machine. It validates a typed
|
|
plan, records durable idempotent jobs, and talks only to dedicated API clients.
|
|
There is no free-form command, Compose content, Docker socket or host path
|
|
input anywhere on the public API.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import uuid
|
|
from typing import Any, Literal
|
|
|
|
from sqlmodel import Session, select
|
|
|
|
from ..connectors.authentik import AuthentikConnector
|
|
from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..connectors.unraid import UnraidConnector
|
|
from ..core.time import utc_now
|
|
from ..db.models import DeploymentJob, Policy
|
|
from ..schemas.deployment import DeploymentPlanV1
|
|
from .audit_logger import log_event
|
|
from .deployment_broker import DeploymentBrokerClient
|
|
from .policy_engine import evaluate_policy
|
|
from .secret_store import get_secret, has_secret, secret_storage_status
|
|
|
|
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
|
_TERMINAL = {"preflight_failed", "succeeded", "failed", "rolled_back", "rollback_failed"}
|
|
_BROKER_EVIDENCE_FIELDS = {"request", "repository", "revision", "status"}
|
|
_OIDC_EVIDENCE_FIELDS = {"success", "status", "applicationId", "providerId", "clientSecretCreated"}
|
|
_PROXY_EVIDENCE_FIELDS = {"success", "status", "hostId", "hostname"}
|
|
_CANONICAL_HUB_REPOSITORY = "jens/itworx_mcp_hub"
|
|
|
|
|
|
class DeploymentControlError(ValueError):
|
|
pass
|
|
|
|
|
|
def _canonical_plan(plan: DeploymentPlanV1) -> tuple[dict[str, Any], str]:
|
|
value = plan.model_dump(mode="json")
|
|
digest = hashlib.sha256(
|
|
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
return value, digest
|
|
|
|
|
|
def _idempotency_digest(value: str) -> str:
|
|
if not _IDEMPOTENCY_KEY.fullmatch(value):
|
|
raise DeploymentControlError("idempotency key must contain 8 to 128 safe characters")
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _check(name: str, status: Literal["passed", "blocked", "warning", "unknown"], detail: str) -> dict[str, str]:
|
|
return {"name": name, "status": status, "detail": detail}
|
|
|
|
|
|
class DeploymentControlPlane:
|
|
"""Durable orchestrator for strict DeploymentPlanV1 requests."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
broker: DeploymentBrokerClient | None = None,
|
|
portainer: PortainerConnector | None = None,
|
|
npm: NginxProxyManagerConnector | None = None,
|
|
authentik: AuthentikConnector | None = None,
|
|
unraid: UnraidConnector | None = None,
|
|
):
|
|
self.broker = broker or DeploymentBrokerClient()
|
|
self.portainer = portainer or PortainerConnector()
|
|
self.npm = npm or NginxProxyManagerConnector()
|
|
self.authentik = authentik or AuthentikConnector()
|
|
self.unraid = unraid or UnraidConnector()
|
|
|
|
def preflight(
|
|
self,
|
|
session: Session,
|
|
plan: DeploymentPlanV1,
|
|
*,
|
|
operation: Literal["preflight", "apply", "rollback"] = "apply",
|
|
) -> dict[str, Any]:
|
|
checks: list[dict[str, str]] = []
|
|
policy_action = "deployment.rollback" if operation == "rollback" else "deployment.apply"
|
|
policy = evaluate_policy(
|
|
session,
|
|
action=policy_action,
|
|
target=plan.application_id,
|
|
source="manual",
|
|
)
|
|
exact_policy = session.exec(
|
|
select(Policy).where(
|
|
(Policy.action == policy_action) & (Policy.target == plan.application_id)
|
|
)
|
|
).first()
|
|
if policy.get("decision") == "allowed" and exact_policy is not None:
|
|
checks.append(_check("deployment_policy", "passed", "Exact-target deployment policy is present."))
|
|
else:
|
|
checks.append(_check("deployment_policy", "blocked", "Deployment requires an explicit allowed exact-target policy."))
|
|
if has_secret(session, plan.broker_secret_ref):
|
|
checks.append(_check("broker_secret_reference", "passed", "Stored controller secret is available by reference."))
|
|
else:
|
|
checks.append(_check("broker_secret_reference", "blocked", "The referenced deployment-broker secret is not configured."))
|
|
|
|
containers = self.portainer.list_containers()
|
|
if containers.get("status") != "ok":
|
|
checks.append(_check("portainer_inventory", "blocked", "Portainer inventory is unavailable for collision preflight."))
|
|
else:
|
|
names = {str(item.get("name") or "").casefold() for item in containers.get("containers", [])}
|
|
collisions = sorted(name for name in plan.topology.containers if name.casefold() in names)
|
|
# Existing expected containers are not unsafe by themselves: an exact
|
|
# controller receipt decides whether an apply is an upgrade or a
|
|
# conflicting takeover.
|
|
detail = "No managed container name is currently present." if not collisions else "Existing containers require controller ownership verification: " + ", ".join(collisions)
|
|
checks.append(_check("container_ownership", "warning" if collisions else "passed", detail))
|
|
|
|
if plan.proxy:
|
|
hosts = self.npm.proxy_hosts()
|
|
if not hosts.get("success"):
|
|
checks.append(_check("proxy_hostname", "blocked", "Nginx Proxy Manager inventory is unavailable."))
|
|
else:
|
|
matches = [
|
|
item
|
|
for item in hosts.get("items", [])
|
|
if plan.proxy.hostname in {str(value).lower() for value in item.get("domainNames", [])}
|
|
]
|
|
if not matches:
|
|
checks.append(_check("proxy_hostname", "passed", "Requested hostname is free in the NPM inventory."))
|
|
elif all(
|
|
str(item.get("forwardHost") or "") == plan.application_id
|
|
and int(item.get("forwardPort") or 0) == plan.topology.internal_port
|
|
for item in matches
|
|
):
|
|
checks.append(_check("proxy_hostname", "passed", "Existing hostname already targets the planned managed container."))
|
|
else:
|
|
checks.append(_check("proxy_hostname", "blocked", "Requested hostname is owned by a different NPM target."))
|
|
|
|
if plan.oidc:
|
|
auth_status = self.authentik.test_connection()
|
|
if auth_status.get("status") != "connected":
|
|
checks.append(_check("oidc_provider", "blocked", "Authentik is unavailable for the requested OIDC plan."))
|
|
else:
|
|
apps = self.authentik.applications()
|
|
if not apps.get("success"):
|
|
checks.append(_check("oidc_provider", "blocked", "Authentik applications cannot be inspected."))
|
|
else:
|
|
found = [item for item in apps.get("items", []) if item.get("slug") == plan.oidc.application_slug]
|
|
checks.append(_check("oidc_provider", "warning" if found else "passed", "Existing Authentik application will be ownership-checked." if found else "OIDC application can be provisioned from the bounded template."))
|
|
storage = secret_storage_status()
|
|
if not (storage.get("productionReady") or storage.get("status") == "ready"):
|
|
checks.append(_check("oidc_secret_reference", "blocked", "Encrypted secret storage is not ready for the referenced OIDC client secret."))
|
|
elif has_secret(session, plan.oidc.client_secret_ref):
|
|
checks.append(_check("oidc_secret_reference", "passed", "The write-only OIDC client secret reference is available for the bounded provider setup."))
|
|
else:
|
|
checks.append(_check("oidc_secret_reference", "blocked", "The referenced OIDC client secret is not configured in encrypted secret storage."))
|
|
|
|
shares = self.unraid.get_share_summary()
|
|
if shares.get("status") in {"ok", "connected"}:
|
|
checks.append(_check("trusted_appdata_root", "passed", f"Appdata path is derived as {plan.appdata_path}."))
|
|
else:
|
|
checks.append(_check("trusted_appdata_root", "warning", f"Appdata is derived as {plan.appdata_path}; Unraid share inventory is currently unavailable."))
|
|
|
|
broker_secret = get_secret(session, plan.broker_secret_ref, touch=False)
|
|
if broker_secret:
|
|
result = self.broker.preflight(plan, secret=broker_secret)
|
|
status = str(result.get("status") or "unknown")
|
|
if status in {"ok", "preflight_ok", "already_deployed"}:
|
|
checks.append(_check("deployment_controller", "passed", "Controller accepted immutable repository and topology preflight."))
|
|
elif status in {"missing_config", "missing_secret"}:
|
|
checks.append(_check("deployment_controller", "blocked", "Deployment controller is not configured for the typed API."))
|
|
else:
|
|
checks.append(_check("deployment_controller", "blocked", "Controller rejected typed deployment preflight."))
|
|
else:
|
|
checks.append(_check("deployment_controller", "blocked", "Controller authentication secret is unavailable."))
|
|
|
|
blockers = [item["name"] for item in checks if item["status"] == "blocked"]
|
|
warnings = [item["name"] for item in checks if item["status"] == "warning"]
|
|
return {
|
|
"status": "blocked" if blockers else "passed",
|
|
"blockers": blockers,
|
|
"warnings": warnings,
|
|
"checks": checks,
|
|
"appdataPath": plan.appdata_path,
|
|
"publicUrl": plan.public_url,
|
|
}
|
|
|
|
def create_job(
|
|
self,
|
|
session: Session,
|
|
*,
|
|
plan: DeploymentPlanV1,
|
|
idempotency_key: str,
|
|
operation: Literal["preflight", "apply", "rollback"],
|
|
) -> DeploymentJob:
|
|
plan_json, plan_digest = _canonical_plan(plan)
|
|
idempotency_digest = _idempotency_digest(f"{operation}:{idempotency_key}")
|
|
existing = session.exec(
|
|
select(DeploymentJob).where(DeploymentJob.idempotency_sha256 == idempotency_digest)
|
|
).first()
|
|
if existing:
|
|
if existing.plan_sha256 != plan_digest:
|
|
raise DeploymentControlError("idempotency key was already used for a different deployment plan")
|
|
return existing
|
|
job = DeploymentJob(
|
|
id=f"deploy-{uuid.uuid4().hex[:16]}",
|
|
application_id=plan.application_id,
|
|
repository=plan.repository,
|
|
revision=plan.revision,
|
|
plan_sha256=plan_digest,
|
|
idempotency_sha256=idempotency_digest,
|
|
status="queued",
|
|
stage="accepted",
|
|
plan_json=plan_json,
|
|
evidence_json={"operation": operation},
|
|
)
|
|
session.add(job)
|
|
session.commit()
|
|
session.refresh(job)
|
|
return job
|
|
|
|
@staticmethod
|
|
def serialize_job(job: DeploymentJob, *, replay: bool = False) -> dict[str, Any]:
|
|
return {
|
|
"id": job.id,
|
|
"applicationId": job.application_id,
|
|
"repository": job.repository,
|
|
"revision": job.revision,
|
|
"status": job.status,
|
|
"stage": job.stage,
|
|
"preflight": job.preflight_json,
|
|
"evidence": job.evidence_json,
|
|
"result": job.result_summary,
|
|
"brokerRequestId": job.broker_request_id,
|
|
"createdAt": job.created_at.isoformat(),
|
|
"updatedAt": job.updated_at.isoformat(),
|
|
"completedAt": job.completed_at.isoformat() if job.completed_at else None,
|
|
"idempotentReplay": replay,
|
|
}
|
|
|
|
def _complete_external_integrations(
|
|
self,
|
|
session: Session,
|
|
plan: DeploymentPlanV1,
|
|
) -> tuple[bool, dict[str, Any], str]:
|
|
"""Complete only plan-derived edge/OIDC integration after live proof.
|
|
|
|
Application environment secret delivery remains an explicit controller
|
|
enrollment responsibility. This code resolves the same write-only
|
|
reference only inside the backend to create/verify the bounded OIDC
|
|
provider, and never writes it to an API result or audit record.
|
|
"""
|
|
evidence: dict[str, Any] = {}
|
|
if plan.oidc:
|
|
client_secret = get_secret(session, plan.oidc.client_secret_ref, touch=False)
|
|
if not client_secret:
|
|
return False, evidence, "Referenced OIDC client secret is unavailable after deployment."
|
|
oidc_result = self.authentik.ensure_deployment_oidc_application(
|
|
application_slug=plan.oidc.application_slug,
|
|
launch_url=plan.public_url or "",
|
|
redirect_uri=f"{plan.public_url}{plan.oidc.redirect_path}",
|
|
client_secret=client_secret,
|
|
administrator_group=plan.oidc.roles.administrator_group,
|
|
support_agent_group=plan.oidc.roles.support_agent_group,
|
|
read_only_group=plan.oidc.roles.read_only_group,
|
|
)
|
|
evidence["oidc"] = {
|
|
key: oidc_result[key] for key in _OIDC_EVIDENCE_FIELDS if key in oidc_result
|
|
}
|
|
if not oidc_result.get("success") or oidc_result.get("status") not in {"created", "already_managed"}:
|
|
return False, evidence, "Typed OIDC provider/application setup was not verified."
|
|
if plan.proxy:
|
|
proxy_result = self.npm.ensure_managed_proxy_host(
|
|
hostname=plan.proxy.hostname,
|
|
forward_host=plan.application_id,
|
|
forward_port=plan.topology.internal_port,
|
|
certificate_mode=plan.proxy.certificate_mode,
|
|
force_ssl=plan.proxy.force_ssl,
|
|
http2=plan.proxy.http2,
|
|
upload_limit_mb=plan.proxy.upload_limit_mb,
|
|
read_timeout_seconds=plan.proxy.read_timeout_seconds,
|
|
)
|
|
evidence["proxy"] = {
|
|
key: proxy_result[key] for key in _PROXY_EVIDENCE_FIELDS if key in proxy_result
|
|
}
|
|
if not proxy_result.get("success") or proxy_result.get("status") not in {"created", "already_managed"}:
|
|
return False, evidence, "Typed reverse-proxy setup was not verified."
|
|
return True, evidence, "External integration is verified."
|
|
|
|
def _complete_apply_job(
|
|
self,
|
|
session: Session,
|
|
job: DeploymentJob,
|
|
plan: DeploymentPlanV1,
|
|
) -> None:
|
|
success, integration_evidence, message = self._complete_external_integrations(session, plan)
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"externalIntegration": integration_evidence,
|
|
}
|
|
if success:
|
|
job.status = "succeeded"
|
|
job.stage = "controller_and_edge_verified"
|
|
job.result_summary = "Deployment controller and required edge integration returned verified evidence."
|
|
job.completed_at = utc_now()
|
|
return
|
|
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"postApplyVerification": {"status": "failed", "detail": message},
|
|
}
|
|
job.status = "failed"
|
|
job.stage = "external_integration_failed"
|
|
if plan.repository.casefold() == _CANONICAL_HUB_REPOSITORY:
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"recovery": {
|
|
"status": "manual_required",
|
|
"reason": "hub_data_aware_recovery_required",
|
|
},
|
|
}
|
|
job.result_summary = (
|
|
f"{message} Automatic image-only rollback is disabled for MCP Hub; "
|
|
"data-aware recovery is required."
|
|
)
|
|
else:
|
|
job.result_summary = message
|
|
job.completed_at = utc_now()
|
|
|
|
def execute(
|
|
self,
|
|
session: Session,
|
|
*,
|
|
plan: DeploymentPlanV1,
|
|
idempotency_key: str,
|
|
operation: Literal["preflight", "apply", "rollback"],
|
|
) -> dict[str, Any]:
|
|
if (
|
|
operation in {"apply", "rollback"}
|
|
and plan.repository.casefold() == _CANONICAL_HUB_REPOSITORY
|
|
):
|
|
raise DeploymentControlError(
|
|
"MCP Hub apply and rollback require the signed protected push workflow"
|
|
)
|
|
job = self.create_job(session, plan=plan, idempotency_key=idempotency_key, operation=operation)
|
|
if job.status != "queued":
|
|
return self.serialize_job(job, replay=True)
|
|
|
|
preflight = self.preflight(session, plan, operation=operation)
|
|
job.preflight_json = preflight
|
|
job.updated_at = utc_now()
|
|
if preflight["status"] != "passed":
|
|
job.status = "preflight_failed"
|
|
job.stage = "preflight"
|
|
job.result_summary = "Deployment preflight was blocked."
|
|
job.completed_at = utc_now()
|
|
session.add(job)
|
|
session.commit()
|
|
log_event(
|
|
session,
|
|
source="deployment-control-plane",
|
|
identity="operator",
|
|
action_id=f"deployment.{operation}",
|
|
target=plan.application_id,
|
|
target_type="deployment",
|
|
risk="high",
|
|
policy_decision="blocked",
|
|
reason="Typed deployment preflight failed closed.",
|
|
result="blocked",
|
|
execution_plan={"planSha256": job.plan_sha256, "operation": operation},
|
|
post_checks={"preflight": preflight},
|
|
)
|
|
return self.serialize_job(job)
|
|
|
|
if operation == "preflight":
|
|
job.status = "preflight_succeeded"
|
|
job.stage = "preflight"
|
|
job.result_summary = "Typed deployment preflight passed."
|
|
job.completed_at = utc_now()
|
|
session.add(job)
|
|
session.commit()
|
|
return self.serialize_job(job)
|
|
|
|
secret = get_secret(session, plan.broker_secret_ref)
|
|
if not secret:
|
|
raise DeploymentControlError("deployment broker secret was unavailable after preflight")
|
|
broker_result = (
|
|
self.broker.apply(plan, secret=secret, idempotency_key=idempotency_key)
|
|
if operation == "apply"
|
|
else self.broker.rollback(plan, secret=secret, idempotency_key=idempotency_key)
|
|
)
|
|
broker_status = str(broker_result.get("status") or "failed")
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"brokerStatus": broker_status,
|
|
"brokerEvidence": {
|
|
key: broker_result[key] for key in _BROKER_EVIDENCE_FIELDS if key in broker_result
|
|
},
|
|
}
|
|
job.broker_request_id = str(broker_result.get("request") or "") or None
|
|
job.updated_at = utc_now()
|
|
if broker_status in {"accepted", "already_queued", "running"}:
|
|
job.status = "applying" if operation == "apply" else "rolling_back"
|
|
job.stage = "controller_queued"
|
|
job.result_summary = "Deployment controller accepted the typed request."
|
|
elif operation == "apply" and broker_status in {"deployed", "already_deployed"}:
|
|
self._complete_apply_job(session, job, plan)
|
|
elif operation == "rollback" and broker_status == "rolled_back":
|
|
job.status = "rolled_back"
|
|
job.stage = "rollback_verified"
|
|
job.result_summary = "Deployment controller returned verified rollback evidence."
|
|
job.completed_at = utc_now()
|
|
else:
|
|
job.status = "failed" if operation == "apply" else "rollback_failed"
|
|
job.stage = "controller_rejected"
|
|
job.result_summary = "Deployment controller rejected the typed request."
|
|
job.completed_at = utc_now()
|
|
session.add(job)
|
|
session.commit()
|
|
log_event(
|
|
session,
|
|
source="deployment-control-plane",
|
|
identity="operator",
|
|
action_id=f"deployment.{operation}",
|
|
target=plan.application_id,
|
|
target_type="deployment",
|
|
risk="high",
|
|
policy_decision="allowed" if job.status not in {"failed", "rollback_failed"} else "blocked",
|
|
reason="Typed deployment request sent to the private controller.",
|
|
result="success" if job.status not in {"failed", "rollback_failed"} else "failed",
|
|
execution_plan={"planSha256": job.plan_sha256, "operation": operation},
|
|
after_state={"status": job.status, "stage": job.stage, "brokerRequestId": job.broker_request_id},
|
|
redacted_metadata={"brokerStatus": broker_status},
|
|
)
|
|
return self.serialize_job(job)
|
|
|
|
def refresh(self, session: Session, job_id: str) -> dict[str, Any]:
|
|
job = session.get(DeploymentJob, job_id)
|
|
if not job:
|
|
raise DeploymentControlError("deployment job was not found")
|
|
if job.status in _TERMINAL or not job.broker_request_id:
|
|
return self.serialize_job(job)
|
|
plan = DeploymentPlanV1.model_validate(job.plan_json)
|
|
secret = get_secret(session, plan.broker_secret_ref)
|
|
if not secret:
|
|
job.status = "failed"
|
|
job.stage = "controller_auth_lost"
|
|
job.result_summary = "Deployment controller secret is no longer available."
|
|
job.completed_at = utc_now()
|
|
else:
|
|
result = self.broker.status(job.broker_request_id, secret=secret)
|
|
status = str(result.get("status") or "unknown")
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"brokerStatus": status,
|
|
"brokerEvidence": {
|
|
key: result[key] for key in _BROKER_EVIDENCE_FIELDS if key in result
|
|
},
|
|
}
|
|
if status in {"deployed", "already_deployed"}:
|
|
auto_rollback = dict(job.evidence_json or {}).get("autoRollback")
|
|
if job.status == "rolling_back":
|
|
job.status = "failed" if isinstance(auto_rollback, dict) else "rollback_failed"
|
|
job.stage = (
|
|
"auto_rollback_failed"
|
|
if isinstance(auto_rollback, dict)
|
|
else "controller_failed"
|
|
)
|
|
job.result_summary = (
|
|
"Rollback request returned a deployed state and was not accepted as rollback proof."
|
|
)
|
|
job.completed_at = utc_now()
|
|
else:
|
|
self._complete_apply_job(session, job, plan)
|
|
elif status == "rolled_back":
|
|
job.status = "rolled_back"
|
|
auto_rollback = dict(job.evidence_json or {}).get("autoRollback")
|
|
if isinstance(auto_rollback, dict):
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"autoRollback": {**auto_rollback, "status": "rolled_back"},
|
|
}
|
|
job.stage = "auto_rollback_verified"
|
|
job.result_summary = (
|
|
"Post-apply verification failed; the controller verified automatic rollback."
|
|
)
|
|
else:
|
|
job.stage = "rollback_verified"
|
|
job.result_summary = "Deployment controller verified rollback evidence."
|
|
job.completed_at = utc_now()
|
|
elif status in {"failed", "non_forward", "rejected"}:
|
|
auto_rollback = dict(job.evidence_json or {}).get("autoRollback")
|
|
if isinstance(auto_rollback, dict) and job.status == "rolling_back":
|
|
job.status = "failed"
|
|
job.stage = "auto_rollback_failed"
|
|
job.evidence_json = {
|
|
**dict(job.evidence_json or {}),
|
|
"autoRollback": {**auto_rollback, "status": status},
|
|
}
|
|
job.result_summary = (
|
|
"Post-apply verification failed and automatic rollback was not verified."
|
|
)
|
|
else:
|
|
job.status = "failed" if job.status == "applying" else "rollback_failed"
|
|
job.stage = "controller_failed"
|
|
job.result_summary = "Deployment controller reported a typed deployment failure."
|
|
job.completed_at = utc_now()
|
|
job.updated_at = utc_now()
|
|
session.add(job)
|
|
session.commit()
|
|
return self.serialize_job(job)
|
|
|
|
def list_jobs(self, session: Session, *, limit: int = 100) -> list[dict[str, Any]]:
|
|
rows = session.exec(
|
|
select(DeploymentJob).order_by(DeploymentJob.created_at.desc()).limit(max(1, min(limit, 250)))
|
|
).all()
|
|
return [self.serialize_job(row) for row in rows]
|