401 lines
13 KiB
Python
401 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
from sqlmodel import Session, SQLModel, create_engine
|
|
|
|
from backend.app.db.models import Policy
|
|
from backend.app.schemas.deployment import DeploymentPlanV1
|
|
from backend.app.services.deployment_control_plane import DeploymentControlPlane
|
|
from backend.app.services.secret_store import set_secret
|
|
|
|
|
|
def _plan() -> DeploymentPlanV1:
|
|
return DeploymentPlanV1.model_validate(
|
|
{
|
|
"application_id": "itworx-mcp-hub",
|
|
"repository": "Jens/ITWorx_MCP_Hub",
|
|
"revision": "a" * 40,
|
|
"release_version": "1.0.0",
|
|
"image_repository": "itworx-mcp-hub",
|
|
"image_tag": "1.0.0",
|
|
"topology": {
|
|
"compose_project": "itworx-mcp-hub",
|
|
"containers": ["itworx-mcp-hub"],
|
|
"proxy_network": "itworx-npm",
|
|
"internal_port": 8080,
|
|
},
|
|
"proxy": {"hostname": "mcp-test.example.com"},
|
|
"oidc": None,
|
|
"broker_secret_ref": "MCP_HUB_TEST_DEPLOYMENT_BROKER",
|
|
}
|
|
)
|
|
|
|
|
|
class _Broker:
|
|
def __init__(self, rollback_status: str = "rolled_back") -> None:
|
|
self.rollback_status = rollback_status
|
|
self.rollback_calls: list[str] = []
|
|
|
|
def preflight(self, plan, *, secret):
|
|
return {"status": "preflight_ok"}
|
|
|
|
def apply(self, plan, *, secret, idempotency_key):
|
|
return {"status": "deployed", "request": "a" * 64}
|
|
|
|
def rollback(self, plan, *, secret, idempotency_key):
|
|
self.rollback_calls.append(idempotency_key)
|
|
return {"status": self.rollback_status, "request": "b" * 64}
|
|
|
|
def status(self, request_id, *, secret):
|
|
return {"status": "rolled_back", "request": request_id}
|
|
|
|
|
|
class _Portainer:
|
|
def list_containers(self):
|
|
return {"status": "ok", "containers": []}
|
|
|
|
|
|
class _FailingNpm:
|
|
def proxy_hosts(self):
|
|
return {"success": True, "items": []}
|
|
|
|
def ensure_managed_proxy_host(self, **kwargs):
|
|
return {"success": False, "status": "failed"}
|
|
|
|
|
|
class _Unraid:
|
|
def get_share_summary(self):
|
|
return {"status": "ok"}
|
|
|
|
|
|
class _Authentik:
|
|
def test_connection(self):
|
|
return {"status": "connected"}
|
|
|
|
def applications(self):
|
|
return {"success": True, "items": []}
|
|
|
|
|
|
def _session(tmp_path) -> Session:
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'auto-rollback.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
return Session(engine)
|
|
|
|
|
|
def _policy(action: str) -> Policy:
|
|
return Policy(
|
|
id=f"allow-mcp-hub-{action.replace('.', '-')}",
|
|
action=action,
|
|
target="itworx-mcp-hub",
|
|
voice="blocked",
|
|
manual="allowed",
|
|
risk="high",
|
|
reason="Exact-target no-regression deployment test policy.",
|
|
)
|
|
|
|
|
|
def _plane(broker: _Broker) -> DeploymentControlPlane:
|
|
return DeploymentControlPlane(
|
|
broker=broker,
|
|
portainer=_Portainer(),
|
|
npm=_FailingNpm(),
|
|
authentik=_Authentik(),
|
|
unraid=_Unraid(),
|
|
)
|
|
|
|
|
|
def _configure(session: Session, *, rollback_policy: bool = True) -> None:
|
|
set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32)
|
|
session.add(_policy("deployment.apply"))
|
|
if rollback_policy:
|
|
session.add(_policy("deployment.rollback"))
|
|
session.commit()
|
|
|
|
|
|
def test_default_compose_file_survives_durable_plan_roundtrip():
|
|
plan = _plan()
|
|
restored = DeploymentPlanV1.model_validate(plan.model_dump(mode="json"))
|
|
assert restored.topology.compose_file == ".itworx/deployment.yml"
|
|
|
|
|
|
def test_post_apply_verification_failure_auto_rolls_back_with_exact_policy(tmp_path):
|
|
broker = _Broker("rolled_back")
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0001",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "rolled_back"
|
|
assert result["stage"] == "auto_rollback_verified"
|
|
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
|
assert result["evidence"]["autoRollback"]["status"] == "rolled_back"
|
|
assert broker.rollback_calls == [f"auto-rollback:{result['id']}"]
|
|
|
|
|
|
def test_post_apply_verification_failure_does_not_rollback_without_exact_policy(tmp_path):
|
|
broker = _Broker("rolled_back")
|
|
with _session(tmp_path) as session:
|
|
_configure(session, rollback_policy=False)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0002",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "external_integration_failed"
|
|
assert result["evidence"]["autoRollback"]["status"] == "blocked"
|
|
assert broker.rollback_calls == []
|
|
|
|
|
|
def test_post_apply_verification_failure_keeps_failure_when_controller_rejects_rollback(tmp_path):
|
|
broker = _Broker("rejected")
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0003",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "auto_rollback_failed"
|
|
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
|
assert result["evidence"]["autoRollback"]["status"] == "rejected"
|
|
assert len(broker.rollback_calls) == 1
|
|
|
|
|
|
def test_queued_auto_rollback_is_polled_and_finishes_as_auto_rollback(tmp_path):
|
|
broker = _Broker("accepted")
|
|
plane = _plane(broker)
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
initial = plane.execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0004",
|
|
operation="apply",
|
|
)
|
|
assert initial["status"] == "rolling_back"
|
|
assert initial["stage"] == "auto_rollback_queued"
|
|
|
|
finished = plane.refresh(session, initial["id"])
|
|
|
|
assert finished["status"] == "rolled_back"
|
|
assert finished["stage"] == "auto_rollback_verified"
|
|
assert finished["evidence"]["autoRollback"]["status"] == "rolled_back"
|
|
|
|
|
|
def test_successful_post_apply_verification_never_invokes_auto_rollback(tmp_path):
|
|
class _HealthyNpm(_FailingNpm):
|
|
def ensure_managed_proxy_host(self, **kwargs):
|
|
return {"success": True, "status": "created", "hostId": 42}
|
|
|
|
broker = _Broker("rolled_back")
|
|
plane = DeploymentControlPlane(
|
|
broker=broker,
|
|
portainer=_Portainer(),
|
|
npm=_HealthyNpm(),
|
|
authentik=_Authentik(),
|
|
unraid=_Unraid(),
|
|
)
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = plane.execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0005",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "succeeded"
|
|
assert result["stage"] == "controller_and_edge_verified"
|
|
assert broker.rollback_calls == []
|
|
|
|
def test_auto_rollback_is_scoped_to_exact_mcp_hub_target(tmp_path):
|
|
broker = _Broker("rolled_back")
|
|
plan = _plan().model_copy(update={"repository": "jens/other-repository"})
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=plan,
|
|
idempotency_key="mcp-hub-apply-0006",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "external_integration_failed"
|
|
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
|
assert "autoRollback" not in result["evidence"]
|
|
assert broker.rollback_calls == []
|
|
|
|
def test_auto_rollback_rejects_same_repository_with_different_application(tmp_path):
|
|
broker = _Broker("rolled_back")
|
|
plan = _plan().model_copy(
|
|
update={
|
|
"application_id": "itworx-mcp-hub-copy",
|
|
"topology": _plan().topology.model_copy(
|
|
update={
|
|
"compose_project": "itworx-mcp-hub-copy",
|
|
"containers": ("itworx-mcp-hub-copy",),
|
|
}
|
|
),
|
|
}
|
|
)
|
|
with _session(tmp_path) as session:
|
|
set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32)
|
|
session.add(
|
|
Policy(
|
|
id="allow-copy-apply",
|
|
action="deployment.apply",
|
|
target="itworx-mcp-hub-copy",
|
|
voice="blocked",
|
|
manual="allowed",
|
|
risk="high",
|
|
reason="Test-only exact target.",
|
|
)
|
|
)
|
|
session.add(
|
|
Policy(
|
|
id="allow-copy-rollback",
|
|
action="deployment.rollback",
|
|
target="itworx-mcp-hub-copy",
|
|
voice="blocked",
|
|
manual="allowed",
|
|
risk="high",
|
|
reason="Test-only exact target.",
|
|
)
|
|
)
|
|
session.commit()
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=plan,
|
|
idempotency_key="mcp-hub-apply-0007",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "external_integration_failed"
|
|
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
|
assert "autoRollback" not in result["evidence"]
|
|
assert broker.rollback_calls == []
|
|
|
|
def test_queued_auto_rollback_rejects_deployed_status_as_rollback_proof(tmp_path):
|
|
class _AmbiguousRollbackBroker(_Broker):
|
|
def status(self, request_id, *, secret):
|
|
return {"status": "deployed", "request": request_id}
|
|
|
|
broker = _AmbiguousRollbackBroker("accepted")
|
|
plane = _plane(broker)
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
initial = plane.execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0008",
|
|
operation="apply",
|
|
)
|
|
assert initial["status"] == "rolling_back"
|
|
|
|
finished = plane.refresh(session, initial["id"])
|
|
|
|
assert finished["status"] == "failed"
|
|
assert finished["stage"] == "auto_rollback_failed"
|
|
assert finished["evidence"]["autoRollback"]["status"] == "accepted"
|
|
assert "deployed state" in finished["result"]
|
|
|
|
def test_manual_rollback_requires_rolled_back_terminal_proof(tmp_path):
|
|
broker = _Broker("deployed")
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-rollback-0009",
|
|
operation="rollback",
|
|
)
|
|
|
|
assert result["status"] == "rollback_failed"
|
|
assert result["stage"] == "controller_rejected"
|
|
|
|
|
|
def test_apply_rejects_rolled_back_terminal_status(tmp_path):
|
|
class _RolledBackApplyBroker(_Broker):
|
|
def apply(self, plan, *, secret, idempotency_key):
|
|
return {"status": "rolled_back", "request": "c" * 64}
|
|
|
|
broker = _RolledBackApplyBroker()
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0010",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "controller_rejected"
|
|
assert broker.rollback_calls == []
|
|
|
|
def test_auto_rollback_refuses_duplicate_exact_rollback_policies(tmp_path):
|
|
broker = _Broker("rolled_back")
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
session.add(
|
|
Policy(
|
|
id="duplicate-rollback-policy",
|
|
action="deployment.rollback",
|
|
target="itworx-mcp-hub",
|
|
voice="blocked",
|
|
manual="allowed",
|
|
risk="high",
|
|
reason="Duplicate exact rollback policy.",
|
|
)
|
|
)
|
|
session.commit()
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0011",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "external_integration_failed"
|
|
assert result["evidence"]["autoRollback"] == {
|
|
"status": "blocked",
|
|
"reason": "exact-target-policy",
|
|
}
|
|
assert broker.rollback_calls == []
|
|
|
|
def test_auto_rollback_rejects_unpollable_accepted_response(tmp_path):
|
|
class _MissingRequestBroker(_Broker):
|
|
def rollback(self, plan, *, secret, idempotency_key):
|
|
self.rollback_calls.append(idempotency_key)
|
|
return {"status": "accepted"}
|
|
|
|
broker = _MissingRequestBroker()
|
|
with _session(tmp_path) as session:
|
|
_configure(session)
|
|
result = _plane(broker).execute(
|
|
session,
|
|
plan=_plan(),
|
|
idempotency_key="mcp-hub-apply-0012",
|
|
operation="apply",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["stage"] == "auto_rollback_failed"
|
|
assert result["evidence"]["autoRollback"] == {
|
|
"status": "accepted",
|
|
"reason": "missing-controller-request",
|
|
}
|
|
assert len(broker.rollback_calls) == 1
|