Publish curated Unraid-AppOps-Gateway source from 2e53f628d284213b90ca2f9c5e121671f5933574
This commit is contained in:
Public Source Publisher committed
1 parent
4f9c5a1106
commit
2facb8de34
13 files changed
+3868
-1749
No files matched your search
@@ -1,10 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import Policy
|
||||
from backend.app.db.models import DeploymentJob, Policy
|
||||
from backend.app.schemas.deployment import DeploymentPlanV1
|
||||
from backend.app.services.deployment_control_plane import DeploymentControlPlane
|
||||
from backend.app.services.deployment_control_plane import (
|
||||
DeploymentControlError,
|
||||
DeploymentControlPlane,
|
||||
)
|
||||
from backend.app.services.secret_store import set_secret
|
||||
|
||||
|
||||
@@ -31,19 +35,22 @@ def _plan() -> DeploymentPlanV1:
|
||||
|
||||
|
||||
class _Broker:
|
||||
def __init__(self, rollback_status: str = "rolled_back") -> None:
|
||||
self.rollback_status = rollback_status
|
||||
def __init__(self) -> None:
|
||||
self.preflight_calls = 0
|
||||
self.apply_calls: list[str] = []
|
||||
self.rollback_calls: list[str] = []
|
||||
|
||||
def preflight(self, plan, *, secret):
|
||||
self.preflight_calls += 1
|
||||
return {"status": "preflight_ok"}
|
||||
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
self.apply_calls.append(idempotency_key)
|
||||
return {"status": "deployed", "request": "a" * 64}
|
||||
|
||||
def rollback(self, plan, *, secret, idempotency_key):
|
||||
self.rollback_calls.append(idempotency_key)
|
||||
return {"status": self.rollback_status, "request": "b" * 64}
|
||||
return {"status": "rolled_back", "request": "b" * 64}
|
||||
|
||||
def status(self, request_id, *, secret):
|
||||
return {"status": "rolled_back", "request": request_id}
|
||||
@@ -103,11 +110,10 @@ def _plane(broker: _Broker) -> DeploymentControlPlane:
|
||||
)
|
||||
|
||||
|
||||
def _configure(session: Session, *, rollback_policy: bool = True) -> None:
|
||||
def _configure(session: Session) -> None:
|
||||
set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32)
|
||||
session.add(_policy("deployment.apply"))
|
||||
if rollback_policy:
|
||||
session.add(_policy("deployment.rollback"))
|
||||
session.add(_policy("deployment.rollback"))
|
||||
session.commit()
|
||||
|
||||
|
||||
@@ -117,284 +123,75 @@ def test_default_compose_file_survives_durable_plan_roundtrip():
|
||||
assert restored.topology.compose_file == ".itworx/deployment.yml"
|
||||
|
||||
|
||||
def test_post_apply_verification_failure_auto_rolls_back_with_exact_policy(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
@pytest.mark.parametrize("operation", ["apply", "rollback"])
|
||||
def test_hub_typed_mutation_is_rejected_before_job_or_broker_queue(
|
||||
tmp_path, operation
|
||||
):
|
||||
broker = _Broker()
|
||||
plane = _plane(broker)
|
||||
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0001",
|
||||
operation="apply",
|
||||
)
|
||||
with pytest.raises(DeploymentControlError, match="signed protected push"):
|
||||
plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key=f"mcp-hub-{operation}-typed-0001",
|
||||
operation=operation,
|
||||
)
|
||||
|
||||
assert result["status"] == "rolled_back"
|
||||
assert result["stage"] == "auto_rollback_verified"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert result["evidence"]["autoRollback"]["status"] == "rolled_back"
|
||||
assert broker.rollback_calls == [f"auto-rollback:{result['id']}"]
|
||||
assert session.exec(select(DeploymentJob)).all() == []
|
||||
|
||||
|
||||
def test_post_apply_verification_failure_does_not_rollback_without_exact_policy(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session, rollback_policy=False)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0002",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["autoRollback"]["status"] == "blocked"
|
||||
assert broker.preflight_calls == 0
|
||||
assert broker.apply_calls == []
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
|
||||
def test_post_apply_verification_failure_keeps_failure_when_controller_rejects_rollback(tmp_path):
|
||||
broker = _Broker("rejected")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0003",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "auto_rollback_failed"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert result["evidence"]["autoRollback"]["status"] == "rejected"
|
||||
assert len(broker.rollback_calls) == 1
|
||||
|
||||
|
||||
def test_queued_auto_rollback_is_polled_and_finishes_as_auto_rollback(tmp_path):
|
||||
broker = _Broker("accepted")
|
||||
def test_hub_read_only_preflight_remains_available(tmp_path):
|
||||
broker = _Broker()
|
||||
plane = _plane(broker)
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
initial = plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0004",
|
||||
operation="apply",
|
||||
)
|
||||
assert initial["status"] == "rolling_back"
|
||||
assert initial["stage"] == "auto_rollback_queued"
|
||||
|
||||
finished = plane.refresh(session, initial["id"])
|
||||
|
||||
assert finished["status"] == "rolled_back"
|
||||
assert finished["stage"] == "auto_rollback_verified"
|
||||
assert finished["evidence"]["autoRollback"]["status"] == "rolled_back"
|
||||
|
||||
|
||||
def test_successful_post_apply_verification_never_invokes_auto_rollback(tmp_path):
|
||||
class _HealthyNpm(_FailingNpm):
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
return {"success": True, "status": "created", "hostId": 42}
|
||||
|
||||
broker = _Broker("rolled_back")
|
||||
plane = DeploymentControlPlane(
|
||||
broker=broker,
|
||||
portainer=_Portainer(),
|
||||
npm=_HealthyNpm(),
|
||||
authentik=_Authentik(),
|
||||
unraid=_Unraid(),
|
||||
)
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0005",
|
||||
operation="apply",
|
||||
idempotency_key="mcp-hub-preflight-0001",
|
||||
operation="preflight",
|
||||
)
|
||||
|
||||
assert result["status"] == "succeeded"
|
||||
assert result["stage"] == "controller_and_edge_verified"
|
||||
assert result["status"] == "preflight_succeeded"
|
||||
assert broker.preflight_calls == 1
|
||||
assert broker.apply_calls == []
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_is_scoped_to_exact_mcp_hub_target(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
plan = _plan().model_copy(update={"repository": "jens/other-repository"})
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=plan,
|
||||
idempotency_key="mcp-hub-apply-0006",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert "autoRollback" not in result["evidence"]
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_rejects_same_repository_with_different_application(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
plan = _plan().model_copy(
|
||||
update={
|
||||
"application_id": "itworx-mcp-hub-copy",
|
||||
"topology": _plan().topology.model_copy(
|
||||
update={
|
||||
"compose_project": "itworx-mcp-hub-copy",
|
||||
"containers": ("itworx-mcp-hub-copy",),
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32)
|
||||
session.add(
|
||||
Policy(
|
||||
id="allow-copy-apply",
|
||||
action="deployment.apply",
|
||||
target="itworx-mcp-hub-copy",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test-only exact target.",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
Policy(
|
||||
id="allow-copy-rollback",
|
||||
action="deployment.rollback",
|
||||
target="itworx-mcp-hub-copy",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test-only exact target.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=plan,
|
||||
idempotency_key="mcp-hub-apply-0007",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert "autoRollback" not in result["evidence"]
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_queued_auto_rollback_rejects_deployed_status_as_rollback_proof(tmp_path):
|
||||
class _AmbiguousRollbackBroker(_Broker):
|
||||
def status(self, request_id, *, secret):
|
||||
return {"status": "deployed", "request": request_id}
|
||||
|
||||
broker = _AmbiguousRollbackBroker("accepted")
|
||||
def test_hub_post_apply_failure_records_data_aware_recovery_without_rollback(tmp_path):
|
||||
broker = _Broker()
|
||||
plane = _plane(broker)
|
||||
plan = _plan()
|
||||
job = DeploymentJob(
|
||||
id="mcp-hub-existing-apply",
|
||||
application_id=plan.application_id,
|
||||
repository=plan.repository,
|
||||
revision=plan.revision,
|
||||
plan_sha256="a" * 64,
|
||||
idempotency_sha256="b" * 64,
|
||||
status="applying",
|
||||
plan_json=plan.model_dump(mode="json"),
|
||||
evidence_json={"brokerStatus": "deployed"},
|
||||
)
|
||||
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
initial = plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0008",
|
||||
operation="apply",
|
||||
)
|
||||
assert initial["status"] == "rolling_back"
|
||||
plane._complete_apply_job(session, job, plan)
|
||||
|
||||
finished = plane.refresh(session, initial["id"])
|
||||
|
||||
assert finished["status"] == "failed"
|
||||
assert finished["stage"] == "auto_rollback_failed"
|
||||
assert finished["evidence"]["autoRollback"]["status"] == "accepted"
|
||||
assert "deployed state" in finished["result"]
|
||||
|
||||
def test_manual_rollback_requires_rolled_back_terminal_proof(tmp_path):
|
||||
broker = _Broker("deployed")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-rollback-0009",
|
||||
operation="rollback",
|
||||
)
|
||||
|
||||
assert result["status"] == "rollback_failed"
|
||||
assert result["stage"] == "controller_rejected"
|
||||
|
||||
|
||||
def test_apply_rejects_rolled_back_terminal_status(tmp_path):
|
||||
class _RolledBackApplyBroker(_Broker):
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
return {"status": "rolled_back", "request": "c" * 64}
|
||||
|
||||
broker = _RolledBackApplyBroker()
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0010",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "controller_rejected"
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_refuses_duplicate_exact_rollback_policies(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
session.add(
|
||||
Policy(
|
||||
id="duplicate-rollback-policy",
|
||||
action="deployment.rollback",
|
||||
target="itworx-mcp-hub",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Duplicate exact rollback policy.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0011",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["autoRollback"] == {
|
||||
"status": "blocked",
|
||||
"reason": "exact-target-policy",
|
||||
assert job.status == "failed"
|
||||
assert job.stage == "external_integration_failed"
|
||||
assert job.evidence_json["postApplyVerification"]["status"] == "failed"
|
||||
assert job.evidence_json["recovery"] == {
|
||||
"status": "manual_required",
|
||||
"reason": "hub_data_aware_recovery_required",
|
||||
}
|
||||
assert "Automatic image-only rollback is disabled" in job.result_summary
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_rejects_unpollable_accepted_response(tmp_path):
|
||||
class _MissingRequestBroker(_Broker):
|
||||
def rollback(self, plan, *, secret, idempotency_key):
|
||||
self.rollback_calls.append(idempotency_key)
|
||||
return {"status": "accepted"}
|
||||
|
||||
broker = _MissingRequestBroker()
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0012",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "auto_rollback_failed"
|
||||
assert result["evidence"]["autoRollback"] == {
|
||||
"status": "accepted",
|
||||
"reason": "missing-controller-request",
|
||||
}
|
||||
assert len(broker.rollback_calls) == 1
|
||||
Reference in new issue
Block a user