1372 lines
52 KiB
Python
1372 lines
52 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from mcp import Client
|
|
from sqlmodel import Session, SQLModel, create_engine, select
|
|
|
|
MCP_TOKEN = "mcp-test-token-that-is-long-and-distinct"
|
|
N8N_TOKEN = "n8n-test-token-that-is-long-and-distinct"
|
|
OPERATOR_TOKEN = "operator-test-token-that-is-long-enough"
|
|
|
|
|
|
@pytest.fixture
|
|
def integration_client(tmp_path, monkeypatch):
|
|
from backend.app import main as app_main
|
|
from backend.app import mcp_server
|
|
from backend.app.core import security
|
|
from backend.app.core.config import settings
|
|
from backend.app.db import session as db_session
|
|
from backend.app.db.init_db import init_db
|
|
|
|
app = app_main.app
|
|
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'integrations.db'}",
|
|
connect_args={"check_same_thread": False},
|
|
)
|
|
previous_engine = db_session.engine
|
|
previous_mcp_engine = mcp_server.engine
|
|
previous = {
|
|
"mcp_enabled": settings.mcp_enabled,
|
|
"mcp_token": settings.mcp_token,
|
|
"n8n_enabled": settings.n8n_enabled,
|
|
"n8n_token": settings.n8n_token,
|
|
"environment": settings.environment,
|
|
"mutation_rate_limit_per_minute": settings.mutation_rate_limit_per_minute,
|
|
}
|
|
db_session.engine = engine
|
|
mcp_server.engine = engine
|
|
# The HTTP session manager is intentionally single-use. Start TestClient with
|
|
# MCP disabled, then enable the auth boundary for request-level assertions.
|
|
settings.mcp_enabled = False
|
|
settings.mcp_token = MCP_TOKEN
|
|
settings.n8n_enabled = True
|
|
settings.n8n_token = N8N_TOKEN
|
|
settings.environment = "production"
|
|
settings.mutation_rate_limit_per_minute = 10
|
|
security.settings = settings
|
|
app_main._MUTATION_WINDOWS.clear()
|
|
|
|
SQLModel.metadata.create_all(engine)
|
|
init_db()
|
|
with TestClient(app) as client:
|
|
settings.mcp_enabled = True
|
|
from backend.app.db.models import GatewayUser
|
|
from backend.app.services.user_access import issue_session
|
|
with Session(engine) as session:
|
|
user = GatewayUser(id="integration-admin", provider="authentik", issuer="test", subject="integration-admin", role="admin")
|
|
session.add(user)
|
|
session.commit()
|
|
client.appops_admin_cookie = issue_session(session, user)
|
|
yield client, engine
|
|
|
|
db_session.engine = previous_engine
|
|
mcp_server.engine = previous_mcp_engine
|
|
for key, value in previous.items():
|
|
setattr(settings, key, value)
|
|
app_main._MUTATION_WINDOWS.clear()
|
|
|
|
|
|
def human_headers(client):
|
|
return {"Cookie": f"appops_user_session={client.appops_admin_cookie}", "X-AppOps-Session-CSRF": "1"}
|
|
|
|
|
|
def test_integrations_use_separate_auth_scopes(integration_client):
|
|
client, _ = integration_client
|
|
|
|
assert client.get("/api/integrations/status").status_code == 401
|
|
assert (
|
|
client.get(
|
|
"/api/integrations/status",
|
|
headers=human_headers(client),
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert client.get("/api/integrations/n8n/events").status_code == 401
|
|
assert (
|
|
client.get(
|
|
"/api/integrations/n8n/events",
|
|
headers={"Authorization": f"Bearer {OPERATOR_TOKEN}"},
|
|
).status_code
|
|
== 403
|
|
)
|
|
assert (
|
|
client.get(
|
|
"/api/integrations/n8n/events",
|
|
headers={"Authorization": f"Bearer {N8N_TOKEN}"},
|
|
).status_code
|
|
== 200
|
|
)
|
|
assert client.post("/mcp/").status_code == 401
|
|
|
|
|
|
def test_deployment_http_surface_requires_operator_and_rejects_raw_fields(integration_client):
|
|
client, _ = integration_client
|
|
assert client.get("/api/deployments").status_code == 401
|
|
response = client.post(
|
|
"/api/deployments",
|
|
headers={
|
|
**human_headers(client),
|
|
"Idempotency-Key": "deployment-schema-test-0001",
|
|
},
|
|
json={
|
|
"operation": "preflight",
|
|
"plan": {
|
|
"api_version": "itworx.deployment/v1",
|
|
"application_id": "wsbc-portal",
|
|
"repository": "jens/windows-support-bundle-collector",
|
|
"revision": "a" * 40,
|
|
"release_version": "0.23.0",
|
|
"image_repository": "wsbc-portal",
|
|
"image_tag": "0.23.0",
|
|
"topology": {
|
|
"compose_project": "wsbc-portal",
|
|
"containers": ["wsbc-portal"],
|
|
"proxy_network": "itworx-npm",
|
|
"internal_port": 8080,
|
|
},
|
|
"broker_secret_ref": "WSBC_PORTAL_DEPLOYMENT_BROKER",
|
|
"command": "docker compose up",
|
|
},
|
|
},
|
|
)
|
|
assert response.status_code == 422
|
|
assert "command" in response.text
|
|
|
|
|
|
def test_n8n_preflight_approval_idempotency_and_no_execute(integration_client):
|
|
client, _ = integration_client
|
|
headers = {"Authorization": f"Bearer {N8N_TOKEN}"}
|
|
|
|
preview = client.post(
|
|
"/api/integrations/n8n/preflight",
|
|
headers=headers,
|
|
json={"action": "stack.redeploy", "target": "appops"},
|
|
)
|
|
assert preview.status_code == 200
|
|
assert preview.json()["policyDecision"] == "approval_required"
|
|
assert preview.json()["executionAvailable"] is False
|
|
|
|
approval_headers = {**headers, "Idempotency-Key": "workflow-run-00000001"}
|
|
first = client.post(
|
|
"/api/integrations/n8n/approvals",
|
|
headers=approval_headers,
|
|
json={
|
|
"action": "stack.redeploy",
|
|
"target": "appops",
|
|
"reason": "Gecontroleerde workflowtest",
|
|
"plan": ["preflight", "menselijke beoordeling"],
|
|
},
|
|
)
|
|
second = client.post(
|
|
"/api/integrations/n8n/approvals",
|
|
headers=approval_headers,
|
|
json={
|
|
"action": "stack.redeploy",
|
|
"target": "appops",
|
|
"reason": "Gecontroleerde workflowtest",
|
|
"plan": ["preflight", "menselijke beoordeling"],
|
|
},
|
|
)
|
|
assert first.status_code == 200
|
|
assert first.json()["created"] is True
|
|
assert first.json()["idempotentReplay"] is False
|
|
assert second.status_code == 200
|
|
assert second.json()["approvalId"] == first.json()["approvalId"]
|
|
assert second.json()["idempotentReplay"] is True
|
|
|
|
status = client.get(
|
|
f"/api/integrations/n8n/approvals/{first.json()['approvalId']}",
|
|
headers=headers,
|
|
)
|
|
assert status.status_code == 200
|
|
assert status.json()["status"] == "pending"
|
|
assert client.post(
|
|
"/api/integrations/n8n/execute",
|
|
headers=headers,
|
|
json={"approvalId": first.json()["approvalId"]},
|
|
).status_code in {404, 405}
|
|
|
|
blocked = client.post(
|
|
"/api/integrations/n8n/approvals",
|
|
headers={**headers, "Idempotency-Key": "workflow-run-00000002"},
|
|
json={
|
|
"action": "volume.delete",
|
|
"target": "important-data",
|
|
"reason": "Moet geblokkeerd blijven",
|
|
"plan": [],
|
|
},
|
|
)
|
|
assert blocked.status_code == 403
|
|
|
|
|
|
def test_n8n_uses_the_shared_grounded_knowledge_service(integration_client):
|
|
client, _ = integration_client
|
|
response = client.post(
|
|
"/api/integrations/n8n/knowledge/query",
|
|
headers={"Authorization": f"Bearer {N8N_TOKEN}"},
|
|
json={"question": "Welke acties vereisen menselijke goedkeuring?", "limit": 4},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.json()["guardrails"]["executionViaPolicyOnly"] is True
|
|
assert response.json()["guardrails"]["secretsExcluded"] is True
|
|
|
|
evaluation = client.get(
|
|
"/api/ai/rag/evaluate",
|
|
headers=human_headers(client),
|
|
)
|
|
assert evaluation.status_code == 200
|
|
assert evaluation.json()["total"] == 5
|
|
assert evaluation.json()["status"] == "passed"
|
|
assert evaluation.json()["passed"] == 5
|
|
assert (
|
|
evaluation.json()["vectorSpace"] == "normalized_binary_word_bigram_char_trigram"
|
|
)
|
|
|
|
|
|
def test_forge_bootstrap_preflight_is_read_only_and_rejects_generic_apply(integration_client):
|
|
client, _engine = integration_client
|
|
from backend.app import mcp_server
|
|
|
|
profile = {
|
|
"api_version": "itworx.forge-bootstrap-preflight/v1",
|
|
"revision": "a" * 40,
|
|
"workspace_image_digest": "sha256:" + "b" * 64,
|
|
"frontdoor_image_digest": "sha256:" + "c" * 64,
|
|
}
|
|
response = client.post(
|
|
"/api/deployments/forge-bootstrap/preflight",
|
|
headers={**human_headers(client), "Idempotency-Key": "forge-api-preflight-0001"},
|
|
json=profile,
|
|
)
|
|
assert response.status_code == 200
|
|
result = response.json()
|
|
assert result["status"] == "preflight_failed"
|
|
assert result["preflight"]["status"] == "blocked"
|
|
assert "deployment_apply_policy" in result["preflight"]["blockers"]
|
|
assert "controller_trust" in result["preflight"]["blockers"]
|
|
|
|
refreshed = client.get(
|
|
f"/api/deployments/{result['id']}", headers=human_headers(client)
|
|
)
|
|
assert refreshed.status_code == 200
|
|
assert refreshed.json()["preflight"]["status"] == "blocked"
|
|
listed = client.get("/api/deployments", headers=human_headers(client))
|
|
assert any(item["id"] == result["id"] for item in listed.json()["items"])
|
|
|
|
generic_apply = client.post(
|
|
"/api/deployments",
|
|
headers={**human_headers(client), "Idempotency-Key": "forge-generic-apply-0001"},
|
|
json={"operation": "apply", "plan": profile},
|
|
)
|
|
assert generic_apply.status_code == 422
|
|
generic_rollback = client.post(
|
|
"/api/deployments",
|
|
headers={**human_headers(client), "Idempotency-Key": "forge-generic-rollback-0001"},
|
|
json={"operation": "rollback", "plan": profile},
|
|
)
|
|
assert generic_rollback.status_code == 422
|
|
|
|
apply_profile = {
|
|
"api_version": "itworx.forge-first-install/v1",
|
|
"revision": profile["revision"],
|
|
"workspace_image_digest": profile["workspace_image_digest"],
|
|
"frontdoor_image_digest": profile["frontdoor_image_digest"],
|
|
"preflight_job_id": result["id"],
|
|
"preflight_sha256": result["preflightSha256"],
|
|
}
|
|
dedicated_apply = client.post(
|
|
"/api/deployments/forge-bootstrap/first-install",
|
|
headers={**human_headers(client), "Idempotency-Key": "forge-api-install-0001"},
|
|
json=apply_profile,
|
|
)
|
|
assert dedicated_apply.status_code == 409
|
|
assert "passed Forge preflight" in dedicated_apply.json()["detail"]
|
|
assert client.get(
|
|
"/api/deployments/forge-bootstrap/first-install/forge-install-" + "0" * 64,
|
|
headers=human_headers(client),
|
|
).status_code == 404
|
|
assert client.get(
|
|
"/api/deployments/forge-install-" + "0" * 64,
|
|
headers=human_headers(client),
|
|
).status_code == 404
|
|
|
|
async def exercise_mcp():
|
|
async with Client(mcp_server.appops_mcp) as mcp_client:
|
|
preflight = await mcp_client.call_tool(
|
|
"deployment.preflight",
|
|
{"plan": profile, "idempotencyKey": "forge-mcp-preflight-0001"},
|
|
)
|
|
assert preflight.structured_content["preflight"]["status"] == "blocked"
|
|
apply = await mcp_client.call_tool(
|
|
"deployment.apply",
|
|
{"plan": profile, "idempotencyKey": "forge-mcp-apply-0001"},
|
|
)
|
|
assert apply.structured_content["status"] == "invalid"
|
|
forge_apply = await mcp_client.call_tool(
|
|
"forge.bootstrap.apply",
|
|
{"profile": apply_profile, "idempotencyKey": "forge-mcp-install-0001"},
|
|
)
|
|
assert forge_apply.structured_content["status"] == "invalid"
|
|
forge_status = await mcp_client.call_tool(
|
|
"forge.bootstrap.status", {"jobId": "forge-install-" + "0" * 64}
|
|
)
|
|
assert forge_status.structured_content["status"] == "not_found"
|
|
generic_forge_status = await mcp_client.call_tool(
|
|
"deployment.status", {"jobId": "forge-install-" + "0" * 64}
|
|
)
|
|
assert generic_forge_status.structured_content["status"] == "invalid"
|
|
forge_resume = await mcp_client.call_tool(
|
|
"forge.bootstrap.resume", {"jobId": "forge-install-" + "0" * 64}
|
|
)
|
|
assert forge_resume.structured_content["status"] == "not_found"
|
|
|
|
asyncio.run(exercise_mcp())
|
|
|
|
|
|
def test_local_rag_indexes_allowlisted_docs_and_redacts_secrets(tmp_path):
|
|
from backend.app.services import knowledge_service
|
|
|
|
root = tmp_path / "knowledge"
|
|
docs = root / "docs"
|
|
docs.mkdir(parents=True)
|
|
(root / "README.md").write_text(
|
|
"# AppOps password=TOPSECRET-HEADING\n\nStack redeploy vereist menselijke goedkeuring. api_key=TOPSECRET-RAG-TOKEN",
|
|
encoding="utf-8",
|
|
)
|
|
(docs / "private_deployment_security.md").write_text(
|
|
"# Private\n\nNEVER-INDEX-THIS-SECRET",
|
|
encoding="utf-8",
|
|
)
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'rag.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
status = knowledge_service.rebuild_index(session, root)
|
|
result = knowledge_service.query(
|
|
session, "Wanneer vereist stack redeploy goedkeuring?", root=root
|
|
)
|
|
|
|
assert status["sources"] == 1
|
|
assert status["redactedMatches"] == 2
|
|
assert status["secretScanStatus"] == "passed_with_redactions"
|
|
assert result["citations"]
|
|
serialized = str(result)
|
|
assert "NEVER-INDEX-THIS-SECRET" not in serialized
|
|
assert "TOPSECRET-RAG-TOKEN" not in serialized
|
|
assert "TOPSECRET-HEADING" not in serialized
|
|
assert "[REDACTED]" in serialized
|
|
assert result["guardrails"]["executionViaPolicyOnly"] is True
|
|
assert result["retrieval"] == {
|
|
"engine": "local_hybrid_lexical_semantic",
|
|
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
|
"externalEmbeddingProvider": False,
|
|
}
|
|
|
|
|
|
def test_local_rag_rebuilds_when_document_manifest_changes(tmp_path):
|
|
from backend.app.services import knowledge_service
|
|
|
|
root = tmp_path / "knowledge-refresh"
|
|
root.mkdir()
|
|
readme = root / "README.md"
|
|
readme.write_text(
|
|
"# Eerste versie\n\nMenselijke approval blijft verplicht.", encoding="utf-8"
|
|
)
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'rag-refresh.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
first = knowledge_service.ensure_index_current(session, root)
|
|
readme.write_text(
|
|
"# Tweede versie\n\nPolicy preflight en menselijke approval blijven verplicht.",
|
|
encoding="utf-8",
|
|
)
|
|
second = knowledge_service.ensure_index_current(session, root)
|
|
|
|
assert first["manifestHash"] != second["manifestHash"]
|
|
assert second["status"] == "ready"
|
|
|
|
|
|
def test_mcp_exposes_governed_server_operations_and_policy_preview(integration_client):
|
|
_, engine = integration_client
|
|
from backend.app import mcp_server
|
|
from backend.app.core.version import APP_VERSION
|
|
|
|
mcp_server.engine = engine
|
|
assert mcp_server.appops_mcp.version == APP_VERSION
|
|
|
|
async def exercise_mcp():
|
|
async with Client(mcp_server.appops_mcp) as client:
|
|
tools = await client.list_tools()
|
|
names = {tool.name for tool in tools.tools}
|
|
assert "query_knowledge" in names
|
|
assert "preview_action" in names
|
|
assert "server.operation.request" in names
|
|
assert "server.operation.status" in names
|
|
assert "server.operation.execute" in names
|
|
assert "server.job.status" in names
|
|
assert "server.job.cancel" in names
|
|
assert "deployment.preflight" in names
|
|
assert "deployment.apply" in names
|
|
assert "deployment.status" in names
|
|
assert "deployment.rollback" in names
|
|
assert "appops.deployment.status" in names
|
|
assert "privatemuse.artifact.install.preflight" in names
|
|
assert "privatemuse.artifact.install" in names
|
|
installer = next(tool for tool in tools.tools if tool.name == "privatemuse.artifact.install")
|
|
assert installer.annotations.destructive_hint is True
|
|
assert "destination" not in installer.input_schema["properties"]
|
|
assert not any(
|
|
"approve" in name or "delete" in name or "shell" in name
|
|
for name in names
|
|
)
|
|
request_tool = next(
|
|
tool for tool in tools.tools if tool.name == "server.operation.request"
|
|
)
|
|
assert request_tool.annotations.read_only_hint is False
|
|
assert request_tool.annotations.destructive_hint is True
|
|
execute_tool = next(
|
|
tool for tool in tools.tools if tool.name == "server.operation.execute"
|
|
)
|
|
assert execute_tool.annotations.read_only_hint is False
|
|
assert execute_tool.annotations.destructive_hint is True
|
|
deployment_preflight = next(
|
|
tool for tool in tools.tools if tool.name == "deployment.preflight"
|
|
)
|
|
assert deployment_preflight.annotations.read_only_hint is True
|
|
deployment_apply = next(
|
|
tool for tool in tools.tools if tool.name == "deployment.apply"
|
|
)
|
|
assert deployment_apply.annotations.destructive_hint is True
|
|
|
|
preview = await client.call_tool(
|
|
"preview_action",
|
|
{"action": "volume.delete", "target": "important-data"},
|
|
)
|
|
assert preview.structured_content["policyDecision"] == "blocked"
|
|
assert preview.structured_content["executed"] is False
|
|
assert preview.structured_content["mcpApprovalRequired"] is False
|
|
assert preview.structured_content["mcpExecutionMode"] == "blocked"
|
|
|
|
registry_result = await client.call_tool("get_action_registry", {})
|
|
registry = registry_result.structured_content
|
|
by_id = {item["id"]: item for item in registry["items"]}
|
|
assert registry["summary"]["mcpApprovalRequired"] == 0
|
|
assert by_id["container.restart"]["mcpApprovalRequired"] is False
|
|
assert by_id["container.restart"]["mcpExecutionMode"] == "direct_typed"
|
|
assert by_id["container.restart"]["mcpExecutionSupported"] is True
|
|
assert by_id["stack.redeploy"]["mcpApprovalRequired"] is False
|
|
assert (
|
|
by_id["stack.redeploy"]["mcpExecutionMode"]
|
|
== "blocked_no_typed_executor"
|
|
)
|
|
assert by_id["stack.redeploy"]["mcpExecutionSupported"] is False
|
|
|
|
asyncio.run(exercise_mcp())
|
|
metrics = mcp_server.runtime_status()["toolMetrics"]["preview_action"]
|
|
assert metrics["calls"] >= 1
|
|
assert metrics["errors"] == 0
|
|
assert metrics["averageMs"] >= 0
|
|
|
|
canary = mcp_server.run_read_only_canary()
|
|
assert canary["status"] == "ok"
|
|
assert mcp_server.runtime_status()["canary"]["lastCheckedAt"]
|
|
|
|
|
|
def test_typed_artifact_mcp_reuses_durable_idempotency_receipt(integration_client, tmp_path, monkeypatch):
|
|
from types import SimpleNamespace
|
|
|
|
from backend.app import mcp_server
|
|
|
|
_, engine = integration_client
|
|
mcp_server.engine = engine
|
|
monkeypatch.setattr(mcp_server, "_artifact_roots", lambda: (tmp_path / "models", tmp_path / "receipts"))
|
|
monkeypatch.setattr(mcp_server, "artifact_preflight", lambda payload, model_root: {"fingerprint": "pinned"})
|
|
calls = []
|
|
|
|
def fake_install(payload, *, model_root, receipt_root):
|
|
calls.append(payload)
|
|
return {"status": "installed", "path": str(model_root / "loras" / payload["filename"]),
|
|
"sha256": payload["sha256"], "bytes": 5, "receipt": str(receipt_root / "receipt.json")}
|
|
|
|
monkeypatch.setattr(mcp_server, "install_artifact_file", fake_install)
|
|
monkeypatch.setattr(mcp_server, "log_event", lambda *args, **kwargs: SimpleNamespace(id="audit-1"))
|
|
args = ("lora", "https://example.com/model.safetensors", "a" * 64,
|
|
"model.safetensors", "source-1", "test-license", "test fixture", "pinned", "repeatable-key")
|
|
first = mcp_server.privatemuse_artifact_install(*args)
|
|
second = mcp_server.privatemuse_artifact_install(*args)
|
|
assert first == second
|
|
assert len(calls) == 1
|
|
|
|
|
|
def test_mcp_operation_executes_directly_without_human_approval_and_is_idempotent(
|
|
integration_client, monkeypatch
|
|
):
|
|
_, engine = integration_client
|
|
from backend.app.db.models import Approval
|
|
from backend.app.services import mcp_operations
|
|
|
|
calls = []
|
|
|
|
def execute_direct(_session, **kwargs):
|
|
calls.append(kwargs)
|
|
return {
|
|
"success": True,
|
|
"status": "success",
|
|
"result": "success",
|
|
"jobId": "job-mcp-direct",
|
|
"auditId": "audit-mcp-direct",
|
|
"policyDecision": "allowed",
|
|
"originalPolicyDecision": kwargs["original_policy_decision"],
|
|
}
|
|
|
|
monkeypatch.setattr(mcp_operations, "execute_governed_action", execute_direct)
|
|
|
|
with Session(engine) as session:
|
|
requested = mcp_operations.request_operation(
|
|
session,
|
|
action="container.restart",
|
|
target="widefrog",
|
|
reason="Recover the unhealthy application through the typed restart executor.",
|
|
idempotency_key="chatgpt-operation-0001",
|
|
)
|
|
replay = mcp_operations.request_operation(
|
|
session,
|
|
action="container.restart",
|
|
target="widefrog",
|
|
reason="Recover the unhealthy application through the typed restart executor.",
|
|
idempotency_key="chatgpt-operation-0001",
|
|
)
|
|
approvals = session.exec(
|
|
select(Approval).where(Approval.target == "widefrog")
|
|
).all()
|
|
|
|
assert requested["created"] is False
|
|
assert requested["executed"] is True
|
|
assert requested["policyDecision"] == "allowed"
|
|
assert requested["originalPolicyDecision"] == "allowed"
|
|
assert requested["executionSupported"] is True
|
|
assert requested["idempotentReplay"] is False
|
|
assert "approvalId" not in requested
|
|
assert replay["jobId"] == requested["jobId"]
|
|
assert replay["idempotentReplay"] is True
|
|
assert len(calls) == 1
|
|
assert approvals == []
|
|
|
|
|
|
def test_mcp_operation_crash_boundary_refuses_duplicate_reexecution(
|
|
integration_client, monkeypatch
|
|
):
|
|
_, engine = integration_client
|
|
from backend.app.services import mcp_operations
|
|
|
|
calls = []
|
|
|
|
def crash_after_boundary(_session, **kwargs):
|
|
calls.append(kwargs)
|
|
raise RuntimeError("simulated crash after side-effect boundary")
|
|
|
|
monkeypatch.setattr(
|
|
mcp_operations, "execute_governed_action", crash_after_boundary
|
|
)
|
|
|
|
with Session(engine) as session:
|
|
with pytest.raises(RuntimeError, match="simulated crash"):
|
|
mcp_operations.request_operation(
|
|
session,
|
|
action="container.restart",
|
|
target="widefrog-crash",
|
|
reason="Exercise durable MCP idempotency boundary.",
|
|
idempotency_key="chatgpt-operation-crash-0001",
|
|
)
|
|
|
|
replay = mcp_operations.request_operation(
|
|
session,
|
|
action="container.restart",
|
|
target="widefrog-crash",
|
|
reason="Exercise durable MCP idempotency boundary.",
|
|
idempotency_key="chatgpt-operation-crash-0001",
|
|
)
|
|
|
|
assert len(calls) == 1
|
|
assert replay["idempotentReplay"] is True
|
|
assert replay["executed"] is False
|
|
assert replay["policyDecision"] == "blocked"
|
|
assert replay["executionState"] == "reconciliation_required"
|
|
assert "approvalId" not in replay
|
|
|
|
|
|
def test_mcp_operation_executes_typed_action_even_when_policy_was_approval_required(
|
|
integration_client, monkeypatch
|
|
):
|
|
_, engine = integration_client
|
|
from backend.app.db.models import Approval, Policy
|
|
from backend.app.services import mcp_operations
|
|
|
|
monkeypatch.setattr(
|
|
mcp_operations,
|
|
"execute_governed_action",
|
|
lambda _session, **kwargs: {
|
|
"success": True,
|
|
"status": "success",
|
|
"result": "success",
|
|
"jobId": "job-mcp-policy",
|
|
"auditId": "audit-mcp-policy",
|
|
"policyDecision": "allowed",
|
|
"originalPolicyDecision": kwargs["original_policy_decision"],
|
|
},
|
|
)
|
|
|
|
with Session(engine) as session:
|
|
session.add(
|
|
Policy(
|
|
id="policy-mcp-no-approval",
|
|
action="container.restart",
|
|
target="widefrog-policy",
|
|
voice="approval_required",
|
|
manual="approval_required",
|
|
risk="medium",
|
|
reason="Historical policy requested approval.",
|
|
)
|
|
)
|
|
session.commit()
|
|
result = mcp_operations.request_operation(
|
|
session,
|
|
action="container.restart",
|
|
target="widefrog-policy",
|
|
reason="Autonomously recover the typed target.",
|
|
idempotency_key="chatgpt-operation-policy-0001",
|
|
)
|
|
approvals = session.exec(
|
|
select(Approval).where(Approval.target == "widefrog-policy")
|
|
).all()
|
|
|
|
assert result["executed"] is True
|
|
assert result["policyDecision"] == "allowed"
|
|
assert result["originalPolicyDecision"] == "approval_required"
|
|
assert "approvalId" not in result
|
|
assert approvals == []
|
|
|
|
|
|
def test_mcp_operation_keeps_blocked_and_read_only_actions_non_executable(
|
|
integration_client,
|
|
):
|
|
_, engine = integration_client
|
|
from backend.app.services import mcp_operations
|
|
|
|
with Session(engine) as session:
|
|
blocked = mcp_operations.request_operation(
|
|
session,
|
|
action="volume.delete",
|
|
target="important-data",
|
|
reason="This must remain blocked.",
|
|
idempotency_key="chatgpt-operation-blocked-0001",
|
|
)
|
|
read_only = mcp_operations.request_operation(
|
|
session,
|
|
action="container.status",
|
|
target="widefrog",
|
|
reason="Read status only.",
|
|
idempotency_key="chatgpt-operation-readonly-0001",
|
|
)
|
|
|
|
assert blocked["policyDecision"] == "blocked"
|
|
assert blocked["created"] is False
|
|
assert blocked["executed"] is False
|
|
assert blocked["executionSupported"] is False
|
|
assert "approvalId" not in blocked
|
|
assert read_only["created"] is False
|
|
assert read_only["executed"] is False
|
|
assert read_only["executionSupported"] is False
|
|
assert "approvalId" not in read_only
|
|
|
|
|
|
def test_mcp_nonblocked_action_without_typed_executor_fails_closed_without_approval(
|
|
integration_client,
|
|
):
|
|
_, engine = integration_client
|
|
from backend.app.db.models import Approval
|
|
from backend.app.services import mcp_operations
|
|
|
|
with Session(engine) as session:
|
|
result = mcp_operations.request_operation(
|
|
session,
|
|
action="stack.redeploy",
|
|
target="appops",
|
|
reason="No generic stack executor is exposed through MCP.",
|
|
idempotency_key="chatgpt-operation-unsupported-0001",
|
|
)
|
|
approvals = session.exec(
|
|
select(Approval).where(Approval.target == "appops")
|
|
).all()
|
|
|
|
assert result["policyDecision"] == "blocked"
|
|
assert result["originalPolicyDecision"] == "approval_required"
|
|
assert result["executionSupported"] is False
|
|
assert result["executed"] is False
|
|
assert "approvalId" not in result
|
|
assert approvals == []
|
|
|
|
|
|
def test_mcp_legacy_operation_cannot_execute_after_approval_expiry(integration_client):
|
|
_, engine = integration_client
|
|
from datetime import timedelta
|
|
|
|
from backend.app.core.time import utc_now
|
|
from backend.app.db.models import Approval
|
|
from backend.app.services import mcp_operations
|
|
from backend.app.services.approval_service import create_approval, update_approval_status
|
|
|
|
with Session(engine) as session:
|
|
created = create_approval(
|
|
session,
|
|
action="container.restart",
|
|
target="widefrog-legacy",
|
|
source="automation",
|
|
risk="medium",
|
|
reason="Legacy approval compatibility test.",
|
|
requester="legacy-test",
|
|
)
|
|
update_approval_status(session, created.id, "approved", actor="test-operator")
|
|
approval = session.get(Approval, created.id)
|
|
assert approval is not None
|
|
approval.expires_at = utc_now() - timedelta(seconds=1)
|
|
session.add(approval)
|
|
session.commit()
|
|
|
|
with pytest.raises(ValueError, match="approval_expired"):
|
|
mcp_operations.execute_operation(
|
|
session,
|
|
approval_id=approval.id,
|
|
expected_action_fingerprint=str(approval.action_fingerprint),
|
|
)
|
|
|
|
|
|
def test_mcp_recursively_redacts_stored_metadata():
|
|
from backend.app.mcp_server import _json
|
|
|
|
serialized = str(
|
|
_json(
|
|
{
|
|
"last_action": "api_key=MCP-SYNTHETIC-SECRET",
|
|
"labels": {"example.secret": "MCP-LABEL-SECRET"},
|
|
}
|
|
)
|
|
)
|
|
|
|
assert "MCP-SYNTHETIC-SECRET" not in serialized
|
|
assert "MCP-LABEL-SECRET" not in serialized
|
|
assert "[REDACTED]" in serialized
|
|
|
|
|
|
def test_n8n_event_and_audit_fields_are_redacted(integration_client):
|
|
client, engine = integration_client
|
|
from backend.app.db.models import AuditEvent
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {N8N_TOKEN}",
|
|
"Idempotency-Key": "workflow-redaction-0001",
|
|
}
|
|
response = client.post(
|
|
"/api/integrations/n8n/approvals",
|
|
headers=headers,
|
|
json={
|
|
"action": "stack.redeploy",
|
|
"target": "appops",
|
|
"reason": "api_key=N8N-REASON-SYNTHETIC",
|
|
"plan": [],
|
|
},
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
with Session(engine) as session:
|
|
approval_audit = session.get(AuditEvent, response.json()["auditId"])
|
|
assert approval_audit is not None
|
|
assert "N8N-REASON-SYNTHETIC" not in str(approval_audit.reason)
|
|
session.add(
|
|
AuditEvent(
|
|
id="audit-secret-target",
|
|
source="manual",
|
|
identity="operator",
|
|
action_id="container.status",
|
|
target="api_key=N8N-TARGET-SYNTHETIC",
|
|
risk="low",
|
|
policy_decision="allowed",
|
|
reason="status",
|
|
result="success",
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
events = client.get(
|
|
"/api/integrations/n8n/events", headers={"Authorization": f"Bearer {N8N_TOKEN}"}
|
|
)
|
|
assert events.status_code == 200
|
|
serialized = str(events.json())
|
|
assert "N8N-TARGET-SYNTHETIC" not in serialized
|
|
assert "[REDACTED]" in serialized
|
|
|
|
|
|
def test_n8n_heartbeat_makes_reachability_truthful(integration_client):
|
|
client, _ = integration_client
|
|
n8n_headers = {"Authorization": f"Bearer {N8N_TOKEN}"}
|
|
operator_headers = human_headers(client)
|
|
|
|
before = client.get("/api/integrations/status", headers=operator_headers).json()
|
|
assert before["n8n"]["state"] == "configured"
|
|
|
|
heartbeat = client.post(
|
|
"/api/integrations/n8n/heartbeat",
|
|
headers=n8n_headers,
|
|
json={
|
|
"workflowId": "appops-test-workflow",
|
|
"workflowVersion": "2",
|
|
"branch": "events",
|
|
"processedEvents": 42,
|
|
"backlogRemaining": True,
|
|
},
|
|
)
|
|
assert heartbeat.status_code == 200
|
|
assert heartbeat.json()["canExecute"] is False
|
|
|
|
knowledge_heartbeat = client.post(
|
|
"/api/integrations/n8n/heartbeat",
|
|
headers=n8n_headers,
|
|
json={
|
|
"workflowId": "appops-test-workflow",
|
|
"workflowVersion": "2",
|
|
"branch": "knowledge",
|
|
"confidence": "high",
|
|
},
|
|
)
|
|
assert knowledge_heartbeat.status_code == 200
|
|
|
|
after = client.get("/api/integrations/status", headers=operator_headers).json()
|
|
assert after["n8n"]["state"] == "reachable"
|
|
assert after["n8n"]["workflow"]["processedEvents"] == 42
|
|
assert after["n8n"]["workflow"]["backlogRemaining"] is True
|
|
assert after["n8n"]["workflow"]["branches"]["events"]["processedEvents"] == 42
|
|
assert after["n8n"]["workflow"]["branches"]["knowledge"]["confidence"] == "high"
|
|
|
|
|
|
def test_audit_endpoint_is_paginated(integration_client):
|
|
client, engine = integration_client
|
|
from backend.app.db.models import AuditEvent
|
|
|
|
with Session(engine) as session:
|
|
for index in range(4):
|
|
session.add(
|
|
AuditEvent(
|
|
id=f"audit-page-{index}",
|
|
source="system",
|
|
identity="test",
|
|
action_id="audit.page",
|
|
target=str(index),
|
|
risk="low",
|
|
policy_decision="allowed",
|
|
result="success",
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
first = client.get("/api/audit?limit=2&offset=0", headers=human_headers(client)).json()
|
|
second = client.get("/api/audit?limit=2&offset=2", headers=human_headers(client)).json()
|
|
assert len(first) == 2
|
|
assert len(second) == 2
|
|
assert {item["id"] for item in first}.isdisjoint({item["id"] for item in second})
|
|
|
|
|
|
def test_n8n_concurrent_idempotency_key_creates_one_approval(integration_client):
|
|
_, engine = integration_client
|
|
from backend.app.db.models import Approval
|
|
from backend.app.services.n8n_integration import request_approval
|
|
|
|
def submit():
|
|
with Session(engine) as session:
|
|
return request_approval(
|
|
session,
|
|
action="stack.redeploy",
|
|
target="appops-concurrent",
|
|
reason="Gecontroleerde gelijktijdige workflowtest",
|
|
plan=["preflight"],
|
|
idempotency_key="workflow-concurrent-0001",
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
results = list(pool.map(lambda _: submit(), range(2)))
|
|
|
|
with Session(engine) as session:
|
|
approvals = session.exec(
|
|
select(Approval).where(Approval.target == "appops-concurrent")
|
|
).all()
|
|
assert len(approvals) == 1
|
|
assert {result["idempotentReplay"] for result in results} == {False, True}
|
|
|
|
|
|
def test_n8n_machine_body_limit_and_mutation_rate_limit(integration_client):
|
|
client, _ = integration_client
|
|
headers = {"Authorization": f"Bearer {N8N_TOKEN}"}
|
|
|
|
too_large = client.post(
|
|
"/api/integrations/n8n/preflight",
|
|
headers=headers,
|
|
content=b"x" * (1_048_576 + 1),
|
|
)
|
|
assert too_large.status_code == 413
|
|
|
|
statuses = [
|
|
client.post(
|
|
"/api/integrations/n8n/preflight",
|
|
headers=headers,
|
|
json={"action": "container.status", "target": f"app-{index}"},
|
|
).status_code
|
|
for index in range(11)
|
|
]
|
|
assert statuses[:10] == [200] * 10
|
|
assert statuses[10] == 429
|
|
|
|
|
|
def test_workmesh_promotion_preflight_creates_sha_bound_approval_and_receipt_job(
|
|
integration_client, monkeypatch
|
|
):
|
|
client, engine = integration_client
|
|
from backend.app.db.models import Approval
|
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
|
from backend.app.services.secret_store import set_secret
|
|
|
|
revision = "a" * 40
|
|
digest = "b" * 64
|
|
backup_inputs_digest = "d" * 64
|
|
receipt_digest = "c" * 64
|
|
backup_receipt_digest = "f" * 64
|
|
broker_secret = "workmesh-controller-secret-" + "x" * 32
|
|
with Session(engine) as session:
|
|
set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", broker_secret)
|
|
|
|
preflight_calls = []
|
|
|
|
def preflight(_self, requested_revision, *, secret):
|
|
assert requested_revision == revision
|
|
assert secret == broker_secret
|
|
preflight_calls.append(requested_revision)
|
|
return {
|
|
"apiVersion": "itworx.workmesh-promotion/v2",
|
|
"status": "preflight_ok",
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"preflightSha256": digest,
|
|
"backupInputsSha256": backup_inputs_digest,
|
|
"checks": {
|
|
"exactMain": "passed",
|
|
"pushValidation": "passed",
|
|
"controllerPreflight": "passed",
|
|
"dataBackup": "passed",
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight)
|
|
preflight_headers = {
|
|
**human_headers(client),
|
|
"Idempotency-Key": "workmesh-promote-preflight-0001",
|
|
}
|
|
direct_action = client.post(
|
|
"/api/actions/execute",
|
|
headers=human_headers(client),
|
|
json={
|
|
"plan": {
|
|
"intent": "deployment.target.promote_workmesh",
|
|
"target": "jens/workmesh",
|
|
"source": "manual",
|
|
"plan": [],
|
|
}
|
|
},
|
|
)
|
|
assert direct_action.status_code == 200
|
|
assert direct_action.json()["result"] == "preflight_required"
|
|
assert client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers=preflight_headers,
|
|
json={"revision": revision, "repository": "jens/other"},
|
|
).status_code == 422
|
|
response = client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers=preflight_headers,
|
|
json={"revision": revision},
|
|
)
|
|
assert response.status_code == 200, response.text
|
|
requested = response.json()
|
|
assert requested["status"] == "approval_required"
|
|
assert requested["revision"] == revision
|
|
assert requested["preflight"]["preflightSha256"] == digest
|
|
assert requested["approvalId"]
|
|
assert requested["promotionId"]
|
|
|
|
with Session(engine) as session:
|
|
approval = session.get(Approval, requested["approvalId"])
|
|
assert approval is not None
|
|
assert approval.action == "deployment.target.promote_workmesh"
|
|
assert approval.target == "jens/workmesh"
|
|
assert approval.request_context_json["revision"] == revision
|
|
assert approval.request_context_json["preflightSha256"] == digest
|
|
assert approval.request_context_json["backupInputsSha256"] == backup_inputs_digest
|
|
assert approval.request_context_json["promotionId"] == requested["promotionId"]
|
|
assert revision in approval.reason
|
|
assert digest in approval.reason
|
|
|
|
listed = client.get(
|
|
"/api/deployments/workmesh/promotions",
|
|
headers=human_headers(client),
|
|
)
|
|
assert listed.status_code == 200
|
|
assert listed.json()["items"][0]["promotionId"] == requested["promotionId"]
|
|
assert listed.json()["items"][0]["approvalStatus"] == "pending"
|
|
|
|
replay = client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers=preflight_headers,
|
|
json={"revision": revision},
|
|
)
|
|
assert replay.status_code == 200
|
|
assert replay.json()["promotionId"] == requested["promotionId"]
|
|
assert replay.json()["approvalId"] == requested["approvalId"]
|
|
assert replay.json()["idempotentReplay"] is True
|
|
assert preflight_calls == [revision]
|
|
changed_revision_replay = client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers=preflight_headers,
|
|
json={"revision": "c" * 40},
|
|
)
|
|
assert changed_revision_replay.status_code == 409
|
|
assert preflight_calls == [revision]
|
|
|
|
expected_request = hashlib.sha256(
|
|
json.dumps(
|
|
{
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"operation": "workmesh_promote",
|
|
"idempotency": f"workmesh-promotion:{requested['promotionId']}",
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
).hexdigest()
|
|
|
|
def apply(_self, requested_revision, *, secret, preflight_sha256, idempotency_key):
|
|
assert requested_revision == revision
|
|
assert secret == broker_secret
|
|
assert preflight_sha256 == digest
|
|
assert idempotency_key == f"workmesh-promotion:{requested['promotionId']}"
|
|
return {
|
|
"status": "accepted",
|
|
"request": expected_request,
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"operation": "workmesh_promote",
|
|
}
|
|
|
|
def status(_self, request_id, *, secret):
|
|
assert request_id == expected_request
|
|
assert secret == broker_secret
|
|
return {
|
|
"status": "deployed",
|
|
"request": expected_request,
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"operation": "workmesh_promote",
|
|
"receiptSha256": receipt_digest,
|
|
"backupReceiptSha256": backup_receipt_digest,
|
|
}
|
|
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply)
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_status", status)
|
|
auth = human_headers(client)
|
|
assert client.patch(
|
|
f"/api/approvals/{requested['approvalId']}",
|
|
headers=auth,
|
|
json={"status": "approved"},
|
|
).status_code == 200
|
|
approved_promotion = client.get(
|
|
f"/api/deployments/workmesh/promotions/{requested['promotionId']}",
|
|
headers=auth,
|
|
)
|
|
assert approved_promotion.status_code == 200
|
|
assert approved_promotion.json()["approvalStatus"] == "approved"
|
|
executed = client.post(
|
|
f"/api/approvals/{requested['approvalId']}/execute", headers=auth
|
|
)
|
|
assert executed.status_code == 200, executed.text
|
|
assert executed.json()["approvalStatus"] == "executed"
|
|
assert executed.json()["status"] == "applying"
|
|
assert executed.json()["jobId"] == requested["promotionId"]
|
|
assert preflight_calls == [revision, revision]
|
|
|
|
reconciled = client.get(
|
|
f"/api/deployments/workmesh/promotions/{requested['promotionId']}",
|
|
headers=auth,
|
|
)
|
|
assert reconciled.status_code == 200, reconciled.text
|
|
assert reconciled.json()["status"] == "succeeded"
|
|
assert reconciled.json()["revision"] == revision
|
|
assert reconciled.json()["evidence"]["receiptSha256"] == receipt_digest
|
|
assert reconciled.json()["evidence"]["backupReceiptSha256"] == backup_receipt_digest
|
|
assert reconciled.json()["approvalStatus"] == "executed"
|
|
|
|
|
|
def test_workmesh_ambiguous_controller_apply_is_consumed_and_only_reconciled(
|
|
integration_client, monkeypatch
|
|
):
|
|
client, engine = integration_client
|
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
|
from backend.app.services.secret_store import set_secret
|
|
|
|
revision = "d" * 40
|
|
broker_secret = "workmesh-controller-secret-" + "y" * 32
|
|
with Session(engine) as session:
|
|
set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", broker_secret)
|
|
|
|
def preflight(_self, requested_revision, *, secret):
|
|
assert requested_revision == revision
|
|
assert secret == broker_secret
|
|
return {
|
|
"apiVersion": "itworx.workmesh-promotion/v2",
|
|
"status": "preflight_ok",
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"preflightSha256": "e" * 64,
|
|
"backupInputsSha256": "f" * 64,
|
|
"checks": {
|
|
"exactMain": "passed",
|
|
"pushValidation": "passed",
|
|
"controllerPreflight": "passed",
|
|
"dataBackup": "passed",
|
|
},
|
|
}
|
|
|
|
apply_calls = []
|
|
status_calls = []
|
|
|
|
def apply(*_args, **_kwargs):
|
|
apply_calls.append(True)
|
|
return {"status": "unreachable", "reason": "connection closed"}
|
|
|
|
def status(*_args, **_kwargs):
|
|
status_calls.append(True)
|
|
return {"status": "not_found"}
|
|
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight)
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply)
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_status", status)
|
|
|
|
auth = human_headers(client)
|
|
requested = client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers={**auth, "Idempotency-Key": "workmesh-promote-uncertain-0001"},
|
|
json={"revision": revision},
|
|
)
|
|
assert requested.status_code == 200, requested.text
|
|
approval_id = requested.json()["approvalId"]
|
|
assert client.patch(
|
|
f"/api/approvals/{approval_id}", headers=auth, json={"status": "approved"}
|
|
).status_code == 200
|
|
|
|
executed = client.post(f"/api/approvals/{approval_id}/execute", headers=auth)
|
|
assert executed.status_code == 200, executed.text
|
|
assert executed.json()["status"] == "outcome_unknown"
|
|
assert executed.json()["approvalStatus"] == "executed"
|
|
assert apply_calls == [True]
|
|
|
|
for _ in range(2):
|
|
reconciled = client.get(
|
|
f"/api/deployments/workmesh/promotions/{requested.json()['promotionId']}",
|
|
headers=auth,
|
|
)
|
|
assert reconciled.status_code == 200, reconciled.text
|
|
assert reconciled.json()["status"] == "outcome_unknown"
|
|
assert reconciled.json()["evidence"] == {"reconciliationRequired": True}
|
|
assert len(status_calls) == 2
|
|
assert apply_calls == [True]
|
|
replay = client.post(f"/api/approvals/{approval_id}/execute", headers=auth)
|
|
assert replay.status_code == 409
|
|
|
|
|
|
def test_mcp_legacy_approval_executor_does_not_expose_workmesh_promotion(integration_client):
|
|
_, engine = integration_client
|
|
from backend.app.services import mcp_operations
|
|
from backend.app.services.approval_service import create_approval, update_approval_status
|
|
|
|
with Session(engine) as session:
|
|
approval = create_approval(
|
|
session,
|
|
action="deployment.target.promote_workmesh",
|
|
target="jens/workmesh",
|
|
source="manual",
|
|
risk="high",
|
|
reason="Fixed WorkMesh promotion contract test.",
|
|
context={"apiVersion": "itworx.workmesh-promotion/v2"},
|
|
)
|
|
update_approval_status(session, approval.id, "approved", actor="test-operator")
|
|
assert mcp_operations.operation_status(session, approval.id)["executionSupported"] is False
|
|
with pytest.raises(ValueError, match="operation_executor_unavailable"):
|
|
mcp_operations.execute_operation(
|
|
session,
|
|
approval_id=approval.id,
|
|
expected_action_fingerprint=str(approval.action_fingerprint),
|
|
)
|
|
|
|
|
|
def test_workmesh_changed_preflight_expires_approval_before_apply(
|
|
integration_client, monkeypatch
|
|
):
|
|
client, engine = integration_client
|
|
from backend.app.db.models import Approval
|
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
|
from backend.app.services.secret_store import set_secret
|
|
|
|
revision = "f" * 40
|
|
with Session(engine) as session:
|
|
set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", "workmesh-controller-secret-" + "z" * 32)
|
|
|
|
preflight_calls = []
|
|
|
|
def preflight(_self, requested_revision, *, secret):
|
|
assert requested_revision == revision
|
|
assert secret.startswith("workmesh-controller-secret-")
|
|
preflight_calls.append(True)
|
|
return {
|
|
"apiVersion": "itworx.workmesh-promotion/v2",
|
|
"status": "preflight_ok",
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"preflightSha256": ("1" if len(preflight_calls) == 1 else "2") * 64,
|
|
"backupInputsSha256": "b" * 64,
|
|
"checks": {
|
|
"exactMain": "passed",
|
|
"pushValidation": "passed",
|
|
"controllerPreflight": "passed",
|
|
"dataBackup": "passed",
|
|
},
|
|
}
|
|
|
|
apply_calls = []
|
|
|
|
def apply(*_args, **_kwargs):
|
|
apply_calls.append(True)
|
|
return {"status": "accepted"}
|
|
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight)
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply)
|
|
auth = human_headers(client)
|
|
requested = client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers={**auth, "Idempotency-Key": "workmesh-promote-drift-0001"},
|
|
json={"revision": revision},
|
|
)
|
|
assert requested.status_code == 200, requested.text
|
|
approval_id = requested.json()["approvalId"]
|
|
assert client.patch(
|
|
f"/api/approvals/{approval_id}", headers=auth, json={"status": "approved"}
|
|
).status_code == 200
|
|
|
|
executed = client.post(f"/api/approvals/{approval_id}/execute", headers=auth)
|
|
assert executed.status_code == 200
|
|
assert executed.json()["status"] == "preflight_changed"
|
|
assert apply_calls == []
|
|
with Session(engine) as session:
|
|
approval = session.get(Approval, approval_id)
|
|
assert approval.status == "expired"
|
|
replay = client.post(f"/api/approvals/{approval_id}/execute", headers=auth)
|
|
assert replay.status_code == 409
|
|
|
|
|
|
def test_controller_preflight_changed_identity_expires_approval_without_unknown_state(
|
|
integration_client, monkeypatch
|
|
):
|
|
client, engine = integration_client
|
|
from backend.app.db.models import Approval, WorkMeshPromotion
|
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
|
from backend.app.services.secret_store import set_secret
|
|
|
|
revision = "9" * 40
|
|
broker_secret = "workmesh-controller-secret-" + "x" * 32
|
|
with Session(engine) as session:
|
|
set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", broker_secret)
|
|
|
|
def preflight(_self, requested_revision, *, secret):
|
|
assert requested_revision == revision
|
|
assert secret == broker_secret
|
|
return {
|
|
"apiVersion": "itworx.workmesh-promotion/v2",
|
|
"status": "preflight_ok",
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"preflightSha256": "a" * 64,
|
|
"backupInputsSha256": "b" * 64,
|
|
"checks": {
|
|
"exactMain": "passed",
|
|
"pushValidation": "passed",
|
|
"controllerPreflight": "passed",
|
|
"dataBackup": "passed",
|
|
},
|
|
}
|
|
|
|
apply_calls = []
|
|
|
|
def apply(_self, requested_revision, *, secret, preflight_sha256, idempotency_key):
|
|
assert requested_revision == revision
|
|
assert secret == broker_secret
|
|
assert preflight_sha256 == "a" * 64
|
|
apply_calls.append(idempotency_key)
|
|
request_id = hashlib.sha256(
|
|
json.dumps(
|
|
{
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"operation": "workmesh_promote",
|
|
"idempotency": idempotency_key,
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
).hexdigest()
|
|
return {
|
|
"status": "preflight_changed",
|
|
"request": request_id,
|
|
"repository": "jens/workmesh",
|
|
"revision": revision,
|
|
"operation": "workmesh_promote",
|
|
"reason": "approved_preflight_no_longer_matches",
|
|
}
|
|
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight)
|
|
monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply)
|
|
auth = human_headers(client)
|
|
requested = client.post(
|
|
"/api/deployments/workmesh/promotions/preflight",
|
|
headers={**auth, "Idempotency-Key": "workmesh-promote-controller-drift-01"},
|
|
json={"revision": revision},
|
|
)
|
|
assert requested.status_code == 200, requested.text
|
|
promotion_id = requested.json()["promotionId"]
|
|
approval_id = requested.json()["approvalId"]
|
|
assert client.patch(
|
|
f"/api/approvals/{approval_id}", headers=auth, json={"status": "approved"}
|
|
).status_code == 200
|
|
|
|
executed = client.post(f"/api/approvals/{approval_id}/execute", headers=auth)
|
|
assert executed.status_code == 200, executed.text
|
|
assert executed.json()["status"] == "preflight_changed"
|
|
assert executed.json()["success"] is False
|
|
assert apply_calls == [f"workmesh-promotion:{promotion_id}"]
|
|
with Session(engine) as session:
|
|
approval = session.get(Approval, approval_id)
|
|
promotion = session.get(WorkMeshPromotion, promotion_id)
|
|
assert approval is not None and approval.status == "expired"
|
|
assert promotion is not None and promotion.status == "preflight_changed"
|