from __future__ import annotations import asyncio import hashlib import json from concurrent.futures import ThreadPoolExecutor import pytest from fastapi.testclient import TestClient from mcp import Client from sqlmodel import Session, SQLModel, create_engine, select MCP_TOKEN = "mcp-test-token-that-is-long-and-distinct" N8N_TOKEN = "n8n-test-token-that-is-long-and-distinct" OPERATOR_TOKEN = "operator-test-token-that-is-long-enough" @pytest.fixture def integration_client(tmp_path, monkeypatch): from backend.app import main as app_main from backend.app import mcp_server from backend.app.core import security from backend.app.core.config import settings from backend.app.db import session as db_session from backend.app.db.init_db import init_db app = app_main.app engine = create_engine( f"sqlite:///{tmp_path / 'integrations.db'}", connect_args={"check_same_thread": False}, ) previous_engine = db_session.engine previous_mcp_engine = mcp_server.engine previous = { "mcp_enabled": settings.mcp_enabled, "mcp_token": settings.mcp_token, "n8n_enabled": settings.n8n_enabled, "n8n_token": settings.n8n_token, "environment": settings.environment, "mutation_rate_limit_per_minute": settings.mutation_rate_limit_per_minute, } db_session.engine = engine mcp_server.engine = engine # The HTTP session manager is intentionally single-use. Start TestClient with # MCP disabled, then enable the auth boundary for request-level assertions. settings.mcp_enabled = False settings.mcp_token = MCP_TOKEN settings.n8n_enabled = True settings.n8n_token = N8N_TOKEN settings.environment = "production" settings.mutation_rate_limit_per_minute = 10 security.settings = settings app_main._MUTATION_WINDOWS.clear() SQLModel.metadata.create_all(engine) init_db() with TestClient(app) as client: settings.mcp_enabled = True from backend.app.db.models import GatewayUser from backend.app.services.user_access import issue_session with Session(engine) as session: user = GatewayUser(id="integration-admin", provider="authentik", issuer="test", subject="integration-admin", role="admin") session.add(user) session.commit() client.appops_admin_cookie = issue_session(session, user) yield client, engine db_session.engine = previous_engine mcp_server.engine = previous_mcp_engine for key, value in previous.items(): setattr(settings, key, value) app_main._MUTATION_WINDOWS.clear() def human_headers(client): return {"Cookie": f"appops_user_session={client.appops_admin_cookie}", "X-AppOps-Session-CSRF": "1"} def test_integrations_use_separate_auth_scopes(integration_client): client, _ = integration_client assert client.get("/api/integrations/status").status_code == 401 assert ( client.get( "/api/integrations/status", headers=human_headers(client), ).status_code == 200 ) assert client.get("/api/integrations/n8n/events").status_code == 401 assert ( client.get( "/api/integrations/n8n/events", headers={"Authorization": f"Bearer {OPERATOR_TOKEN}"}, ).status_code == 403 ) assert ( client.get( "/api/integrations/n8n/events", headers={"Authorization": f"Bearer {N8N_TOKEN}"}, ).status_code == 200 ) assert client.post("/mcp/").status_code == 401 def test_deployment_http_surface_requires_operator_and_rejects_raw_fields(integration_client): client, _ = integration_client assert client.get("/api/deployments").status_code == 401 response = client.post( "/api/deployments", headers={ **human_headers(client), "Idempotency-Key": "deployment-schema-test-0001", }, json={ "operation": "preflight", "plan": { "api_version": "itworx.deployment/v1", "application_id": "wsbc-portal", "repository": "jens/windows-support-bundle-collector", "revision": "a" * 40, "release_version": "0.23.0", "image_repository": "wsbc-portal", "image_tag": "0.23.0", "topology": { "compose_project": "wsbc-portal", "containers": ["wsbc-portal"], "proxy_network": "itworx-npm", "internal_port": 8080, }, "broker_secret_ref": "WSBC_PORTAL_DEPLOYMENT_BROKER", "command": "docker compose up", }, }, ) assert response.status_code == 422 assert "command" in response.text def test_n8n_preflight_approval_idempotency_and_no_execute(integration_client): client, _ = integration_client headers = {"Authorization": f"Bearer {N8N_TOKEN}"} preview = client.post( "/api/integrations/n8n/preflight", headers=headers, json={"action": "stack.redeploy", "target": "appops"}, ) assert preview.status_code == 200 assert preview.json()["policyDecision"] == "approval_required" assert preview.json()["executionAvailable"] is False approval_headers = {**headers, "Idempotency-Key": "workflow-run-00000001"} first = client.post( "/api/integrations/n8n/approvals", headers=approval_headers, json={ "action": "stack.redeploy", "target": "appops", "reason": "Gecontroleerde workflowtest", "plan": ["preflight", "menselijke beoordeling"], }, ) second = client.post( "/api/integrations/n8n/approvals", headers=approval_headers, json={ "action": "stack.redeploy", "target": "appops", "reason": "Gecontroleerde workflowtest", "plan": ["preflight", "menselijke beoordeling"], }, ) assert first.status_code == 200 assert first.json()["created"] is True assert first.json()["idempotentReplay"] is False assert second.status_code == 200 assert second.json()["approvalId"] == first.json()["approvalId"] assert second.json()["idempotentReplay"] is True status = client.get( f"/api/integrations/n8n/approvals/{first.json()['approvalId']}", headers=headers, ) assert status.status_code == 200 assert status.json()["status"] == "pending" assert client.post( "/api/integrations/n8n/execute", headers=headers, json={"approvalId": first.json()["approvalId"]}, ).status_code in {404, 405} blocked = client.post( "/api/integrations/n8n/approvals", headers={**headers, "Idempotency-Key": "workflow-run-00000002"}, json={ "action": "volume.delete", "target": "important-data", "reason": "Moet geblokkeerd blijven", "plan": [], }, ) assert blocked.status_code == 403 def test_n8n_uses_the_shared_grounded_knowledge_service(integration_client): client, _ = integration_client response = client.post( "/api/integrations/n8n/knowledge/query", headers={"Authorization": f"Bearer {N8N_TOKEN}"}, json={"question": "Welke acties vereisen menselijke goedkeuring?", "limit": 4}, ) assert response.status_code == 200 assert response.json()["guardrails"]["executionViaPolicyOnly"] is True assert response.json()["guardrails"]["secretsExcluded"] is True evaluation = client.get( "/api/ai/rag/evaluate", headers=human_headers(client), ) assert evaluation.status_code == 200 assert evaluation.json()["total"] == 5 assert evaluation.json()["status"] == "passed" assert evaluation.json()["passed"] == 5 assert ( evaluation.json()["vectorSpace"] == "normalized_binary_word_bigram_char_trigram" ) def test_forge_bootstrap_preflight_is_read_only_and_rejects_generic_apply(integration_client): client, _engine = integration_client from backend.app import mcp_server profile = { "api_version": "itworx.forge-bootstrap-preflight/v1", "revision": "a" * 40, "workspace_image_digest": "sha256:" + "b" * 64, "frontdoor_image_digest": "sha256:" + "c" * 64, } response = client.post( "/api/deployments/forge-bootstrap/preflight", headers={**human_headers(client), "Idempotency-Key": "forge-api-preflight-0001"}, json=profile, ) assert response.status_code == 200 result = response.json() assert result["status"] == "preflight_failed" assert result["preflight"]["status"] == "blocked" assert "deployment_apply_policy" in result["preflight"]["blockers"] assert "controller_trust" in result["preflight"]["blockers"] refreshed = client.get( f"/api/deployments/{result['id']}", headers=human_headers(client) ) assert refreshed.status_code == 200 assert refreshed.json()["preflight"]["status"] == "blocked" listed = client.get("/api/deployments", headers=human_headers(client)) assert any(item["id"] == result["id"] for item in listed.json()["items"]) generic_apply = client.post( "/api/deployments", headers={**human_headers(client), "Idempotency-Key": "forge-generic-apply-0001"}, json={"operation": "apply", "plan": profile}, ) assert generic_apply.status_code == 422 generic_rollback = client.post( "/api/deployments", headers={**human_headers(client), "Idempotency-Key": "forge-generic-rollback-0001"}, json={"operation": "rollback", "plan": profile}, ) assert generic_rollback.status_code == 422 apply_profile = { "api_version": "itworx.forge-first-install/v1", "revision": profile["revision"], "workspace_image_digest": profile["workspace_image_digest"], "frontdoor_image_digest": profile["frontdoor_image_digest"], "preflight_job_id": result["id"], "preflight_sha256": result["preflightSha256"], } dedicated_apply = client.post( "/api/deployments/forge-bootstrap/first-install", headers={**human_headers(client), "Idempotency-Key": "forge-api-install-0001"}, json=apply_profile, ) assert dedicated_apply.status_code == 409 assert "passed Forge preflight" in dedicated_apply.json()["detail"] assert client.get( "/api/deployments/forge-bootstrap/first-install/forge-install-" + "0" * 64, headers=human_headers(client), ).status_code == 404 assert client.get( "/api/deployments/forge-install-" + "0" * 64, headers=human_headers(client), ).status_code == 404 async def exercise_mcp(): async with Client(mcp_server.appops_mcp) as mcp_client: preflight = await mcp_client.call_tool( "deployment.preflight", {"plan": profile, "idempotencyKey": "forge-mcp-preflight-0001"}, ) assert preflight.structured_content["preflight"]["status"] == "blocked" apply = await mcp_client.call_tool( "deployment.apply", {"plan": profile, "idempotencyKey": "forge-mcp-apply-0001"}, ) assert apply.structured_content["status"] == "invalid" forge_apply = await mcp_client.call_tool( "forge.bootstrap.apply", {"profile": apply_profile, "idempotencyKey": "forge-mcp-install-0001"}, ) assert forge_apply.structured_content["status"] == "invalid" forge_status = await mcp_client.call_tool( "forge.bootstrap.status", {"jobId": "forge-install-" + "0" * 64} ) assert forge_status.structured_content["status"] == "not_found" generic_forge_status = await mcp_client.call_tool( "deployment.status", {"jobId": "forge-install-" + "0" * 64} ) assert generic_forge_status.structured_content["status"] == "invalid" forge_resume = await mcp_client.call_tool( "forge.bootstrap.resume", {"jobId": "forge-install-" + "0" * 64} ) assert forge_resume.structured_content["status"] == "not_found" asyncio.run(exercise_mcp()) def test_local_rag_indexes_allowlisted_docs_and_redacts_secrets(tmp_path): from backend.app.services import knowledge_service root = tmp_path / "knowledge" docs = root / "docs" docs.mkdir(parents=True) (root / "README.md").write_text( "# AppOps password=TOPSECRET-HEADING\n\nStack redeploy vereist menselijke goedkeuring. api_key=TOPSECRET-RAG-TOKEN", encoding="utf-8", ) (docs / "private_deployment_security.md").write_text( "# Private\n\nNEVER-INDEX-THIS-SECRET", encoding="utf-8", ) engine = create_engine(f"sqlite:///{tmp_path / 'rag.db'}") SQLModel.metadata.create_all(engine) with Session(engine) as session: status = knowledge_service.rebuild_index(session, root) result = knowledge_service.query( session, "Wanneer vereist stack redeploy goedkeuring?", root=root ) assert status["sources"] == 1 assert status["redactedMatches"] == 2 assert status["secretScanStatus"] == "passed_with_redactions" assert result["citations"] serialized = str(result) assert "NEVER-INDEX-THIS-SECRET" not in serialized assert "TOPSECRET-RAG-TOKEN" not in serialized assert "TOPSECRET-HEADING" not in serialized assert "[REDACTED]" in serialized assert result["guardrails"]["executionViaPolicyOnly"] is True assert result["retrieval"] == { "engine": "local_hybrid_lexical_semantic", "vectorSpace": "normalized_binary_word_bigram_char_trigram", "externalEmbeddingProvider": False, } def test_local_rag_rebuilds_when_document_manifest_changes(tmp_path): from backend.app.services import knowledge_service root = tmp_path / "knowledge-refresh" root.mkdir() readme = root / "README.md" readme.write_text( "# Eerste versie\n\nMenselijke approval blijft verplicht.", encoding="utf-8" ) engine = create_engine(f"sqlite:///{tmp_path / 'rag-refresh.db'}") SQLModel.metadata.create_all(engine) with Session(engine) as session: first = knowledge_service.ensure_index_current(session, root) readme.write_text( "# Tweede versie\n\nPolicy preflight en menselijke approval blijven verplicht.", encoding="utf-8", ) second = knowledge_service.ensure_index_current(session, root) assert first["manifestHash"] != second["manifestHash"] assert second["status"] == "ready" def test_mcp_exposes_governed_server_operations_and_policy_preview(integration_client): _, engine = integration_client from backend.app import mcp_server from backend.app.core.version import APP_VERSION mcp_server.engine = engine assert mcp_server.appops_mcp.version == APP_VERSION async def exercise_mcp(): async with Client(mcp_server.appops_mcp) as client: tools = await client.list_tools() names = {tool.name for tool in tools.tools} assert "query_knowledge" in names assert "preview_action" in names assert "server.operation.request" in names assert "server.operation.status" in names assert "server.operation.execute" in names assert "server.job.status" in names assert "server.job.cancel" in names assert "deployment.preflight" in names assert "deployment.apply" in names assert "deployment.status" in names assert "deployment.rollback" in names assert "appops.deployment.status" in names assert "privatemuse.artifact.install.preflight" in names assert "privatemuse.artifact.install" in names installer = next(tool for tool in tools.tools if tool.name == "privatemuse.artifact.install") assert installer.annotations.destructive_hint is True assert "destination" not in installer.input_schema["properties"] assert not any( "approve" in name or "delete" in name or "shell" in name for name in names ) request_tool = next( tool for tool in tools.tools if tool.name == "server.operation.request" ) assert request_tool.annotations.read_only_hint is False assert request_tool.annotations.destructive_hint is True execute_tool = next( tool for tool in tools.tools if tool.name == "server.operation.execute" ) assert execute_tool.annotations.read_only_hint is False assert execute_tool.annotations.destructive_hint is True deployment_preflight = next( tool for tool in tools.tools if tool.name == "deployment.preflight" ) assert deployment_preflight.annotations.read_only_hint is True deployment_apply = next( tool for tool in tools.tools if tool.name == "deployment.apply" ) assert deployment_apply.annotations.destructive_hint is True preview = await client.call_tool( "preview_action", {"action": "volume.delete", "target": "important-data"}, ) assert preview.structured_content["policyDecision"] == "blocked" assert preview.structured_content["executed"] is False assert preview.structured_content["mcpApprovalRequired"] is False assert preview.structured_content["mcpExecutionMode"] == "blocked" registry_result = await client.call_tool("get_action_registry", {}) registry = registry_result.structured_content by_id = {item["id"]: item for item in registry["items"]} assert registry["summary"]["mcpApprovalRequired"] == 0 assert by_id["container.restart"]["mcpApprovalRequired"] is False assert by_id["container.restart"]["mcpExecutionMode"] == "direct_typed" assert by_id["container.restart"]["mcpExecutionSupported"] is True assert by_id["stack.redeploy"]["mcpApprovalRequired"] is False assert ( by_id["stack.redeploy"]["mcpExecutionMode"] == "blocked_no_typed_executor" ) assert by_id["stack.redeploy"]["mcpExecutionSupported"] is False asyncio.run(exercise_mcp()) metrics = mcp_server.runtime_status()["toolMetrics"]["preview_action"] assert metrics["calls"] >= 1 assert metrics["errors"] == 0 assert metrics["averageMs"] >= 0 canary = mcp_server.run_read_only_canary() assert canary["status"] == "ok" assert mcp_server.runtime_status()["canary"]["lastCheckedAt"] def test_typed_artifact_mcp_reuses_durable_idempotency_receipt(integration_client, tmp_path, monkeypatch): from types import SimpleNamespace from backend.app import mcp_server _, engine = integration_client mcp_server.engine = engine monkeypatch.setattr(mcp_server, "_artifact_roots", lambda: (tmp_path / "models", tmp_path / "receipts")) monkeypatch.setattr(mcp_server, "artifact_preflight", lambda payload, model_root: {"fingerprint": "pinned"}) calls = [] def fake_install(payload, *, model_root, receipt_root): calls.append(payload) return {"status": "installed", "path": str(model_root / "loras" / payload["filename"]), "sha256": payload["sha256"], "bytes": 5, "receipt": str(receipt_root / "receipt.json")} monkeypatch.setattr(mcp_server, "install_artifact_file", fake_install) monkeypatch.setattr(mcp_server, "log_event", lambda *args, **kwargs: SimpleNamespace(id="audit-1")) args = ("lora", "https://example.com/model.safetensors", "a" * 64, "model.safetensors", "source-1", "test-license", "test fixture", "pinned", "repeatable-key") first = mcp_server.privatemuse_artifact_install(*args) second = mcp_server.privatemuse_artifact_install(*args) assert first == second assert len(calls) == 1 def test_mcp_operation_executes_directly_without_human_approval_and_is_idempotent( integration_client, monkeypatch ): _, engine = integration_client from backend.app.db.models import Approval from backend.app.services import mcp_operations calls = [] def execute_direct(_session, **kwargs): calls.append(kwargs) return { "success": True, "status": "success", "result": "success", "jobId": "job-mcp-direct", "auditId": "audit-mcp-direct", "policyDecision": "allowed", "originalPolicyDecision": kwargs["original_policy_decision"], } monkeypatch.setattr(mcp_operations, "execute_governed_action", execute_direct) with Session(engine) as session: requested = mcp_operations.request_operation( session, action="container.restart", target="widefrog", reason="Recover the unhealthy application through the typed restart executor.", idempotency_key="chatgpt-operation-0001", ) replay = mcp_operations.request_operation( session, action="container.restart", target="widefrog", reason="Recover the unhealthy application through the typed restart executor.", idempotency_key="chatgpt-operation-0001", ) approvals = session.exec( select(Approval).where(Approval.target == "widefrog") ).all() assert requested["created"] is False assert requested["executed"] is True assert requested["policyDecision"] == "allowed" assert requested["originalPolicyDecision"] == "allowed" assert requested["executionSupported"] is True assert requested["idempotentReplay"] is False assert "approvalId" not in requested assert replay["jobId"] == requested["jobId"] assert replay["idempotentReplay"] is True assert len(calls) == 1 assert approvals == [] def test_mcp_operation_crash_boundary_refuses_duplicate_reexecution( integration_client, monkeypatch ): _, engine = integration_client from backend.app.services import mcp_operations calls = [] def crash_after_boundary(_session, **kwargs): calls.append(kwargs) raise RuntimeError("simulated crash after side-effect boundary") monkeypatch.setattr( mcp_operations, "execute_governed_action", crash_after_boundary ) with Session(engine) as session: with pytest.raises(RuntimeError, match="simulated crash"): mcp_operations.request_operation( session, action="container.restart", target="widefrog-crash", reason="Exercise durable MCP idempotency boundary.", idempotency_key="chatgpt-operation-crash-0001", ) replay = mcp_operations.request_operation( session, action="container.restart", target="widefrog-crash", reason="Exercise durable MCP idempotency boundary.", idempotency_key="chatgpt-operation-crash-0001", ) assert len(calls) == 1 assert replay["idempotentReplay"] is True assert replay["executed"] is False assert replay["policyDecision"] == "blocked" assert replay["executionState"] == "reconciliation_required" assert "approvalId" not in replay def test_mcp_operation_executes_typed_action_even_when_policy_was_approval_required( integration_client, monkeypatch ): _, engine = integration_client from backend.app.db.models import Approval, Policy from backend.app.services import mcp_operations monkeypatch.setattr( mcp_operations, "execute_governed_action", lambda _session, **kwargs: { "success": True, "status": "success", "result": "success", "jobId": "job-mcp-policy", "auditId": "audit-mcp-policy", "policyDecision": "allowed", "originalPolicyDecision": kwargs["original_policy_decision"], }, ) with Session(engine) as session: session.add( Policy( id="policy-mcp-no-approval", action="container.restart", target="widefrog-policy", voice="approval_required", manual="approval_required", risk="medium", reason="Historical policy requested approval.", ) ) session.commit() result = mcp_operations.request_operation( session, action="container.restart", target="widefrog-policy", reason="Autonomously recover the typed target.", idempotency_key="chatgpt-operation-policy-0001", ) approvals = session.exec( select(Approval).where(Approval.target == "widefrog-policy") ).all() assert result["executed"] is True assert result["policyDecision"] == "allowed" assert result["originalPolicyDecision"] == "approval_required" assert "approvalId" not in result assert approvals == [] def test_mcp_operation_keeps_blocked_and_read_only_actions_non_executable( integration_client, ): _, engine = integration_client from backend.app.services import mcp_operations with Session(engine) as session: blocked = mcp_operations.request_operation( session, action="volume.delete", target="important-data", reason="This must remain blocked.", idempotency_key="chatgpt-operation-blocked-0001", ) read_only = mcp_operations.request_operation( session, action="container.status", target="widefrog", reason="Read status only.", idempotency_key="chatgpt-operation-readonly-0001", ) assert blocked["policyDecision"] == "blocked" assert blocked["created"] is False assert blocked["executed"] is False assert blocked["executionSupported"] is False assert "approvalId" not in blocked assert read_only["created"] is False assert read_only["executed"] is False assert read_only["executionSupported"] is False assert "approvalId" not in read_only def test_mcp_nonblocked_action_without_typed_executor_fails_closed_without_approval( integration_client, ): _, engine = integration_client from backend.app.db.models import Approval from backend.app.services import mcp_operations with Session(engine) as session: result = mcp_operations.request_operation( session, action="stack.redeploy", target="appops", reason="No generic stack executor is exposed through MCP.", idempotency_key="chatgpt-operation-unsupported-0001", ) approvals = session.exec( select(Approval).where(Approval.target == "appops") ).all() assert result["policyDecision"] == "blocked" assert result["originalPolicyDecision"] == "approval_required" assert result["executionSupported"] is False assert result["executed"] is False assert "approvalId" not in result assert approvals == [] def test_mcp_legacy_operation_cannot_execute_after_approval_expiry(integration_client): _, engine = integration_client from datetime import timedelta from backend.app.core.time import utc_now from backend.app.db.models import Approval from backend.app.services import mcp_operations from backend.app.services.approval_service import create_approval, update_approval_status with Session(engine) as session: created = create_approval( session, action="container.restart", target="widefrog-legacy", source="automation", risk="medium", reason="Legacy approval compatibility test.", requester="legacy-test", ) update_approval_status(session, created.id, "approved", actor="test-operator") approval = session.get(Approval, created.id) assert approval is not None approval.expires_at = utc_now() - timedelta(seconds=1) session.add(approval) session.commit() with pytest.raises(ValueError, match="approval_expired"): mcp_operations.execute_operation( session, approval_id=approval.id, expected_action_fingerprint=str(approval.action_fingerprint), ) def test_mcp_recursively_redacts_stored_metadata(): from backend.app.mcp_server import _json serialized = str( _json( { "last_action": "api_key=MCP-SYNTHETIC-SECRET", "labels": {"example.secret": "MCP-LABEL-SECRET"}, } ) ) assert "MCP-SYNTHETIC-SECRET" not in serialized assert "MCP-LABEL-SECRET" not in serialized assert "[REDACTED]" in serialized def test_n8n_event_and_audit_fields_are_redacted(integration_client): client, engine = integration_client from backend.app.db.models import AuditEvent headers = { "Authorization": f"Bearer {N8N_TOKEN}", "Idempotency-Key": "workflow-redaction-0001", } response = client.post( "/api/integrations/n8n/approvals", headers=headers, json={ "action": "stack.redeploy", "target": "appops", "reason": "api_key=N8N-REASON-SYNTHETIC", "plan": [], }, ) assert response.status_code == 200 with Session(engine) as session: approval_audit = session.get(AuditEvent, response.json()["auditId"]) assert approval_audit is not None assert "N8N-REASON-SYNTHETIC" not in str(approval_audit.reason) session.add( AuditEvent( id="audit-secret-target", source="manual", identity="operator", action_id="container.status", target="api_key=N8N-TARGET-SYNTHETIC", risk="low", policy_decision="allowed", reason="status", result="success", ) ) session.commit() events = client.get( "/api/integrations/n8n/events", headers={"Authorization": f"Bearer {N8N_TOKEN}"} ) assert events.status_code == 200 serialized = str(events.json()) assert "N8N-TARGET-SYNTHETIC" not in serialized assert "[REDACTED]" in serialized def test_n8n_heartbeat_makes_reachability_truthful(integration_client): client, _ = integration_client n8n_headers = {"Authorization": f"Bearer {N8N_TOKEN}"} operator_headers = human_headers(client) before = client.get("/api/integrations/status", headers=operator_headers).json() assert before["n8n"]["state"] == "configured" heartbeat = client.post( "/api/integrations/n8n/heartbeat", headers=n8n_headers, json={ "workflowId": "appops-test-workflow", "workflowVersion": "2", "branch": "events", "processedEvents": 42, "backlogRemaining": True, }, ) assert heartbeat.status_code == 200 assert heartbeat.json()["canExecute"] is False knowledge_heartbeat = client.post( "/api/integrations/n8n/heartbeat", headers=n8n_headers, json={ "workflowId": "appops-test-workflow", "workflowVersion": "2", "branch": "knowledge", "confidence": "high", }, ) assert knowledge_heartbeat.status_code == 200 after = client.get("/api/integrations/status", headers=operator_headers).json() assert after["n8n"]["state"] == "reachable" assert after["n8n"]["workflow"]["processedEvents"] == 42 assert after["n8n"]["workflow"]["backlogRemaining"] is True assert after["n8n"]["workflow"]["branches"]["events"]["processedEvents"] == 42 assert after["n8n"]["workflow"]["branches"]["knowledge"]["confidence"] == "high" def test_audit_endpoint_is_paginated(integration_client): client, engine = integration_client from backend.app.db.models import AuditEvent with Session(engine) as session: for index in range(4): session.add( AuditEvent( id=f"audit-page-{index}", source="system", identity="test", action_id="audit.page", target=str(index), risk="low", policy_decision="allowed", result="success", ) ) session.commit() first = client.get("/api/audit?limit=2&offset=0", headers=human_headers(client)).json() second = client.get("/api/audit?limit=2&offset=2", headers=human_headers(client)).json() assert len(first) == 2 assert len(second) == 2 assert {item["id"] for item in first}.isdisjoint({item["id"] for item in second}) def test_n8n_concurrent_idempotency_key_creates_one_approval(integration_client): _, engine = integration_client from backend.app.db.models import Approval from backend.app.services.n8n_integration import request_approval def submit(): with Session(engine) as session: return request_approval( session, action="stack.redeploy", target="appops-concurrent", reason="Gecontroleerde gelijktijdige workflowtest", plan=["preflight"], idempotency_key="workflow-concurrent-0001", ) with ThreadPoolExecutor(max_workers=2) as pool: results = list(pool.map(lambda _: submit(), range(2))) with Session(engine) as session: approvals = session.exec( select(Approval).where(Approval.target == "appops-concurrent") ).all() assert len(approvals) == 1 assert {result["idempotentReplay"] for result in results} == {False, True} def test_n8n_machine_body_limit_and_mutation_rate_limit(integration_client): client, _ = integration_client headers = {"Authorization": f"Bearer {N8N_TOKEN}"} too_large = client.post( "/api/integrations/n8n/preflight", headers=headers, content=b"x" * (1_048_576 + 1), ) assert too_large.status_code == 413 statuses = [ client.post( "/api/integrations/n8n/preflight", headers=headers, json={"action": "container.status", "target": f"app-{index}"}, ).status_code for index in range(11) ] assert statuses[:10] == [200] * 10 assert statuses[10] == 429 def test_workmesh_promotion_preflight_creates_sha_bound_approval_and_receipt_job( integration_client, monkeypatch ): client, engine = integration_client from backend.app.db.models import Approval from backend.app.services.deployment_broker import DeploymentBrokerClient from backend.app.services.secret_store import set_secret revision = "a" * 40 digest = "b" * 64 backup_inputs_digest = "d" * 64 receipt_digest = "c" * 64 backup_receipt_digest = "f" * 64 broker_secret = "workmesh-controller-secret-" + "x" * 32 with Session(engine) as session: set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", broker_secret) preflight_calls = [] def preflight(_self, requested_revision, *, secret): assert requested_revision == revision assert secret == broker_secret preflight_calls.append(requested_revision) return { "apiVersion": "itworx.workmesh-promotion/v2", "status": "preflight_ok", "repository": "jens/workmesh", "revision": revision, "preflightSha256": digest, "backupInputsSha256": backup_inputs_digest, "checks": { "exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed", }, } monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight) preflight_headers = { **human_headers(client), "Idempotency-Key": "workmesh-promote-preflight-0001", } direct_action = client.post( "/api/actions/execute", headers=human_headers(client), json={ "plan": { "intent": "deployment.target.promote_workmesh", "target": "jens/workmesh", "source": "manual", "plan": [], } }, ) assert direct_action.status_code == 200 assert direct_action.json()["result"] == "preflight_required" assert client.post( "/api/deployments/workmesh/promotions/preflight", headers=preflight_headers, json={"revision": revision, "repository": "jens/other"}, ).status_code == 422 response = client.post( "/api/deployments/workmesh/promotions/preflight", headers=preflight_headers, json={"revision": revision}, ) assert response.status_code == 200, response.text requested = response.json() assert requested["status"] == "approval_required" assert requested["revision"] == revision assert requested["preflight"]["preflightSha256"] == digest assert requested["approvalId"] assert requested["promotionId"] with Session(engine) as session: approval = session.get(Approval, requested["approvalId"]) assert approval is not None assert approval.action == "deployment.target.promote_workmesh" assert approval.target == "jens/workmesh" assert approval.request_context_json["revision"] == revision assert approval.request_context_json["preflightSha256"] == digest assert approval.request_context_json["backupInputsSha256"] == backup_inputs_digest assert approval.request_context_json["promotionId"] == requested["promotionId"] assert revision in approval.reason assert digest in approval.reason listed = client.get( "/api/deployments/workmesh/promotions", headers=human_headers(client), ) assert listed.status_code == 200 assert listed.json()["items"][0]["promotionId"] == requested["promotionId"] assert listed.json()["items"][0]["approvalStatus"] == "pending" replay = client.post( "/api/deployments/workmesh/promotions/preflight", headers=preflight_headers, json={"revision": revision}, ) assert replay.status_code == 200 assert replay.json()["promotionId"] == requested["promotionId"] assert replay.json()["approvalId"] == requested["approvalId"] assert replay.json()["idempotentReplay"] is True assert preflight_calls == [revision] changed_revision_replay = client.post( "/api/deployments/workmesh/promotions/preflight", headers=preflight_headers, json={"revision": "c" * 40}, ) assert changed_revision_replay.status_code == 409 assert preflight_calls == [revision] expected_request = hashlib.sha256( json.dumps( { "repository": "jens/workmesh", "revision": revision, "operation": "workmesh_promote", "idempotency": f"workmesh-promotion:{requested['promotionId']}", }, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() def apply(_self, requested_revision, *, secret, preflight_sha256, idempotency_key): assert requested_revision == revision assert secret == broker_secret assert preflight_sha256 == digest assert idempotency_key == f"workmesh-promotion:{requested['promotionId']}" return { "status": "accepted", "request": expected_request, "repository": "jens/workmesh", "revision": revision, "operation": "workmesh_promote", } def status(_self, request_id, *, secret): assert request_id == expected_request assert secret == broker_secret return { "status": "deployed", "request": expected_request, "repository": "jens/workmesh", "revision": revision, "operation": "workmesh_promote", "receiptSha256": receipt_digest, "backupReceiptSha256": backup_receipt_digest, } monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply) monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_status", status) auth = human_headers(client) assert client.patch( f"/api/approvals/{requested['approvalId']}", headers=auth, json={"status": "approved"}, ).status_code == 200 approved_promotion = client.get( f"/api/deployments/workmesh/promotions/{requested['promotionId']}", headers=auth, ) assert approved_promotion.status_code == 200 assert approved_promotion.json()["approvalStatus"] == "approved" executed = client.post( f"/api/approvals/{requested['approvalId']}/execute", headers=auth ) assert executed.status_code == 200, executed.text assert executed.json()["approvalStatus"] == "executed" assert executed.json()["status"] == "applying" assert executed.json()["jobId"] == requested["promotionId"] assert preflight_calls == [revision, revision] reconciled = client.get( f"/api/deployments/workmesh/promotions/{requested['promotionId']}", headers=auth, ) assert reconciled.status_code == 200, reconciled.text assert reconciled.json()["status"] == "succeeded" assert reconciled.json()["revision"] == revision assert reconciled.json()["evidence"]["receiptSha256"] == receipt_digest assert reconciled.json()["evidence"]["backupReceiptSha256"] == backup_receipt_digest assert reconciled.json()["approvalStatus"] == "executed" def test_workmesh_ambiguous_controller_apply_is_consumed_and_only_reconciled( integration_client, monkeypatch ): client, engine = integration_client from backend.app.services.deployment_broker import DeploymentBrokerClient from backend.app.services.secret_store import set_secret revision = "d" * 40 broker_secret = "workmesh-controller-secret-" + "y" * 32 with Session(engine) as session: set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", broker_secret) def preflight(_self, requested_revision, *, secret): assert requested_revision == revision assert secret == broker_secret return { "apiVersion": "itworx.workmesh-promotion/v2", "status": "preflight_ok", "repository": "jens/workmesh", "revision": revision, "preflightSha256": "e" * 64, "backupInputsSha256": "f" * 64, "checks": { "exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed", }, } apply_calls = [] status_calls = [] def apply(*_args, **_kwargs): apply_calls.append(True) return {"status": "unreachable", "reason": "connection closed"} def status(*_args, **_kwargs): status_calls.append(True) return {"status": "not_found"} monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight) monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply) monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_status", status) auth = human_headers(client) requested = client.post( "/api/deployments/workmesh/promotions/preflight", headers={**auth, "Idempotency-Key": "workmesh-promote-uncertain-0001"}, json={"revision": revision}, ) assert requested.status_code == 200, requested.text approval_id = requested.json()["approvalId"] assert client.patch( f"/api/approvals/{approval_id}", headers=auth, json={"status": "approved"} ).status_code == 200 executed = client.post(f"/api/approvals/{approval_id}/execute", headers=auth) assert executed.status_code == 200, executed.text assert executed.json()["status"] == "outcome_unknown" assert executed.json()["approvalStatus"] == "executed" assert apply_calls == [True] for _ in range(2): reconciled = client.get( f"/api/deployments/workmesh/promotions/{requested.json()['promotionId']}", headers=auth, ) assert reconciled.status_code == 200, reconciled.text assert reconciled.json()["status"] == "outcome_unknown" assert reconciled.json()["evidence"] == {"reconciliationRequired": True} assert len(status_calls) == 2 assert apply_calls == [True] replay = client.post(f"/api/approvals/{approval_id}/execute", headers=auth) assert replay.status_code == 409 def test_mcp_legacy_approval_executor_does_not_expose_workmesh_promotion(integration_client): _, engine = integration_client from backend.app.services import mcp_operations from backend.app.services.approval_service import create_approval, update_approval_status with Session(engine) as session: approval = create_approval( session, action="deployment.target.promote_workmesh", target="jens/workmesh", source="manual", risk="high", reason="Fixed WorkMesh promotion contract test.", context={"apiVersion": "itworx.workmesh-promotion/v2"}, ) update_approval_status(session, approval.id, "approved", actor="test-operator") assert mcp_operations.operation_status(session, approval.id)["executionSupported"] is False with pytest.raises(ValueError, match="operation_executor_unavailable"): mcp_operations.execute_operation( session, approval_id=approval.id, expected_action_fingerprint=str(approval.action_fingerprint), ) def test_workmesh_changed_preflight_expires_approval_before_apply( integration_client, monkeypatch ): client, engine = integration_client from backend.app.db.models import Approval from backend.app.services.deployment_broker import DeploymentBrokerClient from backend.app.services.secret_store import set_secret revision = "f" * 40 with Session(engine) as session: set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", "workmesh-controller-secret-" + "z" * 32) preflight_calls = [] def preflight(_self, requested_revision, *, secret): assert requested_revision == revision assert secret.startswith("workmesh-controller-secret-") preflight_calls.append(True) return { "apiVersion": "itworx.workmesh-promotion/v2", "status": "preflight_ok", "repository": "jens/workmesh", "revision": revision, "preflightSha256": ("1" if len(preflight_calls) == 1 else "2") * 64, "backupInputsSha256": "b" * 64, "checks": { "exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed", }, } apply_calls = [] def apply(*_args, **_kwargs): apply_calls.append(True) return {"status": "accepted"} monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight) monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply) auth = human_headers(client) requested = client.post( "/api/deployments/workmesh/promotions/preflight", headers={**auth, "Idempotency-Key": "workmesh-promote-drift-0001"}, json={"revision": revision}, ) assert requested.status_code == 200, requested.text approval_id = requested.json()["approvalId"] assert client.patch( f"/api/approvals/{approval_id}", headers=auth, json={"status": "approved"} ).status_code == 200 executed = client.post(f"/api/approvals/{approval_id}/execute", headers=auth) assert executed.status_code == 200 assert executed.json()["status"] == "preflight_changed" assert apply_calls == [] with Session(engine) as session: approval = session.get(Approval, approval_id) assert approval.status == "expired" replay = client.post(f"/api/approvals/{approval_id}/execute", headers=auth) assert replay.status_code == 409 def test_controller_preflight_changed_identity_expires_approval_without_unknown_state( integration_client, monkeypatch ): client, engine = integration_client from backend.app.db.models import Approval, WorkMeshPromotion from backend.app.services.deployment_broker import DeploymentBrokerClient from backend.app.services.secret_store import set_secret revision = "9" * 40 broker_secret = "workmesh-controller-secret-" + "x" * 32 with Session(engine) as session: set_secret(session, "WORKMESH_DEPLOYMENT_BROKER", broker_secret) def preflight(_self, requested_revision, *, secret): assert requested_revision == revision assert secret == broker_secret return { "apiVersion": "itworx.workmesh-promotion/v2", "status": "preflight_ok", "repository": "jens/workmesh", "revision": revision, "preflightSha256": "a" * 64, "backupInputsSha256": "b" * 64, "checks": { "exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed", }, } apply_calls = [] def apply(_self, requested_revision, *, secret, preflight_sha256, idempotency_key): assert requested_revision == revision assert secret == broker_secret assert preflight_sha256 == "a" * 64 apply_calls.append(idempotency_key) request_id = hashlib.sha256( json.dumps( { "repository": "jens/workmesh", "revision": revision, "operation": "workmesh_promote", "idempotency": idempotency_key, }, sort_keys=True, separators=(",", ":"), ).encode("utf-8") ).hexdigest() return { "status": "preflight_changed", "request": request_id, "repository": "jens/workmesh", "revision": revision, "operation": "workmesh_promote", "reason": "approved_preflight_no_longer_matches", } monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_preflight", preflight) monkeypatch.setattr(DeploymentBrokerClient, "workmesh_promotion_apply", apply) auth = human_headers(client) requested = client.post( "/api/deployments/workmesh/promotions/preflight", headers={**auth, "Idempotency-Key": "workmesh-promote-controller-drift-01"}, json={"revision": revision}, ) assert requested.status_code == 200, requested.text promotion_id = requested.json()["promotionId"] approval_id = requested.json()["approvalId"] assert client.patch( f"/api/approvals/{approval_id}", headers=auth, json={"status": "approved"} ).status_code == 200 executed = client.post(f"/api/approvals/{approval_id}/execute", headers=auth) assert executed.status_code == 200, executed.text assert executed.json()["status"] == "preflight_changed" assert executed.json()["success"] is False assert apply_calls == [f"workmesh-promotion:{promotion_id}"] with Session(engine) as session: approval = session.get(Approval, approval_id) promotion = session.get(WorkMeshPromotion, promotion_id) assert approval is not None and approval.status == "expired" assert promotion is not None and promotion.status == "preflight_changed" @pytest.mark.parametrize(("name", "argument", "discriminator"), [ ("enrollment.preflight", "plan", None), ("target.enroll", "plan", None), ("deployment.preflight", "plan", None), ("deployment.apply", "plan", None), ("deployment.rollback", "plan", None), ("deployment.preflight", "plan", "itworx.forge-bootstrap-preflight/v1"), ("forge.bootstrap.apply", "profile", None), ]) def test_mcp_typed_schema_rejection_never_echoes_opaque_input( integration_client, monkeypatch, name, argument, discriminator, ): from backend.app import mcp_server canary = "syntheticOpaqueInput_9Z7XQ" payload = {"api_key": canary, canary: {"nested": canary}, "application_id": canary} if discriminator: payload["api_version"] = discriminator arguments = {argument: payload} if name != "enrollment.preflight": arguments["idempotencyKey"] = "schema-redaction-fixture" def forbidden_service(*_args, **_kwargs): pytest.fail("invalid schema must not reach deployment/enrollment services") monkeypatch.setattr(mcp_server, "EnrollmentPlane", forbidden_service) monkeypatch.setattr(mcp_server, "DeploymentControlPlane", forbidden_service) async def exercise(): async with Client(mcp_server.appops_mcp) as client: result = await client.call_tool(name, arguments) assert result.is_error is False assert result.structured_content == { "status": "invalid", "executed": False, "reason": "invalid_typed_plan", } assert canary not in json.dumps(result.model_dump(mode="json")) asyncio.run(exercise()) assert mcp_server._MCP_EXECUTION_LOCK.locked() is False