Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020

tests/test_wp230_policy_simulate.py

@@ -0,0 +1,555 @@
+"""WP-230B regression tests: /api/policies/simulate wired to the real engine.
+
+Exercises services/runtime/control_api.py::simulate_policy() directly (no
+FastAPI app needed -- it's a module-level function precisely so it can be
+tested this way) against this repository's own real catalog/ directory and
+the same _policy_engine() production traffic uses
+(services/runtime/dataplane.py), plus a lower-level PolicyEngine test for
+the production deny/default-deny rules and the exact v1.1 Gitea R2 exception.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+from packages.hub_clients import ClientKind, ClientRecord, ClientStatus
+from packages.hub_policy import Decision, InvocationContext
+from services.control_api.auth.models import AuthenticatedPrincipal, ManagementRole, PrincipalKind
+from services.runtime.control_api import simulate_policy
+from services.runtime.dataplane import _policy_engine
+
+ROOT = Path(__file__).resolve().parents[1]
+CATALOG_ROOT = ROOT / "catalog"
+_NOW = datetime.now(UTC)
+
+
+class FakeClientLister:
+    def __init__(self, records: list[ClientRecord]) -> None:
+        self._records = records
+        self.list_call_count = 0
+
+    def list(self) -> list[ClientRecord]:
+        self.list_call_count += 1
+        return self._records
+
+
+def _client_record(status: ClientStatus = ClientStatus.ENABLED) -> ClientRecord:
+    return ClientRecord(
+        id=uuid4(),
+        name="test-client",
+        project="ludarium",
+        environment="prod",
+        kind=ClientKind.NATIVE_2026,
+        status=status,
+        credential_reference="authentik://mcp-client/test",
+        created_at=_NOW,
+    )
+
+
+def _principal(role: ManagementRole = ManagementRole.VIEWER) -> AuthenticatedPrincipal:
+    return AuthenticatedPrincipal(
+        subject="test-admin",
+        issuer="https://fake.oidc.test",
+        audience=("itworx-mcp-hub",),
+        kind=PrincipalKind.MANAGEMENT_USER,
+        expires_at=_NOW,
+        issued_at=_NOW,
+        roles=frozenset({role}),
+    )
+
+
+def test_allow_for_a_real_registered_project_environment_tool_binding() -> None:
+    # knowledge.document.search -> connectorType "knowledge" -> ludarium/prod
+    # has a real "knowledge-default" binding covering "prod" (see
+    # catalog/projects/ludarium.yaml) -> binding_present is derived True.
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
+        _principal(),
+    )
+    assert result["allowed"] is True
+    assert result["decision"] == "allow_if_conditions"
+    assert "040-read-tool-allow" in result["matchedPolicies"]
+    assert result["reasonCode"] == "ALLOW_RULE"
+    assert result["steps"], "a real evaluator produces a non-empty explainability trace"
+
+
+def test_deny_precedence_a_privileged_tool_name_is_denied_before_any_allow_check() -> None:
+    # 010-deny-generic-privileged-tools is terminal and evaluated first -- it
+    # must win even though this project/environment/binding would otherwise
+    # be perfectly valid (proves deny precedence, not just deny). Tested at
+    # the engine level directly: no such tool can ever be a *registered*
+    # catalog tool (validate_pack.py's FORBIDDEN_TOOL_PREFIXES rejects it
+    # outright at catalog-validation time), so through simulate_policy()'s
+    # own known-tool gate it would correctly, but less interestingly, deny
+    # as UNKNOWN_TOOL before ever reaching this rule -- this test isolates
+    # the specific behavior it exists to prove.
+    engine = _policy_engine(CATALOG_ROOT)
+    context = InvocationContext(
+        workspace="itworx",
+        project="ludarium",
+        environment="prod",
+        principal="test-admin",
+        tool="shell.exec",
+        connector="unraid",
+        risk_class="R1",
+        binding_present=True,
+        client_enabled=True,
+    )
+    trace = engine.evaluate(context)
+    assert trace.decision == Decision.DENY_AND_ALERT
+    assert trace.reason_code == "GLOBAL_DENY"
+    assert "010-deny-generic-privileged-tools" in trace.matched_rule_ids
+
+
+def test_missing_binding_denied_even_for_an_otherwise_valid_read_tool() -> None:
+    # project.status.get -> connectorType "control" -- ludarium/prod has no
+    # registered binding of that connector type (see catalog/projects/ludarium.yaml),
+    # so binding_present is correctly derived False, and the engine's
+    # top-level binding gate denies before any policy rule is even consulted.
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "ludarium", "environment": "prod", "tool": "project.status.get"},
+        _principal(),
+    )
+    assert result["allowed"] is False
+    assert result["reasonCode"] == "MISSING_BINDING"
+
+
+def test_explicit_binding_present_override_is_honoured() -> None:
+    # The caller can override the catalog-derived default -- e.g. "what if
+    # this connector were bound" -- and get a real allow for it, still
+    # through the real engine (not a hardcoded response).
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {
+            "project": "ludarium",
+            "environment": "prod",
+            "tool": "project.status.get",
+            "bindingPresent": True,
+        },
+        _principal(),
+    )
+    assert result["allowed"] is True
+    assert result["reasonCode"] == "ALLOW_RULE"
+
+
+def test_unknown_project_is_denied_by_the_simulator_known_entity_check() -> None:
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "does-not-exist", "environment": "prod", "tool": "knowledge.document.search"},
+        _principal(),
+    )
+    assert result["allowed"] is False
+    assert result["reasonCode"] == "UNKNOWN_PROJECT"
+
+
+def test_unknown_tool_is_denied_by_the_simulator_known_entity_check() -> None:
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "ludarium", "environment": "prod", "tool": "not.a.real.tool"},
+        _principal(),
+    )
+    assert result["allowed"] is False
+    assert result["reasonCode"] == "UNKNOWN_TOOL"
+
+
+def test_wrong_environment_has_no_registered_binding_and_is_denied() -> None:
+    # ludarium only registers bindings for "prod" -- an unregistered
+    # environment correctly derives binding_present=False.
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "ludarium", "environment": "staging", "tool": "knowledge.document.search"},
+        _principal(),
+    )
+    assert result["allowed"] is False
+    assert result["reasonCode"] == "MISSING_BINDING"
+
+
+def test_invalid_client_binding_unknown_client_id_is_denied() -> None:
+    known = _client_record()
+    unknown_client_id = uuid4()
+    assert unknown_client_id != known.id
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([known]),
+        {
+            "project": "ludarium",
+            "environment": "prod",
+            "tool": "knowledge.document.search",
+            "clientId": str(unknown_client_id),
+        },
+        _principal(),
+    )
+    assert result["allowed"] is False
+    assert result["reasonCode"] == "UNKNOWN_CLIENT"
+
+
+def test_known_enabled_client_id_does_not_block_an_otherwise_valid_allow() -> None:
+    known = _client_record(status=ClientStatus.ENABLED)
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([known]),
+        {
+            "project": "ludarium",
+            "environment": "prod",
+            "tool": "knowledge.document.search",
+            "clientId": str(known.id),
+        },
+        _principal(),
+    )
+    assert result["allowed"] is True
+
+
+def test_malformed_client_id_is_treated_as_absent_not_a_crash() -> None:
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {
+            "project": "ludarium",
+            "environment": "prod",
+            "tool": "knowledge.document.search",
+            "clientId": "not-a-uuid",
+        },
+        _principal(),
+    )
+    # Malformed input must not 500 -- and must not be silently treated as a
+    # valid, unscoped bypass either.
+    assert result["decision"] in {"allow_if_conditions", "deny"}
+
+
+def test_simulation_never_mutates_client_state() -> None:
+    lister = FakeClientLister([_client_record()])
+    before = list(lister.list())
+    simulate_policy(
+        CATALOG_ROOT,
+        lister,
+        {"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
+        _principal(),
+    )
+    after = list(lister.list())
+    assert before == after
+    assert lister.list_call_count >= 1  # it did read, just never wrote
+
+
+def test_repeated_simulation_is_deterministic_except_trace_id() -> None:
+    payload = {"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"}
+    first = simulate_policy(CATALOG_ROOT, FakeClientLister([]), payload, _principal())
+    second = simulate_policy(CATALOG_ROOT, FakeClientLister([]), payload, _principal())
+    assert first["decision"] == second["decision"]
+    assert first["matchedPolicies"] == second["matchedPolicies"]
+    assert first["reasonCode"] == second["reasonCode"]
+    # Each call gets its own real trace ID -- not the old hardcoded
+    # UUID(int=0).hex constant repeated on every request.
+    assert first["traceId"] != second["traceId"]
+    assert first["traceId"] != "0" * 32
+
+
+def test_required_scopes_are_surfaced_from_the_tool_manifest() -> None:
+    result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
+        _principal(),
+    )
+    assert result["requiredScopes"], "knowledge.document.search declares requiredScopes"
+
+
+def test_principal_identity_flows_into_the_evaluated_context() -> None:
+    # Not asserted via a private attribute -- proven behaviorally: the trace
+    # is produced per-principal (trace_id differs), and simulate_policy does
+    # not silently fall back to an anonymous/default principal string.
+    admin_result = simulate_policy(
+        CATALOG_ROOT,
+        FakeClientLister([]),
+        {"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
+        _principal(ManagementRole.ADMIN),
+    )
+    assert admin_result["allowed"] is True
+
+
+# --- Lower-level engine tests prove the exact bounded Gitea R2 exception and
+# default denial of every other R2 operation against the same engine production
+# traffic uses.
+
+
+def test_unallowlisted_r2_operations_remain_default_denied_in_production() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    context = InvocationContext(
+        workspace="itworx",
+        project="ludarium",
+        environment="prod",
+        principal="test-admin",
+        tool="hypothetical.future.write_tool",
+        connector="unraid",
+        risk_class="R2",
+        environment_classification="production",
+        binding_present=True,
+        client_enabled=True,
+    )
+    trace = engine.evaluate(context)
+    assert trace.decision == Decision.DENY
+    assert trace.reason_code == "DEFAULT_DENY"
+
+
+def test_only_registered_gitea_r2_write_set_is_allowed_in_production() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    for tool, arguments in (
+        ("gitea.repository.create", {"name": "new-project", "private": True}),
+        (
+            "gitea.branch.create",
+            {"repositoryRef": "hub", "branch": "chatgpt/test", "fromRef": "main"},
+        ),
+    ):
+        context = InvocationContext(
+            workspace="itworx",
+            project="gitea-all",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool=tool,
+            connector="gitea",
+            risk_class="R2",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+            arguments=arguments,
+        )
+        trace = engine.evaluate(context)
+        assert trace.decision == Decision.ALLOW_IF_CONDITIONS
+        assert trace.matched_rule_ids == ("035-gitea-controlled-writes-v1-1",)
+        assert trace.reason_code == "ALLOW_RULE"
+
+
+def test_analysis_profile_allows_only_its_two_bounded_r2_mutations() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    for tool in ("projectbrain.analysis.start", "projectbrain.analysis.cancel"):
+        context = InvocationContext(
+            workspace="itworx",
+            project="gitea-analysis",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool=tool,
+            connector="projectbrain",
+            risk_class="R2",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+        )
+        trace = engine.evaluate(context)
+        assert trace.decision == Decision.ALLOW_IF_CONDITIONS
+        assert trace.matched_rule_ids == ("038-projectbrain-analysis-operations",)
+        assert trace.reason_code == "ALLOW_RULE"
+
+    denied = engine.evaluate(
+        InvocationContext(
+            workspace="itworx",
+            project="gitea-analysis",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool="projectbrain.delivery.create",
+            connector="projectbrain",
+            risk_class="R2",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+        )
+    )
+    assert denied.decision == Decision.DENY
+    assert denied.reason_code == "DEFAULT_DENY"
+
+
+def test_closed_loop_profile_allows_governed_finding_and_delivery_mutations() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    repository_create = engine.evaluate(
+        InvocationContext(
+            workspace="itworx",
+            project="gitea-loop",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool="gitea.repository.create",
+            connector="gitea",
+            risk_class="R2",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+            arguments={"name": "new-project", "private": True},
+        )
+    )
+    assert repository_create.decision == Decision.ALLOW_IF_CONDITIONS
+    assert repository_create.matched_rule_ids == ("039-projectbrain-finding-loop-operations",)
+
+    for tool in (
+        "gitea.workspace.run",
+        "projectbrain.finding.issue.sync",
+        "projectbrain.finding.investigation.start",
+        "projectbrain.finding.investigation.save",
+        "projectbrain.finding.delivery.create",
+        "projectbrain.delivery.file.write",
+        "projectbrain.delivery.file.delete",
+        "projectbrain.delivery.complete",
+        "projectbrain.finding.verify",
+        "projectbrain.brainstorm.save",
+        "projectbrain.knowledge.save",
+        "projectbrain.documentation.save",
+        "projectbrain.memory.save",
+        "projectbrain.contextpack.create",
+        "projectbrain.feedback.record",
+    ):
+        trace = engine.evaluate(
+            InvocationContext(
+                workspace="itworx",
+                project="gitea-loop",
+                environment="prod",
+                principal="chatgpt-gitea",
+                tool=tool,
+                connector="gitea" if tool == "gitea.workspace.run" else "projectbrain",
+                risk_class="R2",
+                environment_classification="production",
+                binding_present=True,
+                client_enabled=True,
+            )
+        )
+        assert trace.decision == Decision.ALLOW_IF_CONDITIONS
+        assert trace.matched_rule_ids == ("039-projectbrain-finding-loop-operations",)
+
+
+def test_only_reviewed_gitea_r3_lifecycle_set_is_allowed_in_production() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    for tool in (
+        "gitea.pullrequest.sync",
+        "gitea.pullrequest.merge",
+        "gitea.branch.delete",
+    ):
+        context = InvocationContext(
+            workspace="itworx",
+            project="gitea-all",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool=tool,
+            connector="gitea",
+            risk_class="R3",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+            arguments={"repositoryRef": "hub", "expectedHeadSha": "e" * 40},
+        )
+        trace = engine.evaluate(context)
+        assert trace.decision == Decision.ALLOW_IF_CONDITIONS
+        assert trace.matched_rule_ids == ("036-gitea-reviewed-lifecycle-v1",)
+        assert trace.reason_code == "ALLOW_RULE"
+
+    for tool in ("gitea.pullrequest.sync", "gitea.pullrequest.merge"):
+        loop_trace = engine.evaluate(
+            InvocationContext(
+            workspace="itworx",
+            project="gitea-loop",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool=tool,
+            connector="gitea",
+            risk_class="R3",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+            )
+        )
+        assert loop_trace.decision == Decision.ALLOW_IF_CONDITIONS
+        assert loop_trace.matched_rule_ids == ("036-gitea-loop-reviewed-lifecycle-v1",)
+    denied = engine.evaluate(
+        InvocationContext(
+            workspace="itworx", project="gitea-loop", environment="prod",
+            principal="chatgpt-gitea", tool="gitea.pullrequest.merge",
+            connector="gitea", risk_class="R3",
+            environment_classification="production", binding_present=False,
+            client_enabled=True,
+        )
+    )
+    assert denied.decision == Decision.DENY
+
+
+def test_server_ops_allows_only_catalogued_governed_mutations() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    for tool, risk in (
+        ("server.operation.request", "R2"),
+        ("server.operation.execute", "R3"),
+        ("server.job.cancel", "R2"),
+    ):
+        trace = engine.evaluate(
+            InvocationContext(
+                workspace="itworx",
+                project="server-ops",
+                environment="prod",
+                principal="chatgpt-gitea",
+                tool=tool,
+                connector="appops",
+                risk_class=risk,
+                environment_classification="production",
+                binding_present=True,
+                client_enabled=True,
+            )
+        )
+        assert trace.decision == Decision.ALLOW_IF_CONDITIONS
+        assert trace.matched_rule_ids == ("041-appops-governed-operations",)
+
+    denied = engine.evaluate(
+        InvocationContext(
+            workspace="itworx",
+            project="server-ops",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool="server.shell.execute",
+            connector="appops",
+            risk_class="R3",
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+        )
+    )
+    assert denied.decision == Decision.DENY
+    assert denied.reason_code == "DEFAULT_DENY"
+
+
+def test_unregistered_r3_and_all_r4_operations_remain_denied_in_production() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    for risk_class, expected_reason in (("R3", "DEFAULT_DENY"), ("R4", "GLOBAL_DENY")):
+        context = InvocationContext(
+            workspace="itworx",
+            project="gitea-all",
+            environment="prod",
+            principal="chatgpt-gitea",
+            tool="gitea.repository.delete",
+            connector="gitea",
+            risk_class=risk_class,
+            environment_classification="production",
+            binding_present=True,
+            client_enabled=True,
+        )
+        trace = engine.evaluate(context)
+        assert trace.decision in {Decision.DENY, Decision.DENY_AND_ALERT}
+        assert trace.reason_code == expected_reason
+
+
+def test_production_restrictions_do_not_apply_outside_production_classification() -> None:
+    engine = _policy_engine(CATALOG_ROOT)
+    context = InvocationContext(
+        workspace="itworx",
+        project="ludarium",
+        environment="dev",
+        principal="test-admin",
+        tool="hypothetical.future.write_tool",
+        connector="unraid",
+        risk_class="R2",
+        environment_classification="development",
+        binding_present=True,
+        client_enabled=True,
+    )
+    trace = engine.evaluate(context)
+    assert "030-production-destructive-deny-v1-1" not in trace.matched_rule_ids