556 lines
20 KiB
Python
556 lines
20 KiB
Python
"""WP-230B regression tests: /api/policies/simulate wired to the real engine.
|
|
|
|
Exercises services/runtime/control_api.py::simulate_policy() directly (no
|
|
FastAPI app needed -- it's a module-level function precisely so it can be
|
|
tested this way) against this repository's own real catalog/ directory and
|
|
the same _policy_engine() production traffic uses
|
|
(services/runtime/dataplane.py), plus a lower-level PolicyEngine test for
|
|
the production deny/default-deny rules and the exact v1.1 Gitea R2 exception.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from packages.hub_clients import ClientKind, ClientRecord, ClientStatus
|
|
from packages.hub_policy import Decision, InvocationContext
|
|
from services.control_api.auth.models import AuthenticatedPrincipal, ManagementRole, PrincipalKind
|
|
from services.runtime.control_api import simulate_policy
|
|
from services.runtime.dataplane import _policy_engine
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CATALOG_ROOT = ROOT / "catalog"
|
|
_NOW = datetime.now(UTC)
|
|
|
|
|
|
class FakeClientLister:
|
|
def __init__(self, records: list[ClientRecord]) -> None:
|
|
self._records = records
|
|
self.list_call_count = 0
|
|
|
|
def list(self) -> list[ClientRecord]:
|
|
self.list_call_count += 1
|
|
return self._records
|
|
|
|
|
|
def _client_record(status: ClientStatus = ClientStatus.ENABLED) -> ClientRecord:
|
|
return ClientRecord(
|
|
id=uuid4(),
|
|
name="test-client",
|
|
project="ludarium",
|
|
environment="prod",
|
|
kind=ClientKind.NATIVE_2026,
|
|
status=status,
|
|
credential_reference="authentik://mcp-client/test",
|
|
created_at=_NOW,
|
|
)
|
|
|
|
|
|
def _principal(role: ManagementRole = ManagementRole.VIEWER) -> AuthenticatedPrincipal:
|
|
return AuthenticatedPrincipal(
|
|
subject="test-admin",
|
|
issuer="https://fake.oidc.test",
|
|
audience=("itworx-mcp-hub",),
|
|
kind=PrincipalKind.MANAGEMENT_USER,
|
|
expires_at=_NOW,
|
|
issued_at=_NOW,
|
|
roles=frozenset({role}),
|
|
)
|
|
|
|
|
|
def test_allow_for_a_real_registered_project_environment_tool_binding() -> None:
|
|
# knowledge.document.search -> connectorType "knowledge" -> ludarium/prod
|
|
# has a real "knowledge-default" binding covering "prod" (see
|
|
# catalog/projects/ludarium.yaml) -> binding_present is derived True.
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is True
|
|
assert result["decision"] == "allow_if_conditions"
|
|
assert "040-read-tool-allow" in result["matchedPolicies"]
|
|
assert result["reasonCode"] == "ALLOW_RULE"
|
|
assert result["steps"], "a real evaluator produces a non-empty explainability trace"
|
|
|
|
|
|
def test_deny_precedence_a_privileged_tool_name_is_denied_before_any_allow_check() -> None:
|
|
# 010-deny-generic-privileged-tools is terminal and evaluated first -- it
|
|
# must win even though this project/environment/binding would otherwise
|
|
# be perfectly valid (proves deny precedence, not just deny). Tested at
|
|
# the engine level directly: no such tool can ever be a *registered*
|
|
# catalog tool (validate_pack.py's FORBIDDEN_TOOL_PREFIXES rejects it
|
|
# outright at catalog-validation time), so through simulate_policy()'s
|
|
# own known-tool gate it would correctly, but less interestingly, deny
|
|
# as UNKNOWN_TOOL before ever reaching this rule -- this test isolates
|
|
# the specific behavior it exists to prove.
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="ludarium",
|
|
environment="prod",
|
|
principal="test-admin",
|
|
tool="shell.exec",
|
|
connector="unraid",
|
|
risk_class="R1",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert trace.decision == Decision.DENY_AND_ALERT
|
|
assert trace.reason_code == "GLOBAL_DENY"
|
|
assert "010-deny-generic-privileged-tools" in trace.matched_rule_ids
|
|
|
|
|
|
def test_missing_binding_denied_even_for_an_otherwise_valid_read_tool() -> None:
|
|
# project.status.get -> connectorType "control" -- ludarium/prod has no
|
|
# registered binding of that connector type (see catalog/projects/ludarium.yaml),
|
|
# so binding_present is correctly derived False, and the engine's
|
|
# top-level binding gate denies before any policy rule is even consulted.
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "ludarium", "environment": "prod", "tool": "project.status.get"},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is False
|
|
assert result["reasonCode"] == "MISSING_BINDING"
|
|
|
|
|
|
def test_explicit_binding_present_override_is_honoured() -> None:
|
|
# The caller can override the catalog-derived default -- e.g. "what if
|
|
# this connector were bound" -- and get a real allow for it, still
|
|
# through the real engine (not a hardcoded response).
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{
|
|
"project": "ludarium",
|
|
"environment": "prod",
|
|
"tool": "project.status.get",
|
|
"bindingPresent": True,
|
|
},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is True
|
|
assert result["reasonCode"] == "ALLOW_RULE"
|
|
|
|
|
|
def test_unknown_project_is_denied_by_the_simulator_known_entity_check() -> None:
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "does-not-exist", "environment": "prod", "tool": "knowledge.document.search"},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is False
|
|
assert result["reasonCode"] == "UNKNOWN_PROJECT"
|
|
|
|
|
|
def test_unknown_tool_is_denied_by_the_simulator_known_entity_check() -> None:
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "ludarium", "environment": "prod", "tool": "not.a.real.tool"},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is False
|
|
assert result["reasonCode"] == "UNKNOWN_TOOL"
|
|
|
|
|
|
def test_wrong_environment_has_no_registered_binding_and_is_denied() -> None:
|
|
# ludarium only registers bindings for "prod" -- an unregistered
|
|
# environment correctly derives binding_present=False.
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "ludarium", "environment": "staging", "tool": "knowledge.document.search"},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is False
|
|
assert result["reasonCode"] == "MISSING_BINDING"
|
|
|
|
|
|
def test_invalid_client_binding_unknown_client_id_is_denied() -> None:
|
|
known = _client_record()
|
|
unknown_client_id = uuid4()
|
|
assert unknown_client_id != known.id
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([known]),
|
|
{
|
|
"project": "ludarium",
|
|
"environment": "prod",
|
|
"tool": "knowledge.document.search",
|
|
"clientId": str(unknown_client_id),
|
|
},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is False
|
|
assert result["reasonCode"] == "UNKNOWN_CLIENT"
|
|
|
|
|
|
def test_known_enabled_client_id_does_not_block_an_otherwise_valid_allow() -> None:
|
|
known = _client_record(status=ClientStatus.ENABLED)
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([known]),
|
|
{
|
|
"project": "ludarium",
|
|
"environment": "prod",
|
|
"tool": "knowledge.document.search",
|
|
"clientId": str(known.id),
|
|
},
|
|
_principal(),
|
|
)
|
|
assert result["allowed"] is True
|
|
|
|
|
|
def test_malformed_client_id_is_treated_as_absent_not_a_crash() -> None:
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{
|
|
"project": "ludarium",
|
|
"environment": "prod",
|
|
"tool": "knowledge.document.search",
|
|
"clientId": "not-a-uuid",
|
|
},
|
|
_principal(),
|
|
)
|
|
# Malformed input must not 500 -- and must not be silently treated as a
|
|
# valid, unscoped bypass either.
|
|
assert result["decision"] in {"allow_if_conditions", "deny"}
|
|
|
|
|
|
def test_simulation_never_mutates_client_state() -> None:
|
|
lister = FakeClientLister([_client_record()])
|
|
before = list(lister.list())
|
|
simulate_policy(
|
|
CATALOG_ROOT,
|
|
lister,
|
|
{"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
|
|
_principal(),
|
|
)
|
|
after = list(lister.list())
|
|
assert before == after
|
|
assert lister.list_call_count >= 1 # it did read, just never wrote
|
|
|
|
|
|
def test_repeated_simulation_is_deterministic_except_trace_id() -> None:
|
|
payload = {"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"}
|
|
first = simulate_policy(CATALOG_ROOT, FakeClientLister([]), payload, _principal())
|
|
second = simulate_policy(CATALOG_ROOT, FakeClientLister([]), payload, _principal())
|
|
assert first["decision"] == second["decision"]
|
|
assert first["matchedPolicies"] == second["matchedPolicies"]
|
|
assert first["reasonCode"] == second["reasonCode"]
|
|
# Each call gets its own real trace ID -- not the old hardcoded
|
|
# UUID(int=0).hex constant repeated on every request.
|
|
assert first["traceId"] != second["traceId"]
|
|
assert first["traceId"] != "0" * 32
|
|
|
|
|
|
def test_required_scopes_are_surfaced_from_the_tool_manifest() -> None:
|
|
result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
|
|
_principal(),
|
|
)
|
|
assert result["requiredScopes"], "knowledge.document.search declares requiredScopes"
|
|
|
|
|
|
def test_principal_identity_flows_into_the_evaluated_context() -> None:
|
|
# Not asserted via a private attribute -- proven behaviorally: the trace
|
|
# is produced per-principal (trace_id differs), and simulate_policy does
|
|
# not silently fall back to an anonymous/default principal string.
|
|
admin_result = simulate_policy(
|
|
CATALOG_ROOT,
|
|
FakeClientLister([]),
|
|
{"project": "ludarium", "environment": "prod", "tool": "knowledge.document.search"},
|
|
_principal(ManagementRole.ADMIN),
|
|
)
|
|
assert admin_result["allowed"] is True
|
|
|
|
|
|
# --- Lower-level engine tests prove the exact bounded Gitea R2 exception and
|
|
# default denial of every other R2 operation against the same engine production
|
|
# traffic uses.
|
|
|
|
|
|
def test_unallowlisted_r2_operations_remain_default_denied_in_production() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="ludarium",
|
|
environment="prod",
|
|
principal="test-admin",
|
|
tool="hypothetical.future.write_tool",
|
|
connector="unraid",
|
|
risk_class="R2",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert trace.decision == Decision.DENY
|
|
assert trace.reason_code == "DEFAULT_DENY"
|
|
|
|
|
|
def test_only_registered_gitea_r2_write_set_is_allowed_in_production() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
for tool, arguments in (
|
|
("gitea.repository.create", {"name": "new-project", "private": True}),
|
|
(
|
|
"gitea.branch.create",
|
|
{"repositoryRef": "hub", "branch": "chatgpt/test", "fromRef": "main"},
|
|
),
|
|
):
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-all",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool=tool,
|
|
connector="gitea",
|
|
risk_class="R2",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
arguments=arguments,
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert trace.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert trace.matched_rule_ids == ("035-gitea-controlled-writes-v1-1",)
|
|
assert trace.reason_code == "ALLOW_RULE"
|
|
|
|
|
|
def test_analysis_profile_allows_only_its_two_bounded_r2_mutations() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
for tool in ("projectbrain.analysis.start", "projectbrain.analysis.cancel"):
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-analysis",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool=tool,
|
|
connector="projectbrain",
|
|
risk_class="R2",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert trace.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert trace.matched_rule_ids == ("038-projectbrain-analysis-operations",)
|
|
assert trace.reason_code == "ALLOW_RULE"
|
|
|
|
denied = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-analysis",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool="projectbrain.delivery.create",
|
|
connector="projectbrain",
|
|
risk_class="R2",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
)
|
|
assert denied.decision == Decision.DENY
|
|
assert denied.reason_code == "DEFAULT_DENY"
|
|
|
|
|
|
def test_closed_loop_profile_allows_governed_finding_and_delivery_mutations() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
repository_create = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-loop",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool="gitea.repository.create",
|
|
connector="gitea",
|
|
risk_class="R2",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
arguments={"name": "new-project", "private": True},
|
|
)
|
|
)
|
|
assert repository_create.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert repository_create.matched_rule_ids == ("039-projectbrain-finding-loop-operations",)
|
|
|
|
for tool in (
|
|
"gitea.workspace.run",
|
|
"projectbrain.finding.issue.sync",
|
|
"projectbrain.finding.investigation.start",
|
|
"projectbrain.finding.investigation.save",
|
|
"projectbrain.finding.delivery.create",
|
|
"projectbrain.delivery.file.write",
|
|
"projectbrain.delivery.file.delete",
|
|
"projectbrain.delivery.complete",
|
|
"projectbrain.finding.verify",
|
|
"projectbrain.brainstorm.save",
|
|
"projectbrain.knowledge.save",
|
|
"projectbrain.documentation.save",
|
|
"projectbrain.memory.save",
|
|
"projectbrain.contextpack.create",
|
|
"projectbrain.feedback.record",
|
|
):
|
|
trace = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-loop",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool=tool,
|
|
connector="gitea" if tool == "gitea.workspace.run" else "projectbrain",
|
|
risk_class="R2",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
)
|
|
assert trace.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert trace.matched_rule_ids == ("039-projectbrain-finding-loop-operations",)
|
|
|
|
|
|
def test_only_reviewed_gitea_r3_lifecycle_set_is_allowed_in_production() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
for tool in (
|
|
"gitea.pullrequest.sync",
|
|
"gitea.pullrequest.merge",
|
|
"gitea.branch.delete",
|
|
):
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-all",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool=tool,
|
|
connector="gitea",
|
|
risk_class="R3",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
arguments={"repositoryRef": "hub", "expectedHeadSha": "e" * 40},
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert trace.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert trace.matched_rule_ids == ("036-gitea-reviewed-lifecycle-v1",)
|
|
assert trace.reason_code == "ALLOW_RULE"
|
|
|
|
for tool in ("gitea.pullrequest.sync", "gitea.pullrequest.merge"):
|
|
loop_trace = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-loop",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool=tool,
|
|
connector="gitea",
|
|
risk_class="R3",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
)
|
|
assert loop_trace.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert loop_trace.matched_rule_ids == ("036-gitea-loop-reviewed-lifecycle-v1",)
|
|
denied = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx", project="gitea-loop", environment="prod",
|
|
principal="chatgpt-gitea", tool="gitea.pullrequest.merge",
|
|
connector="gitea", risk_class="R3",
|
|
environment_classification="production", binding_present=False,
|
|
client_enabled=True,
|
|
)
|
|
)
|
|
assert denied.decision == Decision.DENY
|
|
|
|
|
|
def test_server_ops_allows_only_catalogued_governed_mutations() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
for tool, risk in (
|
|
("server.operation.request", "R2"),
|
|
("server.operation.execute", "R3"),
|
|
("server.job.cancel", "R2"),
|
|
):
|
|
trace = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx",
|
|
project="server-ops",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool=tool,
|
|
connector="appops",
|
|
risk_class=risk,
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
)
|
|
assert trace.decision == Decision.ALLOW_IF_CONDITIONS
|
|
assert trace.matched_rule_ids == ("041-appops-governed-operations",)
|
|
|
|
denied = engine.evaluate(
|
|
InvocationContext(
|
|
workspace="itworx",
|
|
project="server-ops",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool="server.shell.execute",
|
|
connector="appops",
|
|
risk_class="R3",
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
)
|
|
assert denied.decision == Decision.DENY
|
|
assert denied.reason_code == "DEFAULT_DENY"
|
|
|
|
|
|
def test_unregistered_r3_and_all_r4_operations_remain_denied_in_production() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
for risk_class, expected_reason in (("R3", "DEFAULT_DENY"), ("R4", "GLOBAL_DENY")):
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="gitea-all",
|
|
environment="prod",
|
|
principal="chatgpt-gitea",
|
|
tool="gitea.repository.delete",
|
|
connector="gitea",
|
|
risk_class=risk_class,
|
|
environment_classification="production",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert trace.decision in {Decision.DENY, Decision.DENY_AND_ALERT}
|
|
assert trace.reason_code == expected_reason
|
|
|
|
|
|
def test_production_restrictions_do_not_apply_outside_production_classification() -> None:
|
|
engine = _policy_engine(CATALOG_ROOT)
|
|
context = InvocationContext(
|
|
workspace="itworx",
|
|
project="ludarium",
|
|
environment="dev",
|
|
principal="test-admin",
|
|
tool="hypothetical.future.write_tool",
|
|
connector="unraid",
|
|
risk_class="R2",
|
|
environment_classification="development",
|
|
binding_present=True,
|
|
client_enabled=True,
|
|
)
|
|
trace = engine.evaluate(context)
|
|
assert "030-production-destructive-deny-v1-1" not in trace.matched_rule_ids
|