Files
ITWorx_MCP_Hub-Public/tests/test_operator_workspaces.py
T

520 lines
22 KiB
Python

from __future__ import annotations
import os
from collections.abc import Iterator
from contextlib import ExitStack
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from http.cookies import SimpleCookie
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlparse
from uuid import UUID, uuid4
import pytest
from fastapi.routing import APIRoute
from fastapi.testclient import TestClient
from services.control_api.auth.config import OIDCSettings
from services.control_api.auth.fake_provider import FakeOIDCProvider
from services.control_api.auth.models import (
AuthenticatedPrincipal,
ManagementRole,
PrincipalKind,
)
from services.control_api.contextforge.fake import FakeContextForge
from services.control_api.contextforge.models import ContextForgeSettings
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.control_api.persistence.machine_client import provision_machine_client
from services.control_api.persistence.models import (
AuditEventRow,
ClientBindingRow,
ClientRow,
ConnectorRow,
EnvironmentRow,
ProjectRow,
VirtualServerRow,
WorkspaceRow,
)
from services.runtime import control_api
from services.runtime.control_api import build_app
from services.runtime.state_sync import sync_runtime_state
ROOT = Path(__file__).resolve().parents[1]
ISSUER = "https://fake.operator-workspaces.test"
ROLE_GROUPS = {
ManagementRole.VIEWER: "hub-viewer",
ManagementRole.OPERATOR: "hub-operator",
ManagementRole.ADMIN: "hub-admin",
}
@dataclass
class RoleSession:
app: Any
client: TestClient
csrf: str
def get(self, path: str) -> Any:
return self.client.get(path)
def post(self, path: str, **kwargs: Any) -> Any:
headers = dict(kwargs.pop("headers", {}) or {})
headers["X-CSRF-Token"] = self.csrf
return self.client.post(path, headers=headers, **kwargs)
@pytest.fixture(scope="module", autouse=True)
def client_bindings_env() -> Iterator[None]:
previous_bindings = os.environ.get("MCP_CLIENT_BINDINGS_JSON")
previous_token = os.environ.get("OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN")
os.environ["MCP_CLIENT_BINDINGS_JSON"] = "[]"
os.environ["OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN"] = "operator-workspaces-fake-token" # noqa: S105
yield
if previous_bindings is None:
os.environ.pop("MCP_CLIENT_BINDINGS_JSON", None)
else:
os.environ["MCP_CLIENT_BINDINGS_JSON"] = previous_bindings
if previous_token is None:
os.environ.pop("OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN", None)
else:
os.environ["OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN"] = previous_token
@pytest.fixture(scope="module")
def db_path(tmp_path_factory: pytest.TempPathFactory) -> Path:
path = tmp_path_factory.mktemp("operator-workspaces") / "hub.db"
engine = create_engine_from_url(f"sqlite:///{path}")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", ISSUER)
engine.dispose()
return path
def _login(client: TestClient, provider: FakeOIDCProvider) -> str:
login = client.get("/auth/login", follow_redirects=False)
query = parse_qs(urlparse(login.headers["location"]).query)
code = provider.authorize(login.headers["location"])
callback = client.get(
"/auth/callback",
params={"code": code, "state": query["state"][0]},
follow_redirects=False,
)
cookie = SimpleCookie()
cookie.load(callback.headers["set-cookie"])
cookie_name = next(iter(cookie))
client.cookies.set(cookie_name, cookie[cookie_name].value)
return str(client.get("/auth/session").json()["csrf_token"])
@pytest.fixture(scope="module")
def role_sessions(db_path: Path) -> Iterator[dict[ManagementRole, RoleSession]]:
with ExitStack() as stack:
result: dict[ManagementRole, RoleSession] = {}
for role, group in ROLE_GROUPS.items():
provider = FakeOIDCProvider(issuer=ISSUER, user_groups=(group,))
oidc = OIDCSettings(
issuer=ISSUER,
audience=provider.audience,
client_id="operator-workspaces-hub",
redirect_uri="https://hub.test/auth/callback",
secure_cookie=True,
role_group_map={value: key for key, value in ROLE_GROUPS.items()},
)
engine = create_engine_from_url(f"sqlite:///{db_path}")
app = build_app(
oidc=oidc,
provider_transport=provider.transport(),
contextforge_settings=ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN", # noqa: S106
),
contextforge_transport=FakeContextForge().transport(),
engine=engine,
catalog_root=ROOT / "catalog",
)
client = stack.enter_context(TestClient(app))
csrf = _login(client, provider)
result[role] = RoleSession(app=app, client=client, csrf=csrf)
yield result
def _ready_probe(_: str) -> dict[str, Any]:
return {"reachable": True, "status": "ready", "detail": "Direct connector readiness verified."}
def test_project_workspace_returns_registered_server_and_source_metadata(
role_sessions: dict[ManagementRole, RoleSession],
) -> None:
response = role_sessions[ManagementRole.VIEWER].get("/api/projects/ludarium")
assert response.status_code == 200, response.text
body = response.json()
assert body["mcpServers"]
assert body["mcpServers"][0]["endpointAlias"] == "/mcp/ludarium/prod"
assert body["mcpServers"][0]["stateSource"] in {"persisted", "catalog-only"}
assert body["knowledgeSources"]
source = body["knowledgeSources"][0]
assert source["uri"] == "discovery://repository/ludarium"
assert source["indexesSecrets"] is False
assert "scoped Knowledge connector" in source["accessBoundary"]
assert "README*" in source["include"]
assert "credentialReferences" not in response.text
def test_connector_workspace_is_scoped_and_secret_free(
role_sessions: dict[ManagementRole, RoleSession], monkeypatch: pytest.MonkeyPatch
) -> None:
probe_calls: list[str] = []
def record_probe(slug: str) -> dict[str, Any]:
probe_calls.append(slug)
return _ready_probe(slug)
monkeypatch.setattr(control_api, "_direct_connector_readiness", record_probe)
viewer = role_sessions[ManagementRole.VIEWER]
response = viewer.get("/api/connectors/knowledge-default?project=ludarium&environment=prod")
assert response.status_code == 200, response.text
body = response.json()
assert body["id"] == "knowledge-default"
assert body["connectorType"] == "knowledge"
assert body["registration"]["state"] == "registered"
assert body["readiness"]["source"] in {"persisted", "unavailable"}
if body["readiness"]["source"] == "unavailable":
assert body["readiness"]["observedAt"] is None
assert probe_calls == []
assert body["bindings"] == [
{"project": "ludarium", "environment": "prod", "verification": "verified"}
]
assert body["relatedTools"]
for sensitive_key in ("credentialReferences", "credentialReference", "endpoint", "token"):
assert sensitive_key not in body
assert "credentialReferences" not in response.text
assert "service_token" not in response.text
assert (
viewer.get("/api/connectors/knowledge-default?project=missing&environment=prod").status_code
== 404
)
assert (
viewer.get(
"/api/connectors/knowledge-default?project=ludarium&environment=staging"
).status_code
== 404
)
assert viewer.get("/api/connectors/knowledge-default?project=ludarium").status_code == 422
def test_connector_workspace_marks_old_persisted_health_unknown_without_probing(
role_sessions: dict[ManagementRole, RoleSession],
db_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
probe_calls: list[str] = []
def record_probe(slug: str) -> dict[str, Any]:
probe_calls.append(slug)
return _ready_probe(slug)
monkeypatch.setattr(control_api, "_direct_connector_readiness", record_probe)
engine = create_engine_from_url(f"sqlite:///{db_path}")
sessions = session_factory(engine)
observed_at = datetime.now(UTC) - timedelta(minutes=5)
with sessions() as session, session.begin():
workspace = session.query(WorkspaceRow).filter_by(slug="itworx").one()
project = (
session.query(ProjectRow).filter_by(workspace_id=workspace.id, slug="ludarium").one()
)
environment = (
session.query(EnvironmentRow)
.filter_by(workspace_id=workspace.id, project_id=project.id, name="prod")
.one()
)
connector = (
session.query(ConnectorRow)
.filter_by(
workspace_id=workspace.id,
project_id=project.id,
environment_scope="prod",
type="knowledge",
slug="knowledge-default",
)
.one_or_none()
)
if connector is None:
connector = ConnectorRow(
id=uuid4(),
workspace_id=workspace.id,
project_id=project.id,
environment_id=environment.id,
environment_scope="prod",
type="knowledge",
slug="knowledge-default",
)
session.add(connector)
connector.health_state = "healthy"
connector.last_check_at = observed_at
connector.endpoint = "https://knowledge.example.invalid/health"
connector.credential_reference = "vault://private-reference"
engine.dispose()
response = role_sessions[ManagementRole.VIEWER].get(
"/api/connectors/knowledge-default?project=ludarium&environment=prod"
)
assert response.status_code == 200, response.text
readiness = response.json()["readiness"]
assert readiness["source"] == "persisted"
assert readiness["stale"] is True
assert readiness["reachable"] is None
assert readiness["status"] == "unknown"
assert readiness["observedAt"]
assert "Stale observation" in readiness["detail"]
assert "knowledge.example.invalid" not in response.text
assert "vault://private-reference" not in response.text
assert probe_calls == []
def test_connector_and_onboarding_routes_deny_machine_principals_even_with_admin_role(
role_sessions: dict[ManagementRole, RoleSession],
) -> None:
app = role_sessions[ManagementRole.VIEWER].app
route = next(
route
for route in app.routes
if isinstance(route, APIRoute) and route.path == "/api/connectors/{slug}"
)
dependency_calls: list[Any] = []
def visit(dependant: Any) -> None:
if dependant.call is not None:
dependency_calls.append(dependant.call)
for child in dependant.dependencies:
visit(child)
visit(route.dependant)
authenticate = next(
call for call in dependency_calls if getattr(call, "__name__", "") == "authenticate"
)
now = datetime.now(UTC)
machine = AuthenticatedPrincipal(
subject="machine-connector-reader",
issuer=ISSUER,
audience=("operator-workspaces-hub",),
kind=PrincipalKind.MACHINE_CLIENT,
client_id=uuid4(),
roles=frozenset({ManagementRole.ADMIN}),
expires_at=now + timedelta(minutes=5),
issued_at=now,
)
app.dependency_overrides[authenticate] = lambda: machine
try:
client = role_sessions[ManagementRole.VIEWER].client
connector = client.get("/api/connectors/knowledge-default")
onboarding = client.get(f"/api/client-management/{uuid4()}/onboarding-status")
finally:
app.dependency_overrides.pop(authenticate, None)
assert connector.status_code == 403
assert onboarding.status_code == 403
def test_onboarding_verification_records_server_checks_without_enabling_client_or_binding(
role_sessions: dict[ManagementRole, RoleSession], db_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
probe_calls: list[str] = []
def record_probe(slug: str) -> dict[str, Any]:
probe_calls.append(slug)
return _ready_probe(slug)
monkeypatch.setattr(control_api, "_direct_connector_readiness", record_probe)
admin = role_sessions[ManagementRole.ADMIN]
operator = role_sessions[ManagementRole.OPERATOR]
viewer = role_sessions[ManagementRole.VIEWER]
created = admin.post(
"/api/client-management",
json={
"name": "resume-me",
"project": "gitea-loop",
"environment": "prod",
"kind": "native-2026",
},
)
assert created.status_code == 201, created.text
client_id = created.json()["client"]["id"]
operator_status = operator.get(f"/api/client-management/{client_id}/onboarding-status")
assert operator_status.status_code == 200, operator_status.text
assert operator_status.json()["checks"]["connector"]["state"] == "unknown"
assert probe_calls == []
status = viewer.get(f"/api/client-management/{client_id}/onboarding-status")
assert status.status_code == 200, status.text
before = status.json()
assert before["clientStatus"] == "pending"
assert before["checks"]["externalSubject"]["state"] == "pending"
assert before["checks"]["externalScope"]["state"] == "unavailable"
assert before["checks"]["catalog"]["state"] == "ready"
assert before["checks"]["hubBinding"]["state"] == "pending"
assert before["checks"]["connector"]["state"] == "unknown"
assert before["checks"]["connector"]["observedAt"] is None
assert before["checks"]["mcpProof"]["state"] == "unavailable"
handoff = before["provisioning"]
assert handoff == {
"status": "pending-authentik-service-account",
"subject": None,
"issuer": ISSUER,
"oauthUsername": f"svc-mcp-codex-gitea-loop-{client_id.replace('-', '')[:8]}",
"tokenEndpoint": f"{ISSUER}/application/o/token/",
"oauthClientId": "operator-workspaces-hub",
"steps": [
"Provision the named Authentik service account and app password.",
"Activate this pending Hub client with the exact Authentik user UUID.",
"Install the generated Codex stdio profile using a user-only credential file.",
"Run MCP discovery, one read-only invocation and one cross-project denial.",
],
}
assert not {"credentialReference", "bootstrapReference", "credential", "secret"} & set(handoff)
assert "credentialReference" not in status.text
assert "bootstrapReference" not in status.text
assert "authentik-bootstrap://" not in status.text
assert "authentik://service-account/" not in status.text
assert probe_calls == []
denied_verify = viewer.post(f"/api/client-management/{client_id}/verify")
assert denied_verify.status_code == 403
assert probe_calls == []
checked_pending = admin.post(f"/api/client-management/{client_id}/verify")
assert checked_pending.status_code == 200, checked_pending.text
assert probe_calls
pending_verification_probe_count = len(probe_calls)
assert checked_pending.json()["verification"]["checkedByManagement"] is True
assert checked_pending.json()["verification"]["verifiedByManagement"] is False
assert checked_pending.json()["clientStatus"] == "pending"
engine = create_engine_from_url(f"sqlite:///{db_path}")
sessions = session_factory(engine)
with sessions() as session, session.begin():
server = session.query(VirtualServerRow).filter_by(slug="gitea-loop-prod").one()
server.gateway_external_id = "fixture-contextforge-server"
with sessions() as session, session.begin():
provision_machine_client(
session,
issuer=ISSUER,
subject="authentik-user-uuid-1234",
oauth_username="svc-mcp-codex-gitea-loop-test1234",
name="resume-me",
project_slug="gitea-loop",
environment_name="prod",
public_host="hub.test",
client_id=UUID(client_id),
)
resumed = viewer.get(f"/api/client-management/{client_id}/onboarding-status")
assert resumed.status_code == 200, resumed.text
resumed_body = resumed.json()
assert resumed_body["clientStatus"] == "enabled"
assert resumed_body["checks"]["externalSubject"]["state"] == "ready"
assert resumed_body["checks"]["hubBinding"]["state"] == "ready"
assert resumed_body["provisioning"]["status"] == "authentik-service-account-provisioned"
assert resumed_body["provisioning"]["subject"] == "authentik-user-uuid-1234"
assert resumed_body["provisioning"]["oauthUsername"] == "svc-mcp-codex-gitea-loop-test1234"
assert resumed_body["provisioning"]["issuer"] == ISSUER
assert resumed_body["provisioning"]["tokenEndpoint"] == f"{ISSUER}/application/o/token/"
assert resumed_body["provisioning"]["oauthClientId"] == "operator-workspaces-hub"
assert len(probe_calls) == pending_verification_probe_count
assert resumed_body["checks"]["externalScope"]["state"] == "unavailable"
assert resumed_body["checks"]["mcpProof"]["state"] == "unavailable"
with sessions() as session, session.begin():
client_row = session.get(ClientRow, UUID(client_id))
assert client_row is not None
client_row.issuer = "https://foreign-identity.example.invalid"
probes_before_foreign_issuer_read = len(probe_calls)
foreign_issuer_status = viewer.get(f"/api/client-management/{client_id}/onboarding-status")
assert foreign_issuer_status.status_code == 200, foreign_issuer_status.text
foreign_issuer_body = foreign_issuer_status.json()
assert foreign_issuer_body["checks"]["externalSubject"]["state"] == "pending"
assert foreign_issuer_body["provisioning"]["subject"] is None
assert foreign_issuer_body["verification"]["verifiedByManagement"] is False
assert len(probe_calls) == probes_before_foreign_issuer_read
foreign_issuer_verification = admin.post(f"/api/client-management/{client_id}/verify")
assert foreign_issuer_verification.status_code == 200, foreign_issuer_verification.text
assert foreign_issuer_verification.json()["checks"]["externalSubject"]["state"] == "pending"
assert foreign_issuer_verification.json()["verification"]["verifiedByManagement"] is False
assert len(probe_calls) > probes_before_foreign_issuer_read
with sessions() as session, session.begin():
client_row = session.get(ClientRow, UUID(client_id))
assert client_row is not None
client_row.issuer = ISSUER
verified = admin.post(f"/api/client-management/{client_id}/verify")
assert verified.status_code == 200, verified.text
assert len(probe_calls) > pending_verification_probe_count
after = verified.json()
assert after["verification"]["checkedByManagement"] is True
assert after["verification"]["verifiedByManagement"] is True
assert after["verification"]["hubChecksPassed"] is True
assert after["checks"]["externalScope"]["state"] == "unavailable"
assert after["checks"]["mcpProof"]["state"] == "unavailable"
verified_probe_count = len(probe_calls)
persisted_status = viewer.get(f"/api/client-management/{client_id}/onboarding-status")
assert persisted_status.status_code == 200, persisted_status.text
assert persisted_status.json()["checks"]["connector"]["state"] == "healthy"
assert persisted_status.json()["verification"]["verifiedByManagement"] is True
assert len(probe_calls) == verified_probe_count
for stale_checked_at in (
datetime.now(UTC) - timedelta(minutes=5),
datetime.now(UTC) + timedelta(minutes=5),
):
stale_timestamp = stale_checked_at.isoformat()
with sessions() as session, session.begin():
client_row = session.get(ClientRow, UUID(client_id))
assert client_row is not None
metadata = dict(client_row.record_metadata or {})
verification = dict(metadata["onboardingVerification"])
verification["checkedAt"] = stale_timestamp
metadata["onboardingVerification"] = verification
client_row.record_metadata = metadata
stale_status = viewer.get(f"/api/client-management/{client_id}/onboarding-status")
assert stale_status.status_code == 200, stale_status.text
stale_body = stale_status.json()
assert stale_body["checks"]["connector"]["state"] == "unknown"
assert stale_body["checks"]["connector"]["observedAt"] is None
assert stale_body["verification"]["checkedAt"] == stale_timestamp
assert stale_body["verification"]["checkedByManagement"] is True
assert stale_body["verification"]["verifiedByManagement"] is False
assert stale_body["verification"]["hubChecksPassed"] is False
assert len(probe_calls) == verified_probe_count
with sessions() as session:
client_row = session.get(ClientRow, UUID(client_id))
workspace = session.query(WorkspaceRow).filter_by(slug="itworx").one()
binding = session.query(ClientBindingRow).filter_by(client_id=UUID(client_id)).one()
audit_event = (
session.query(AuditEventRow)
.filter_by(client_id=UUID(client_id), event_type="client.onboarding.verify")
.order_by(AuditEventRow.occurred_at.desc())
.first()
)
engine.dispose()
assert client_row is not None and client_row.status == "active"
assert binding.workspace_id == workspace.id
assert binding.enabled is True
assert client_row.record_metadata["onboardingVerification"]["verifiedByManagement"] is True
assert "credential" not in str(client_row.record_metadata["onboardingVerification"]).lower()
assert audit_event is not None
assert audit_event.principal_subject == "user-management-1"
assert audit_event.sanitized_arguments["hubChecksPassed"] is True
def test_unauthed_client_is_not_exposed_by_connector_management_route(
role_sessions: dict[ManagementRole, RoleSession],
) -> None:
unauthenticated = TestClient(role_sessions[ManagementRole.VIEWER].app)
unauthenticated.cookies.clear()
response = unauthenticated.get("/api/connectors/knowledge-default")
assert response.status_code == 401