from __future__ import annotations import os from collections.abc import Iterator from contextlib import ExitStack from dataclasses import dataclass from datetime import UTC, datetime, timedelta from http.cookies import SimpleCookie from pathlib import Path from typing import Any from urllib.parse import parse_qs, urlparse from uuid import UUID, uuid4 import pytest from fastapi.routing import APIRoute from fastapi.testclient import TestClient from services.control_api.auth.config import OIDCSettings from services.control_api.auth.fake_provider import FakeOIDCProvider from services.control_api.auth.models import ( AuthenticatedPrincipal, ManagementRole, PrincipalKind, ) from services.control_api.contextforge.fake import FakeContextForge from services.control_api.contextforge.models import ContextForgeSettings from services.control_api.persistence import Base, create_engine_from_url, session_factory from services.control_api.persistence.machine_client import provision_machine_client from services.control_api.persistence.models import ( AuditEventRow, ClientBindingRow, ClientRow, ConnectorRow, EnvironmentRow, ProjectRow, VirtualServerRow, WorkspaceRow, ) from services.runtime import control_api from services.runtime.control_api import build_app from services.runtime.state_sync import sync_runtime_state ROOT = Path(__file__).resolve().parents[1] ISSUER = "https://fake.operator-workspaces.test" ROLE_GROUPS = { ManagementRole.VIEWER: "hub-viewer", ManagementRole.OPERATOR: "hub-operator", ManagementRole.ADMIN: "hub-admin", } @dataclass class RoleSession: app: Any client: TestClient csrf: str def get(self, path: str) -> Any: return self.client.get(path) def post(self, path: str, **kwargs: Any) -> Any: headers = dict(kwargs.pop("headers", {}) or {}) headers["X-CSRF-Token"] = self.csrf return self.client.post(path, headers=headers, **kwargs) @pytest.fixture(scope="module", autouse=True) def client_bindings_env() -> Iterator[None]: previous_bindings = os.environ.get("MCP_CLIENT_BINDINGS_JSON") previous_token = os.environ.get("OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN") os.environ["MCP_CLIENT_BINDINGS_JSON"] = "[]" os.environ["OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN"] = "operator-workspaces-fake-token" # noqa: S105 yield if previous_bindings is None: os.environ.pop("MCP_CLIENT_BINDINGS_JSON", None) else: os.environ["MCP_CLIENT_BINDINGS_JSON"] = previous_bindings if previous_token is None: os.environ.pop("OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN", None) else: os.environ["OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN"] = previous_token @pytest.fixture(scope="module") def db_path(tmp_path_factory: pytest.TempPathFactory) -> Path: path = tmp_path_factory.mktemp("operator-workspaces") / "hub.db" engine = create_engine_from_url(f"sqlite:///{path}") Base.metadata.create_all(engine) sessions = session_factory(engine) with sessions() as session, session.begin(): sync_runtime_state(session, ROOT / "catalog", ISSUER) engine.dispose() return path def _login(client: TestClient, provider: FakeOIDCProvider) -> str: login = client.get("/auth/login", follow_redirects=False) query = parse_qs(urlparse(login.headers["location"]).query) code = provider.authorize(login.headers["location"]) callback = client.get( "/auth/callback", params={"code": code, "state": query["state"][0]}, follow_redirects=False, ) cookie = SimpleCookie() cookie.load(callback.headers["set-cookie"]) cookie_name = next(iter(cookie)) client.cookies.set(cookie_name, cookie[cookie_name].value) return str(client.get("/auth/session").json()["csrf_token"]) @pytest.fixture(scope="module") def role_sessions(db_path: Path) -> Iterator[dict[ManagementRole, RoleSession]]: with ExitStack() as stack: result: dict[ManagementRole, RoleSession] = {} for role, group in ROLE_GROUPS.items(): provider = FakeOIDCProvider(issuer=ISSUER, user_groups=(group,)) oidc = OIDCSettings( issuer=ISSUER, audience=provider.audience, client_id="operator-workspaces-hub", redirect_uri="https://hub.test/auth/callback", secure_cookie=True, role_group_map={value: key for key, value in ROLE_GROUPS.items()}, ) engine = create_engine_from_url(f"sqlite:///{db_path}") app = build_app( oidc=oidc, provider_transport=provider.transport(), contextforge_settings=ContextForgeSettings( base_url="https://contextforge.test", auth_token_env="OPERATOR_WORKSPACES_FAKE_CONTEXTFORGE_TOKEN", # noqa: S106 ), contextforge_transport=FakeContextForge().transport(), engine=engine, catalog_root=ROOT / "catalog", ) client = stack.enter_context(TestClient(app)) csrf = _login(client, provider) result[role] = RoleSession(app=app, client=client, csrf=csrf) yield result def _ready_probe(_: str) -> dict[str, Any]: return {"reachable": True, "status": "ready", "detail": "Direct connector readiness verified."} def test_project_workspace_returns_registered_server_and_source_metadata( role_sessions: dict[ManagementRole, RoleSession], ) -> None: response = role_sessions[ManagementRole.VIEWER].get("/api/projects/ludarium") assert response.status_code == 200, response.text body = response.json() assert body["mcpServers"] assert body["mcpServers"][0]["endpointAlias"] == "/mcp/ludarium/prod" assert body["mcpServers"][0]["stateSource"] in {"persisted", "catalog-only"} assert body["knowledgeSources"] source = body["knowledgeSources"][0] assert source["uri"] == "discovery://repository/ludarium" assert source["indexesSecrets"] is False assert "scoped Knowledge connector" in source["accessBoundary"] assert "README*" in source["include"] assert "credentialReferences" not in response.text def test_connector_workspace_is_scoped_and_secret_free( role_sessions: dict[ManagementRole, RoleSession], monkeypatch: pytest.MonkeyPatch ) -> None: probe_calls: list[str] = [] def record_probe(slug: str) -> dict[str, Any]: probe_calls.append(slug) return _ready_probe(slug) monkeypatch.setattr(control_api, "_direct_connector_readiness", record_probe) viewer = role_sessions[ManagementRole.VIEWER] response = viewer.get("/api/connectors/knowledge-default?project=ludarium&environment=prod") assert response.status_code == 200, response.text body = response.json() assert body["id"] == "knowledge-default" assert body["connectorType"] == "knowledge" assert body["registration"]["state"] == "registered" assert body["readiness"]["source"] in {"persisted", "unavailable"} if body["readiness"]["source"] == "unavailable": assert body["readiness"]["observedAt"] is None assert probe_calls == [] assert body["bindings"] == [ {"project": "ludarium", "environment": "prod", "verification": "verified"} ] assert body["relatedTools"] for sensitive_key in ("credentialReferences", "credentialReference", "endpoint", "token"): assert sensitive_key not in body assert "credentialReferences" not in response.text assert "service_token" not in response.text assert ( viewer.get("/api/connectors/knowledge-default?project=missing&environment=prod").status_code == 404 ) assert ( viewer.get( "/api/connectors/knowledge-default?project=ludarium&environment=staging" ).status_code == 404 ) assert viewer.get("/api/connectors/knowledge-default?project=ludarium").status_code == 422 def test_connector_workspace_marks_old_persisted_health_unknown_without_probing( role_sessions: dict[ManagementRole, RoleSession], db_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: probe_calls: list[str] = [] def record_probe(slug: str) -> dict[str, Any]: probe_calls.append(slug) return _ready_probe(slug) monkeypatch.setattr(control_api, "_direct_connector_readiness", record_probe) engine = create_engine_from_url(f"sqlite:///{db_path}") sessions = session_factory(engine) observed_at = datetime.now(UTC) - timedelta(minutes=5) with sessions() as session, session.begin(): workspace = session.query(WorkspaceRow).filter_by(slug="itworx").one() project = ( session.query(ProjectRow).filter_by(workspace_id=workspace.id, slug="ludarium").one() ) environment = ( session.query(EnvironmentRow) .filter_by(workspace_id=workspace.id, project_id=project.id, name="prod") .one() ) connector = ( session.query(ConnectorRow) .filter_by( workspace_id=workspace.id, project_id=project.id, environment_scope="prod", type="knowledge", slug="knowledge-default", ) .one_or_none() ) if connector is None: connector = ConnectorRow( id=uuid4(), workspace_id=workspace.id, project_id=project.id, environment_id=environment.id, environment_scope="prod", type="knowledge", slug="knowledge-default", ) session.add(connector) connector.health_state = "healthy" connector.last_check_at = observed_at connector.endpoint = "https://knowledge.example.invalid/health" connector.credential_reference = "vault://private-reference" engine.dispose() response = role_sessions[ManagementRole.VIEWER].get( "/api/connectors/knowledge-default?project=ludarium&environment=prod" ) assert response.status_code == 200, response.text readiness = response.json()["readiness"] assert readiness["source"] == "persisted" assert readiness["stale"] is True assert readiness["reachable"] is None assert readiness["status"] == "unknown" assert readiness["observedAt"] assert "Stale observation" in readiness["detail"] assert "knowledge.example.invalid" not in response.text assert "vault://private-reference" not in response.text assert probe_calls == [] def test_connector_and_onboarding_routes_deny_machine_principals_even_with_admin_role( role_sessions: dict[ManagementRole, RoleSession], ) -> None: app = role_sessions[ManagementRole.VIEWER].app route = next( route for route in app.routes if isinstance(route, APIRoute) and route.path == "/api/connectors/{slug}" ) dependency_calls: list[Any] = [] def visit(dependant: Any) -> None: if dependant.call is not None: dependency_calls.append(dependant.call) for child in dependant.dependencies: visit(child) visit(route.dependant) authenticate = next( call for call in dependency_calls if getattr(call, "__name__", "") == "authenticate" ) now = datetime.now(UTC) machine = AuthenticatedPrincipal( subject="machine-connector-reader", issuer=ISSUER, audience=("operator-workspaces-hub",), kind=PrincipalKind.MACHINE_CLIENT, client_id=uuid4(), roles=frozenset({ManagementRole.ADMIN}), expires_at=now + timedelta(minutes=5), issued_at=now, ) app.dependency_overrides[authenticate] = lambda: machine try: client = role_sessions[ManagementRole.VIEWER].client connector = client.get("/api/connectors/knowledge-default") onboarding = client.get(f"/api/client-management/{uuid4()}/onboarding-status") finally: app.dependency_overrides.pop(authenticate, None) assert connector.status_code == 403 assert onboarding.status_code == 403 def test_onboarding_verification_records_server_checks_without_enabling_client_or_binding( role_sessions: dict[ManagementRole, RoleSession], db_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: probe_calls: list[str] = [] def record_probe(slug: str) -> dict[str, Any]: probe_calls.append(slug) return _ready_probe(slug) monkeypatch.setattr(control_api, "_direct_connector_readiness", record_probe) admin = role_sessions[ManagementRole.ADMIN] operator = role_sessions[ManagementRole.OPERATOR] viewer = role_sessions[ManagementRole.VIEWER] created = admin.post( "/api/client-management", json={ "name": "resume-me", "project": "gitea-loop", "environment": "prod", "kind": "native-2026", }, ) assert created.status_code == 201, created.text client_id = created.json()["client"]["id"] operator_status = operator.get(f"/api/client-management/{client_id}/onboarding-status") assert operator_status.status_code == 200, operator_status.text assert operator_status.json()["checks"]["connector"]["state"] == "unknown" assert probe_calls == [] status = viewer.get(f"/api/client-management/{client_id}/onboarding-status") assert status.status_code == 200, status.text before = status.json() assert before["clientStatus"] == "pending" assert before["checks"]["externalSubject"]["state"] == "pending" assert before["checks"]["externalScope"]["state"] == "unavailable" assert before["checks"]["catalog"]["state"] == "ready" assert before["checks"]["hubBinding"]["state"] == "pending" assert before["checks"]["connector"]["state"] == "unknown" assert before["checks"]["connector"]["observedAt"] is None assert before["checks"]["mcpProof"]["state"] == "unavailable" handoff = before["provisioning"] assert handoff == { "status": "pending-authentik-service-account", "subject": None, "issuer": ISSUER, "oauthUsername": f"svc-mcp-codex-gitea-loop-{client_id.replace('-', '')[:8]}", "tokenEndpoint": f"{ISSUER}/application/o/token/", "oauthClientId": "operator-workspaces-hub", "steps": [ "Provision the named Authentik service account and app password.", "Activate this pending Hub client with the exact Authentik user UUID.", "Install the generated Codex stdio profile using a user-only credential file.", "Run MCP discovery, one read-only invocation and one cross-project denial.", ], } assert not {"credentialReference", "bootstrapReference", "credential", "secret"} & set(handoff) assert "credentialReference" not in status.text assert "bootstrapReference" not in status.text assert "authentik-bootstrap://" not in status.text assert "authentik://service-account/" not in status.text assert probe_calls == [] denied_verify = viewer.post(f"/api/client-management/{client_id}/verify") assert denied_verify.status_code == 403 assert probe_calls == [] checked_pending = admin.post(f"/api/client-management/{client_id}/verify") assert checked_pending.status_code == 200, checked_pending.text assert probe_calls pending_verification_probe_count = len(probe_calls) assert checked_pending.json()["verification"]["checkedByManagement"] is True assert checked_pending.json()["verification"]["verifiedByManagement"] is False assert checked_pending.json()["clientStatus"] == "pending" engine = create_engine_from_url(f"sqlite:///{db_path}") sessions = session_factory(engine) with sessions() as session, session.begin(): server = session.query(VirtualServerRow).filter_by(slug="gitea-loop-prod").one() server.gateway_external_id = "fixture-contextforge-server" with sessions() as session, session.begin(): provision_machine_client( session, issuer=ISSUER, subject="authentik-user-uuid-1234", oauth_username="svc-mcp-codex-gitea-loop-test1234", name="resume-me", project_slug="gitea-loop", environment_name="prod", public_host="hub.test", client_id=UUID(client_id), ) resumed = viewer.get(f"/api/client-management/{client_id}/onboarding-status") assert resumed.status_code == 200, resumed.text resumed_body = resumed.json() assert resumed_body["clientStatus"] == "enabled" assert resumed_body["checks"]["externalSubject"]["state"] == "ready" assert resumed_body["checks"]["hubBinding"]["state"] == "ready" assert resumed_body["provisioning"]["status"] == "authentik-service-account-provisioned" assert resumed_body["provisioning"]["subject"] == "authentik-user-uuid-1234" assert resumed_body["provisioning"]["oauthUsername"] == "svc-mcp-codex-gitea-loop-test1234" assert resumed_body["provisioning"]["issuer"] == ISSUER assert resumed_body["provisioning"]["tokenEndpoint"] == f"{ISSUER}/application/o/token/" assert resumed_body["provisioning"]["oauthClientId"] == "operator-workspaces-hub" assert len(probe_calls) == pending_verification_probe_count assert resumed_body["checks"]["externalScope"]["state"] == "unavailable" assert resumed_body["checks"]["mcpProof"]["state"] == "unavailable" with sessions() as session, session.begin(): client_row = session.get(ClientRow, UUID(client_id)) assert client_row is not None client_row.issuer = "https://foreign-identity.example.invalid" probes_before_foreign_issuer_read = len(probe_calls) foreign_issuer_status = viewer.get(f"/api/client-management/{client_id}/onboarding-status") assert foreign_issuer_status.status_code == 200, foreign_issuer_status.text foreign_issuer_body = foreign_issuer_status.json() assert foreign_issuer_body["checks"]["externalSubject"]["state"] == "pending" assert foreign_issuer_body["provisioning"]["subject"] is None assert foreign_issuer_body["verification"]["verifiedByManagement"] is False assert len(probe_calls) == probes_before_foreign_issuer_read foreign_issuer_verification = admin.post(f"/api/client-management/{client_id}/verify") assert foreign_issuer_verification.status_code == 200, foreign_issuer_verification.text assert foreign_issuer_verification.json()["checks"]["externalSubject"]["state"] == "pending" assert foreign_issuer_verification.json()["verification"]["verifiedByManagement"] is False assert len(probe_calls) > probes_before_foreign_issuer_read with sessions() as session, session.begin(): client_row = session.get(ClientRow, UUID(client_id)) assert client_row is not None client_row.issuer = ISSUER verified = admin.post(f"/api/client-management/{client_id}/verify") assert verified.status_code == 200, verified.text assert len(probe_calls) > pending_verification_probe_count after = verified.json() assert after["verification"]["checkedByManagement"] is True assert after["verification"]["verifiedByManagement"] is True assert after["verification"]["hubChecksPassed"] is True assert after["checks"]["externalScope"]["state"] == "unavailable" assert after["checks"]["mcpProof"]["state"] == "unavailable" verified_probe_count = len(probe_calls) persisted_status = viewer.get(f"/api/client-management/{client_id}/onboarding-status") assert persisted_status.status_code == 200, persisted_status.text assert persisted_status.json()["checks"]["connector"]["state"] == "healthy" assert persisted_status.json()["verification"]["verifiedByManagement"] is True assert len(probe_calls) == verified_probe_count for stale_checked_at in ( datetime.now(UTC) - timedelta(minutes=5), datetime.now(UTC) + timedelta(minutes=5), ): stale_timestamp = stale_checked_at.isoformat() with sessions() as session, session.begin(): client_row = session.get(ClientRow, UUID(client_id)) assert client_row is not None metadata = dict(client_row.record_metadata or {}) verification = dict(metadata["onboardingVerification"]) verification["checkedAt"] = stale_timestamp metadata["onboardingVerification"] = verification client_row.record_metadata = metadata stale_status = viewer.get(f"/api/client-management/{client_id}/onboarding-status") assert stale_status.status_code == 200, stale_status.text stale_body = stale_status.json() assert stale_body["checks"]["connector"]["state"] == "unknown" assert stale_body["checks"]["connector"]["observedAt"] is None assert stale_body["verification"]["checkedAt"] == stale_timestamp assert stale_body["verification"]["checkedByManagement"] is True assert stale_body["verification"]["verifiedByManagement"] is False assert stale_body["verification"]["hubChecksPassed"] is False assert len(probe_calls) == verified_probe_count with sessions() as session: client_row = session.get(ClientRow, UUID(client_id)) workspace = session.query(WorkspaceRow).filter_by(slug="itworx").one() binding = session.query(ClientBindingRow).filter_by(client_id=UUID(client_id)).one() audit_event = ( session.query(AuditEventRow) .filter_by(client_id=UUID(client_id), event_type="client.onboarding.verify") .order_by(AuditEventRow.occurred_at.desc()) .first() ) engine.dispose() assert client_row is not None and client_row.status == "active" assert binding.workspace_id == workspace.id assert binding.enabled is True assert client_row.record_metadata["onboardingVerification"]["verifiedByManagement"] is True assert "credential" not in str(client_row.record_metadata["onboardingVerification"]).lower() assert audit_event is not None assert audit_event.principal_subject == "user-management-1" assert audit_event.sanitized_arguments["hubChecksPassed"] is True def test_unauthed_client_is_not_exposed_by_connector_management_route( role_sessions: dict[ManagementRole, RoleSession], ) -> None: unauthenticated = TestClient(role_sessions[ManagementRole.VIEWER].app) unauthenticated.cookies.clear() response = unauthenticated.get("/api/connectors/knowledge-default") assert response.status_code == 401