from __future__ import annotations import base64 import json import os import stat from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, Protocol, cast from urllib.parse import urlparse from uuid import NAMESPACE_URL, UUID, uuid5 import httpx import yaml # type: ignore[import-untyped] from fastapi import Depends, FastAPI, HTTPException, Query, Request, Response, status from fastapi.responses import JSONResponse from jsonschema import Draft202012Validator # type: ignore[import-untyped] from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy import func, select, text from sqlalchemy.engine import Engine from packages.hub_clients import ClientRecord, ClientStatus from packages.hub_policy import Decision, Governance, PolicyDenied, PolicySimulator from services.control_api.auth import BFFService, OIDCProviderClient, OIDCSettings, PrincipalMapper from services.control_api.auth.errors import AuthError from services.control_api.auth.models import AuthenticatedPrincipal from services.control_api.auth.router import build_auth_router from services.control_api.authz import InsufficientRole, check_role from services.control_api.catalog import CatalogLoader from services.control_api.clients import build_client_router from services.control_api.contextforge import ContextForgeClient, ContextForgeSettings from services.control_api.persistence import create_engine_from_url, session_factory from services.control_api.persistence.models import AuditEventRow, ProjectRow from .audit import SqlAlchemyAuditLog from .client_service import PersistentClientService from .common import hub_database_url, identity_assertion, load_connector_identity_key, run_asgi from .dataplane import _policy_engine from .state_sync import CONNECTOR_GATEWAYS, reconcile_runtime_state class ClientLister(Protocol): """The minimal shape simulate_policy() needs -- lets tests pass a plain in-memory stand-in instead of a real database-backed ClientService.""" def list(self) -> Iterable[ClientRecord]: ... RESOURCE_FOLDERS = { "projects": "projects", "servers": "projects", "tools": "tools", "connectors": "connectors", "policies": "policies", } CONNECTOR_GATEWAY_BY_TYPE = { slug.removeprefix("itworx-"): slug for slug, _url in CONNECTOR_GATEWAYS } CONNECTOR_READINESS_URL_BY_SLUG = { slug: f"{url.removesuffix('/mcp')}/health/ready" for slug, url in CONNECTOR_GATEWAYS } CONNECTOR_URL_BY_TYPE = {slug.removeprefix("itworx-"): url for slug, url in CONNECTOR_GATEWAYS} class ToolReadinessRequest(BaseModel): """Bounded management-plane request; never a general MCP invocation.""" model_config = ConfigDict(extra="forbid") project: str = Field(pattern=r"^[a-z0-9][a-z0-9-]{0,62}$") environment: str = Field(pattern=r"^(dev|test|staging|prod)$") tool: str = Field(min_length=1, max_length=256) arguments: dict[str, Any] = Field(default_factory=dict) @field_validator("arguments") @classmethod def bound_arguments(cls, value: dict[str, Any]) -> dict[str, Any]: if len(value) > 64: raise ValueError("arguments contain too many top-level properties") serialized = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8") if len(serialized) > 32768: raise ValueError("arguments exceed the readiness-check size limit") return value class ToolInvokeRequest(ToolReadinessRequest): """The management console can invoke only cataloged read-only tools.""" def read_backup_status(path: Path, *, now: datetime | None = None) -> dict[str, Any]: """Read a bounded operator-produced status projection, never backup contents.""" unknown = { "state": "unknown", "lastSuccessfulAt": None, "detail": "No verified backup status has been published yet.", "restoreVerified": False, "artifactCount": 0, } try: info = path.lstat() if not stat.S_ISREG(info.st_mode) or path.is_symlink() or info.st_size > 65536: return unknown value = json.loads(path.read_text(encoding="utf-8")) if not isinstance(value, dict) or value.get("schemaVersion") != "1": return unknown completed = datetime.fromisoformat(str(value.get("completedAt", ""))) if completed.tzinfo is None or completed.utcoffset() is None: return unknown restore_verified = value.get("restoreVerified") is True artifact_count = int(value.get("artifactCount", 0)) if not restore_verified or not 1 <= artifact_count <= 1000: return { **unknown, "state": "degraded", "detail": "The latest backup status is incomplete or lacks restore verification.", "lastSuccessfulAt": completed.astimezone(UTC).isoformat(), "artifactCount": max(0, artifact_count), } age = (now or datetime.now(UTC)) - completed.astimezone(UTC) stale_after = timedelta(hours=int(os.environ.get("HUB_BACKUP_STALE_AFTER_HOURS", "26"))) stale = age > stale_after or age < timedelta(minutes=-5) return { "state": "degraded" if stale else "healthy", "lastSuccessfulAt": completed.astimezone(UTC).isoformat(), "detail": ( "Latest backup is restore-verified but older than the freshness target." if stale else "Latest backup and isolated restore verification succeeded." ), "restoreVerified": True, "artifactCount": artifact_count, } except (OSError, ValueError, TypeError, json.JSONDecodeError): return unknown def _documents(root: Path, folder: str) -> list[dict[str, Any]]: values: list[dict[str, Any]] = [] for path in sorted((root / folder).glob("*.yaml")): document = yaml.safe_load(path.read_text(encoding="utf-8")) if isinstance(document, dict): values.append(document) return values def _catalog_registration_health(document: dict[str, Any]) -> tuple[str, str]: """Whether a catalog document is a well-formed, registered entry. This aggregate list signal is registration validity, not an on-demand readiness probe. Operators can run the bounded readiness workflow for individual tool paths. Shared between resource rows and the dashboard overview so those surfaces do not disagree. """ if document.get("kind"): return ( "healthy", "Catalog entry is registered and well-formed; " "use tool readiness for a live path check.", ) return "unknown", "Catalog entry is missing or malformed." def _row(document: dict[str, Any], kind: str) -> dict[str, Any]: metadata = cast( dict[str, Any], document.get("metadata") if isinstance(document.get("metadata"), dict) else {}, ) spec = cast( dict[str, Any], document.get("spec") if isinstance(document.get("spec"), dict) else {}, ) slug = str(metadata.get("slug", "unknown"))[:128] name = str(metadata.get("name", slug))[:256] summary = str(spec.get("description", document.get("kind", kind)))[:1000] status, _detail = _catalog_registration_health(document) return { "id": slug, "name": name, "status": status, "summary": summary, "project": str(spec.get("project", "")) or None, "environment": None, "risk": str(spec.get("riskClass", "R0")) if kind == "tools" else None, # The catalog slug (`id` above) and canonicalName are different # strings (e.g. "deployment-project-status" vs # "deployment.project.status") -- callers that need to invoke or # simulate this tool (not just display it) need the latter. "canonicalName": str(spec.get("canonicalName")) if kind == "tools" else None, } def _verify_contextforge_dependency(contextforge: ContextForgeClient) -> None: health = contextforge.health() if health.get("status") not in {"ready", "healthy", "ok"}: raise RuntimeError("gateway not healthy") contextforge.list_gateways(limit=1) def measure_connector_drift(contextforge: ContextForgeClient) -> dict[str, Any]: """Compare ContextForge with the connector state production reconciliation owns. Project virtual servers are a separate resource type and must not be compared with connector gateways. Keeping this desired state sourced from ``CONNECTOR_GATEWAYS`` makes the dashboard and ``reconcile_runtime_state`` describe the same operational reality. """ desired = {slug: url.rstrip("/") for slug, url in CONNECTOR_GATEWAYS} actual = {gateway.slug: gateway for gateway in contextforge.list_gateways(limit=100)} affected = { slug for slug, expected_url in desired.items() if (gateway := actual.get(slug)) is None or not gateway.enabled or gateway.url.rstrip("/") != expected_url } affected.update(set(actual) - set(desired)) ordered = sorted(affected) return { "count": len(ordered), "projects": ordered, "known": True, "detail": ( "No connector gateway drift measured." if not ordered else "Measured connector gateway drift requires review before reconciliation." ), } _CURSOR_TTL_SECONDS = 3600 _AUDIT_FETCH_CAP = 1000 # v1 scale bound: bounded audit browsing, not unbounded history search def encode_page_cursor(offset: int) -> str: """Opaque cursor: offset + issue time, so an expired cursor can be rejected outright instead of silently wrapping around or resetting to page 1 (which would look like success while returning the wrong page).""" payload = f"{offset}:{int(datetime.now(UTC).timestamp())}" return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii") def decode_page_cursor(cursor: str) -> int: """Raises HTTPException(400) for a malformed or expired cursor -- fails safely (visible 400, never a 500, never a silent wraparound to page 1).""" try: decoded = base64.urlsafe_b64decode(cursor.encode("ascii")).decode("utf-8") offset_text, issued_text = decoded.split(":", 1) offset = int(offset_text) issued_at = int(issued_text) except Exception as exc: raise HTTPException(status_code=400, detail="invalid pagination cursor") from exc if offset < 0: raise HTTPException(status_code=400, detail="invalid pagination cursor") if datetime.now(UTC).timestamp() - issued_at > _CURSOR_TTL_SECONDS: raise HTTPException( status_code=400, detail="pagination cursor has expired; restart from the first page", ) return offset _ALLOW_DECISIONS = frozenset( { Decision.ALLOW, Decision.ALLOW_WITH_PREVIEW, Decision.ALLOW_WITH_APPROVAL, Decision.ALLOW_IF_CONDITIONS, } ) def simulate_policy( catalog_root: Path, clients: ClientLister, payload: dict[str, Any], principal: AuthenticatedPrincipal, ) -> dict[str, Any]: """Real policy simulation: the exact evaluator production traffic uses (`_policy_engine` from services/runtime/dataplane.py, unmodified), against catalog-derived context -- not a hand-maintained stub. Never mutates runtime state: PolicySimulator only ever calls PolicyEngine.evaluate(), which reads policy rules and returns a DecisionTrace; nothing here writes to the catalog, the database, or any external system. Module-level (not a nested closure inside build_app()) specifically so it's unit-testable without constructing a full FastAPI app -- see tests/test_wp230_policy_simulate.py. """ project_slug = str(payload.get("project", "")) environment = str(payload.get("environment", "")) tool_name = str(payload.get("tool", "")) projects_by_slug = { str(doc.get("metadata", {}).get("slug", "")): doc for doc in _documents(catalog_root, "projects") } tools_by_name = { str(doc.get("spec", {}).get("canonicalName", "")): doc for doc in _documents(catalog_root, "tools") } connector_type_by_slug = { str(doc.get("metadata", {}).get("slug", "")): str( doc.get("spec", {}).get("connectorType", "") ) for doc in _documents(catalog_root, "connectors") } project_doc = projects_by_slug.get(project_slug) project_spec = cast(dict[str, Any], project_doc.get("spec", {})) if project_doc else {} tool_doc = tools_by_name.get(tool_name) tool_spec = cast(dict[str, Any], tool_doc.get("spec", {})) if tool_doc else {} connector_type = str(tool_spec.get("connectorType", "unknown")) risk_class = str(tool_spec.get("riskClass", "R1")) required_scopes = list(tool_spec.get("requiredScopes", [])) environments_spec = cast(dict[str, Any], project_spec.get("environments") or {}) environment_spec = cast(dict[str, Any], environments_spec.get(environment) or {}) environment_classification = environment_spec.get("classification") # Real capability-pack/binding lookup: is there a registered connector # binding, for this project, of this tool's connector type, covering # this environment? This is exactly what "capability pack or binding" # in the WP-230B acceptance criteria means -- not a caller-supplied flag. binding_present = any( connector_type_by_slug.get(str(cast(dict[str, Any], binding).get("ref", ""))) == connector_type and environment in (cast(dict[str, Any], binding).get("environments") or []) for binding in project_spec.get("connectorBindings", []) if isinstance(binding, dict) ) client_id: UUID | None = None raw_client_id = payload.get("clientId") if raw_client_id: try: client_id = UUID(str(raw_client_id)) except ValueError: client_id = None # invalid input -> simulated as an absent/unknown client, not a 400 known_clients = {record.id for record in clients.list()} client_enabled = False if client_id is not None: matching = next((record for record in clients.list() if record.id == client_id), None) client_enabled = bool(matching and matching.status == ClientStatus.ENABLED) # Caller-supplied overrides are honored (a deliberate "what if" simulation # capability -- e.g. "what would happen if this client were bound") but # every value has a real, catalog/database-derived default rather than an # unconditional True the way the old stub effectively assumed. context = Governance.context( workspace="itworx", project=project_slug, environment=environment, principal=principal.subject, principal_type=principal.kind.value, tool=tool_name, connector=connector_type, arguments={"requestedScopes": payload.get("scopes", required_scopes)}, client_id=client_id, binding_present=bool(payload.get("bindingPresent", binding_present)), client_enabled=bool(payload.get("clientEnabled", client_enabled)), capability=payload.get("capability"), risk_class=risk_class, environment_classification=environment_classification, ) engine = _policy_engine(catalog_root) simulator = PolicySimulator( engine, known_projects=set(projects_by_slug), known_clients=known_clients, known_tools=set(tools_by_name), ) trace = simulator.simulate(context) return { "decision": trace.decision.value, "allowed": trace.decision in _ALLOW_DECISIONS, "matchedPolicies": list(trace.matched_rule_ids), "reasons": [trace.reason_code], "reasonCode": trace.reason_code, "traceId": trace.trace_id, "requiredScopes": required_scopes, "steps": [step.model_dump(mode="json") for step in trace.steps], } def _tool_input_errors(schema: dict[str, Any], arguments: dict[str, Any]) -> list[str]: """Return bounded, non-value-bearing JSON Schema diagnostics.""" errors = sorted( Draft202012Validator(schema).iter_errors(arguments), key=lambda error: tuple(str(part) for part in error.absolute_path), ) diagnostics: list[str] = [] for error in errors[:10]: path = ".".join(str(part) for part in error.absolute_path) or "$" diagnostics.append(f"{path}: {error.validator or 'schema'} validation failed") return diagnostics def _direct_connector_readiness(gateway_slug: str) -> dict[str, Any]: """Use an allowlisted internal probe when ContextForge has no test route.""" url = CONNECTOR_READINESS_URL_BY_SLUG.get(gateway_slug) if url is None: return { "reachable": False, "status": "unsupported", "detail": "No bounded connector readiness route is registered.", } try: response = httpx.get(url, timeout=5.0, follow_redirects=False) if response.status_code != 200 or len(response.content) > 65536: raise ValueError("connector readiness response was not acceptable") payload = response.json() status_value = ( str(payload.get("status", "unknown")) if isinstance(payload, dict) else "unknown" ) reachable = status_value in {"ready", "healthy"} return { "reachable": reachable, "status": status_value if reachable else "unavailable", "detail": ( "Direct connector readiness verified." if reachable else "Connector readiness did not report ready." ), } except Exception: return { "reachable": False, "status": "unavailable", "detail": "Connector readiness could not be verified.", } def evaluate_tool_readiness( catalog_root: Path, clients: ClientLister, contextforge: ContextForgeClient, request: ToolReadinessRequest, principal: AuthenticatedPrincipal, ) -> dict[str, Any]: """Validate a tool path without invoking the tool or returning input values.""" projects_by_slug = { str(doc.get("metadata", {}).get("slug", "")): doc for doc in _documents(catalog_root, "projects") } tools_by_name = { str(doc.get("spec", {}).get("canonicalName", "")): doc for doc in _documents(catalog_root, "tools") } connector_type_by_slug = { str(doc.get("metadata", {}).get("slug", "")): str( doc.get("spec", {}).get("connectorType", "") ) for doc in _documents(catalog_root, "connectors") } project_doc = projects_by_slug.get(request.project) if project_doc is None: raise HTTPException(status_code=404, detail="project is not registered") tool_doc = tools_by_name.get(request.tool) if tool_doc is None: raise HTTPException(status_code=404, detail="tool is not registered") project_spec = cast(dict[str, Any], project_doc.get("spec", {})) environments = cast(dict[str, Any], project_spec.get("environments") or {}) if request.environment not in environments: raise HTTPException(status_code=422, detail="environment is not registered for project") tool_spec = cast(dict[str, Any], tool_doc.get("spec", {})) connector_type = str(tool_spec.get("connectorType", "")) read_only = bool(tool_spec.get("readOnly", False)) binding_present = any( connector_type_by_slug.get(str(binding.get("ref", ""))) == connector_type and request.environment in (binding.get("environments") or []) and binding.get("verification") == "verified" for binding in project_spec.get("connectorBindings", []) if isinstance(binding, dict) ) input_schema = cast(dict[str, Any], tool_spec.get("inputSchema") or {}) input_errors = _tool_input_errors(input_schema, request.arguments) policy = simulate_policy( catalog_root, clients, { "project": request.project, "environment": request.environment, "tool": request.tool, "bindingPresent": binding_present, "clientEnabled": True, }, principal, ) gateway_slug = CONNECTOR_GATEWAY_BY_TYPE.get(connector_type) if gateway_slug is None: connector_check = { "reachable": False, "status": "not-deployed", "detail": "No production connector gateway is registered for this connector type.", } else: try: result = contextforge.test_gateway_by_slug(gateway_slug) connector_check = result.model_dump(mode="json") if connector_check["status"] == "unsupported": connector_check = _direct_connector_readiness(gateway_slug) except Exception: connector_check = { "reachable": False, "status": "unavailable", "detail": "Connector readiness could not be verified.", } ready = bool( read_only and not input_errors and binding_present and policy["allowed"] and connector_check["reachable"] ) return { "ready": ready, "tool": request.tool, "project": request.project, "environment": request.environment, "checks": { "catalog": {"valid": True, "readOnly": read_only, "connectorType": connector_type}, "input": {"valid": not input_errors, "errors": input_errors}, "binding": { "valid": binding_present, "detail": ( "Project connector binding covers this environment." if binding_present else "Project connector binding is missing for this environment." ), }, "policy": { "allowed": policy["allowed"], "decision": policy["decision"], "reasonCode": policy["reasonCode"], "traceId": policy["traceId"], }, "connector": connector_check, }, "note": "Readiness only; no MCP tool invocation was performed.", } def _schema_projection(value: Any, schema: dict[str, Any]) -> Any: if schema.get("type") == "object" and isinstance(value, dict): properties = schema.get("properties") if not isinstance(properties, dict): return value projected = { name: _schema_projection(value[name], cast(dict[str, Any], child_schema)) for name, child_schema in properties.items() if name in value and isinstance(child_schema, dict) } additional = schema.get("additionalProperties", True) if additional is not False: additional_schema = additional if isinstance(additional, dict) else {} projected.update( { name: _schema_projection(item, additional_schema) for name, item in value.items() if name not in properties } ) return projected if schema.get("type") == "array" and isinstance(value, list): item_schema = schema.get("items") if isinstance(item_schema, dict): return [_schema_projection(item, item_schema) for item in value] return value def _project_connector_output( structured_content: Any, output_schema: dict[str, Any] ) -> tuple[Any, list[str]]: """Project a standard connector envelope onto its registered public contract.""" if not isinstance(structured_content, dict) or "data" not in structured_content: return structured_content, [] meta = structured_content.get("meta") warnings = structured_content.get("warnings") if ( not isinstance(meta, dict) or not isinstance(warnings, list) or len(warnings) > 20 or any(not isinstance(item, str) or len(item) > 500 for item in warnings) ): raise ValueError("connector envelope is invalid") data = structured_content["data"] properties = output_schema.get("properties") if isinstance(data, dict) and isinstance(properties, dict): data = { **data, **{name: meta[name] for name in properties if name not in data and name in meta}, } return _schema_projection(data, output_schema), warnings def invoke_read_only_tool( catalog_root: Path, clients: ClientLister, contextforge: ContextForgeClient, sessions: Any, request: ToolInvokeRequest, principal: AuthenticatedPrincipal, signing_key: bytes, *, transport: httpx.BaseTransport | None = None, ) -> dict[str, Any]: """Invoke one cataloged read-only connector through a bounded operator path.""" readiness = evaluate_tool_readiness(catalog_root, clients, contextforge, request, principal) if not readiness["ready"]: raise HTTPException(status_code=409, detail="tool path is not ready") tool_doc = next( doc for doc in _documents(catalog_root, "tools") if str(doc.get("spec", {}).get("canonicalName", "")) == request.tool ) tool_spec = cast(dict[str, Any], tool_doc.get("spec", {})) if tool_spec.get("readOnly") is not True: raise HTTPException(status_code=403, detail="management console permits read-only tools") connector_type = str(tool_spec.get("connectorType", "")) connector_url = CONNECTOR_URL_BY_TYPE.get(connector_type) if connector_url is None: raise HTTPException(status_code=409, detail="connector is not registered") console_client_id = uuid5( NAMESPACE_URL, f"https://hub.example.com/management-console/{principal.subject}" ) governance = Governance( _policy_engine(catalog_root), SqlAlchemyAuditLog(sessions, issuer=principal.issuer), ) context = Governance.context( workspace="itworx", project=request.project, environment=request.environment, principal=principal.subject, principal_type=principal.kind.value, client_id=console_client_id, binding_present=True, client_enabled=True, tool=request.tool, connector=connector_type, arguments=request.arguments, resource_type="tool", risk_class=str(tool_spec.get("riskClass", "R1")), environment_classification=( "production" if request.environment == "prod" else "development" ), ) try: trace, started = governance.begin(context) except PolicyDenied as exc: raise HTTPException( status_code=403, detail="policy denied management-console invocation" ) from exc headers = { "Accept": "application/json", "Content-Type": "application/json", "MCP-Protocol-Version": "2025-11-25", "x-itworx-project": request.project, "x-itworx-environment": request.environment, "x-itworx-client-id": str(console_client_id), "x-itworx-trace-id": trace.trace_id, "x-itworx-identity-assertion": identity_assertion( signing_key, project=request.project, environment=request.environment, client_id=str(console_client_id), trace_id=trace.trace_id, ), } body = { "jsonrpc": "2.0", "id": trace.trace_id, "method": "tools/call", "params": {"name": request.tool, "arguments": request.arguments}, } max_output = min(int(tool_spec.get("limits", {}).get("maxOutputBytes", 131072)), 262144) timeout_seconds = min( max(float(tool_spec.get("limits", {}).get("timeoutMs", 10000)) / 1000, 1.0), 20.0, ) try: with httpx.Client( transport=transport, timeout=timeout_seconds, follow_redirects=False ) as client: response = client.post(connector_url, headers=headers, json=body) if response.status_code != 200 or len(response.content) > max_output: governance.finish( context, trace, started, outcome="error", error_code="CONNECTOR_RESPONSE_REJECTED", ) raise HTTPException(status_code=502, detail="connector response was not acceptable") document = response.json() if not isinstance(document, dict): raise ValueError("connector response is not an object") result = document.get("result") error = document.get("error") if error is not None or not isinstance(result, dict): governance.finish(context, trace, started, outcome="error", error_code="MCP_TOOL_ERROR") return { "ok": False, "traceId": trace.trace_id, "tool": request.tool, "error": {"code": "MCP_TOOL_ERROR", "message": "The tool returned an error."}, } is_error = result.get("isError") is True output_schema = cast(dict[str, Any], tool_spec.get("outputSchema") or {}) structured_content = result.get("structuredContent") try: contract_content, connector_warnings = _project_connector_output( structured_content, output_schema ) except ValueError as exc: governance.finish( context, trace, started, outcome="error", error_code="OUTPUT_ENVELOPE_INVALID", ) raise HTTPException( status_code=502, detail="connector output envelope was not acceptable" ) from exc output_errors = list(Draft202012Validator(output_schema).iter_errors(contract_content)) if not is_error and output_errors: governance.finish( context, trace, started, outcome="error", error_code="OUTPUT_SCHEMA_INVALID", ) raise HTTPException( status_code=502, detail="connector output failed the registered contract" ) governance.finish( context, trace, started, outcome="error" if is_error else "success", error_code="MCP_TOOL_ERROR" if is_error else None, result_summary=( (contract_content.get("data") or {}).get("auditEvidence") if not is_error and request.tool == "gitea.pullrequest.merge" else None ), ) return { "ok": not is_error, "traceId": trace.trace_id, "tool": request.tool, "structuredContent": contract_content, "warnings": connector_warnings, } except HTTPException: raise except (httpx.HTTPError, ValueError, json.JSONDecodeError) as exc: governance.finish( context, trace, started, outcome="error", error_code="CONNECTOR_UNAVAILABLE" ) raise HTTPException(status_code=502, detail="connector invocation failed safely") from exc def operations_snapshot(sessions: Any, backup_path: Path) -> dict[str, Any]: """Return aggregated, bounded operational signals without audit payloads.""" now = datetime.now(UTC) since = now - timedelta(hours=24) with sessions() as session: rows = session.execute( select(AuditEventRow.occurred_at, AuditEventRow.outcome, AuditEventRow.tool_name) .where(AuditEventRow.occurred_at >= since) .order_by(AuditEventRow.occurred_at) .limit(5001) ).all() truncated = len(rows) > 5000 rows = rows[:5000] buckets: dict[str, dict[str, int]] = {} for occurred_at, outcome, _tool in rows: key = occurred_at.astimezone(UTC).replace(minute=0, second=0, microsecond=0).isoformat() bucket = buckets.setdefault(key, {"success": 0, "denied": 0, "error": 0, "rate_limited": 0}) if outcome in bucket: bucket[outcome] += 1 totals = {"success": 0, "denied": 0, "error": 0, "rate_limited": 0} for bucket in buckets.values(): for outcome in totals: totals[outcome] += bucket[outcome] backup = read_backup_status(backup_path, now=now) alerts: list[dict[str, str]] = [] if backup["state"] != "healthy": alerts.append( {"severity": "warning", "code": "BACKUP_ATTENTION", "message": backup["detail"]} ) if totals["error"]: alerts.append( { "severity": "warning", "code": "TOOL_ERRORS", "message": f"{totals['error']} governed call errors occurred in the last 24 hours.", } ) if totals["denied"] + totals["rate_limited"] >= 10: alerts.append( { "severity": "info", "code": "AUTHORIZATION_ACTIVITY", "message": "Authorization denials or rate limits need review.", } ) if truncated: alerts.append( { "severity": "info", "code": "AUDIT_AGGREGATE_TRUNCATED", "message": "The 24-hour aggregate reached its 5,000-event display bound.", } ) return { "generatedAt": now.isoformat(), "backup": backup, "audit24h": totals, "auditTruncated": truncated, "trend": [{"hour": hour, **values} for hour, values in sorted(buckets.items())], "alerts": alerts, } def build_app( *, oidc: OIDCSettings | None = None, provider_transport: httpx.BaseTransport | None = None, contextforge_settings: ContextForgeSettings | None = None, contextforge_transport: httpx.BaseTransport | None = None, engine: Engine | None = None, catalog_root: Path | None = None, ) -> FastAPI: """Build the production ASGI app. Every keyword argument is an optional injection point for tests: with none given, behavior is identical to before (settings from environment, real network transports, a real database engine). Tests that need a fully real app -- to exercise actual route wiring and RBAC end to end, not just the underlying matrix in isolation -- pass a fake OIDC/ContextForge transport (`services/control_api/auth/fake_provider.py`, `services/control_api/contextforge/fake.py`) and a throwaway SQLite engine. """ catalog_root = ( catalog_root or Path(os.environ.get("HUB_CATALOG_ROOT", "/opt/hub/catalog")).resolve() ) if not catalog_root.is_dir(): raise ValueError("catalog root is unavailable") oidc = oidc or OIDCSettings.from_env() provider = OIDCProviderClient(oidc, transport=provider_transport) mapper = PrincipalMapper(role_group_map=oidc.role_group_map) bff = BFFService(oidc, provider, principal_mapper=mapper.map_claims) contextforge = ContextForgeClient( contextforge_settings or ContextForgeSettings.from_env(), transport=contextforge_transport ) engine = engine or create_engine_from_url(hub_database_url()) sessions = session_factory(engine) loader = CatalogLoader( catalog_root, public_base_url=os.environ.get("HUB_PUBLIC_MCP_URL", "https://mcp.example.com"), ) public_mcp_url = os.environ.get("HUB_PUBLIC_MCP_URL", "https://mcp.example.com") public_host = urlparse(public_mcp_url).hostname if not public_host: raise ValueError("HUB_PUBLIC_MCP_URL must contain a hostname") clients = PersistentClientService( sessions, issuer=str(oidc.issuer), public_host=public_host, ) backup_status_path = Path( os.environ.get("HUB_BACKUP_STATUS_FILE", "/var/lib/itworx/backup-status.json") ) @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: try: yield finally: provider.close() contextforge.close() engine.dispose() app = FastAPI( title="ITWorx MCP Hub control API", version=os.environ.get("HUB_RELEASE_VERSION", "1.0.0-rc"), docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan, ) app.include_router(build_auth_router(bff)) @app.get("/healthz") @app.get("/health") def health() -> dict[str, str]: return {"status": "healthy", "component": "control-api"} @app.get("/ready") @app.get("/health/ready") def ready(response: Response) -> dict[str, str]: try: with sessions() as session: session.execute(text("SELECT 1")).scalar_one() _verify_contextforge_dependency(contextforge) except Exception: response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE return {"status": "not-ready", "component": "control-api"} return {"status": "ready", "component": "control-api"} @app.get("/version") def version() -> dict[str, str]: return {"version": os.environ.get("HUB_RELEASE_VERSION", "1.0.0-rc")} def authenticate(request: Request) -> AuthenticatedPrincipal: """Verify the session is valid and carries at least one recognized management role. This is authentication plus a coarse baseline gate, not authorization for any specific action -- see `require()` below for the actual per-endpoint permission check.""" try: principal = bff.require_session(request.cookies.get(oidc.cookie_name)) except AuthError as exc: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail={"code": exc.code, "message": exc.public_message}, ) from exc if not principal.roles: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="management role is required", ) return principal # Bound once, reused as route defaults below: ruff (flake8-bugbear B008) # flags calling `Depends(...)` directly inside a function signature's # default value, the same reason the pre-WP-230 code bound # `authorized = Depends(authorize)` once instead of inlining it per route. authenticated = Depends(authenticate) def require(action: str) -> Any: """FastAPI dependency: authenticate, then enforce the per-endpoint permission matrix (services/control_api/authz.py) for `action`.""" def dependency(principal: AuthenticatedPrincipal = authenticated) -> AuthenticatedPrincipal: try: check_role(principal, action) except InsufficientRole as exc: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"role does not permit {action}", ) from exc return principal return Depends(dependency) require_dashboard_read = require("dashboard.read") require_resource_read = require("resource.read") require_policy_simulate = require("policy.simulate") require_tool_test = require("tool.test") require_tool_invoke = require("tool.invoke") require_operations_check = require("operations.check") require_catalog_preview = require("catalog.preview") require_catalog_apply = require("catalog.apply") def authorize_action(principal: AuthenticatedPrincipal, action: str) -> None: """Non-dependency form of the same check, for routers (e.g. the client router) that need to select an action per-request rather than per-route.""" try: check_role(principal, action) except InsufficientRole as exc: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"role does not permit {action}", ) from exc app.include_router( build_client_router( clients, # type: ignore[arg-type] authenticate=authenticate, authorize=authorize_action, prefix="/client-management", token_endpoint=( f"{urlparse(str(oidc.issuer)).scheme}://" f"{urlparse(str(oidc.issuer)).netloc}/application/o/token/" ), oauth_client_id=oidc.client_id, ), prefix="/api", ) @app.middleware("http") async def response_security(request: Request, call_next: Any) -> Any: if request.url.path.startswith("/api/") and request.method not in { "GET", "HEAD", "OPTIONS", }: try: bff.verify_csrf( request.cookies.get(oidc.cookie_name), request.headers.get("X-CSRF-Token") ) except AuthError as exc: return JSONResponse( status_code=status.HTTP_403_FORBIDDEN, content={"detail": {"code": exc.code, "message": exc.public_message}}, ) response = await call_next(request) response.headers["Cache-Control"] = "no-store" response.headers["X-Content-Type-Options"] = "nosniff" response.headers["Referrer-Policy"] = "no-referrer" response.headers["X-Frame-Options"] = "DENY" return response @app.get("/api/dashboard") def dashboard(_: Any = require_dashboard_read) -> dict[str, Any]: gateway = "unavailable" gateway_detail = "ContextForge unavailable" try: health = contextforge.health() gateway = "healthy" if health.get("status") else "degraded" gateway_detail = "ContextForge health verified" except Exception: gateway = "unavailable" projects = [] for document in _documents(catalog_root, "projects"): metadata = document.get("metadata", {}) spec = document.get("spec", {}) registration_health, registration_detail = _catalog_registration_health(document) for environment in spec.get("environments") or {}: projects.append( { "slug": metadata.get("slug", "unknown"), "name": metadata.get("name", "Unknown"), "environment": environment, "health": registration_health, "detail": registration_detail, } ) connectors = [] for item in _documents(catalog_root, "connectors"): registration_state, registration_detail = _catalog_registration_health(item) connectors.append( { "name": item.get("metadata", {}).get("name", "Unknown"), "project": "registered", "state": registration_state, "detail": registration_detail, } ) with sessions() as session: denial_count = session.scalar( select(func.count(AuditEventRow.id)).where( AuditEventRow.outcome.in_(("denied", "rate_limited")) ) ) drift: dict[str, Any] = { "count": 0, "projects": [], "known": False, "detail": "Catalog drift could not be measured because the gateway is unavailable.", } try: drift = measure_connector_drift(contextforge) except Exception: # A dashboard read must stay available when ContextForge is down; # report that the result is unmeasured rather than implying zero drift. drift["detail"] = ( "Catalog drift could not be measured because the gateway is unavailable." ) return { "generatedAt": datetime.now(UTC).isoformat(), "staleAfterSeconds": 60, "health": { "hub": "healthy", "gateway": gateway, "detail": gateway_detail, }, "projects": projects, "connectors": connectors, "denials": {"count": int(denial_count or 0), "window": "retained audit window"}, "drift": drift, "activeClients": len([item for item in clients.list() if item.status == "enabled"]), "backup": read_backup_status(backup_status_path), "version": { "application": os.environ.get("HUB_RELEASE_VERSION", "1.0.0-rc"), "schema": os.environ.get("HUB_SCHEMA_VERSION", "0004"), }, } @app.get("/api/projects/{slug}") def project_detail(slug: str, _: Any = require_resource_read) -> dict[str, Any]: """Real, catalog-derived project detail -- this is what the project detail page's tabs actually render, replacing the old hardcoded "prod" / "3 servers" placeholders that showed for every project slug regardless of what was actually registered. """ project_doc = next( ( doc for doc in _documents(catalog_root, "projects") if str(doc.get("metadata", {}).get("slug", "")) == slug ), None, ) if project_doc is None: raise HTTPException(status_code=404, detail="project is not registered") spec = cast(dict[str, Any], project_doc.get("spec", {})) metadata = cast(dict[str, Any], project_doc.get("metadata", {})) connector_type_by_slug = { str(doc.get("metadata", {}).get("slug", "")): str( doc.get("spec", {}).get("connectorType", "") ) for doc in _documents(catalog_root, "connectors") } environments = cast(dict[str, Any], spec.get("environments") or {}) connector_bindings = [ { "ref": binding.get("ref"), "connectorType": connector_type_by_slug.get(str(binding.get("ref", ""))), "environments": binding.get("environments", []), "verification": binding.get("verification"), } for binding in spec.get("connectorBindings", []) if isinstance(binding, dict) ] project_clients = [item for item in clients.list() if item.project == slug] virtual_servers = [ server for server in spec.get("virtualServers", []) if isinstance(server, dict) ] return { "slug": slug, "name": str(metadata.get("name", slug)), "description": str(spec.get("description", "")), "environments": [ {"name": name, "classification": cast(dict[str, Any], env).get("classification")} for name, env in environments.items() ], "capabilityPacks": list(spec.get("capabilityPacks", [])), "connectorBindings": connector_bindings, "virtualServerCount": len(virtual_servers), "clientCount": len(project_clients), "activeClientCount": len( [item for item in project_clients if item.status == "enabled"] ), } @app.get("/api/tool-definitions/{canonical_name:path}") def tool_definition(canonical_name: str, _: Any = require_resource_read) -> dict[str, Any]: document = next( ( item for item in _documents(catalog_root, "tools") if str(item.get("spec", {}).get("canonicalName", "")) == canonical_name ), None, ) if document is None: raise HTTPException(status_code=404, detail="tool is not registered") spec = cast(dict[str, Any], document.get("spec", {})) return { "canonicalName": canonical_name, "description": str(spec.get("description", ""))[:1000], "readOnly": spec.get("readOnly") is True, "riskClass": str(spec.get("riskClass", "R1")), "inputSchema": cast(dict[str, Any], spec.get("inputSchema") or {}), "limits": cast(dict[str, Any], spec.get("limits") or {}), } @app.get("/api/operations/summary") def operation_summary(_: Any = require_dashboard_read) -> dict[str, Any]: return operations_snapshot(sessions, backup_status_path) @app.post("/api/operations/check") def operation_check(_: Any = require_operations_check) -> dict[str, Any]: checks = { slug: _direct_connector_readiness(slug) for slug in sorted(CONNECTOR_READINESS_URL_BY_SLUG) } return { "generatedAt": datetime.now(UTC).isoformat(), "healthy": all(item["reachable"] for item in checks.values()), "connectors": checks, } @app.get("/api/{kind}") def resource_page( kind: str, q: str = Query(default="", max_length=200), limit: int = Query(default=25, ge=1, le=100), cursor: str | None = Query(default=None, max_length=128), principal: AuthenticatedPrincipal = authenticated, ) -> dict[str, Any]: # A single handler serves several resource kinds with different # sensitivity -- audit reads require auditor+, everything else here # is viewer+ -- so the permission check happens here, keyed on # `kind`, rather than as a single fixed per-route dependency. authorize_action(principal, "audit.read" if kind == "audit" else "resource.read") offset = decode_page_cursor(cursor) if cursor else 0 documents: list[dict[str, Any]] = [] rows: list[dict[str, Any]] if kind == "clients": rows = [ { "id": str(item.id), "name": item.name, "status": ( "healthy" if item.status == "enabled" else "unavailable" if item.status == "revoked" else "degraded" ), "summary": ( f"{item.kind.value}; last seen " f"{item.last_seen_at.isoformat() if item.last_seen_at else 'never'}" ), "project": item.project, "environment": item.environment, "risk": None, } for item in clients.list() ] elif kind == "audit": # Fetched up to a fixed v1 scale bound, then paginated the same # way as every other kind below -- bounded audit browsing, not # unbounded history search (see _AUDIT_FETCH_CAP). with sessions() as session: audit_rows = session.execute( select(AuditEventRow, ProjectRow.slug) .join(ProjectRow, ProjectRow.id == AuditEventRow.project_id) .order_by(AuditEventRow.occurred_at.desc(), AuditEventRow.id.desc()) .limit(_AUDIT_FETCH_CAP) ).all() rows = [ { "id": str(event.id), "name": event.event_type, "status": "healthy" if event.outcome == "success" else "degraded", "summary": ( f"{event.tool_name or 'mcp.request'}; {event.outcome}; " f"{event.occurred_at.isoformat()}" ), "project": project_slug, "environment": None, "decision": event.policy_decision, "traceId": event.trace_id, } for event, project_slug in audit_rows ] else: folder = RESOURCE_FOLDERS.get(kind) if folder is None: raise HTTPException(status_code=404, detail="resource kind is unavailable") documents = _documents(catalog_root, folder) if kind not in {"clients", "audit"}: rows = [_row(document, kind) for document in documents] if q: needle = q.casefold() rows = [ item for item in rows if needle in f"{item['name']} {item['summary']}".casefold() ] total = len(rows) page_rows = rows[offset : offset + limit] next_cursor = encode_page_cursor(offset + limit) if offset + limit < total else None return { "kind": kind, "generatedAt": datetime.now(UTC).isoformat(), "rows": page_rows, "page": {"total": total, "nextCursor": next_cursor}, } @app.post("/api/policies/simulate") def policy_simulation( payload: dict[str, Any], principal: AuthenticatedPrincipal = require_policy_simulate ) -> dict[str, Any]: return simulate_policy(catalog_root, clients, payload, principal) @app.post("/api/tools/test") def tool_test( payload: ToolReadinessRequest, principal: AuthenticatedPrincipal = require_tool_test, ) -> dict[str, Any]: return evaluate_tool_readiness(catalog_root, clients, contextforge, payload, principal) @app.post("/api/tools/invoke") def tool_invoke( payload: ToolInvokeRequest, principal: AuthenticatedPrincipal = require_tool_invoke, ) -> dict[str, Any]: return invoke_read_only_tool( catalog_root, clients, contextforge, sessions, payload, principal, load_connector_identity_key(), ) @app.post("/api/catalog/preview") def catalog_preview( payload: dict[str, Any], __: Any = require_catalog_preview ) -> dict[str, Any]: project = str(payload.get("project", "")) projects = { str(item.get("metadata", {}).get("slug", "")) for item in _documents(catalog_root, "projects") } report = loader.load() errors = [issue.message for issue in report.issues] if project not in projects: errors.append("requested project is not registered") return { "valid": report.valid and project in projects, "changes": [f"reconcile registered project {project}"] if not errors else [], "errors": errors, } @app.post("/api/catalog/apply") def catalog_apply(payload: dict[str, Any], __: Any = require_catalog_apply) -> dict[str, Any]: project = str(payload.get("project", "")) projects = { str(item.get("metadata", {}).get("slug", "")) for item in _documents(catalog_root, "projects") } if project not in projects: raise HTTPException(status_code=404, detail="requested project is not registered") reconcile_runtime_state( contextforge, sessions, catalog_root, str(oidc.issuer), ) return {"accepted": True, "message": "Catalog reconciliation completed and verified."} return app def run() -> None: run_asgi(build_app())