Files
ITWorx_MCP_Hub-Public/services/runtime/state_sync.py
T

621 lines
27 KiB
Python

from __future__ import annotations
import fnmatch
import hashlib
import json
import os
from pathlib import Path
from typing import Any, Literal
from uuid import UUID
import yaml # type: ignore[import-untyped]
from pydantic import AnyHttpUrl, BaseModel, ConfigDict, Field, TypeAdapter
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from services.control_api.catalog import CatalogLoader
from services.control_api.contextforge import ContextForgeClient
from services.control_api.contextforge.models import (
GatewayDesiredState,
GatewayHandle,
GatewayPrompt,
GatewayTool,
VirtualServerDesiredState,
)
from services.control_api.persistence.models import (
ClientBindingRow,
ClientRow,
EnvironmentRow,
ProjectRow,
VirtualServerRow,
)
from services.control_api.persistence.seed import seed_database
from .common import deterministic_id, load_json_environment
from .mcp_server import WORKFLOW_PROMPTS
PASSTHROUGH_HEADERS = (
"x-itworx-project",
"x-itworx-environment",
"x-itworx-client-id",
"x-itworx-trace-id",
"x-itworx-identity-assertion",
)
INGESTION_CONNECTOR_ENABLED = (
os.environ.get("INGESTION_CONNECTOR_ENABLED", "false").lower() == "true"
)
BASE_CONNECTOR_GATEWAYS = (
("itworx-knowledge", "http://connector-knowledge:8080/mcp"),
("itworx-gitea", "http://connector-gitea:8080/mcp"),
("itworx-unraid", "http://connector-unraid:8080/mcp"),
("itworx-mobilityops", "http://connector-mobilityops:8080/mcp"),
("itworx-projectbrain", "http://connector-projectbrain:8080/mcp"),
("itworx-appops", "http://connector-appops:8080/mcp"),
("itworx-stitch", "http://connector-stitch:8080/mcp"),
("itworx-opencode", "http://openchamber-mcp:8080/mcp"),
("itworx-opencode-readonly", "http://openchamber-mcp-readonly:8080/mcp"),
)
CONNECTOR_GATEWAYS = BASE_CONNECTOR_GATEWAYS + (
(("itworx-ingestion", "http://connector-ingestion:8080/mcp"),)
if INGESTION_CONNECTOR_ENABLED
else ()
)
class MachineBinding(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
client_id: UUID = Field(alias="clientId")
subject: str = Field(min_length=1, max_length=256)
issuer: AnyHttpUrl | None = None
project: str = Field(pattern=r"^[a-z0-9][a-z0-9-]{0,62}$")
environment: Literal["dev", "test", "staging", "prod"]
enabled: bool = True
def configured_bindings() -> tuple[MachineBinding, ...]:
raw = load_json_environment("MCP_CLIENT_BINDINGS_JSON", list)
values = tuple(MachineBinding.model_validate(item) for item in raw)
identities = {(item.client_id, item.project, item.environment) for item in values}
if len(identities) != len(values):
raise ValueError("MCP client binding identities must be unique")
return values
def _project_documents(catalog_root: Path) -> tuple[dict[str, Any], ...]:
documents: list[dict[str, Any]] = []
for path in sorted((catalog_root / "projects").glob("*.yaml")):
value = yaml.safe_load(path.read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise ValueError("project catalog document is invalid")
if (
not INGESTION_CONNECTOR_ENABLED
and value.get("metadata", {}).get("slug") == "knowledge-hub"
):
continue
documents.append(value)
return tuple(documents)
def _expected_tools_by_gateway(catalog_root: Path) -> dict[str, set[str]]:
expected: dict[str, set[str]] = {}
for path in sorted((catalog_root / "tools").glob("*.yaml")):
value = yaml.safe_load(path.read_text(encoding="utf-8"))
spec = value.get("spec", {}) if isinstance(value, dict) else {}
connector_type = spec.get("connectorType")
canonical_name = spec.get("canonicalName")
connector_tool_name = spec.get("connectorToolName", canonical_name)
if isinstance(connector_type, str) and isinstance(canonical_name, str):
if not isinstance(connector_tool_name, str):
raise ValueError("connector tool name is invalid")
expected.setdefault(f"itworx-{connector_type}", set()).add(connector_tool_name)
return expected
def _catalog_revisions_by_gateway(catalog_root: Path) -> dict[str, str]:
"""Return stable connector-scoped revisions for cached gateway metadata.
ContextForge caches the complete MCP tool descriptor, not just tool names.
Including a digest of the governed tool manifests in the gateway description
lets reconciliation refresh that cache once after a schema/catalog change,
while leaving the normal 30-second reconciliation loop read-only.
"""
specifications: dict[str, list[dict[str, Any]]] = {}
for path in sorted((catalog_root / "tools").glob("*.yaml")):
value = yaml.safe_load(path.read_text(encoding="utf-8"))
spec = value.get("spec", {}) if isinstance(value, dict) else {}
connector_type = spec.get("connectorType")
if isinstance(connector_type, str) and isinstance(spec, dict):
specifications.setdefault(f"itworx-{connector_type}", []).append(spec)
return {
gateway: hashlib.sha256(
json.dumps(items, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()[:16]
for gateway, items in specifications.items()
}
def _expected_required_arguments_by_gateway(
catalog_root: Path,
) -> dict[str, dict[str, frozenset[str]]]:
expected: dict[str, dict[str, frozenset[str]]] = {}
for path in sorted((catalog_root / "tools").glob("*.yaml")):
value = yaml.safe_load(path.read_text(encoding="utf-8"))
spec = value.get("spec", {}) if isinstance(value, dict) else {}
connector_type = spec.get("connectorType")
canonical_name = spec.get("connectorToolName", spec.get("canonicalName"))
input_schema = spec.get("inputSchema")
if (
isinstance(connector_type, str)
and isinstance(canonical_name, str)
and isinstance(input_schema, dict)
):
required = input_schema.get("required", [])
if not isinstance(required, list) or not all(
isinstance(item, str) for item in required
):
raise ValueError("tool catalog required arguments are invalid")
expected.setdefault(f"itworx-{connector_type}", {})[canonical_name] = frozenset(
required
)
return expected
def _required_arguments(tool: GatewayTool) -> frozenset[str] | None:
if tool.input_schema is None:
return None
required = tool.input_schema.get("required", [])
if not isinstance(required, list) or not all(isinstance(item, str) for item in required):
return frozenset()
return frozenset(required)
def _required_arguments_drifted(
tools_by_name: dict[str, GatewayTool],
governed: dict[str, frozenset[str]],
) -> bool:
"""Compare the governed contract with the connector's live input schema.
A governed manifest may intentionally be stricter than the connector schema
(for example by requiring an explicit confirmation flag that the runtime
defaults to false). That is compatible only when the connector actually
accepts every governed field. A connector that requires an undeclared
field is always drifted.
"""
for tool_name, governed_required in governed.items():
tool = tools_by_name.get(tool_name)
if tool is None:
continue
actual_required = _required_arguments(tool)
if actual_required is None:
continue
properties = tool.input_schema.get("properties", {}) if tool.input_schema else {}
accepted = frozenset(properties) if isinstance(properties, dict) else actual_required
if not actual_required.issubset(governed_required):
return True
if not governed_required.issubset(accepted):
return True
stricter = governed_required - actual_required
if any(
name != "confirm"
or not isinstance(properties.get(name), dict)
or properties[name].get("default") is not False
for name in stricter
):
return True
return False
def _inherit_derived_profile_bindings(
session: Session,
*,
workspace_id: UUID,
servers: dict[tuple[str, str], VirtualServerRow],
) -> None:
"""Give full-profile clients their explicitly smaller derived surfaces."""
profiles = (
("gitea-all", ("gitea-analysis", "gitea-loop", "server-ops")),
("openchamber-agent", ("openchamber-analysis",)),
)
for source_slug, target_slugs in profiles:
source = servers.get((source_slug, "prod"))
if source is None:
continue
source_bindings = session.scalars(
select(ClientBindingRow).where(
ClientBindingRow.workspace_id == workspace_id,
ClientBindingRow.virtual_server_id == source.id,
ClientBindingRow.enabled.is_(True),
)
).all()
for target_slug in target_slugs:
target = servers.get((target_slug, "prod"))
if target is None:
continue
for source_binding in source_bindings:
target_binding = session.scalar(
select(ClientBindingRow).where(
ClientBindingRow.client_id == source_binding.client_id,
ClientBindingRow.virtual_server_id == target.id,
ClientBindingRow.capability_pack == source_binding.capability_pack,
)
)
inherited_from = f"{source_slug}/prod"
inherited_constraints = {
**dict(source_binding.constraints or {}),
"inheritedFrom": inherited_from,
}
if target_binding is None:
capability = source_binding.capability_pack or "default"
target_binding = ClientBindingRow(
id=deterministic_id(
"client-binding",
f"{source_binding.client_id}/{target_slug}/prod/{capability}",
),
workspace_id=workspace_id,
project_id=target.project_id,
client_id=source_binding.client_id,
virtual_server_id=target.id,
capability_pack=source_binding.capability_pack,
constraints=inherited_constraints,
)
session.add(target_binding)
elif target_binding.constraints.get("inheritedFrom") != inherited_from:
continue
target_binding.constraints = inherited_constraints
target_binding.enabled = True
def sync_runtime_state(session: Session, catalog_root: Path, issuer: str) -> UUID:
report = CatalogLoader(catalog_root).load()
if not report.valid or report.desired_hash is None:
raise ValueError("catalog validation failed")
seed = seed_database(session)
servers: dict[tuple[str, str], VirtualServerRow] = {}
for document in _project_documents(catalog_root):
metadata = document.get("metadata", {})
spec = document.get("spec", {})
slug = str(metadata.get("slug", ""))
project_id = deterministic_id("project", slug)
project = session.get(ProjectRow, project_id)
if project is None:
project = ProjectRow(
id=project_id,
workspace_id=seed.workspace_id,
slug=slug,
name=str(metadata.get("name", slug)),
)
session.add(project)
project.name = str(metadata.get("name", slug))
project.description = str(spec.get("description", ""))[:1000] or None
project.status = "active"
session.flush()
environment_specs = spec.get("environments", {})
for environment, environment_spec in environment_specs.items():
environment_id = deterministic_id("environment", f"{slug}/{environment}")
row = session.get(EnvironmentRow, environment_id)
if row is None:
row = EnvironmentRow(
id=environment_id,
workspace_id=seed.workspace_id,
project_id=project_id,
name=environment,
classification=str(environment_spec.get("classification", "internal")),
)
session.add(row)
row.status = "active"
session.flush()
for definition in spec.get("virtualServers", []):
environment = str(definition["environment"])
environment_id = deterministic_id("environment", f"{slug}/{environment}")
server_id = deterministic_id("virtual-server", f"{slug}/{environment}")
server = session.get(VirtualServerRow, server_id)
if server is None:
server = VirtualServerRow(
id=server_id,
workspace_id=seed.workspace_id,
project_id=project_id,
environment_id=environment_id,
slug=str(definition["slug"]),
desired_hash=report.desired_hash,
)
session.add(server)
server.enabled = True
server.desired_hash = report.desired_hash
server.record_metadata = dict(definition)
servers[(slug, environment)] = server
if not INGESTION_CONNECTOR_ENABLED:
intake_server = session.get(
VirtualServerRow,
deterministic_id("virtual-server", "knowledge-hub/prod"),
)
if intake_server is not None:
intake_server.enabled = False
session.flush()
session.execute(
update(ClientBindingRow)
.where(ClientBindingRow.workspace_id == seed.workspace_id)
.values(enabled=False)
)
configured_ids: set[UUID] = set()
for binding in configured_bindings():
server = servers.get((binding.project, binding.environment))
if server is None:
if (
binding.project == "knowledge-hub"
and binding.environment == "prod"
and not INGESTION_CONNECTOR_ENABLED
):
continue
raise ValueError("MCP client binding targets an unknown virtual server")
configured_ids.add(binding.client_id)
binding_issuer = str(binding.issuer) if binding.issuer is not None else issuer
client = session.get(ClientRow, binding.client_id)
if client is None:
client = ClientRow(
id=binding.client_id,
workspace_id=seed.workspace_id,
subject=binding.subject,
issuer=binding_issuer,
record_metadata={
"managedBy": "catalog-binding",
"operatorDisabled": False,
},
)
session.add(client)
client.subject = binding.subject
client.issuer = binding_issuer
metadata = dict(client.record_metadata or {})
metadata.setdefault("managedBy", "catalog-binding")
operator_disabled = bool(metadata.get("operatorDisabled", False))
metadata["configuredEnabled"] = binding.enabled
client.record_metadata = metadata
if client.status != "revoked":
client.status = "active" if binding.enabled and not operator_disabled else "disabled"
session.flush()
existing = session.scalar(
select(ClientBindingRow).where(
ClientBindingRow.client_id == binding.client_id,
ClientBindingRow.virtual_server_id == server.id,
ClientBindingRow.capability_pack.is_(None),
)
)
if existing is None:
existing = ClientBindingRow(
id=deterministic_id(
"client-binding",
f"{binding.client_id}/{binding.project}/{binding.environment}",
),
workspace_id=seed.workspace_id,
project_id=server.project_id,
client_id=binding.client_id,
virtual_server_id=server.id,
)
session.add(existing)
existing.enabled = binding.enabled and client.status == "active"
# Bindings for clients outside the static catalog config (line ~137's
# blanket `enabled=False` sweep) are only re-enabled above for clients in
# `configured_ids`. Clients provisioned through a different, still
# deliberately-managed path (runtime onboarding, the Fleet Ops demo
# token) are exempt from the disable sweep below, but their bindings
# still need to be explicitly re-enabled here each cycle -- otherwise
# they'd stay wiped by that same blanket update forever, since nothing
# else ever flips a binding back to enabled for a non-catalog client.
exempt_managed_by = {
"runtime-onboarding",
"authentik-machine-provisioning",
"demo-token-provisioning",
}
for client in session.scalars(
select(ClientRow).where(ClientRow.workspace_id == seed.workspace_id)
).all():
metadata = client.record_metadata if isinstance(client.record_metadata, dict) else {}
exempt = metadata.get("managedBy") in exempt_managed_by
operator_disabled = bool(metadata.get("operatorDisabled", False))
if client.id not in configured_ids and not exempt:
client.status = "disabled"
elif exempt and client.status == "active" and not operator_disabled:
for exempt_binding in session.scalars(
select(ClientBindingRow).where(ClientBindingRow.client_id == client.id)
).all():
exempt_binding.enabled = True
session.flush()
_inherit_derived_profile_bindings(
session,
workspace_id=seed.workspace_id,
servers=servers,
)
session.flush()
return seed.workspace_id
def reconcile_runtime_state(
client: ContextForgeClient, sessions: Any, catalog_root: Path, issuer: str
) -> UUID:
with sessions() as session:
with session.begin():
workspace_id = sync_runtime_state(session, catalog_root, issuer)
try:
tools: list[GatewayTool] = []
prompts: list[GatewayPrompt] = []
expected_tools = _expected_tools_by_gateway(catalog_root)
expected_required_arguments = _expected_required_arguments_by_gateway(catalog_root)
catalog_revisions = _catalog_revisions_by_gateway(catalog_root)
existing_gateways = {item.slug: item for item in client.list_gateways(limit=100)}
staged_gateways: list[tuple[str, GatewayHandle, bool]] = []
# Phase 1 is deliberately mutation-light: create/reactivate the managed
# records and prove every connector is reachable before refreshing any
# cached tool inventory. A rolling deployment can therefore not refresh
# half the gateways and leave virtual servers linked to obsolete tool IDs.
for name, url in CONNECTOR_GATEWAYS:
revision = catalog_revisions.get(name)
if revision is None:
raise ValueError(f"ContextForge gateway {name} has no catalog revision")
desired = GatewayDesiredState(
slug=name,
url=TypeAdapter(AnyHttpUrl).validate_python(url),
description=(
f"ITWorx managed {name} production connector; catalog revision {revision}"
),
transport="STREAMABLEHTTP",
enabled=True,
passthrough_headers=PASSTHROUGH_HEADERS,
)
existing = existing_gateways.get(name)
descriptor_changed = existing is not None and (
existing.description != desired.description
or existing.url != str(desired.url)
or existing.transport != desired.transport
)
handle = client.ensure_gateway(desired)
probe = client.test_gateway(handle)
# ContextForge v1.0.6 does not expose the optional gateway test
# endpoint. ``unsupported`` is therefore not an outage: phase 2
# immediately performs a real tool/prompt inventory request, which
# is the authoritative reachability and parity check for that
# runtime. Genuine offline/error results still fail closed.
if not probe.reachable and probe.status != "unsupported":
raise ValueError(
f"ContextForge gateway {name} failed connector preflight: {probe.status}"
)
staged_gateways.append((name, handle, descriptor_changed))
# Phase 2 validates exact catalog/handler parity before any virtual
# server is published healthy. This is the runtime startup gate.
for name, handle, descriptor_changed in staged_gateways:
gateway_tools = client.ensure_canonical_tool_names(client.list_gateway_tools(handle))
gateway_prompts = client.list_gateway_prompts(handle)
profile = name.removeprefix("itworx-")
required_tools = expected_tools.get(name, set())
allowed_tools = required_tools
required_prompts = {item[0] for item in WORKFLOW_PROMPTS.get(profile, ())}
discovered_tools = {item.name for item in gateway_tools}
discovered_prompts = {item.name for item in gateway_prompts}
gateway_tools_by_name = {item.name: item for item in gateway_tools}
governed_required = expected_required_arguments.get(name, {})
schema_drifted = _required_arguments_drifted(
gateway_tools_by_name,
governed_required,
)
if (
descriptor_changed
or schema_drifted
or discovered_tools != allowed_tools
or not required_prompts.issubset(discovered_prompts)
):
client.refresh_gateway(handle)
gateway_tools = client.ensure_canonical_tool_names(
client.list_gateway_tools(handle)
)
gateway_prompts = client.list_gateway_prompts(handle)
discovered_tools = {item.name for item in gateway_tools}
discovered_prompts = {item.name for item in gateway_prompts}
gateway_tools_by_name = {item.name: item for item in gateway_tools}
schema_drifted = _required_arguments_drifted(
gateway_tools_by_name,
governed_required,
)
missing = required_tools - discovered_tools
extra = discovered_tools - allowed_tools
if missing or extra:
raise ValueError(
f"ContextForge gateway {name} tool inventory parity failed "
f"(missing={len(missing)}, extra={len(extra)})"
)
if schema_drifted:
raise ValueError(
f"ContextForge gateway {name} input schema parity failed after refresh"
)
if not required_prompts.issubset(discovered_prompts):
raise ValueError(f"ContextForge gateway {name} is missing workflow prompts")
tools.extend(gateway_tools)
prompts.extend(gateway_prompts)
if not tools:
raise ValueError("ContextForge connector discovery returned no tools")
with sessions() as session:
with session.begin():
servers = session.scalars(
select(VirtualServerRow).where(
VirtualServerRow.workspace_id == workspace_id,
VirtualServerRow.enabled.is_(True),
)
).all()
for server in servers:
metadata = (
server.record_metadata if isinstance(server.record_metadata, dict) else {}
)
patterns = metadata.get("toolAllowPatterns", [])
if not isinstance(patterns, list) or not all(
isinstance(pattern, str) for pattern in patterns
):
raise ValueError("virtual server tool allowlist is invalid")
selected = tuple(
tool.external_id
for tool in tools
if any(fnmatch.fnmatchcase(tool.name, pattern) for pattern in patterns)
)
if not selected:
raise ValueError("virtual server resolved to an empty tool allowlist")
prompt_patterns = metadata.get("promptAllowPatterns", [])
if not isinstance(prompt_patterns, list) or not all(
isinstance(pattern, str) for pattern in prompt_patterns
):
raise ValueError("virtual server prompt allowlist is invalid")
selected_prompts = tuple(
prompt.external_id
for prompt in prompts
if any(
fnmatch.fnmatchcase(prompt.name, pattern) for pattern in prompt_patterns
)
)
if prompt_patterns and not selected_prompts:
raise ValueError("virtual server resolved to an empty prompt allowlist")
external = client.ensure_virtual_server(
VirtualServerDesiredState(
external_id=server.id,
name=server.slug,
description="ITWorx managed project and environment scoped MCP server",
associated_tool_ids=selected,
associated_prompt_ids=selected_prompts,
)
)
server.gateway_external_id = external.external_id
server.health_state = "healthy" if external.enabled else "disabled"
except Exception:
# Preserve routes that were already published successfully. A transient
# connector or ContextForge failure must not turn a stable public MCP
# endpoint into RESOURCE_NOT_REGISTERED: clients interpret that route
# disappearance as a terminated session and cannot recover cleanly.
#
# Servers without a published ContextForge route still fail closed as
# unavailable. Previously published routes become degraded until the
# next successful reconciliation proves them healthy again.
with sessions() as session:
with session.begin():
session.execute(
update(VirtualServerRow)
.where(
VirtualServerRow.workspace_id == workspace_id,
VirtualServerRow.enabled.is_(True),
VirtualServerRow.gateway_external_id.is_not(None),
)
.values(health_state="degraded")
)
session.execute(
update(VirtualServerRow)
.where(
VirtualServerRow.workspace_id == workspace_id,
VirtualServerRow.enabled.is_(True),
VirtualServerRow.gateway_external_id.is_(None),
)
.values(health_state="unavailable")
)
raise
return workspace_id