Files
ITWorx_MCP_Hub-Public/tests/test_production_runtime.py
T

1322 lines
56 KiB
Python

from __future__ import annotations
import importlib
import json
import os
import tempfile
import unittest
from datetime import UTC, datetime, timedelta
from pathlib import Path
from unittest.mock import Mock, patch
from uuid import UUID
import yaml
from fastapi.testclient import TestClient
ROOT = Path(__file__).resolve().parents[1]
class ProductionRuntimeContractTests(unittest.TestCase):
def test_gitea_protected_resource_advertises_refresh_scope(self) -> None:
from services.runtime.web import build_app
with tempfile.TemporaryDirectory() as directory:
Path(directory, "index.html").write_text("<!doctype html>", encoding="utf-8")
with patch.dict(os.environ, {"HUB_WEB_ROOT": directory}):
response = TestClient(build_app()).get(
"/.well-known/oauth-protected-resource/mcp/gitea-all/prod"
)
self.assertEqual(response.status_code, 200)
self.assertIn("offline_access", response.json()["scopes_supported"])
def test_analysis_profile_advertises_its_exact_resource(self) -> None:
from services.runtime.web import build_app
with tempfile.TemporaryDirectory() as directory:
Path(directory, "index.html").write_text("<!doctype html>", encoding="utf-8")
with patch.dict(os.environ, {"HUB_WEB_ROOT": directory}):
response = TestClient(build_app()).get(
"/.well-known/oauth-protected-resource/mcp/gitea-analysis/prod"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json()["resource"],
"https://mcp.example.com/mcp/gitea-analysis/prod",
)
def test_closed_loop_profile_advertises_its_exact_resource(self) -> None:
from services.runtime.web import build_app
with tempfile.TemporaryDirectory() as directory:
Path(directory, "index.html").write_text("<!doctype html>", encoding="utf-8")
with patch.dict(os.environ, {"HUB_WEB_ROOT": directory}):
response = TestClient(build_app()).get(
"/.well-known/oauth-protected-resource/mcp/gitea-loop/prod"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()["resource"], "https://mcp.example.com/mcp/gitea-loop/prod")
def test_server_ops_profile_advertises_its_exact_resource(self) -> None:
from services.runtime.web import build_app
with tempfile.TemporaryDirectory() as directory:
Path(directory, "index.html").write_text("<!doctype html>", encoding="utf-8")
with patch.dict(os.environ, {"HUB_WEB_ROOT": directory}):
response = TestClient(build_app()).get(
"/.well-known/oauth-protected-resource/mcp/server-ops/prod"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json()["resource"],
"https://mcp.example.com/mcp/server-ops/prod",
)
def test_openchamber_profile_advertises_its_exact_resource(self) -> None:
from services.runtime.web import build_app
with tempfile.TemporaryDirectory() as directory:
Path(directory, "index.html").write_text("<!doctype html>", encoding="utf-8")
with patch.dict(os.environ, {"HUB_WEB_ROOT": directory}):
response = TestClient(build_app()).get(
"/.well-known/oauth-protected-resource/mcp/openchamber-agent/prod"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json()["resource"],
"https://mcp.example.com/mcp/openchamber-agent/prod",
)
def test_openchamber_analysis_profile_advertises_its_exact_resource(self) -> None:
from services.runtime.web import build_app
with tempfile.TemporaryDirectory() as directory:
Path(directory, "index.html").write_text("<!doctype html>", encoding="utf-8")
with patch.dict(os.environ, {"HUB_WEB_ROOT": directory}):
response = TestClient(build_app()).get(
"/.well-known/oauth-protected-resource/mcp/openchamber-analysis/prod"
)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json()["resource"],
"https://mcp.example.com/mcp/openchamber-analysis/prod",
)
def test_dataplane_maps_contextforge_names_to_canonical_policy_tools(self) -> None:
from services.runtime.dataplane import _tool_governance_metadata
metadata = _tool_governance_metadata(ROOT / "catalog")
self.assertEqual(
metadata["itworx-gitea-gitea-repository-list"],
("gitea", "R1", "gitea.repository.list"),
)
self.assertEqual(metadata["search"], ("gitea", "R1", "gitea.research.search"))
self.assertEqual(metadata["itworx-gitea-fetch"], ("gitea", "R1", "gitea.research.fetch"))
self.assertIn(("appops", "R3", "server.operation.execute"), metadata.values())
for canonical_name in (
"gitea.capabilities.get",
"gitea.connection.check",
"gitea.workspace.status",
"gitea.portfolio.health",
"gitea.branchprotection.get",
"gitea.branchprotection.preview",
"gitea.portfolio.governance.preview",
"gitea.portfolio.governance.verify",
"gitea.orchestration.plan",
):
published_name = f"itworx-gitea-{canonical_name.replace('.', '-')}"
self.assertEqual(metadata[published_name], ("gitea", "R1", canonical_name))
for canonical_name in (
"gitea.repository.create",
"gitea.branchprotection.ensure",
"gitea.portfolio.governance.apply",
"gitea.branch.create",
"gitea.issue.create",
"gitea.issue.update",
"gitea.issue.comment.create",
"gitea.file.create",
"gitea.file.update",
"gitea.file.delete",
"gitea.pullrequest.create",
"gitea.pullrequest.close",
"gitea.pullrequest.validation.ensure",
"gitea.pullrequest.update",
"gitea.changeset.apply",
"gitea.validation.run",
"gitea.workspace.run",
"gitea.orchestration.start",
):
published_name = f"itworx-gitea-{canonical_name.replace('.', '-')}"
self.assertEqual(metadata[published_name], ("gitea", "R2", canonical_name))
for canonical_name in (
"gitea.pullrequest.sync",
"gitea.pullrequest.merge",
"gitea.branch.delete",
):
published_name = f"itworx-gitea-{canonical_name.replace('.', '-')}"
self.assertEqual(metadata[published_name], ("gitea", "R3", canonical_name))
def test_gitea_workspace_reads_are_in_the_project_reader_pack(self) -> None:
pack = json.loads(
(ROOT / "catalog/capability-packs/project-reader.yaml").read_text(encoding="utf-8")
)
patterns = set(pack["spec"]["toolAllowPatterns"])
self.assertTrue(
{
"gitea.capabilities.get",
"gitea.connection.check",
"gitea.selftest.run",
"gitea.validation.failure.get",
"gitea.change.reconcile",
"gitea.workspace.status",
"gitea.portfolio.health",
"gitea.orchestration.plan",
"gitea.research.search",
"gitea.research.fetch",
}.issubset(patterns)
)
@patch("services.runtime.synthetic_monitor.urllib.request.urlopen")
def test_gitea_synthetic_monitor_detects_capability_drift(self, urlopen: Mock) -> None:
from services.runtime.synthetic_monitor import GiteaSyntheticMonitor
expected_count = 58
first = Mock(status=200)
first.read.return_value = json.dumps(
{
"status": "ready",
"upstream": "reachable",
"toolCount": expected_count,
"capabilityFingerprint": "a" * 64,
}
).encode()
second = Mock(status=200)
second.read.return_value = json.dumps(
{
"status": "ready",
"upstream": "reachable",
"toolCount": expected_count,
"capabilityFingerprint": "b" * 64,
}
).encode()
first_context = Mock()
first_context.__enter__ = Mock(return_value=first)
first_context.__exit__ = Mock(return_value=False)
second_context = Mock()
second_context.__enter__ = Mock(return_value=second)
second_context.__exit__ = Mock(return_value=False)
urlopen.side_effect = [first_context, second_context]
monitor = GiteaSyntheticMonitor(ROOT / "catalog")
self.assertEqual(monitor.probe().tool_count, expected_count)
with self.assertRaisesRegex(RuntimeError, "fingerprint changed"):
monitor.probe()
@patch("services.runtime.control_api.httpx.get")
def test_bounded_connector_readiness_fallback_handles_contextforge_without_test_route(
self, request: Mock
) -> None:
from services.runtime.control_api import _direct_connector_readiness
response = Mock(status_code=200, content=b'{"status":"ready"}')
response.json.return_value = {"status": "ready", "credential": "not-returned"}
request.return_value = response
self.assertEqual(
_direct_connector_readiness("itworx-knowledge"),
{
"reachable": True,
"status": "ready",
"detail": "Direct connector readiness verified.",
},
)
request.assert_called_once_with(
"http://connector-knowledge:8080/health/ready",
timeout=5.0,
follow_redirects=False,
)
@patch("services.runtime.control_api.httpx.get")
def test_bounded_connector_readiness_fallback_fails_closed_and_sanitized(
self, request: Mock
) -> None:
from services.runtime.control_api import _direct_connector_readiness
request.side_effect = RuntimeError("upstream-secret")
result = _direct_connector_readiness("itworx-gitea")
self.assertFalse(result["reachable"])
self.assertEqual(result["status"], "unavailable")
self.assertNotIn("upstream-secret", json.dumps(result))
self.assertEqual(_direct_connector_readiness("not-registered")["status"], "unsupported")
def test_control_readiness_uses_scoped_contextforge_surfaces(self) -> None:
from services.runtime.control_api import _verify_contextforge_dependency
contextforge = Mock()
contextforge.health.return_value = {"status": "healthy"}
_verify_contextforge_dependency(contextforge)
contextforge.health.assert_called_once_with()
contextforge.list_gateways.assert_called_once_with(limit=1)
contextforge.readiness.assert_not_called()
contextforge.version.assert_not_called()
contextforge.health.return_value = {"status": "degraded"}
with self.assertRaisesRegex(RuntimeError, "not healthy"):
_verify_contextforge_dependency(contextforge)
def test_owned_image_installs_locked_runtime_and_dispatches_real_services(self) -> None:
dockerfile = (ROOT / "infrastructure/images/python-service/Dockerfile").read_text(
encoding="utf-8"
)
self.assertIn("uv sync --locked --no-dev", dockerfile)
self.assertIn("COPY connectors /opt/hub/connectors", dockerfile)
self.assertIn("COPY contracts /opt/hub/contracts", dockerfile)
self.assertIn("COPY apps/web", dockerfile)
self.assertIn('ENTRYPOINT ["python", "-m", "services.runtime"]', dockerfile)
self.assertNotIn('ENTRYPOINT ["python", "/opt/hub/health_server.py"]', dockerfile)
def test_runtime_dispatch_has_no_health_only_fallback(self) -> None:
from services.runtime.dispatch import service_module
expected = {
"control-api": "services.runtime.control_api",
"worker": "services.runtime.worker",
"dataplane-router": "services.runtime.dataplane",
"web": "services.runtime.web",
"connector-knowledge": "services.runtime.connector_knowledge",
"connector-gitea": "services.runtime.connector_gitea",
"connector-unraid": "services.runtime.connector_unraid",
"connector-mobilityops": "services.runtime.connector_mobilityops",
"connector-projectbrain": "services.runtime.connector_projectbrain",
"connector-appops": "services.runtime.connector_appops",
"connector-stitch": "services.runtime.connector_stitch",
"database-migrate": "services.runtime.migrate",
}
self.assertEqual({name: service_module(name) for name in expected}, expected)
with self.assertRaises(ValueError):
service_module("unknown-service")
for module in expected.values():
self.assertTrue(callable(importlib.import_module(module).run))
def test_required_https_clients_have_isolated_egress_only(self) -> None:
compose = yaml.safe_load(
(ROOT / "infrastructure/compose/docker-compose.yml").read_text(encoding="utf-8")
)
networks = compose["networks"]
self.assertIn("hub-egress", networks)
self.assertFalse(networks["hub-egress"].get("internal", False))
egress_users = {
name
for name, service in compose["services"].items()
if "hub-egress" in service.get("networks", [])
}
self.assertEqual(
egress_users,
{
"control-api",
"dataplane-router",
"connector-gitea",
"connector-mobilityops",
"connector-projectbrain",
"connector-appops",
"connector-stitch",
"connector-ingestion",
"ingestion-clamd",
},
)
for name in egress_users - {"dataplane-router"}:
self.assertNotIn("ports", compose["services"][name])
self.assertEqual(
compose["services"]["dataplane-router"]["ports"],
["${MCP_BIND_IP:-127.0.0.1}:${MCP_PORT:-18442}:8080"],
)
def test_contextforge_internal_network_exception_is_exact_and_scoped(self) -> None:
compose = yaml.safe_load(
(ROOT / "infrastructure/compose/docker-compose.yml").read_text(encoding="utf-8")
)
contextforge = compose["services"]["contextforge"]
self.assertEqual(
contextforge["environment"]["SSRF_ALLOWED_NETWORKS"], '["203.0.113.0/24"]'
)
self.assertEqual(
compose["networks"]["hub-connectors"]["ipam"]["config"],
[{"subnet": "203.0.113.0/24"}],
)
self.assertTrue(compose["networks"]["hub-connectors"]["internal"])
self.assertEqual(contextforge["environment"]["API_ALLOW_BASIC_AUTH"], "false")
self.assertEqual(contextforge["environment"]["ENABLE_HEADER_PASSTHROUGH"], "true")
self.assertEqual(
contextforge["environment"]["PLATFORM_ADMIN_EMAIL"],
"${CONTEXTFORGE_PLATFORM_ADMIN_EMAIL:-contextforge-admin@example.com}",
)
for name in ("control-api", "worker", "dataplane-router"):
service = compose["services"][name]
self.assertEqual(service["environment"]["HUB_CONTEXTFORGE_AUTH_SCHEME"], "Bearer")
self.assertEqual(
service["environment"]["HUB_CONTEXTFORGE_TOKEN_FILE"],
"/run/secrets/contextforge_api_token",
)
self.assertIn("contextforge_api_token", service["secrets"])
self.assertNotIn("contextforge_admin_credential", service["secrets"])
env_example = (ROOT / "infrastructure/compose/.env.example").read_text(encoding="utf-8")
self.assertIn("HUB_PORT=1100", env_example)
identity_secret_users = {
name
for name, service in compose["services"].items()
if "connector_identity_signing_key" in service.get("secrets", [])
}
self.assertEqual(
identity_secret_users,
{
"dataplane-router",
"control-api",
"connector-knowledge",
"connector-gitea",
"connector-unraid",
"connector-mobilityops",
"connector-projectbrain",
"connector-appops",
"connector-stitch",
},
)
self.assertIn("hub-connectors", compose["services"]["control-api"]["networks"])
self.assertNotIn("/var/run/docker.sock", str(compose["services"]["control-api"]))
def test_all_owned_networks_have_target_verified_explicit_subnets(self) -> None:
compose = yaml.safe_load(
(ROOT / "infrastructure/compose/docker-compose.yml").read_text(encoding="utf-8")
)
expected = {
"hub-edge": "203.0.113.0/24",
"hub-egress": "203.0.113.0/24",
"hub-control": "203.0.113.0/24",
"hub-broker": "203.0.113.0/24",
"hub-data": "203.0.113.0/24",
"hub-observe": "203.0.113.0/24",
"hub-connectors": "203.0.113.0/24",
}
actual = {
name: network["ipam"]["config"][0]["subnet"]
for name, network in compose["networks"].items()
}
self.assertEqual(actual, expected)
self.assertEqual(len(set(actual.values())), len(actual))
def test_postgres_projects_private_host_secrets_into_runtime_tmpfs(self) -> None:
compose = yaml.safe_load(
(ROOT / "infrastructure/compose/docker-compose.yml").read_text(encoding="utf-8")
)
postgres = compose["services"]["postgres"]
self.assertEqual(postgres["entrypoint"], ["sh", "/opt/itworx/postgres-entrypoint.sh"])
self.assertEqual(postgres["command"], ["postgres"])
self.assertIn(
"../images/postgres/entrypoint.sh:/opt/itworx/postgres-entrypoint.sh:ro",
postgres["volumes"],
)
self.assertEqual(postgres["tmpfs"], ["/run/itworx-secrets:size=1m,mode=0700"])
wrapper = (ROOT / "infrastructure/images/postgres/entrypoint.sh").read_text(
encoding="utf-8"
)
self.assertIn('install -o "$runtime_uid" -g "$runtime_gid" -m 0400', wrapper)
self.assertIn('exec /usr/local/bin/docker-entrypoint.sh "$@"', wrapper)
self.assertNotIn("printf '%s' \"$value\"", wrapper)
def test_contextforge_uses_the_production_secret_owner_identity(self) -> None:
compose = yaml.safe_load(
(ROOT / "infrastructure/compose/docker-compose.yml").read_text(encoding="utf-8")
)
self.assertEqual(
compose["services"]["contextforge"]["user"],
"${PUID:-1000}:${PGID:-1000}",
)
def test_runtime_state_sync_is_idempotent_and_binding_scoped(self) -> None:
from services.control_api.contextforge import (
ContextForgeClient,
ContextForgeSettings,
FakeContextForge,
)
from services.control_api.persistence import (
Base,
SqlAlchemyAliasSource,
create_engine_from_url,
session_factory,
)
from services.runtime.state_sync import reconcile_runtime_state, sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
binding = [
{
"clientId": "11111111-1111-4111-8111-111111111111",
"subject": "svc-ludarium",
"project": "ludarium",
"environment": "prod",
"enabled": True,
}
]
with patch.dict(
"os.environ", {"MCP_CLIENT_BINDINGS_JSON": json.dumps(binding)}, clear=False
):
for _ in range(2):
with sessions() as session, session.begin():
workspace_id = sync_runtime_state(
session, ROOT / "catalog", "https://auth.itworx.test/application/o/itworx/"
)
fake = FakeContextForge()
contextforge = ContextForgeClient(
ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="CTX_TOKEN", # noqa: S106
),
transport=fake.transport(),
environ={"CTX_TOKEN": "fixture-token"},
)
try:
workspace_id = reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
finally:
contextforge.close()
with sessions() as session:
aliases = SqlAlchemyAliasSource(
session,
workspace_id=workspace_id,
gateway_base_url="http://contextforge-control:4444/mcp",
).aliases()
self.assertEqual(len(aliases), 14)
ludarium = next(item for item in aliases if item.project_slug == "ludarium")
self.assertEqual(len(ludarium.bindings), 1)
self.assertEqual(
ludarium.bindings[0].issuer,
"https://auth.itworx.test/application/o/itworx/",
)
general = next(item for item in aliases if item.project_slug == "general-infrastructure")
self.assertEqual(general.alias, "/mcp/general/prod")
self.assertRegex(
str(general.upstream_url),
r"^http://contextforge-control:4444/servers/[0-9a-f]{32}/mcp$",
)
general_server = next(
item
for item in fake.servers.values()
if item["name"] == "general-infrastructure-prod-readonly"
)
selected_names = {
tool["name"]
for gateway_tools in fake.tools.values()
for tool in gateway_tools
if tool["id"] in general_server["associated_tool_ids"]
}
self.assertIn("unraid.host.status", selected_names)
self.assertNotIn("knowledge.document.search", selected_names)
self.assertEqual(general_server["associated_prompt_ids"], [])
ludarium_server = next(
item for item in fake.servers.values() if item["name"] == "ludarium-prod-readonly"
)
ludarium_prompt_names = {
prompt["name"]
for gateway_prompts in fake.prompts.values()
for prompt in gateway_prompts
if prompt["id"] in ludarium_server["associated_prompt_ids"]
}
self.assertEqual(
ludarium_prompt_names,
{
"project-context-pack",
"pre-change-impact-review",
"repository-change-review",
"release-readiness",
"production-incident-triage",
"deployment-health-review",
},
)
self.assertIn(
"x-itworx-identity-assertion",
next(iter(fake.gateways.values()))["passthrough_headers"],
)
engine.dispose()
def test_gitea_clients_inherit_only_the_smaller_analysis_profile(self) -> None:
from sqlalchemy import select
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.control_api.persistence.models import ClientBindingRow, ProjectRow
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
client_id = UUID("11111111-1111-4111-8111-111111111111")
binding = [
{
"clientId": str(client_id),
"subject": "svc-chatgpt-gitea",
"project": "gitea-all",
"environment": "prod",
"enabled": True,
}
]
with patch.dict(
"os.environ", {"MCP_CLIENT_BINDINGS_JSON": json.dumps(binding)}, clear=False
):
for _ in range(2):
with sessions() as session, session.begin():
sync_runtime_state(
session,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
with sessions() as session:
rows = session.execute(
select(ClientBindingRow, ProjectRow.slug)
.join(ProjectRow, ProjectRow.id == ClientBindingRow.project_id)
.where(ClientBindingRow.client_id == client_id)
.order_by(ProjectRow.slug)
).all()
self.assertEqual(
[slug for _, slug in rows],
["gitea-all", "gitea-analysis", "gitea-loop", "server-ops"],
)
for profile in ("gitea-analysis", "gitea-loop", "server-ops"):
inherited = next(row for row, slug in rows if slug == profile)
self.assertTrue(inherited.enabled)
self.assertEqual(inherited.constraints["inheritedFrom"], "gitea-all/prod")
engine.dispose()
def test_openchamber_clients_inherit_the_optional_readonly_profile(self) -> None:
from sqlalchemy import select
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.control_api.persistence.models import ClientBindingRow, ProjectRow
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
client_id = UUID("22222222-2222-4222-8222-222222222222")
binding = [
{
"clientId": str(client_id),
"subject": "svc-chatgpt-openchamber",
"project": "openchamber-agent",
"environment": "prod",
"enabled": True,
}
]
with patch.dict(
"os.environ", {"MCP_CLIENT_BINDINGS_JSON": json.dumps(binding)}, clear=False
):
for _ in range(2):
with sessions() as session, session.begin():
sync_runtime_state(
session,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
with sessions() as session:
rows = session.execute(
select(ClientBindingRow, ProjectRow.slug)
.join(ProjectRow, ProjectRow.id == ClientBindingRow.project_id)
.where(ClientBindingRow.client_id == client_id)
.order_by(ProjectRow.slug)
).all()
self.assertEqual([slug for _, slug in rows], ["openchamber-agent", "openchamber-analysis"])
inherited = next(row for row, slug in rows if slug == "openchamber-analysis")
self.assertTrue(inherited.enabled)
self.assertEqual(inherited.constraints["inheritedFrom"], "openchamber-agent/prod")
engine.dispose()
def test_reconciliation_refreshes_changed_gateway_descriptor_once(self) -> None:
from services.control_api.contextforge import (
ContextForgeClient,
ContextForgeSettings,
FakeContextForge,
)
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.runtime.state_sync import reconcile_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
fake = FakeContextForge()
contextforge = ContextForgeClient(
ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="CTX_TOKEN", # noqa: S106
),
transport=fake.transport(),
environ={"CTX_TOKEN": "fixture-token"},
)
try:
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
refresh_path = "/gateways/{}/tools/refresh"
self.assertFalse(
any(
method == "POST" and path.endswith("/tools/refresh")
for method, path, _ in fake.requests
)
)
gitea_gateway = next(
item for item in fake.gateways.values() if item["name"] == "itworx-gitea"
)
gitea_gateway["description"] = "legacy descriptor without catalog revision"
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
expected_refresh_path = refresh_path.format(gitea_gateway["id"])
refreshes = [
request
for request in fake.requests
if request[0] == "POST" and request[1] == expected_refresh_path
]
self.assertEqual(len(refreshes), 1)
repository_create = next(
tool
for tool in fake.tools[gitea_gateway["id"]]
if tool["name"] == "gitea.repository.create"
)
repository_create["inputSchema"]["required"] = ["name"]
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
refreshes = [
request
for request in fake.requests
if request[0] == "POST" and request[1] == expected_refresh_path
]
self.assertEqual(len(refreshes), 2)
refreshed_repository_create = next(
tool
for tool in fake.tools[gitea_gateway["id"]]
if tool["name"] == "gitea.repository.create"
)
self.assertEqual(
set(refreshed_repository_create["inputSchema"]["required"]),
{"name", "description", "private", "autoInit", "defaultBranch"},
)
self.assertIn("catalog revision", gitea_gateway["description"])
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
refreshes = [
request
for request in fake.requests
if request[0] == "POST" and request[1] == expected_refresh_path
]
self.assertEqual(len(refreshes), 2)
finally:
contextforge.close()
engine.dispose()
def test_reconciliation_failure_preserves_last_known_good_public_aliases(self) -> None:
from sqlalchemy import select
from services.control_api.contextforge import (
ContextForgeClient,
ContextForgeSettings,
FakeContextForge,
)
from services.control_api.contextforge.models import GatewayTestResult
from services.control_api.persistence import (
Base,
SqlAlchemyAliasSource,
create_engine_from_url,
session_factory,
)
from services.control_api.persistence.models import VirtualServerRow
from services.runtime.state_sync import reconcile_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
fake = FakeContextForge()
contextforge = ContextForgeClient(
ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="CTX_TOKEN", # noqa: S106
),
transport=fake.transport(),
environ={"CTX_TOKEN": "fixture-token"},
)
try:
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
workspace_id = reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
with patch.object(
contextforge,
"test_gateway",
return_value=GatewayTestResult(
reachable=False,
status="offline",
detail="simulated connector outage",
),
):
with self.assertRaisesRegex(ValueError, "failed connector preflight"):
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
with sessions() as session:
states = set(session.scalars(select(VirtualServerRow.health_state)).all())
aliases = SqlAlchemyAliasSource(
session,
workspace_id=workspace_id,
gateway_base_url="http://contextforge-control:4444/mcp",
).aliases()
self.assertEqual(states, {"degraded"})
self.assertTrue(aliases)
self.assertTrue(all(alias.enabled for alias in aliases))
finally:
contextforge.close()
engine.dispose()
def test_initial_reconciliation_failure_keeps_unpublished_aliases_withdrawn(self) -> None:
from sqlalchemy import select
from services.control_api.contextforge import (
ContextForgeClient,
ContextForgeSettings,
FakeContextForge,
)
from services.control_api.contextforge.models import GatewayTestResult
from services.control_api.persistence import (
Base,
SqlAlchemyAliasSource,
create_engine_from_url,
session_factory,
)
from services.control_api.persistence.models import VirtualServerRow
from services.runtime.state_sync import reconcile_runtime_state, sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
fake = FakeContextForge()
contextforge = ContextForgeClient(
ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="CTX_TOKEN", # noqa: S106
),
transport=fake.transport(),
environ={"CTX_TOKEN": "fixture-token"},
)
try:
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
with sessions() as session, session.begin():
workspace_id = sync_runtime_state(
session,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
with patch.object(
contextforge,
"test_gateway",
return_value=GatewayTestResult(
reachable=False,
status="offline",
detail="simulated initial connector outage",
),
):
with self.assertRaisesRegex(ValueError, "failed connector preflight"):
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
with sessions() as session:
states = set(session.scalars(select(VirtualServerRow.health_state)).all())
aliases = SqlAlchemyAliasSource(
session,
workspace_id=workspace_id,
gateway_base_url="http://contextforge-control:4444/mcp",
).aliases()
self.assertEqual(states, {"unavailable"})
self.assertEqual(aliases, ())
finally:
contextforge.close()
engine.dispose()
def test_reconciliation_rejects_schema_drift_that_refresh_does_not_repair(self) -> None:
from services.control_api.contextforge import (
ContextForgeClient,
ContextForgeSettings,
FakeContextForge,
)
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.runtime.state_sync import reconcile_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
fake = FakeContextForge()
contextforge = ContextForgeClient(
ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="CTX_TOKEN", # noqa: S106
),
transport=fake.transport(),
environ={"CTX_TOKEN": "fixture-token"},
)
try:
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
gateway = next(
item for item in fake.gateways.values() if item["name"] == "itworx-gitea"
)
repository_create = next(
tool
for tool in fake.tools[gateway["id"]]
if tool["name"] == "gitea.repository.create"
)
repository_create["inputSchema"]["required"] = ["name", "undeclared"]
with (
patch.object(contextforge, "refresh_gateway", return_value=None),
self.assertRaisesRegex(ValueError, "input schema parity failed after refresh"),
):
reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
finally:
contextforge.close()
engine.dispose()
def test_reconciliation_uses_inventory_when_gateway_test_is_unsupported(self) -> None:
from services.control_api.contextforge import (
ContextForgeClient,
ContextForgeSettings,
FakeContextForge,
)
from services.control_api.contextforge.models import GatewayTestResult
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.runtime.state_sync import reconcile_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
fake = FakeContextForge()
contextforge = ContextForgeClient(
ContextForgeSettings(
base_url="https://contextforge.test",
auth_token_env="CTX_TOKEN", # noqa: S106
),
transport=fake.transport(),
environ={"CTX_TOKEN": "fixture-token"},
)
try:
with (
patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False),
patch.object(
contextforge,
"test_gateway",
return_value=GatewayTestResult(
reachable=False,
status="unsupported",
detail="Gateway test endpoint unavailable.",
),
),
):
workspace_id = reconcile_runtime_state(
contextforge,
sessions,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
self.assertIsNotNone(workspace_id)
finally:
contextforge.close()
engine.dispose()
def test_runtime_sync_preserves_authentik_provisioned_client(self) -> None:
from services.control_api.persistence import (
Base,
create_engine_from_url,
session_factory,
)
from services.control_api.persistence.machine_client import provision_machine_client
from services.control_api.persistence.models import (
ClientBindingRow,
ClientRow,
ProjectRow,
VirtualServerRow,
)
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
issuer = "https://auth.itworx.test/application/o/itworx/"
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
project = session.query(ProjectRow).filter_by(slug="ludarium").one()
server = (
session.query(VirtualServerRow)
.filter_by(project_id=project.id, enabled=True)
.one()
)
server.gateway_external_id = "a" * 32
session.flush()
result = provision_machine_client(
session,
issuer=issuer,
subject="6f865098-6d67-4ee6-b0fe-b0173cc74e23",
oauth_username="svc-mcp-codex-ludarium",
name="Codex workstation",
project_slug="ludarium",
)
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
with sessions() as session:
client = session.get(ClientRow, result.client_id)
binding = session.get(ClientBindingRow, result.binding_id)
self.assertIsNotNone(client)
self.assertIsNotNone(binding)
self.assertEqual(client.status, "active")
self.assertTrue(binding.enabled)
engine.dispose()
def test_machine_authentication_preserves_exact_issuer_boundary(self) -> None:
from services.control_api.auth.errors import UnknownMachinePrincipal
from services.control_api.auth.models import ValidatedClaims
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.runtime.dataplane import DatabaseMachineAuthenticator
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
issuer = "https://auth.itworx.test/application/o/itworx/"
subject = "svc-ludarium"
binding = [
{
"clientId": "11111111-1111-4111-8111-111111111111",
"subject": subject,
"project": "ludarium",
"environment": "prod",
"enabled": True,
}
]
with patch.dict(
"os.environ", {"MCP_CLIENT_BINDINGS_JSON": json.dumps(binding)}, clear=False
):
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
now = datetime.now(UTC)
provider = Mock()
provider.validate_token.return_value = ValidatedClaims.model_validate(
{
"sub": subject,
"iss": issuer,
"aud": ["itworx-mcp-hub"],
"iat": now,
"exp": now + timedelta(minutes=5),
}
)
authenticator = DatabaseMachineAuthenticator(provider, sessions)
principal = authenticator("Bearer fixture-token")
self.assertEqual(principal.issuer, issuer)
provider.validate_token.return_value = provider.validate_token.return_value.model_copy(
update={"issuer": issuer.rstrip("/")}
)
with self.assertRaises(UnknownMachinePrincipal):
authenticator("Bearer fixture-token")
engine.dispose()
def test_production_client_lifecycle_persists_and_reconciliation_preserves_revocation(
self,
) -> None:
from sqlalchemy import select
from packages.hub_clients import ClientKind, ClientStatus
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.control_api.persistence.models import ClientBindingRow, ClientRow
from services.runtime.client_service import PersistentClientService
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
issuer = "https://auth.itworx.test/application/o/itworx/"
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
first_process = PersistentClientService(
sessions, issuer=issuer, public_host="mcp.itworx.test"
)
created, bootstrap, config = first_process.create(
name="production-codex",
project="ludarium",
environment="prod",
kind=ClientKind.NATIVE_2026,
)
self.assertEqual(created.status, ClientStatus.PENDING)
self.assertEqual(config.endpoint, "https://mcp.itworx.test/mcp/ludarium/prod")
self.assertNotIn("token", config.model_dump_json().lower())
self.assertTrue(bootstrap.reference.startswith("authentik-bootstrap://"))
restarted_process = PersistentClientService(
sessions, issuer=issuer, public_host="mcp.itworx.test"
)
self.assertEqual(restarted_process.list()[0].id, created.id)
revoked = restarted_process.revoke(created.id)
self.assertEqual(revoked.status, ClientStatus.REVOKED)
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
with sessions() as session:
row = session.get(ClientRow, UUID(str(created.id)))
binding_enabled = session.scalar(
select(ClientBindingRow.enabled).where(ClientBindingRow.client_id == created.id)
)
self.assertIsNotNone(row)
assert row is not None
self.assertEqual(row.status, "revoked")
self.assertEqual(row.issuer, issuer)
self.assertFalse(binding_enabled)
engine.dispose()
def test_demo_client_binding_survives_reconciliation_but_operator_disable_persists(
self,
) -> None:
from sqlalchemy import select
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.control_api.persistence.demo_client import (
DEMO_CLIENT_ID,
provision_demo_client,
)
from services.control_api.persistence.models import (
ClientBindingRow,
ClientRow,
ProjectRow,
VirtualServerRow,
)
from services.runtime.client_service import PersistentClientService
from services.runtime.common import deterministic_id
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
issuer = "https://auth.itworx.test/application/o/itworx/"
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
# sync_runtime_state seeds the mobilityops virtual server
# from the catalog but only ever sets gateway_external_id via
# a real ContextForge round trip -- fake that one field, the
# way reconcile_runtime_state's second half would for real.
project_id = deterministic_id("project", "mobilityops")
server_id = deterministic_id("virtual-server", "mobilityops/prod")
project = session.get(ProjectRow, project_id)
server = session.get(VirtualServerRow, server_id)
assert project is not None
assert server is not None
server.gateway_external_id = "a" * 32
with sessions() as session, session.begin():
provision_demo_client(session)
with sessions() as session:
binding = session.scalar(
select(ClientBindingRow).where(ClientBindingRow.client_id == DEMO_CLIENT_ID)
)
assert binding is not None
self.assertTrue(binding.enabled)
# sync_runtime_state's own catalog-binding loop unconditionally
# disables every ClientBindingRow first, then only re-enables
# ones for clients it recognizes -- this is the exact production
# bug: the demo client's binding must survive this, repeatedly,
# not just once.
for _ in range(3):
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
with sessions() as session:
client = session.get(ClientRow, DEMO_CLIENT_ID)
binding = session.scalar(
select(ClientBindingRow).where(ClientBindingRow.client_id == DEMO_CLIENT_ID)
)
assert client is not None
assert binding is not None
self.assertEqual(client.status, "active")
self.assertTrue(binding.enabled)
# An operator-disabled demo client must NOT be silently
# re-enabled by the next reconciliation cycle.
service = PersistentClientService(
sessions, issuer=issuer, public_host="mcp.itworx.test"
)
service.disable(DEMO_CLIENT_ID)
with sessions() as session, session.begin():
sync_runtime_state(session, ROOT / "catalog", issuer)
with sessions() as session:
client = session.get(ClientRow, DEMO_CLIENT_ID)
binding = session.scalar(
select(ClientBindingRow).where(ClientBindingRow.client_id == DEMO_CLIENT_ID)
)
assert client is not None
assert binding is not None
self.assertEqual(client.status, "disabled")
self.assertFalse(binding.enabled)
engine.dispose()
def test_production_governance_uses_catalog_rate_limit_and_persistent_sanitized_audit(
self,
) -> None:
from sqlalchemy import select
from packages.hub_policy import (
Governance,
InvocationContext,
RateLimitExceeded,
SlidingWindowRateLimiter,
)
from services.control_api.persistence import Base, create_engine_from_url, session_factory
from services.control_api.persistence.models import AuditEventRow
from services.runtime.audit import SqlAlchemyAuditLog
from services.runtime.dataplane import _policy_engine
from services.runtime.state_sync import sync_runtime_state
engine = create_engine_from_url("sqlite:///:memory:")
Base.metadata.create_all(engine)
sessions = session_factory(engine)
with patch.dict("os.environ", {"MCP_CLIENT_BINDINGS_JSON": "[]"}, clear=False):
with sessions() as session, session.begin():
workspace_id = sync_runtime_state(
session,
ROOT / "catalog",
"https://auth.itworx.test/application/o/itworx/",
)
governance = Governance(
_policy_engine(ROOT / "catalog"),
SqlAlchemyAuditLog(sessions, issuer="https://auth.itworx.test/application/o/itworx/"),
rate_limiter=SlidingWindowRateLimiter(limit=1),
)
context = InvocationContext(
workspace=str(workspace_id),
project="ludarium",
environment="prod",
principal="itworx-mcp-client-fixture",
client_id=UUID("11111111-1111-4111-8111-111111111111"),
binding_present=True,
client_enabled=True,
tool="unraid.host.status",
connector="dataplane-router",
risk_class="R1",
resource_type="tool",
environment_classification="production",
arguments={"authorization": "Bearer never-store-this"},
)
governance.invoke(context, lambda: {"ok": True})
with self.assertRaises(RateLimitExceeded):
governance.invoke(context, lambda: {"ok": True})
with sessions() as session:
events = session.scalars(
select(AuditEventRow).order_by(AuditEventRow.occurred_at)
).all()
self.assertEqual([event.outcome for event in events], ["success", "rate_limited"])
self.assertTrue(
all(
event.principal_issuer == "https://auth.itworx.test/application/o/itworx/"
for event in events
)
)
self.assertEqual(events[0].sanitized_arguments["authorization"], "<redacted>")
self.assertNotIn("never-store-this", str(events))
engine.dispose()
def test_connector_identity_assertion_is_short_lived_and_tamper_evident(self) -> None:
from services.runtime.common import connector_context, identity_assertion
key = b"fixture-connector-identity-key"
now = 1_800_000_000
headers = {
"x-itworx-project": "ludarium",
"x-itworx-environment": "prod",
"x-itworx-client-id": "11111111-1111-4111-8111-111111111111",
"x-itworx-trace-id": "trace-fixture",
}
headers["x-itworx-identity-assertion"] = identity_assertion(
key,
project=headers["x-itworx-project"],
environment=headers["x-itworx-environment"],
client_id=headers["x-itworx-client-id"],
trace_id=headers["x-itworx-trace-id"],
now=now,
)
context = connector_context(headers, {"ludarium"}, signing_key=key, now=now)
self.assertEqual(str(context.client_id), headers["x-itworx-client-id"])
with self.assertRaises(PermissionError):
connector_context(
headers | {"x-itworx-project": "geointel"},
{"ludarium", "geointel"},
signing_key=key,
now=now,
)
with self.assertRaises(PermissionError):
connector_context(headers, {"ludarium"}, signing_key=key, now=now + 61)
if __name__ == "__main__":
unittest.main()