"""Contract and adversarial tests for WP-180 production promotion preflight.""" from __future__ import annotations import hashlib import json import os import shutil import subprocess from pathlib import Path from unittest.mock import patch import pytest import yaml from packages.hub_runtime import SecretFileError, read_secret_file, validate_secret_file from scripts.promotion_preflight import ( BASE_COMPOSE, CONTEXTFORGE_CONTRACT, GENERATED_SECRET_FILES, IMAGE_ENVIRONMENT, PROD_COMPOSE, REQUIRED_SECRET_FILES, _ComposeMergeLoader, _deep_merge, _rendered_topology, load_env_file, run, ) from services.control_api.auth.bff import _hash from services.control_api.auth.config import OIDCSettings from services.control_api.auth.errors import ConfigurationError from services.control_api.contextforge.errors import ContextForgeError from services.control_api.contextforge.models import ContextForgeSettings def _fixture(root: Path) -> tuple[dict[str, str], dict[str, bytes]]: directories = {} for name in ("appdata", "secrets", "sources", "backups"): path = root / name path.mkdir() path.chmod(0o700) directories[name] = path registrations = root / "registered-targets.json" registrations.write_text('{"targets":[]}', encoding="utf-8") demo_directory = root / "demo-token-revocations" demo_directory.mkdir() demo_directory.chmod(0o700) demo_revocations = demo_directory / "demo-token-revocations.json" demo_revocations.write_text('{"schemaVersion":"1.0","revokedJtis":[]}\n', encoding="utf-8") demo_revocations.chmod(0o600) (directories["sources"] / "ludarium").mkdir() uid = directories["secrets"].stat().st_uid gid = directories["secrets"].stat().st_gid secret_values: dict[str, bytes] = {} for index, name in enumerate(REQUIRED_SECRET_FILES): value = f"wp180-secret-{index:02d}-{name}".encode() path = directories["secrets"] / name path.write_bytes(value) path.chmod(0o600) secret_values[name] = value digest = "a" * 64 contextforge_contract_digest = hashlib.sha256(CONTEXTFORGE_CONTRACT.read_bytes()).hexdigest() environment = { "UNRAID_HOST": "unraid.itworx.test", "APPDATA_ROOT": str(directories["appdata"]), "SECRETS_DIR": str(directories["secrets"]), "PROJECTS_READONLY_ROOT": str(directories["sources"]), "KNOWLEDGE_SOURCES_JSON": json.dumps( [ { "project": "ludarium", "sourceRef": "repository", "path": "ludarium", "sourceType": "documentation", "revision": "a" * 40, } ] ), "BACKUP_ROOT": str(directories["backups"]), "UNRAID_REGISTRATIONS_FILE": str(registrations), "DEMO_TOKEN_REVOCATIONS_DIR_HOST": str(demo_directory), "HUB_BIND_IP": "192.0.2.10", "MCP_BIND_IP": "192.0.2.10", "INSPECTOR_BIND_IP": "192.0.2.10", "HUB_PORT": "1100", "MCP_PORT": "18442", "INSPECTOR_PORT": "18443", "PUID": str(uid), "PGID": str(gid), "DOCKER_SOCKET_GID": str(gid), "REGISTRY_HOST": "registry.itworx.test", "REGISTRY_NAMESPACE": "platform/itworx-mcp-hub", "REGISTRY_CREDENTIAL_REF": "docker-credential-helper-itworx", "HUB_OIDC_ISSUER": "https://auth.itworx.test/application/o/itworx/", "HUB_OIDC_AUDIENCE": "itworx-mcp-hub", "HUB_OIDC_CLIENT_ID": "itworx-mcp-hub-production", "HUB_OIDC_REDIRECT_URI": "https://mcp.itworx.test/auth/callback", "HUB_OIDC_ROLE_GROUP_MAP": json.dumps({"hub-admin": "admin", "hub-operator": "operator"}), "HUB_MCP_ADDITIONAL_OIDC_ISSUER": ( "https://auth.itworx.test/application/o/chatgpt-itworx-gitea/" ), "HUB_MCP_ADDITIONAL_OIDC_AUDIENCE": "chatgpt-itworx-gitea", "HUB_MCP_ADDITIONAL_OIDC_CLIENT_ID": "chatgpt-itworx-gitea", "HUB_MCP_ADDITIONAL_OIDC_REDIRECT_URI": ( "https://chatgpt.com/connector/oauth/test-connector" ), "HUB_PUBLIC_MCP_URL": "https://mcp.itworx.test", "HUB_RATE_LIMIT_PER_MINUTE": "600", "HUB_UPSTREAM_MAX_CONCURRENCY": "24", "HUB_UPSTREAM_QUEUE_TIMEOUT_SECONDS": "5", "AUTHENTIK_PROVIDER_ID": "provider-itworx-hub", "AUTHENTIK_HUB_APPLICATION_ID": "application-itworx-hub", "AUTHENTIK_SCOPES": "openid,profile,email,groups,offline_access", "AUTHENTIK_ALLOWED_GROUPS": "hub-admin,hub-operator", "AUTHENTIK_SERVICE_ACCOUNTS": json.dumps(["svc-mcp-inspector"]), "MCP_CLIENT_BINDINGS_JSON": json.dumps( [ { "clientId": "11111111-1111-4111-8111-111111111111", "subject": "svc-mcp-inspector", "project": "ludarium", "environment": "prod", "enabled": True, } ] ), "HUB_CONTEXTFORGE_URL": "http://contextforge-control:4444", "CONTEXTFORGE_DB_HOST": "postgres", "CONTEXTFORGE_DB_NAME": "contextforge", "CONTEXTFORGE_DB_USER": "contextforge", "CONTEXTFORGE_CONTRACT_SHA256": contextforge_contract_digest, "REDIS_AUTH_ENABLED": "true", "GITEA_ENDPOINT": "https://git.itworx.test", "GITEA_REPOSITORIES_JSON": json.dumps( [{"project": "hub", "owner": "itworx", "name": "mcp-hub"}] ), "GITEA_AUTO_DISCOVER_PROJECT": "hub", "GITEA_TOKEN_SCOPE": "read:repository", "GITEA_WRITE_TOKEN_SCOPE": "write:repository,write:issue", "GITEA_ENABLE_PR_WRITES": "true", "MOBILITYOPS_ENDPOINT": "https://mobilityops.itworx.test", "MOBILITYOPS_PROJECTS_JSON": json.dumps(["mobilityops"]), "PROJECTBRAIN_ENDPOINT": "https://projectbrain.itworx.test", "PROJECTBRAIN_PROJECTS": "gitea-all", "APPOPS_ENDPOINT": "http://198.18.10.150:1216", "APPOPS_PROJECTS": "server-ops", "NPM_NETWORK_NAME": "npm-production", # Single public hostname (see infrastructure/nginx-proxy-manager/ROUTING.md): # the *_HUB_* variables are the management-surface half of the one # path-routed NPM proxy host, so they equal the *_MCP_* variables. "NPM_HUB_PROXY_HOST": "mcp.itworx.test", "NPM_MCP_PROXY_HOST": "mcp.itworx.test", "NPM_CERTIFICATE_REF": "npm-certificate-itworx", "DNS_HUB_HOSTNAME": "mcp.itworx.test", "DNS_MCP_HOSTNAME": "mcp.itworx.test", "DNS_EXPECTED_ADDRESSES_JSON": json.dumps({"mcp.itworx.test": ["192.0.2.20"]}), "NETWORK_TARGETS_JSON": json.dumps( {"registry": "registry.itworx.test:443", "gitea": "git.itworx.test:443"} ), } for service, variable in IMAGE_ENVIRONMENT.items(): image_name = service.replace("-router", "").replace("-api", "") environment[variable] = ( f"registry.itworx.test/platform/itworx-mcp-hub/{image_name}:v1.0.0-rc2@sha256:{digest}" ) return environment, secret_values def _failed(result: dict[str, object], check_id: str) -> bool: return any( item["id"] == check_id and item["status"] == "fail" for item in result["checks"] # type: ignore[union-attr] ) def test_valid_structural_preflight_passes_without_live_access(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) result = run(environment, "predeploy") assert result["passed"] is True assert result["failureCount"] == 0 assert result["deferredCount"] == 19 def test_single_hostname_hub_and_mcp_routing_passes_coherence(tmp_path: Path) -> None: """Production's real architecture is one public hostname (`mcp.example.com`) routed by path -- the management-surface `*_HUB_*` variables and the dataplane `*_MCP_*` variables intentionally hold the identical hostname. This must pass, not merely be untested: a prior production incident (`evidence/WP-240/REDEPLOYMENT_RECORD_c4a0f6d.md` section 8) was stale `hub.example.com` config drift going undetected because nothing asserted the single-host case explicitly. """ environment, _ = _fixture(tmp_path) result = run(environment, "predeploy") assert result["passed"] is True assert not _failed(result, "auth.coherence") assert not _failed(result, "auth.single_host") def test_distinct_hub_hostname_is_rejected(tmp_path: Path) -> None: """`hub.example.com` (or any hostname distinct from the single public MCP hostname) must never validate as production configuration again.""" environment, _ = _fixture(tmp_path) environment["NPM_HUB_PROXY_HOST"] = "hub.example.com" environment["DNS_HUB_HOSTNAME"] = "hub.example.com" environment["HUB_OIDC_REDIRECT_URI"] = "https://hub.example.com/auth/callback" result = run(environment, "predeploy") assert result["passed"] is False assert _failed(result, "auth.single_host") def test_contextforge_api_token_is_generated_then_required(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) predeploy = run(environment, "predeploy") assert predeploy["passed"] is True poststart_missing = run(environment, "poststart") assert poststart_missing["passed"] is False assert _failed(poststart_missing, "secret.contextforge_api_token") token = Path(environment["SECRETS_DIR"]) / "contextforge_api_token" token.write_text("scoped-runtime-api-token", encoding="utf-8") token.chmod(0o600) poststart = run(environment, "poststart") assert poststart["passed"] is True def test_default_predeploy_never_contacts_external_or_local_runtime(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) with ( patch("scripts.promotion_preflight.subprocess.run", side_effect=AssertionError), patch("scripts.promotion_preflight.socket.create_connection", side_effect=AssertionError), patch("scripts.promotion_preflight.socket.getaddrinfo", side_effect=AssertionError), patch("scripts.promotion_preflight.socket.socket", side_effect=AssertionError), ): result = run(environment, "predeploy") assert result["passed"] is True def test_predeploy_bind_check_requires_ports_to_be_available(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) with patch("scripts.promotion_preflight.socket.socket") as socket_factory: result = run(environment, "predeploy", verify_bind_ports=True) assert result["passed"] is True assert socket_factory.return_value.__enter__.return_value.bind.call_count == 2 def test_poststart_bind_check_requires_exact_owned_container_bindings(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) token = Path(environment["SECRETS_DIR"]) / "contextforge_api_token" token.write_text("scoped-runtime-api-token", encoding="utf-8") token.chmod(0o600) responses = [ subprocess.CompletedProcess([], 0, stdout="web-id\n", stderr=""), subprocess.CompletedProcess( [], 0, stdout='{"8080/tcp":[{"HostIp":"192.0.2.10","HostPort":"1100"}]}\n', stderr="" ), subprocess.CompletedProcess([], 0, stdout="router-id\n", stderr=""), subprocess.CompletedProcess( [], 0, stdout='{"8080/tcp":[{"HostIp":"192.0.2.10","HostPort":"18442"}]}\n', stderr="" ), ] with ( patch("scripts.promotion_preflight.shutil.which", return_value="/usr/bin/docker"), patch("scripts.promotion_preflight.subprocess.run", side_effect=responses) as docker_run, patch("scripts.promotion_preflight.socket.socket", side_effect=AssertionError), ): result = run(environment, "poststart", verify_bind_ports=True) assert result["passed"] is True assert docker_run.call_count == 4 def test_poststart_bind_check_rejects_wrong_or_ambiguous_owner(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) token = Path(environment["SECRETS_DIR"]) / "contextforge_api_token" token.write_text("scoped-runtime-api-token", encoding="utf-8") token.chmod(0o600) responses = [ subprocess.CompletedProcess([], 0, stdout="wrong-a\nwrong-b\n", stderr=""), subprocess.CompletedProcess([], 0, stdout="router-id\n", stderr=""), subprocess.CompletedProcess( [], 0, stdout='{"8080/tcp":[{"HostIp":"0.0.0.0","HostPort":"18442"}]}\n', stderr="" ), ] with ( patch("scripts.promotion_preflight.shutil.which", return_value="/usr/bin/docker"), patch("scripts.promotion_preflight.subprocess.run", side_effect=responses), ): result = run(environment, "poststart", verify_bind_ports=True) assert result["passed"] is False assert _failed(result, "ports.hub") assert _failed(result, "ports.mcp") def test_poststart_bind_check_rejects_unrendered_extra_binding(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) token = Path(environment["SECRETS_DIR"]) / "contextforge_api_token" token.write_text("scoped-runtime-api-token", encoding="utf-8") token.chmod(0o600) responses = [ subprocess.CompletedProcess([], 0, stdout="web-id\n", stderr=""), subprocess.CompletedProcess( [], 0, stdout=( '{"8080/tcp":[{"HostIp":"192.0.2.10","HostPort":"1100"}],' '"9090/tcp":[{"HostIp":"192.0.2.10","HostPort":"1190"}]}\n' ), stderr="", ), subprocess.CompletedProcess([], 0, stdout="router-id\n", stderr=""), subprocess.CompletedProcess( [], 0, stdout='{"8080/tcp":[{"HostIp":"192.0.2.10","HostPort":"18442"}]}\n', stderr="", ), ] with ( patch("scripts.promotion_preflight.shutil.which", return_value="/usr/bin/docker"), patch("scripts.promotion_preflight.subprocess.run", side_effect=responses), ): result = run(environment, "poststart", verify_bind_ports=True) assert result["passed"] is False assert _failed(result, "ports.hub") @pytest.mark.skipif(shutil.which("docker") is None, reason="Docker CLI is unavailable") def test_docker_compose_accepts_production_overlay_without_secret_interpolation( tmp_path: Path, ) -> None: environment, secret_values = _fixture(tmp_path) completed = subprocess.run( # noqa: S603 [ shutil.which("docker") or "docker", "compose", "-f", "infrastructure/compose/docker-compose.yml", "-f", "infrastructure/compose/docker-compose.prod.yml", "config", "--quiet", ], cwd=Path(__file__).resolve().parents[1], env={**os.environ, **environment}, capture_output=True, text=True, check=False, ) output = completed.stdout + completed.stderr assert completed.returncode == 0, output assert all(value.decode() not in output for value in secret_values.values()) def test_env_parser_rejects_malformed_input(tmp_path: Path) -> None: path = tmp_path / "production.env" path.write_text("GOOD=value\nnot-an-assignment\n", encoding="utf-8") with pytest.raises(ValueError): load_env_file(path) @pytest.mark.parametrize("secret_name", ["gitea_mcp_token", "gitea_mcp_write_token"]) @pytest.mark.parametrize("mutation", ["missing", "empty"]) def test_missing_or_empty_gitea_secret_file_fails( tmp_path: Path, mutation: str, secret_name: str ) -> None: environment, _ = _fixture(tmp_path) secret = Path(environment["SECRETS_DIR"]) / secret_name if mutation == "missing": secret.unlink() else: secret.write_bytes(b"") result = run(environment, "predeploy") assert not result["passed"] assert _failed(result, f"secret.{secret_name}") @pytest.mark.parametrize( "scope", ["write:repository", "write:issue", "all", "write:repository,write:issue,write:admin"], ) def test_gitea_write_scope_must_be_exactly_bounded(tmp_path: Path, scope: str) -> None: environment, _ = _fixture(tmp_path) environment["GITEA_WRITE_TOKEN_SCOPE"] = scope result = run(environment, "predeploy") assert not result["passed"] assert _failed(result, "gitea.configuration") @pytest.mark.skipif(os.name == "nt", reason="POSIX mode contract is enforced on deployment host") def test_permissive_secret_mode_fails(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) (Path(environment["SECRETS_DIR"]) / "hub_session_signing_key").chmod(0o644) result = run(environment, "predeploy") assert _failed(result, "secret.hub_session_signing_key") def test_secret_runtime_helper_rejects_wrong_owner_and_never_projects_value( tmp_path: Path, ) -> None: secret = tmp_path / "token" secret.write_text("must-not-appear-in-error", encoding="utf-8") secret.chmod(0o600) with pytest.raises(SecretFileError) as captured: validate_secret_file(secret, root=tmp_path, expected_uid=secret.stat().st_uid + 1) assert "must-not-appear-in-error" not in str(captured.value) def test_secret_runtime_helper_accepts_private_single_line_file(tmp_path: Path) -> None: secret = tmp_path / "token" secret.write_text("bounded-secret\n", encoding="utf-8") secret.chmod(0o600) assert read_secret_file(secret, root=tmp_path) == "bounded-secret" def test_placeholder_credential_reference_fails(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) environment["REGISTRY_CREDENTIAL_REF"] = "REPLACE_WITH_CREDENTIAL" result = run(environment, "predeploy") assert _failed(result, "inputs.required") def test_secret_interpolation_leakage_fails(tmp_path: Path) -> None: environment, values = _fixture(tmp_path) topology = _rendered_topology(environment) topology["services"]["web"]["environment"] = { "ACCIDENTAL_TOKEN": values["gitea_mcp_token"].decode() } result = run(environment, "predeploy", topology_override=topology) assert _failed(result, "compose.secret-redaction") def test_missing_gitea_service_wiring_fails(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) topology = _rendered_topology(environment) del topology["services"]["connector-gitea"] result = run(environment, "predeploy", topology_override=topology) assert _failed(result, "compose.required-services") assert _failed(result, "compose.connector-gitea") def test_mutable_or_development_image_fails(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) environment["CONNECTOR_GITEA_IMAGE"] = "local/connector-gitea:dev" result = run(environment, "predeploy") assert not result["passed"] assert _failed(result, "inputs.required") or _failed(result, "image.connector-gitea") @pytest.mark.parametrize( ("name", "value", "check_id"), [ ("HUB_OIDC_ISSUER", "http://auth.itworx.test/issuer", "url.hub_oidc_issuer"), ("HUB_CONTEXTFORGE_URL", "ftp://contextforge", "url.contextforge"), ("PROJECTBRAIN_ENDPOINT", "http://projectbrain.itworx.test", "url.projectbrain_endpoint"), ], ) def test_malformed_identity_or_gateway_endpoint_fails( tmp_path: Path, name: str, value: str, check_id: str ) -> None: environment, _ = _fixture(tmp_path) environment[name] = value result = run(environment, "predeploy") assert _failed(result, check_id) def test_duplicate_or_internal_exposed_port_fails(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) topology = _rendered_topology(environment) topology["services"]["connector-gitea"]["ports"] = ["192.0.2.10:18441:8080"] result = run(environment, "predeploy", topology_override=topology) assert _failed(result, "compose.ports") assert _failed(result, "compose.connector-gitea") def test_accidental_docker_socket_access_fails(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) topology = _rendered_topology(environment) topology["services"]["connector-gitea"]["volumes"] = [ "/var/run/docker.sock:/var/run/docker.sock:ro" ] result = run(environment, "predeploy", topology_override=topology) assert _failed(result, "compose.docker-socket") def test_production_image_cannot_fall_back_to_development_default(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) topology = _rendered_topology(environment) topology["services"]["web"]["image"] = "local/itworx-mcp-hub-web:dev" result = run(environment, "predeploy", topology_override=topology) assert _failed(result, "image.web") def test_oidc_and_contextforge_file_credentials_are_consumed_at_boundary() -> None: oidc = OIDCSettings( issuer="https://auth.itworx.test/issuer", audience="hub", client_id="hub", redirect_uri="https://hub.itworx.test/auth/callback", client_secret_file=Path("/run/secrets/oidc"), session_signing_key_file=Path("/run/secrets/session"), ) gateway = ContextForgeSettings( base_url="http://contextforge:4444", auth_token_file=Path("/run/secrets/contextforge"), ) with patch("services.control_api.auth.config.read_secret_file", return_value="oidc-secret"): assert oidc.client_secret() == "oidc-secret" assert oidc.session_signing_key() == b"oidc-secret" with patch( "services.control_api.contextforge.models.read_secret_file", return_value="gateway-secret" ): assert gateway.credential() == "gateway-secret" def test_file_credential_errors_are_sanitized() -> None: oidc = OIDCSettings( issuer="https://auth.itworx.test/issuer", audience="hub", client_id="hub", redirect_uri="https://hub.itworx.test/auth/callback", client_secret_file=Path("/run/secrets/oidc"), ) gateway = ContextForgeSettings( base_url="http://contextforge:4444", auth_token_file=Path("/run/secrets/contextforge"), ) with ( patch( "services.control_api.auth.config.read_secret_file", side_effect=SecretFileError("unsafe-secret-value"), ), pytest.raises(ConfigurationError) as oidc_error, ): oidc.client_secret() with ( patch( "services.control_api.contextforge.models.read_secret_file", side_effect=SecretFileError("unsafe-secret-value"), ), pytest.raises(ContextForgeError) as gateway_error, ): gateway.credential() assert "unsafe-secret-value" not in str(oidc_error.value) assert "unsafe-secret-value" not in str(gateway_error.value) def test_session_references_use_keyed_hash_when_signing_key_is_configured() -> None: assert ( _hash("session-reference", b"server-key") != hashlib.sha256(b"session-reference").hexdigest() ) def test_prod_overlay_reset_tag_removes_build_key() -> None: # _ComposeMergeLoader subclasses yaml.SafeLoader and only adds the !reset tag. base = yaml.load(BASE_COMPOSE.read_text(encoding="utf-8"), Loader=_ComposeMergeLoader) # noqa: S506 production = yaml.load(PROD_COMPOSE.read_text(encoding="utf-8"), Loader=_ComposeMergeLoader) # noqa: S506 merged = _deep_merge(base, production) # Derived from the base compose file's own `build:` keys (not a hand-maintained # duplicate list) so this test automatically covers any future service that # gains a `build:` section, rather than silently skipping it. See # evidence/WP-200/ADVERSARIAL_REVIEW.md item (c). owned_with_build = tuple( name for name, service in base["services"].items() if "build" in service ) assert owned_with_build, "no service in the base compose file defines build:" for name in owned_with_build: assert "build" not in merged["services"][name], name def test_malformed_image_digest_fails_preflight(tmp_path: Path) -> None: environment, _ = _fixture(tmp_path) topology = { "services": { service: { "image": f"registry.itworx.test/platform/itworx-mcp-hub/{service}@sha256:short" } for service in IMAGE_ENVIRONMENT } } result = run(environment, "predeploy", topology_override=topology) assert result["passed"] is False image_checks = [item for item in result["checks"] if item["id"].startswith("image.")] assert image_checks and all(item["status"] == "fail" for item in image_checks) def test_secret_inventory_is_explicit() -> None: assert "redis_password" in REQUIRED_SECRET_FILES assert "gitea_mcp_token" in REQUIRED_SECRET_FILES assert "gitea_mcp_write_token" in REQUIRED_SECRET_FILES assert "connector_identity_signing_key" in REQUIRED_SECRET_FILES assert "mobilityops_service_token" in REQUIRED_SECRET_FILES # Hard build_app() startup dependency of dataplane-router, not optional. assert "demo_token_signing_key" in REQUIRED_SECRET_FILES assert "projectbrain_hub_service_token" in REQUIRED_SECRET_FILES assert "appops_hub_service_token" in REQUIRED_SECRET_FILES assert "stitch_api_key" in REQUIRED_SECRET_FILES assert len(REQUIRED_SECRET_FILES) == 18 assert GENERATED_SECRET_FILES == ("contextforge_api_token",)