from __future__ import annotations import hashlib import importlib.util import io import json import copy import os import stat import subprocess import sys import tarfile import types from datetime import datetime, timedelta, timezone from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" / "deploy.py" def _load_controller(): if "fcntl" not in sys.modules: fcntl = types.ModuleType("fcntl") fcntl.LOCK_EX = 2 fcntl.LOCK_UN = 8 fcntl.flock = lambda *_args: None sys.modules["fcntl"] = fcntl spec = importlib.util.spec_from_file_location("mcp_hub_topology_controller_test", CONTROLLER) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module @pytest.fixture def controller(): return _load_controller() def _legacy_target(controller): return { "branch": "main", "slug": controller.MCP_HUB_PROJECT, "compose_project": controller.MCP_HUB_PROJECT, "containers": list(controller.MCP_HUB_LEGACY_CONTAINERS), "exact_source": True, "verify_oci_revision": True, "manual": False, "deploy_script": "", "archive_deploy_script": "", "live_probe": None, "managed_plan_version": "", "source_dir": (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix(), } @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="backup ownership contract requires root-owned POSIX fixtures", ) def test_hub_backup_child_is_created_with_controller_owner_and_private_mode( controller, tmp_path, monkeypatch ): state_root = tmp_path / "controller-state" backup_root = state_root / "mcp-hub-backups" state_root.mkdir(mode=0o700) backup_root.mkdir(mode=0o700) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) attempt = backup_root / "attempt" controller._mcp_hub_private_directory(attempt, create=True, root=backup_root) metadata = attempt.stat() assert (metadata.st_uid, metadata.st_gid) == (0, 0) assert stat.S_IMODE(metadata.st_mode) == 0o700 controller._mcp_hub_private_directory(attempt, root=backup_root) @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="backup root contract requires fixed-owner POSIX fixtures", ) def test_hub_backup_root_rejects_group_readable_root(controller, tmp_path, monkeypatch): state_root = tmp_path / "controller-state" backup_root = state_root / "mcp-hub-backups" state_root.mkdir(mode=0o700) backup_root.mkdir(mode=0o750) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) with pytest.raises(controller.DeployError, match="not private and canonical"): controller._mcp_hub_private_directory(backup_root, root=backup_root) @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="fixed Hub backup owner contract requires root-owned POSIX fixtures", ) def test_hub_backup_root_rejects_runtime_owner_even_when_it_matches_parent( controller, tmp_path, monkeypatch ): state_root = tmp_path / "controller-state" backup_root = state_root / "mcp-hub-backups" state_root.mkdir(mode=0o700) backup_root.mkdir(mode=0o700) os.chown(state_root, 99, 100) os.chown(backup_root, 99, 100) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) with pytest.raises(controller.DeployError, match="owner differs"): controller._mcp_hub_private_directory(backup_root, root=backup_root) @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="descriptor-relative private storage requires root-owned POSIX fixtures", ) def test_hub_backup_anchor_is_created_under_the_pinned_controller_state( controller, tmp_path, monkeypatch ): state_root = tmp_path / "controller-state" state_root.mkdir(mode=0o755) backup_root = state_root / "mcp-hub-backups" monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) controller._mcp_hub_private_directory(backup_root, create=True) metadata = backup_root.stat() assert (metadata.st_uid, metadata.st_gid) == (0, 0) assert stat.S_IMODE(metadata.st_mode) == 0o700 assert stat.S_IMODE(state_root.stat().st_mode) == 0o755 @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="descriptor-relative path replacement requires root-owned POSIX fixtures", ) def test_hub_backup_child_rejects_parent_replacement_during_creation( controller, tmp_path, monkeypatch ): state_root = tmp_path / "controller-state" state_root.mkdir(mode=0o700) backup_root = state_root / "mcp-hub-backups" backup_root.mkdir(mode=0o700) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) real_mkdir = controller.os.mkdir def replace_parent(path, mode=0o777, *, dir_fd=None): real_mkdir(path, mode, dir_fd=dir_fd) if path == "attempt" and dir_fd is not None: backup_root.rename(state_root / "replaced-backups") real_mkdir(backup_root, 0o700) monkeypatch.setattr(controller.os, "mkdir", replace_parent) with pytest.raises(controller.DeployError, match="could not be created safely"): controller._mcp_hub_private_directory(backup_root / "attempt", create=True) assert not (backup_root / "attempt").exists() @pytest.mark.skipif(os.name == "nt", reason="release receipt storage requires POSIX mode and ACL metadata") def test_hub_backup_receipt_directory_is_guarded_by_its_release_volume_parent( controller, tmp_path, monkeypatch ): volume_root = tmp_path / "controller-hub-releases" state_root = volume_root / "state" receipt_root = state_root / "backup-receipts" receipt_root.mkdir(mode=0o700, parents=True) volume_root.chmod(0o700) state_root.chmod(0o700) monkeypatch.setattr(controller, "MCP_HUB_RELEASE_STORE_STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_RECEIPT_ROOT", receipt_root) controller._mcp_hub_private_directory(receipt_root, root=state_root) with pytest.raises(controller.DeployError, match="escapes its fixed parent"): controller._mcp_hub_private_directory(receipt_root, root=controller.STATE_ROOT) def _valid_model(controller): services = {} for service_name in controller.MCP_HUB_SERVICES: if service_name == "contextforge": image = "itworx-mcp-hub/contextforge-schema:0123456789ab" elif service_name == "postgres": image = controller.MCP_HUB_POSTGRES_IMAGE elif service_name == "redis": image = controller.MCP_HUB_REDIS_IMAGE elif service_name == "ingestion-clamd": image = controller.MCP_HUB_INTAKE_CLAMAV_IMAGE else: image = f"registry.example/{service_name}@sha256:{'a' * 64}" service = {"image": image, "volumes": [], "secrets": [], "tmpfs": []} if service_name == "postgres": service["environment"] = {"PGDATA": controller.MCP_HUB_POSTGRES_PGDATA} if service_name in controller.MCP_HUB_INTAKE_SERVICES: service["profiles"] = ["knowledge-intake"] if service_name in {"control-api", "worker"}: service["environment"] = {"INGESTION_CONNECTOR_ENABLED": "true"} if service_name == "contextforge": service["environment"] = {"MCPGATEWAY_SKIP_MIGRATIONS": "true"} services[service_name] = service return {"services": services, "volumes": {}, "secrets": {}} def _observed_candidate_model(controller): model = _valid_model(controller) for service_name, local_repository in controller.MCP_HUB_LOCAL_BUILD_IMAGE_REPOSITORIES.items(): model["services"][service_name]["image"] = f"{local_repository}:bbbbbbbbbbbb" model["services"]["connector-ingestion"]["image"] = ( f"{controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES['connector-ingestion']}@sha256:{'b' * 64}" ) return model def test_contextforge_candidate_model_accepts_only_the_frozen_full_revision(controller): revision = "a" * 40 project = { "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "_deployment_metadata": {"revision": revision}, } model = _valid_model(controller) model["services"]["contextforge"]["image"] = ( f"itworx-mcp-hub/contextforge-schema:{revision}" ) assert controller.validate_mcp_hub_compose_model(project, model) is model model["services"]["contextforge"]["image"] = ( f"itworx-mcp-hub/contextforge-schema:{'b' * 40}" ) with pytest.raises(controller.DeployError, match="ContextForge"): controller.validate_mcp_hub_compose_model(project, model) @pytest.mark.parametrize("mutation", ["revision", "tree", "epoch", "root", "tag"]) def test_frozen_contextforge_provenance_rejects_changed_build_fields(controller, mutation): revision = "a" * 40 build = {"tree": "b" * 40, "id": revision[:12], "builtAt": "2026-10-02T00:00:00.000Z"} epoch = "1790888863" reference = f"itworx-mcp-hub/contextforge-schema:{revision}" image = {"Config": {"User": "10001:10001", "Labels": { "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, "org.opencontainers.image.revision": revision, "org.opencontainers.image.source-tree": build["tree"], "org.opencontainers.image.version": build["id"], "org.opencontainers.image.created": build["builtAt"], "org.opencontainers.image.source-date-epoch": epoch, }}} controller._mcp_hub_verify_frozen_image_provenance("contextforge", reference, image, revision, build, epoch) if mutation == "root": image["Config"]["User"] = "root:10001" elif mutation == "tag": reference = "itworx-mcp-hub/contextforge-schema:12d4a0c7789c" else: key = {"revision": "revision", "tree": "source-tree", "epoch": "source-date-epoch"}[mutation] image["Config"]["Labels"][f"org.opencontainers.image.{key}"] = "changed" with pytest.raises(controller.DeployError, match="provenance|ContextForge"): controller._mcp_hub_verify_frozen_image_provenance("contextforge", reference, image, revision, build, epoch) def test_exact_legacy_target_derives_only_the_fixed_17_service_contract(controller): target = _legacy_target(controller) result = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, target) assert set(result["containers"]) == set(controller.MCP_HUB_CONTAINERS) assert result["compose_files"] == list(controller.MCP_HUB_COMPOSE_FILES) assert result["_compose_profiles"] == ["knowledge-intake"] assert result["legacy_containers"] == [] assert target["containers"] == list(controller.MCP_HUB_LEGACY_CONTAINERS) assert controller.effective_mcp_hub_project("jens/other", target) is target def test_db_candidate_pins_and_observed_transition_refs_are_exact(controller): postgres = ( "gitea.example.com/jens/itworx-mcp-hub/postgres:" "wp400-gosu-v2-20261002-a94e0b4d2029@sha256:" "a60d81e06987671ae903757f1b71cdfbbff4d481b42610e4de2b7dcfa32aad73" ) redis = ( "gitea.example.com/jens/itworx-mcp-hub/redis:" "wp400-remediation-20261002-4e18f61c@sha256:" "7d826b44f245ad32e8fe4bf40d42eae95855c3942c5f2e42784ee8fd3f9547a2" ) assert controller.MCP_HUB_POSTGRES_IMAGE == postgres assert controller.MCP_HUB_REDIS_IMAGE == redis assert controller.MCP_HUB_POSTGRES_PREVIOUS_IMAGE == ( "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04" ) assert controller.MCP_HUB_REDIS_PREVIOUS_IMAGE == ( "redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48" ) assert controller.MCP_HUB_OBSERVED_EXTERNAL_IMAGE_TRANSITIONS == { "postgres": ( (controller.MCP_HUB_POSTGRES_LEGACY_IMAGE, controller.MCP_HUB_POSTGRES_PREVIOUS_IMAGE), postgres, ), "redis": ( (controller.MCP_HUB_REDIS_LEGACY_IMAGE, controller.MCP_HUB_REDIS_PREVIOUS_IMAGE), redis, ), } @pytest.mark.parametrize("service", ["postgres", "redis"]) def test_db_transition_accepts_only_new_forward_pin_against_exact_observed_refs(controller, service): observed_refs, candidate_ref = controller.MCP_HUB_OBSERVED_EXTERNAL_IMAGE_TRANSITIONS[service] assert controller._mcp_hub_observed_external_ref_allowed(service, candidate_ref, candidate_ref) for old_ref in observed_refs: assert controller._mcp_hub_observed_external_ref_allowed(service, candidate_ref, old_ref) assert not controller._mcp_hub_observed_external_ref_allowed(service, old_ref, old_ref) assert not controller._mcp_hub_observed_external_ref_allowed(service, candidate_ref, old_ref + "-unexpected") assert not controller._mcp_hub_observed_external_ref_allowed(service, candidate_ref + "-unexpected", candidate_ref) other_candidate = ( controller.MCP_HUB_REDIS_IMAGE if service == "postgres" else controller.MCP_HUB_POSTGRES_IMAGE ) assert not controller._mcp_hub_observed_external_ref_allowed(service, other_candidate, candidate_ref) def test_installed_legacy_registry_shape_is_adapted_only_in_memory(controller): target = { "branch": "main", "slug": controller.MCP_HUB_PROJECT, "compose_project": controller.MCP_HUB_PROJECT, "containers": list(controller.MCP_HUB_LEGACY_CONTAINERS), "source_dir": "/srv/example/appdata/itworx-mcp-hub/deployment/repository", "compose_dir": "infrastructure/compose", "compose_files": [ "docker-compose.yml", "docker-compose.prod.yml", "docker-compose.autodeploy.yml", ], "manual": False, "deploy_script": "", "archive_deploy_script": "", "live_probe": None, "managed_plan_version": "", "env_source": "/srv/example/appdata/itworx-mcp-hub/deploy.env", } original = copy.deepcopy(target) result = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, target) assert result["source_dir"] == (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix() assert result["exact_source"] is True assert result["compose_files"] == list(controller.MCP_HUB_COMPOSE_FILES) assert set(result["containers"]) == set(controller.MCP_HUB_CONTAINERS) assert target == original for field, value in ( ("source_dir", "/srv/example/appdata/other/repository"), ("compose_files", ["docker-compose.yml", "unexpected.yml"]), ("exact_source", None), ("exact_source", False), ("exact_source", True), ("verify_oci_revision", None), ("verify_oci_revision", False), ("verify_oci_revision", True), ("clean_untracked_paths", None), ("clean_untracked_paths", False), ("clean_untracked_paths", []), ("env_source", "/tmp/untrusted.env"), ): changed = {**original, field: value} with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, changed) def test_hub_verified_build_date_cannot_be_overridden_by_target_environment( controller, monkeypatch ): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) project["environment"] = { "APPOPS_BUILD_DATE": "untrusted-override", "APPOPS_SOURCE_TREE": "untrusted-tree", "APPOPS_BUILD_ID": "untrusted-id", "SOURCE_DATE_EPOCH": "untrusted-epoch", } project["_mcp_hub_source_date_epoch"] = "1790888863" metadata = { "repository": controller.MCP_HUB_REPOSITORY, "revision": "a" * 40, "builtAt": "2026-10-01T18:00:00.000Z", "tree": "b" * 40, "id": "a" * 12, } monkeypatch.setenv("APPOPS_BUILD_DATE", "process-override") environment = controller.deployment_environment( project, "a" * 40, controller.MCP_HUB_REPOSITORY, metadata ) assert environment["APPOPS_BUILD_DATE"] == metadata["builtAt"] assert environment["APPOPS_SOURCE_TREE"] == metadata["tree"] assert environment["APPOPS_BUILD_ID"] == metadata["id"] assert environment["SOURCE_DATE_EPOCH"] == "1790888863" def test_hub_compose_command_preserves_candidate_build_metadata_and_fixed_intake_profile( controller, monkeypatch, tmp_path ): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) project["source_dir"] = str(tmp_path) project["compose_dir"] = "infrastructure/compose" built_at = "2026-10-01T19:04:44Z" project["_deployment_metadata"] = { "repository": controller.MCP_HUB_REPOSITORY, "revision": "a" * 40, "builtAt": built_at, } compose_directory = tmp_path / "infrastructure" / "compose" compose_directory.mkdir(parents=True) for filename in controller.MCP_HUB_COMPOSE_FILES: (compose_directory / filename).write_text("services: {}\n", encoding="utf-8") overlay = tmp_path / "private-overlay.yaml" monkeypatch.setattr( controller, "_mcp_hub_compose_override", lambda _project, _compose_dir, _base_command: overlay, ) monkeypatch.setenv("COMPOSE_PROFILES", "observability,inspector") _, command = controller.compose_command(project) environment = controller.deployment_environment( project, "a" * 40, controller.MCP_HUB_REPOSITORY ) profile_arguments = [ command[index + 1] for index, argument in enumerate(command[:-1]) if argument == "--profile" ] assert profile_arguments == ["knowledge-intake"] assert environment["COMPOSE_PROFILES"] == "knowledge-intake" assert environment["GITEA_COMMIT_SHA"] == "a" * 40 assert environment["APPOPS_BUILD_DATE"] == built_at project_directory_index = command.index("--project-directory") assert command[project_directory_index + 1] == str( compose_directory ) assert command[-2:] == ["--file", str(overlay)] @pytest.mark.skipif(os.name == "nt", reason="release publication requires POSIX ownership and directory fsync") def test_hub_release_source_stage_is_immutable_and_idempotent( controller, monkeypatch, tmp_path ): state_root = tmp_path / "controller-state" state_root.mkdir(mode=0o700) monkeypatch.setattr(controller, "STATE_ROOT", state_root) releases = _patch_release_volume_paths(controller, monkeypatch, tmp_path) revision = "a" * 40 project = { "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "slug": controller.MCP_HUB_PROJECT, "branch": "main", } monkeypatch.setattr(controller, "validate_commit", lambda *_args: revision) monkeypatch.setattr(controller, "output", lambda *_args: revision) extracted = [] def extract(_repo, _revision, target): extracted.append(target) (target / "catalog").mkdir() (target / "catalog" / "items.json").write_text("{}", encoding="utf-8") monkeypatch.setattr(controller, "extract_revision", extract) monkeypatch.setattr( controller, "exact_source_matches_revision", lambda _repo, _revision, target: ( (target / "catalog" / "items.json").is_file() and (target / "catalog" / "items.json").read_text(encoding="utf-8") == "{}" ), ) release = controller.mcp_hub_stage_release_source(project, tmp_path / "hub.git", revision) assert release == releases / revision assert release.joinpath("catalog", "items.json").read_text(encoding="utf-8") == "{}" assert stat.S_IMODE(release.stat().st_mode) == 0o555 assert stat.S_IMODE(release.joinpath("catalog").stat().st_mode) == 0o555 assert stat.S_IMODE(release.joinpath("catalog", "items.json").stat().st_mode) == 0o444 controller.mcp_hub_stage_release_source(project, tmp_path / "hub.git", revision) assert len(extracted) == 1 release.joinpath("catalog", "items.json").write_text("tampered", encoding="utf-8") with pytest.raises(controller.DeployError, match="differs from its protected Git tree"): controller.mcp_hub_stage_release_source(project, tmp_path / "hub.git", revision) @pytest.mark.skipif(os.name == "nt", reason="release publication requires POSIX directory fsync") def test_hub_release_stage_syncs_tree_before_rename_and_parent_after( controller, monkeypatch, tmp_path ): releases = _patch_release_volume_paths(controller, monkeypatch, tmp_path) revision = "a" * 40 project = { "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "slug": controller.MCP_HUB_PROJECT, "branch": "main", } monkeypatch.setattr(controller, "validate_commit", lambda *_args: revision) monkeypatch.setattr(controller, "output", lambda *_args: revision) def extract(_repo, _revision, target): catalog = target / "catalog" catalog.mkdir() (catalog / "items.json").write_text("{}", encoding="utf-8") monkeypatch.setattr(controller, "extract_revision", extract) monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True) events = [] original_fsync = controller.os.fsync original_directory_sync = controller._mcp_hub_fsync_release_directory original_rename = Path.rename original_parent_sync = controller._fsync_directory def record_fsync(descriptor): if stat.S_ISREG(controller.os.fstat(descriptor).st_mode): events.append(("file", "")) original_fsync(descriptor) def record_directory_sync(path): events.append(("directory", Path(path).name)) original_directory_sync(path) def record_rename(path, target): events.append(("rename", str(target))) return original_rename(path, target) def record_parent_sync(path): if Path(path) == releases: events.append(("release-parent", "")) original_parent_sync(path) monkeypatch.setattr(controller.os, "fsync", record_fsync) monkeypatch.setattr(controller, "_mcp_hub_fsync_release_directory", record_directory_sync) monkeypatch.setattr(Path, "rename", record_rename) monkeypatch.setattr(controller, "_fsync_directory", record_parent_sync) controller.mcp_hub_stage_release_source(project, tmp_path / "hub.git", revision) rename_index = next(index for index, event in enumerate(events) if event[0] == "rename") assert any(event[0] == "file" for event in events[:rename_index]) assert any(event == ("directory", "catalog") for event in events[:rename_index]) assert events.index(("release-parent", "")) > rename_index @pytest.mark.skipif(os.name == "nt", reason="release publication requires POSIX ownership and directory fsync") def test_prior_release_can_only_be_staged_from_protected_main_ancestry( controller, monkeypatch, tmp_path ): releases = _patch_release_volume_paths(controller, monkeypatch, tmp_path) prior_revision = "a" * 40 protected_head = "b" * 40 project = { "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "slug": controller.MCP_HUB_PROJECT, "branch": "main", } monkeypatch.setattr(controller, "validate_commit", lambda _repo, _branch, revision: revision) monkeypatch.setattr(controller, "output", lambda *_args: protected_head) monkeypatch.setattr( controller, "extract_revision", lambda _repo, _revision, target: (target / "catalog").mkdir(), ) monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True) monkeypatch.setattr( controller.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 0), ) staged = controller._mcp_hub_stage_protected_release_source( project, tmp_path / "hub.git", prior_revision, protected_head=protected_head ) assert staged == releases / prior_revision assert not (controller.MCP_HUB_APPDATA_ROOT / "deployment" / "releases").exists() monkeypatch.setattr( controller.subprocess, "run", lambda *_args, **_kwargs: subprocess.CompletedProcess([], 1), ) with pytest.raises(controller.DeployError, match="not an ancestor of exact protected main"): controller._mcp_hub_stage_protected_release_source( project, tmp_path / "hub.git", "c" * 40, protected_head=protected_head ) @pytest.mark.skipif(os.name == "nt", reason="release metadata checks require POSIX mode bits") def test_hub_release_anchor_and_tree_reject_writable_metadata( controller, monkeypatch, tmp_path ): volume_root = tmp_path / "protected-volume" releases = volume_root / "releases" release = releases / ("a" * 40) release.mkdir(parents=True) (release / "source.py").write_text("safe", encoding="utf-8") volume_root.chmod(0o700) releases.chmod(0o700) release.chmod(0o555) monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases) monkeypatch.setattr(controller, "_root_owned", lambda _path: True) with pytest.raises(controller.DeployError, match="outside its fixed Docker-local volume path"): controller._mcp_hub_require_trusted_release_anchor() releases.chmod(0o700) monkeypatch.setattr( controller, "_mcp_hub_require_trusted_release_anchor", lambda **_kwargs: releases, ) (release / "source.py").chmod(0o644) with pytest.raises(controller.DeployError, match="writable or not canonical"): controller._mcp_hub_verify_release_tree_metadata(release) def test_hub_rollback_mount_contract_ignores_only_release_generation_sha(controller): old = [{ "type": "bind", "source": (controller.MCP_HUB_RELEASES_ROOT / ("a" * 40) / "services/runtime/worker.py").as_posix(), "destination": "/opt/hub/worker.py", "rw": False, "name": "", }] candidate = [{ **old[0], "source": (controller.MCP_HUB_RELEASES_ROOT / ("b" * 40) / "services/runtime/worker.py").as_posix(), }] assert controller._mcp_hub_mount_contract_equal(old, candidate) candidate[0]["source"] = ( controller.MCP_HUB_RELEASES_ROOT / ("b" * 40) / "services/runtime/other.py" ).as_posix() assert not controller._mcp_hub_mount_contract_equal(old, candidate) candidate[0]["source"] = "/srv/example/appdata/other/worker.py" assert not controller._mcp_hub_mount_contract_equal(old, candidate) def test_legacy_release_source_is_only_a_canonical_captured_path_identity( controller, monkeypatch, tmp_path ): appdata = tmp_path / "runtime-writable-appdata" monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata) revision = "a" * 40 legacy = (appdata / "deployment" / "releases" / revision / "catalog").as_posix() assert controller._mcp_hub_legacy_release_source(legacy) == (revision, "catalog") assert controller._mcp_hub_legacy_release_source(legacy + "/../catalog") is None assert controller._mcp_hub_release_source(legacy) is None def test_live_mount_release_generation_is_bound_to_each_service_image(controller, monkeypatch, tmp_path): image_revision = "a" * 40 candidate_revision = "c" * 40 monkeypatch.setattr(controller, "_mcp_hub_release_tree_verified", lambda *_args: True) monkeypatch.setattr( controller, "_mcp_hub_is_ancestor", lambda _p, ancestor, descendant: ancestor in {image_revision, "b" * 40} and descendant == candidate_revision, ) monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True) image_id = "sha256:" + "a" * 64 image = { "Id": image_id, "Config": {"Labels": { "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, "org.opencontainers.image.revision": image_revision, }}, } monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) model = { "services": {"control-api": { "volumes": [{ "type": "bind", "source": f"{controller._MCP_HUB_RELEASE_STORE.logical_release_path(candidate_revision)}/catalog", "target": "/opt/hub/catalog", "read_only": True, }], "secrets": [], "tmpfs": [], }}, "volumes": {}, "secrets": {}, } container = { "Image": image_id, "Mounts": [{ "Type": "bind", "Source": f"{controller._MCP_HUB_RELEASE_STORE.logical_release_path(image_revision)}/catalog", "Destination": "/opt/hub/catalog", "RW": False, }], } project = { "_repository_path": "/fixed/hub.git", "_deployment_metadata": {"revision": candidate_revision}, } attested = controller.mcp_hub_attested_live_mounts( model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) assert attested[0]["source"] == ( f"{controller._MCP_HUB_RELEASE_STORE.logical_release_path(image_revision)}/catalog" ) container["Mounts"][0]["Source"] = f"{controller._MCP_HUB_RELEASE_STORE.logical_release_path('b' * 40)}/catalog" with pytest.raises(controller.DeployError, match="source tree differs from its OCI image revision"): controller.mcp_hub_attested_live_mounts( model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) def test_legacy_live_release_mount_is_rebased_to_protected_tree_only_in_transition( controller, monkeypatch, tmp_path ): image_revision = "a" * 40 candidate_revision = "c" * 40 monkeypatch.setattr(controller, "_mcp_hub_release_tree_verified", lambda *_args: True) monkeypatch.setattr( controller, "_mcp_hub_is_ancestor", lambda _project, ancestor, descendant: ancestor in {image_revision, "b" * 40} and descendant == candidate_revision, ) image_id = "sha256:" + "a" * 64 monkeypatch.setattr( controller, "inspect_image", lambda _image_id: { "Id": image_id, "Config": {"Labels": { "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, "org.opencontainers.image.revision": image_revision, }}, }, ) model = { "services": {"control-api": { "volumes": [{ "type": "bind", "source": f"{controller._MCP_HUB_RELEASE_STORE.logical_release_path(candidate_revision)}/catalog", "target": "/opt/hub/catalog", "read_only": True, }], "secrets": [], "tmpfs": [], }}, "volumes": {}, "secrets": {}, } captured_legacy_source = "/srv/example/appdata/itworx-mcp-hub/deployment/releases/" + image_revision + "/catalog" monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", Path("/srv/example/appdata/itworx-mcp-hub")) container = { "Image": image_id, "Mounts": [{ "Type": "bind", "Source": captured_legacy_source, "Destination": "/opt/hub/catalog", "RW": False, }], } project = { "_repository_path": "/fixed/hub.git", "_deployment_metadata": {"revision": candidate_revision}, "_mcp_hub_allow_legacy_live_mounts": True, } attested = controller.mcp_hub_attested_live_mounts( model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) assert attested[0]["source"] == captured_legacy_source assert not (controller.MCP_HUB_APPDATA_ROOT / "deployment" / "releases").exists() container["Mounts"][0]["Source"] = ( controller.MCP_HUB_APPDATA_ROOT / "deployment" / "releases" / ("b" * 40) / "catalog" ).as_posix() with pytest.raises(controller.DeployError, match="source tree differs from its OCI image revision"): controller.mcp_hub_attested_live_mounts( model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) container["Mounts"][0]["Source"] = captured_legacy_source project.pop("_mcp_hub_allow_legacy_live_mounts") with pytest.raises(controller.DeployError, match="live mount set|release bind|release tree"): controller.mcp_hub_attested_live_mounts( model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) @pytest.mark.parametrize( "field,value", [ ("branch", "release"), ("compose_project", "other-project"), ("source_dir", "/tmp/untrusted-source"), ("manual", True), ("deploy_script", "run-anything.sh"), ("live_probe", {"type": "unexpected"}), ("containers", ["itworx-mcp-hub-postgres-1"]), ], ) def test_hub_topology_derivation_rejects_any_target_outside_the_fixed_legacy_contract( controller, field, value ): target = _legacy_target(controller) target[field] = value with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, target) def test_rendered_model_requires_the_exact_17_services_and_intake_safety_gates(controller): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) model = _valid_model(controller) assert controller.validate_mcp_hub_compose_model(project, model) is model missing_intake = _valid_model(controller) missing_intake["services"].pop("connector-ingestion") with pytest.raises(controller.DeployError, match="fixed 17-service contract"): controller.validate_mcp_hub_compose_model(project, missing_intake) wrong_profile = _valid_model(controller) wrong_profile["services"]["connector-ingestion"]["profiles"] = [] with pytest.raises(controller.DeployError, match="knowledge-intake"): controller.validate_mcp_hub_compose_model(project, wrong_profile) intake_disabled_in_core = _valid_model(controller) intake_disabled_in_core["services"]["worker"]["environment"][ "INGESTION_CONNECTOR_ENABLED" ] = "false" with pytest.raises(controller.DeployError, match="preserve the existing enabled"): controller.validate_mcp_hub_compose_model(project, intake_disabled_in_core) mutable_database = _valid_model(controller) mutable_database["services"]["postgres"]["image"] = "postgres:latest" with pytest.raises(controller.DeployError, match="PostgreSQL image"): controller.validate_mcp_hub_compose_model(project, mutable_database) def test_live_enabled_connector_flags_must_match_all_17_verified_services( controller, monkeypatch ): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) model = _valid_model(controller) containers = {} for name in controller.MCP_HUB_CONTAINERS: service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") service = model["services"][service_name] environment = ( ["INGESTION_CONNECTOR_ENABLED=true"] if service_name in {"control-api", "worker"} else [] ) containers[name] = { "Id": "c" * 64, "Image": "sha256:" + "a" * 64, "Config": { "Image": service["image"], "Env": environment, "Labels": { "com.docker.compose.project": controller.MCP_HUB_PROJECT, "com.docker.compose.service": service_name, }, }, "State": {"Running": True, "Health": {"Status": "healthy"}}, } monkeypatch.setattr( controller, "compose_project_container_names", lambda _project: set(containers) ) monkeypatch.setattr(controller, "inspect_container", lambda name: containers.get(name)) monkeypatch.setattr(controller, "mcp_hub_attested_live_mounts", lambda *_args, **_kwargs: []) result = controller.verify_mcp_hub_live_topology(project, model) assert set(result) == set(controller.MCP_HUB_CONTAINERS) containers["itworx-mcp-hub-worker-1"]["Config"]["Env"] = [ "INGESTION_CONNECTOR_ENABLED=false" ] with pytest.raises(controller.DeployError, match="live connector state differs"): controller.verify_mcp_hub_live_topology(project, model) controller._mcp_hub_require_connector_state_preserved( {"environment": {"INGESTION_CONNECTOR_ENABLED": "true"}}, {"environment": {"INGESTION_CONNECTOR_ENABLED": "true"}}, ) with pytest.raises(controller.DeployError, match="changes the existing connector enablement"): controller._mcp_hub_require_connector_state_preserved( {"environment": {"INGESTION_CONNECTOR_ENABLED": "true"}}, {"environment": {"INGESTION_CONNECTOR_ENABLED": "false"}}, ) def test_backup_inputs_match_the_secret_free_live_contract(controller): recent_status = { "schemaVersion": "1", "completedAt": datetime.now(timezone.utc).isoformat(), "restoreVerified": True, "artifactCount": 1, "artifacts": [{"name": "hub.dump", "bytes": 42, "sha256": "a" * 64}], } controller._mcp_hub_validate_backup_status(json.dumps(recent_status).encode()) controller._mcp_hub_validate_revocations( json.dumps({"schemaVersion": "1.0", "revokedJtis": ["a" * 32]}).encode() ) stale_status = {**recent_status, "completedAt": (datetime.now(timezone.utc) - timedelta(hours=27)).isoformat()} with pytest.raises(controller.DeployError, match="stale"): controller._mcp_hub_validate_backup_status(json.dumps(stale_status).encode()) unverified_status = {**recent_status, "restoreVerified": False} with pytest.raises(controller.DeployError, match="secret-free schema"): controller._mcp_hub_validate_backup_status(json.dumps(unverified_status).encode()) malformed_revocations = {"schemaVersion": "1.0", "revokedJtis": ["not-a-jti"]} with pytest.raises(controller.DeployError, match="revocation state is invalid"): controller._mcp_hub_validate_revocations(json.dumps(malformed_revocations).encode()) def test_registered_target_archive_projection_accepts_only_the_broker_schema(controller): registrations = { "schemaVersion": "1.0", "containers": [ { "registeredId": "hub-control-api", "project": "itworx-mcp-hub", "environment": "prod", "match": {"composeProject": "itworx-mcp-hub", "service": "control-api"}, "allowedReads": ["inspect", "health"], } ], "networks": [], "defaultDecision": "deny", } controller._mcp_hub_validate_registered_targets(json.dumps(registrations).encode()) with_token = json.loads(json.dumps(registrations)) with_token["containers"][0]["token"] = "must-not-be-archived" with pytest.raises(controller.DeployError, match="registration schema"): controller._mcp_hub_validate_registered_targets(json.dumps(with_token).encode()) with_unknown_scope = json.loads(json.dumps(registrations)) with_unknown_scope["containers"][0]["allowedReads"] = ["exec"] with pytest.raises(controller.DeployError, match="registration schema"): controller._mcp_hub_validate_registered_targets(json.dumps(with_unknown_scope).encode()) def test_backup_mount_inventory_includes_exact_registry_source(controller, monkeypatch, tmp_path): config_root = tmp_path / "appdata" / "config" config_root.mkdir(parents=True) monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", config_root.parent) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_STATUS_SOURCE", config_root / "backup-status.json") monkeypatch.setattr(controller, "MCP_HUB_REVOCATIONS_SOURCE", config_root / "demo-token-revocations.json") monkeypatch.setattr(controller, "MCP_HUB_REGISTERED_TARGETS_SOURCE", config_root / "registered-targets.json") model = _valid_model(controller) model["services"]["control-api"]["volumes"] = [ { "type": "bind", "source": str(config_root / "backup-status.json"), "target": controller.MCP_HUB_BACKUP_STATUS_TARGET, "read_only": True, } ] model["services"]["dataplane-router"]["volumes"] = [ { "type": "bind", "source": str(config_root), "target": "/etc/itworx", "read_only": True, } ] registry_mount = { "type": "bind", "source": str(config_root / "registered-targets.json"), "target": "/etc/itworx/registered-targets.json", "read_only": True, } model["services"]["docker-read-broker"]["volumes"] = [registry_mount] model["services"]["connector-unraid"]["volumes"] = [registry_mount] inputs = controller.mcp_hub_fixed_backup_inputs(model) assert inputs["registered_targets"] == config_root / "registered-targets.json" model["services"]["connector-unraid"]["volumes"][0]["source"] = str(config_root / "other-targets.json") with pytest.raises(controller.DeployError, match="registration mount"): controller.mcp_hub_fixed_backup_inputs(model) def test_all_secret_file_mounts_are_fixed_private_metadata_only_inputs( controller, monkeypatch, tmp_path ): appdata_root = tmp_path / "appdata" secrets_root = appdata_root / "secrets" secrets_root.mkdir(parents=True) secrets_root.chmod(0o700) monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata_root) monkeypatch.setattr(controller, "MCP_HUB_SECRETS_ROOT", secrets_root) model = _valid_model(controller) model["secrets"] = {} secret_owner = secrets_root.stat() model["services"]["control-api"]["environment"].update( { "SECRET_EXPECTED_UID": str(getattr(secret_owner, "st_uid", 1000)), "SECRET_EXPECTED_GID": str(getattr(secret_owner, "st_gid", 1000)), } ) for name in controller.MCP_HUB_REQUIRED_SECRET_NAMES: secret_file = secrets_root / name secret_file.write_bytes(b"fixture-only-secret-value") secret_file.chmod(0o600) model["secrets"][name] = {"file": str(secret_file)} assert controller.mcp_hub_validate_secret_files(model) == len( controller.MCP_HUB_REQUIRED_SECRET_NAMES ) wrong_path = secrets_root / "alternate" wrong_path.write_bytes(b"fixture-only-secret-value") wrong_path.chmod(0o600) model["secrets"][controller.MCP_HUB_REQUIRED_SECRET_NAMES[0]]["file"] = str(wrong_path) with pytest.raises(controller.DeployError, match="fixed secret-file contract"): controller.mcp_hub_validate_secret_files(model) model["secrets"][controller.MCP_HUB_REQUIRED_SECRET_NAMES[0]]["file"] = str( secrets_root / controller.MCP_HUB_REQUIRED_SECRET_NAMES[0] ) missing_name = controller.MCP_HUB_REQUIRED_SECRET_NAMES[-1] (secrets_root / missing_name).unlink() with pytest.raises(controller.DeployError, match="secret file is unavailable"): controller.mcp_hub_validate_secret_files(model) missing_path = secrets_root / missing_name missing_path.write_bytes(b"fixture-only-secret-value") missing_path.chmod(0o600) if os.name != "nt": valid_secret = secrets_root / controller.MCP_HUB_REQUIRED_SECRET_NAMES[0] valid_secret.chmod(0o640) with pytest.raises(controller.DeployError, match="metadata is unsafe"): controller.mcp_hub_validate_secret_files(model) metadata = types.SimpleNamespace( st_mode=stat.S_IFREG | 0o600, st_size=1, st_uid=1000, st_gid=1000, ) assert controller._mcp_hub_secret_metadata_matches( metadata, 1000, 1000, platform_name="posix" ) assert not controller._mcp_hub_secret_metadata_matches( metadata, 1001, 1000, platform_name="posix" ) def test_database_backup_capacity_uses_dump_and_restore_size_headroom(controller, monkeypatch): sizes = {"itworx_mcp_hub": 1024**3, "contextforge": 2 * 1024**3} required_before = ( sum(sizes.values()) * controller.MCP_HUB_BACKUP_SIZE_MULTIPLIER + controller.MCP_HUB_MIN_BACKUP_FREE_BYTES ) monkeypatch.setattr( controller.shutil, "disk_usage", lambda _path: types.SimpleNamespace(free=required_before - 1), ) with pytest.raises(controller.DeployError, match="dump and restore capacity"): controller._mcp_hub_require_backup_space(sizes) controller._mcp_hub_require_backup_space(sizes, after_dumps=True) monkeypatch.setattr( controller.shutil, "disk_usage", lambda _path: types.SimpleNamespace( free=(sum(sizes.values()) * 2 + controller.MCP_HUB_MIN_BACKUP_FREE_BYTES) ), ) controller._mcp_hub_require_backup_space(sizes, after_dumps=True) def test_database_size_query_is_limited_to_two_fixed_database_names(controller, monkeypatch): captured = {} def docker(*arguments, timeout): captured["arguments"] = arguments captured["timeout"] = timeout return types.SimpleNamespace( stdout=b"contextforge:23\nitworx_mcp_hub:42\n", returncode=0, ) monkeypatch.setattr(controller, "_mcp_hub_docker", docker) assert controller._mcp_hub_live_database_sizes() == { "contextforge": 23, "itworx_mcp_hub": 42, } query = captured["arguments"][-1] assert "contextforge" in query and "itworx_mcp_hub" in query assert "pg_database_size" in query assert captured["timeout"] == 30 @pytest.mark.skipif( os.name != "nt" and (not hasattr(os, "geteuid") or os.geteuid() != 0), reason="backup cleanup ownership fixture requires root on POSIX", ) def test_failed_backup_attempt_removes_only_its_unique_private_directory( controller, monkeypatch, tmp_path ): appdata_root = tmp_path / "hub-appdata" backup_root = appdata_root / "backups" appdata_root.mkdir(mode=0o700) backup_root.mkdir(mode=0o700) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "STATE_ROOT", appdata_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata_root) private_directory = controller._mcp_hub_private_directory monkeypatch.setattr( controller, "_mcp_hub_private_directory", lambda path, **kwargs: private_directory(path, **{"root": backup_root, **kwargs}), ) monkeypatch.setattr(controller, "mcp_hub_validate_secret_files", lambda _model: 21) monkeypatch.setattr( controller, "mcp_hub_fixed_backup_inputs", lambda _model: { "backup_status": Path("status"), "demo_token_revocations": Path("revocations"), "registered_targets": Path("registered-targets"), }, ) monkeypatch.setattr(controller, "_mcp_hub_fixed_input_file", lambda *_args, **_kwargs: b"safe") monkeypatch.setattr(controller, "_mcp_hub_validate_backup_status", lambda _data: None) monkeypatch.setattr(controller, "_mcp_hub_validate_revocations", lambda _data: None) monkeypatch.setattr(controller, "_mcp_hub_validate_registered_targets", lambda _data: None) monkeypatch.setattr(controller, "_mcp_hub_live_database_sizes", lambda: { "contextforge": 1, "itworx_mcp_hub": 1, }) monkeypatch.setattr(controller, "_mcp_hub_require_backup_space", lambda *_args, **_kwargs: None) monkeypatch.setattr( controller, "_capture_mcp_hub_backup_attempt", lambda *_args, **_kwargs: (_ for _ in ()).throw(controller.DeployError("fixture failure")), ) project = {"_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT} with pytest.raises(controller.DeployError, match="fixture failure"): controller.capture_mcp_hub_backup( project, "a" * 40, {}, {}, runtime={}, image_facts={}, ) assert list(backup_root.iterdir()) == [] def test_rollback_keeps_digest_refs_immutable_and_only_tags_local_refs(controller, monkeypatch): image_id = "sha256:" + "a" * 64 digest = "sha256:" + "b" * 64 image = { "Id": image_id, "RepoDigests": [f"registry.example/hub/service@{digest}"], } tagged = [] monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) monkeypatch.setattr(controller, "run", lambda command: tagged.append(command)) controller._mcp_hub_prepare_rollback_image( image_id, f"registry.example/hub/service@{digest}" ) assert tagged == [] controller._mcp_hub_prepare_rollback_image(image_id, "itworx-mcp-hub/service:old") assert tagged == [["docker", "image", "tag", image_id, "itworx-mcp-hub/service:old"]] def test_digest_proof_normalizes_docker_hub_aliases_without_losing_registry_ports( controller, ): digest = "sha256:" + "a" * 64 assert controller._mcp_hub_digest_ref_proven( {"RepoDigests": [f"docker.io/library/postgres@{digest}"]}, f"postgres:18.6@{digest}", ) assert controller._mcp_hub_digest_ref_proven( {"RepoDigests": [f"redis@{digest}"]}, f"redis:8.10.2@{digest}", ) assert controller._mcp_hub_digest_ref_proven( {"RepoDigests": [f"registry.example:5000/team/service@{digest}"]}, f"registry.example:5000/team/service:release@{digest}", ) assert not controller._mcp_hub_digest_ref_proven( {"RepoDigests": [f"foreign/postgres@{digest}"]}, f"postgres:18.6@{digest}", ) assert not controller._mcp_hub_digest_ref_proven( {"RepoDigests": ["postgres@sha256:bad"]}, "postgres:18.6@sha256:bad" ) def test_connector_ingestion_requires_both_digest_and_hub_source_ancestry(controller, monkeypatch): image_id = "sha256:" + "a" * 64 digest = "sha256:" + "b" * 64 image_ref = f"registry.example/hub/connector-ingestion@{digest}" revision = "1" * 40 candidate = "2" * 40 image = { "Id": image_id, "RepoDigests": [f"registry.example/hub/connector-ingestion@{digest}"], "Config": { "Labels": { "org.opencontainers.image.source": "https://untrusted.example/hub", "org.opencontainers.image.revision": revision, } }, } container = { "Id": "c" * 64, "Image": image_id, "Config": {"Image": image_ref, "Labels": {}}, } project = {"_repository_path": "/fixed/hub.git"} monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) with pytest.raises(controller.DeployError, match="provenance"): controller._mcp_hub_live_image_proven( project, "itworx-mcp-hub-connector-ingestion-1", container, image_ref, revision, candidate, require_current_provenance=True, ) image["Config"]["Labels"]["org.opencontainers.image.source"] = ( "https://gitea.example.com/jens/itworx-mcp-hub" ) result = controller._mcp_hub_live_image_proven( project, "itworx-mcp-hub-connector-ingestion-1", container, image_ref, revision, candidate, require_current_provenance=True, ) assert result["image_revision"] == revision def test_owned_image_revision_may_be_older_than_the_mixed_controller_receipt( controller, monkeypatch ): image_id = "sha256:" + "a" * 64 digest = "sha256:" + "b" * 64 image_ref = f"registry.example/hub/control-api@{digest}" older_service_revision = "a" * 40 controller_receipt_revision = "b" * 40 candidate_revision = "c" * 40 image = { "Id": image_id, "RepoDigests": [f"registry.example/hub/control-api@{digest}"], "Config": {"Labels": { "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, "org.opencontainers.image.revision": older_service_revision, }}, } container = { "Id": "d" * 64, "Image": image_id, "Config": {"Image": image_ref, "Labels": {}}, } ancestor_checks = [] monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) monkeypatch.setattr( controller, "_mcp_hub_is_ancestor", lambda _project, ancestor, descendant: ( ancestor_checks.append((ancestor, descendant)) or ancestor == older_service_revision and descendant == candidate_revision ), ) proven = controller._mcp_hub_live_image_proven( {"_repository_path": "/fixed/hub.git"}, "itworx-mcp-hub-control-api-1", container, image_ref, controller_receipt_revision, candidate_revision, require_current_provenance=True, ) assert proven["image_revision"] == older_service_revision assert ancestor_checks == [(older_service_revision, candidate_revision)] def test_catalog_archiver_accepts_only_bounded_regular_catalog_members(controller, monkeypatch): stream = io.BytesIO() with tarfile.open(fileobj=stream, mode="w:") as archive: item = tarfile.TarInfo("catalog/approved.json") item.size = 2 archive.addfile(item, io.BytesIO(b"{}")) monkeypatch.setattr(controller, "repository_path_from_project", lambda _project: "/fixed/hub.git") monkeypatch.setattr( controller.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0, stdout=stream.getvalue()), ) payload = controller._mcp_hub_git_catalog_archive({"slug": "itworx-mcp-hub"}, "a" * 40) assert payload == stream.getvalue() unsafe = io.BytesIO() with tarfile.open(fileobj=unsafe, mode="w:") as archive: item = tarfile.TarInfo("catalog/../outside") item.size = 1 archive.addfile(item, io.BytesIO(b"x")) monkeypatch.setattr( controller.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0, stdout=unsafe.getvalue()), ) with pytest.raises(controller.DeployError, match="unsafe member"): controller._mcp_hub_git_catalog_archive({"slug": "itworx-mcp-hub"}, "a" * 40) def test_backup_catalog_archives_bind_exact_mount_and_active_image_revisions( controller, monkeypatch, tmp_path ): legacy_appdata_root = Path("C:/runtime-appdata") if os.name == "nt" else Path("/srv/example/appdata/itworx-mcp-hub") monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", legacy_appdata_root) source_dir = tmp_path / "source" candidate_revision = "b" * 40 candidate_catalog = (controller.MCP_HUB_RELEASES_ROOT / candidate_revision / "catalog").as_posix() model = {"services": {}, "volumes": {}, "secrets": {}} runtime = {} image_facts = {} active_revisions = { "control-api": "8" * 40, "worker": "8" * 40, "dataplane-router": "a" * 40, } for service in controller.MCP_HUB_CATALOG_SERVICES: container = f"itworx-mcp-hub-{service}-1" rendered_mount = { "type": "bind", "source": candidate_catalog, "target": "/opt/hub/catalog", "read_only": True, } mount = { "type": "bind", "source": ( controller.MCP_HUB_APPDATA_ROOT / "deployment" / "releases" / active_revisions[service] / "catalog" ).as_posix(), "destination": "/opt/hub/catalog", "rw": False, "name": "", } model["services"][service] = {"volumes": [rendered_mount], "secrets": [], "tmpfs": []} runtime[container] = {"_appops_attested_mounts": [mount]} image_facts[container] = { "image_revision": active_revisions[service], "image_source": "https://gitea.example.com/jens/itworx-mcp-hub", } monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) monkeypatch.setattr(controller, "_mcp_hub_git_catalog_archive", lambda *_args: b"fixed-tar") backup = tmp_path / "backup" backup.mkdir() archives = controller._mcp_hub_capture_catalog_archives( {"source_dir": str(source_dir), "_mcp_hub_allow_legacy_live_mounts": True}, candidate_revision, model, runtime, image_facts, backup, ) assert {item["revision"] for item in archives} == { active_revisions["control-api"], active_revisions["dataplane-router"], candidate_revision } assert all((backup / item["file"]).read_bytes() == b"fixed-tar" for item in archives) model["services"]["worker"]["volumes"][0]["read_only"] = False runtime["itworx-mcp-hub-worker-1"]["_appops_attested_mounts"][0]["rw"] = True with pytest.raises(controller.DeployError, match="catalog source is not bound"): controller._mcp_hub_capture_catalog_archives( {"source_dir": str(source_dir), "_mcp_hub_allow_legacy_live_mounts": True}, candidate_revision, model, runtime, image_facts, tmp_path / "negative-backup", ) def test_pg_restore_streams_private_dump_over_stdin_without_mounting_backup( controller, monkeypatch, tmp_path ): backup_root = tmp_path / "backups" backup_root.mkdir(mode=0o700) backup = backup_root / "fixed-run" backup.mkdir(mode=0o700) dump = backup / "contextforge.dump" dump.write_bytes(b"private dump payload") dump.chmod(0o600) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) captured = {} def run(command, **kwargs): captured["command"] = command captured["stdin"] = kwargs["stdin"].read() return types.SimpleNamespace(returncode=0) monkeypatch.setattr(controller.subprocess, "run", run) controller._mcp_hub_pg_restore_dump("a" * 64, "contextforge", backup) assert captured["stdin"] == b"private dump payload" assert captured["command"] == [ "docker", "exec", "--interactive", "--user", "postgres", "a" * 64, "pg_restore", "--host=/run/postgresql", "--username=postgres", "--no-password", "--no-owner", "--no-privileges", "--exit-on-error", "--single-transaction", "--dbname=contextforge", "-", ] def test_private_backup_artifacts_publish_atomically_without_overwrite( controller, tmp_path ): artifact = tmp_path / "fixed-backup.json" controller._mcp_hub_write_private_new(artifact, b"first") assert artifact.read_bytes() == b"first" assert list(tmp_path.glob("*.tmp")) == [] with pytest.raises(controller.DeployError, match="already exists"): controller._mcp_hub_write_private_new(artifact, b"second") assert artifact.read_bytes() == b"first" assert list(tmp_path.glob("*.tmp")) == [] @pytest.mark.parametrize( "image_ref_name", ["MCP_HUB_POSTGRES_IMAGE", "MCP_HUB_POSTGRES_PREVIOUS_IMAGE", "MCP_HUB_POSTGRES_LEGACY_IMAGE"], ) def test_live_mount_inventory_includes_volume_identity_source_and_read_only_state( controller, monkeypatch, image_ref_name ): image_ref = getattr(controller, image_ref_name) config_source = (Path("/srv/example/appdata/itworx-mcp-hub/config") if sys.platform != "win32" else Path("C:/appdata/itworx-mcp-hub/config")) model = { "services": { "postgres": { "image": image_ref, "environment": {"PGDATA": controller.MCP_HUB_POSTGRES_PGDATA}, "volumes": [ {"type": "volume", "source": "pgdata", "target": "/var/lib/postgresql/data", "read_only": False}, {"type": "bind", "source": str(config_source), "target": "/etc/itworx", "read_only": True}, ], "secrets": [], "tmpfs": ["/run:rw,nosuid"], } }, "volumes": {"pgdata": {"name": "itworx-mcp-hub-postgres"}}, "secrets": {}, } anon_name = "f" * 64 mountpoint = "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data" anon_mountpoint = f"/var/lib/docker/volumes/{anon_name}/_data" image_id = "sha256:" + "a" * 64 live = { "Image": image_id, "Config": {"Image": image_ref}, "Mounts": [ {"Type": "volume", "Name": "itworx-mcp-hub-postgres", "Source": mountpoint, "Driver": "local", "Destination": "/var/lib/postgresql/data", "RW": True}, {"Type": "volume", "Name": anon_name, "Source": anon_mountpoint, "Driver": "local", "Destination": "/var/lib/postgresql", "RW": True}, {"Type": "bind", "Source": str(config_source), "Destination": "/etc/itworx", "RW": False}, {"Type": "tmpfs", "Source": "", "Destination": "/run", "RW": True}, ] } volume_data = { "itworx-mcp-hub-postgres": mountpoint, anon_name: anon_mountpoint, } created_at = "2020-01-01T10:00:00Z" monkeypatch.setattr( controller, "_mcp_hub_docker", lambda _kind, _operation, name, **_kwargs: subprocess.CompletedProcess( [], 0, stdout=( f'[{json.dumps({"Name": name, "Driver": "local", "Mountpoint": volume_data[name], "Options": {}, "CreatedAt": created_at})}]' ).encode("utf-8"), ), ) monkeypatch.setattr( controller, "inspect_image", lambda image: { "Id": image_id, "RepoDigests": [controller._mcp_hub_image_repository(image_ref) + "@" + image_ref.split("@", 1)[1]], "Config": {"Volumes": {"/var/lib/postgresql": {}}}, }, ) attested = controller.mcp_hub_attested_live_mounts( model, "postgres", controller.MCP_HUB_PROJECT, live ) volume = next( item for item in attested if item["type"] == "volume" and item["destination"] == "/var/lib/postgresql/data" ) assert volume == { "type": "volume", "source": mountpoint, "destination": "/var/lib/postgresql/data", "rw": True, "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, "created_at": created_at, } anonymous_volume = next(item for item in attested if item["destination"] == "/var/lib/postgresql") assert anonymous_volume == { "type": "volume", "source": anon_mountpoint, "destination": "/var/lib/postgresql", "rw": True, "name": anon_name, "driver": "local", "options": {}, "created_at": created_at, } assert next(item for item in attested if item["destination"] == "/etc/itworx")["rw"] is False live["Mounts"][0]["Name"] = "unexpected-postgres-volume" with pytest.raises(controller.DeployError, match="volume name"): controller.mcp_hub_attested_live_mounts( model, "postgres", controller.MCP_HUB_PROJECT, live ) @pytest.mark.parametrize( "mutate,match", [ (lambda mount, _image: mount.update(Destination="/var/lib/postgresql/other"), "anonymous volume is missing"), (lambda mount, _image: mount.update(Name="bad-anonymous-volume"), "anonymous"), (lambda mount, _image: mount.update(RW=False), "anonymous volume identity"), (lambda _mount, image: image.update(Config={"Volumes": {"/unexpected": {}}}), "image-declared"), ], ) def test_postgres_anonymous_volume_is_fixed_to_the_pinned_image_and_path( controller, monkeypatch, mutate, match ): volume_name = "f" * 64 image_id = "sha256:" + "a" * 64 mountpoint = f"/var/lib/docker/volumes/{volume_name}/_data" live_mount = { "Type": "volume", "Name": volume_name, "Source": mountpoint, "Driver": "local", "Destination": "/var/lib/postgresql", "RW": True, } image = { "Id": image_id, "RepoDigests": [controller.MCP_HUB_POSTGRES_IMAGE.split("@", 1)[0].split(":", 1)[0] + "@" + controller.MCP_HUB_POSTGRES_IMAGE.split("@", 1)[1]], "Config": {"Volumes": {"/var/lib/postgresql": {}}}, } mutate(live_mount, image) volume_data = { "itworx-mcp-hub-postgres": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", volume_name: mountpoint, } def inspect_volume(_kind, _operation, name, **_kwargs): payload = { "Name": name, "Driver": "local", "Mountpoint": volume_data[name], "Options": {}, "CreatedAt": "2020-01-01T10:00:00Z", } return subprocess.CompletedProcess([], 0, stdout=json.dumps([payload]).encode("utf-8")) monkeypatch.setattr(controller, "_mcp_hub_docker", inspect_volume) monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) model = { "services": { "postgres": { "image": controller.MCP_HUB_POSTGRES_IMAGE, "environment": {"PGDATA": controller.MCP_HUB_POSTGRES_PGDATA}, "volumes": [{ "type": "volume", "source": "pgdata", "target": "/var/lib/postgresql/data", "read_only": False, }], "secrets": [], "tmpfs": [], } }, "volumes": {"pgdata": {"name": "itworx-mcp-hub-postgres"}}, "secrets": {}, } container = { "Image": image_id, "Config": {"Image": controller.MCP_HUB_POSTGRES_IMAGE}, "Mounts": [ { "Type": "volume", "Name": "itworx-mcp-hub-postgres", "Source": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", "Driver": "local", "Destination": "/var/lib/postgresql/data", "RW": True, }, live_mount, ], } with pytest.raises(controller.DeployError, match=match): controller.mcp_hub_attested_live_mounts( model, "postgres", controller.MCP_HUB_PROJECT, container ) def test_postgres_staged_deploy_must_preserve_both_volume_identities(controller): data_volume = { "type": "volume", "source": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, "rw": True, "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:00Z", } anonymous_name = "f" * 64 anonymous_volume = { "type": "volume", "source": f"/var/lib/docker/volumes/{anonymous_name}/_data", "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, "rw": True, "name": anonymous_name, "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:01Z", } snapshot = { "containers": [{ "name": controller.MCP_HUB_POSTGRES_CONTAINER, "mounts": [data_volume, anonymous_volume], }] } runtime = { controller.MCP_HUB_POSTGRES_CONTAINER: { "_appops_attested_mounts": [data_volume, anonymous_volume] } } controller._mcp_hub_require_postgres_volumes_preserved(snapshot, runtime) changed = json.loads(json.dumps(runtime)) changed[controller.MCP_HUB_POSTGRES_CONTAINER]["_appops_attested_mounts"][1][ "name" ] = "e" * 64 with pytest.raises(controller.DeployError, match="changed a PostgreSQL data volume identity"): controller._mcp_hub_require_postgres_volumes_preserved(snapshot, changed) changed = json.loads(json.dumps(runtime)) changed[controller.MCP_HUB_POSTGRES_CONTAINER]["_appops_attested_mounts"][0][ "created_at" ] = "2021-01-01T09:00:00Z" with pytest.raises(controller.DeployError, match="changed a PostgreSQL data volume identity"): controller._mcp_hub_require_postgres_volumes_preserved(snapshot, changed) @pytest.mark.parametrize( "created_at", [None, "not-a-timestamp", "2020-01-01T10:00:00", "2999-01-01T00:00:00Z"], ) def test_docker_volume_metadata_requires_valid_nonfuture_creation_time( controller, monkeypatch, created_at ): payload = { "Name": "itworx-mcp-hub-postgres", "Driver": "local", "Mountpoint": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", "Options": {}, } if created_at is not None: payload["CreatedAt"] = created_at monkeypatch.setattr( controller, "_mcp_hub_docker", lambda *_args, **_kwargs: subprocess.CompletedProcess( [], 0, stdout=json.dumps([payload]).encode("utf-8") ), ) with pytest.raises(controller.DeployError, match="creation timestamp"): controller._mcp_hub_volume_metadata("itworx-mcp-hub-postgres", {}) @pytest.mark.parametrize("baseline_kind", ["observed", "normal_upgrade"]) def test_observed_postgres_volume_continuity_rechecks_created_at_independently( controller, monkeypatch, baseline_kind ): baseline = [ { "type": "volume", "source": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, "rw": True, "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:00Z", }, { "type": "volume", "source": f"/var/lib/docker/volumes/{'f' * 64}/_data", "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, "rw": True, "name": "f" * 64, "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:01Z", }, ] current = json.loads(json.dumps(baseline)) project = { "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "_mcp_hub_observed_postgres_volume_baseline": baseline, "_mcp_hub_rendered_model": _valid_model(controller), "compose_project": controller.MCP_HUB_PROJECT, } if baseline_kind == "normal_upgrade": project.pop("_mcp_hub_observed_postgres_volume_baseline") project["_compose_rollback"] = { "containers": [{"name": controller.MCP_HUB_POSTGRES_CONTAINER, "mounts": baseline}] } postgres = { "Id": "a" * 64, "Image": "sha256:" + "b" * 64, "Config": {"Image": controller.MCP_HUB_POSTGRES_IMAGE, "Labels": { "com.docker.compose.project": controller.MCP_HUB_PROJECT, "com.docker.compose.service": "postgres", }}, "State": {"Running": True, "Health": {"Status": "healthy"}}, } monkeypatch.setattr(controller, "inspect_container", lambda _name: postgres) monkeypatch.setattr( controller, "mcp_hub_attested_live_mounts", lambda *_args, **_kwargs: current ) controller._mcp_hub_require_observed_postgres_volume_continuity(project) # Docker can recreate a named volume with the same name and mountpoint. # Creation-time evidence must still detect that replacement. current[0]["created_at"] = "2021-01-01T10:00:00Z" with pytest.raises(controller.DeployError, match="PostgreSQL volume identity changed"): controller._mcp_hub_require_observed_postgres_volume_continuity(project) @pytest.mark.parametrize("intent_status", ["started", "complete", None]) def test_explicit_hub_rollback_is_blocked_before_repository_or_runtime_actions( controller, monkeypatch, intent_status ): target = _legacy_target(controller) monkeypatch.setattr( controller, "load_projects", lambda: {controller.MCP_HUB_REPOSITORY: target}, ) intent = ( ({"status": intent_status}, b"private-intent") if intent_status is not None else None ) monkeypatch.setattr(controller, "_mcp_hub_read_migration_intent", lambda: intent) monkeypatch.setattr( controller, "repository_path", lambda *_args: pytest.fail("rollback guard must precede repository/source actions"), ) monkeypatch.setattr( sys, "argv", ["deploy.py", "rollback", controller.MCP_HUB_REPOSITORY, "a" * 40], ) with pytest.raises(controller.DeployError, match="image-only rollback is unsupported"): controller.main() def test_explicit_rollback_guard_does_not_block_unrelated_targets(controller, monkeypatch): class ReachedRepositoryResolution(Exception): pass target = {"slug": "unraid-appops-gateway", "branch": "main", "containers": []} monkeypatch.setattr( controller, "load_projects", lambda: {"jens/unraid-appops-gateway": target} ) monkeypatch.setattr( controller, "repository_path", lambda *_args: (_ for _ in ()).throw(ReachedRepositoryResolution()), ) monkeypatch.setattr( sys, "argv", ["deploy.py", "rollback", "jens/unraid-appops-gateway", "a" * 40], ) with pytest.raises(ReachedRepositoryResolution): controller.main() def test_started_migration_marker_requires_manual_recovery_without_rereading_intent( controller, monkeypatch ): project = { "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "_mcp_hub_migration_started": True, } monkeypatch.setattr( controller, "_mcp_hub_read_migration_intent", lambda: pytest.fail("durable pre-apply marker already requires manual recovery"), ) assert controller._mcp_hub_intent_requires_manual_recovery(project) is True def test_restore_probe_is_offline_and_uses_only_temporary_database_storage( controller, monkeypatch, tmp_path ): backup_root = tmp_path / "backups" backup_root.mkdir(mode=0o700) backup = backup_root / "approved-backup" backup.mkdir(mode=0o700) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_mcp_hub_private_directory", lambda path, **_kwargs: None) monkeypatch.setattr(controller.time, "sleep", lambda _seconds: None) calls = [] def docker(*arguments, timeout): calls.append((arguments, timeout)) stdout = ( b"a" * 64 if arguments[0] == "run" else b"7" if arguments[0] == "exec" and "psql" in arguments else b"" ) return subprocess.CompletedProcess(arguments, 0, stdout=stdout) monkeypatch.setattr(controller, "_mcp_hub_docker", docker) restored_dumps = [] monkeypatch.setattr( controller, "_mcp_hub_pg_restore_dump", lambda container_id, database, source: restored_dumps.append( (container_id, database, source) ), ) result = controller._mcp_hub_pg_restore_probe(backup) run_args = calls[0][0] assert result["status"] == "verified" assert result["auditEvents"] == { "database": "itworx_mcp_hub", "table": "audit_events", "restoredRows": 7, } assert run_args[0] == "run" assert "--network" in run_args and run_args[run_args.index("--network") + 1] == "none" assert "--read-only" in run_args and "--pull=never" in run_args mounts = [item for item in run_args if item.startswith("type=bind,")] assert len(mounts) == 1 assert "target=/var/lib/postgresql/data" in mounts[0] assert "target=/backup" not in mounts[0] and "production" not in mounts[0] assert [database for _, database, _ in restored_dumps] == list(controller.MCP_HUB_DATABASES) assert all(identifier == "a" * 64 and source == backup for identifier, _, source in restored_dumps) assert [call[0][0] for call in calls].count("rm") == 1 def _legacy_migration_fixture(controller, monkeypatch, tmp_path): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) candidate = "b" * 40 previous = "a" * 40 model = _valid_model(controller) state_path = tmp_path / f"{controller.MCP_HUB_PROJECT}.json" image_metadata = { "container_revision": "", "image_revision": "", "image_source": "", "image_tree": "", "image_version": "", "image_created": "", } old_artifacts = {} receipt_containers = {} runtime = {} for index, name in enumerate(controller.MCP_HUB_CONTAINERS): service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") image_id = "sha256:" + f"{index + 1:064x}" container_id = f"{index + 1:064x}" image_ref = model["services"][service_name]["image"] artifact = { "container_id": container_id, "image_id": image_id, "image_ref": image_ref, **image_metadata, } runtime[name] = { "Id": container_id, "Image": image_id, "Config": {"Image": image_ref}, "_appops_attested_mounts": [], } if name in controller.MCP_HUB_LEGACY_CONTAINERS: old_artifacts[name] = dict(artifact) receipt_containers[name] = dict(artifact) tree = "c" * 40 receipt = { "schema": "gitea-deploy-receipt.v1", "repository": controller.MCP_HUB_REPOSITORY, "revision": previous, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "build": { "id": previous[:12], "repository": controller.MCP_HUB_REPOSITORY, "revision": previous, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-09-30T10:00:00.000Z", }, "containers": receipt_containers, "live": None, } prior = { "schema_version": 3, "repository": controller.MCP_HUB_REPOSITORY, "slug": controller.MCP_HUB_PROJECT, "compose_project": controller.MCP_HUB_PROJECT, "revision": previous, "live_revision": previous, "live_status": "healthy", "live_legacy_provenance": False, "live_artifacts": {"containers": old_artifacts}, "live_receipt": receipt, "status": "success", } state_path.write_text(json.dumps(prior), encoding="utf-8") state_path.chmod(0o600) monkeypatch.setattr(controller, "STATE_ROOT", tmp_path) monkeypatch.setattr(controller, "_root_owned", lambda _path: True) monkeypatch.setattr(controller, "validate_recorded_receipt", lambda *_args: {}) monkeypatch.setattr(controller, "_mcp_hub_git_output", lambda *_args: candidate) monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) monkeypatch.setattr( controller, "_mcp_hub_live_image_proven", lambda _project, _name, container, image_ref, *_args, **_kwargs: { "container_id": str(container["Id"]), "image_id": str(container["Image"]), "image_ref": image_ref, **image_metadata, }, ) backup_receipt = { "restoreVerified": True, "repository": controller.MCP_HUB_REPOSITORY, "revision": candidate, "manifestSha256": "d" * 64, "composeSha256": "e" * 64, } return project, candidate, model, backup_receipt, runtime, state_path, prior def _baseline_image_facts(controller, runtime): facts = {} for name, container in runtime.items(): service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") image_source = ( controller.MCP_HUB_OCI_IMAGE_SOURCE if service_name not in {"postgres", "redis", "contextforge", "ingestion-clamd"} else "" ) image_revision = "a" * 40 if image_source else "" proof_kind = ( "owned_git" if image_source else "contextforge_compat" if service_name == "contextforge" else "external_contract" ) facts[name] = { "container_id": str(container["Id"]), "image_id": str(container["Image"]), "image_ref": str(container["Config"]["Image"]), "container_revision": "", "image_revision": image_revision, "image_source": image_source, "image_tree": "", "image_tree_label_present": False, "image_git_tree": "b" * 40 if image_source else "", "proof_kind": proof_kind, "mounts": list(container["_appops_attested_mounts"]), } return facts def _observed_live_fixture(controller, monkeypatch, tmp_path): project, candidate, _model, backup, runtime, state_path, prior = _legacy_migration_fixture( controller, monkeypatch, tmp_path ) model = _observed_candidate_model(controller) prior["status"] = "failed" prior["live_status"] = "unknown" state_path.write_text(json.dumps(prior), encoding="utf-8") state_path.chmod(0o600) project["_deployment_metadata"] = {"revision": candidate} project["_repository_path"] = str(tmp_path / "hub.git") monkeypatch.setattr(controller, "_mcp_hub_git_output", lambda _project, _cmd, *args: ( candidate if args == ("refs/heads/main",) else "c" * 40 )) monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) monkeypatch.setattr(controller, "compose_project_container_names", lambda _project: set(runtime)) monkeypatch.setattr(controller, "inspect_container", lambda name: runtime.get(name)) image_inventory = {} for index, name in enumerate(controller.MCP_HUB_CONTAINERS): service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") container = runtime[name] container["Id"] = f"{index + 101:064x}" container["Image"] = "sha256:" + f"{index + 101:064x}" container["State"] = {"Running": True, "Health": {"Status": "healthy"}} container["Config"]["Labels"] = { "com.docker.compose.project": controller.MCP_HUB_PROJECT, "com.docker.compose.service": service_name, } container["Config"]["Env"] = ( ["INGESTION_CONNECTOR_ENABLED=true"] if service_name in {"control-api", "worker"} else [] ) if service_name == "postgres": container["Config"]["Env"].append( f"PGDATA={controller.MCP_HUB_POSTGRES_PGDATA}" ) if service_name == "contextforge": container["Config"]["Env"].append("MCPGATEWAY_SKIP_MIGRATIONS=true") image_revision = ("a" if index % 2 == 0 else "d") * 40 image_labels = {} if service_name not in {"postgres", "redis", "contextforge", "ingestion-clamd"}: image_labels = { "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, "org.opencontainers.image.revision": image_revision, } if service_name in controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES: image_repository = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES[service_name] image_ref = f"{image_repository}@sha256:{index + 1:064x}" elif service_name == "postgres": image_ref = controller.MCP_HUB_POSTGRES_LEGACY_IMAGE elif service_name == "redis": image_ref = controller.MCP_HUB_REDIS_LEGACY_IMAGE else: image_ref = str(model["services"][service_name]["image"]) container["Config"]["Image"] = image_ref if service_name == "postgres": data_source = "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data" anonymous_name = "f" * 64 container["_test_attested_mounts"] = [ { "type": "volume", "source": data_source, "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, "rw": True, "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:00Z", }, { "type": "volume", "source": f"/var/lib/docker/volumes/{anonymous_name}/_data", "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, "rw": True, "name": anonymous_name, "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:01Z", }, ] repo_digests = [] if "@" in image_ref: repo_digests = [ f"{controller._mcp_hub_image_repository(image_ref)}@{image_ref.split('@', 1)[1]}" ] image_inventory[str(container["Image"])] = { "Id": str(container["Image"]), "RepoDigests": repo_digests, "Config": {"Labels": image_labels}, } monkeypatch.setattr(controller, "inspect_image", lambda image_id: image_inventory.get(image_id)) def observed_live_image_proof(_project, _name, container, image_ref, *_args, **_kwargs): image = image_inventory[str(container["Image"])] labels = dict(image["Config"]["Labels"]) return { "container_id": str(container["Id"]), "image_id": str(container["Image"]), "image_ref": image_ref, "container_revision": str( dict(container.get("Config") or {}).get("Labels", {}).get( "org.opencontainers.image.revision", "" ) ), "image_revision": str(labels.get("org.opencontainers.image.revision") or ""), "image_source": str(labels.get("org.opencontainers.image.source") or ""), "image_tree": str(labels.get("org.opencontainers.image.source-tree") or ""), "image_version": str(labels.get("org.opencontainers.image.version") or ""), "image_created": str(labels.get("org.opencontainers.image.created") or ""), } monkeypatch.setattr(controller, "_mcp_hub_live_image_proven", observed_live_image_proof) monkeypatch.setattr(controller, "_mcp_hub_contextforge_compat_proven", lambda *_args: True) monkeypatch.setattr(controller, "_mcp_hub_dataplane_compat_proven", lambda *_args: True) monkeypatch.setattr( controller, "mcp_hub_attested_live_mounts", lambda _model, _service, _project, container, **_kwargs: list( container.get("_test_attested_mounts", []) ), ) return project, candidate, model, backup, runtime, image_inventory, state_path, prior def _trusted_backup_receipt_fixture(controller, monkeypatch, tmp_path, candidate): state_root = tmp_path backup_root = state_root / "mcp-hub-backups" backup_root.mkdir(mode=0o700, exist_ok=True) receipt_root = state_root / "mcp-hub-backup-receipts" receipt_root.mkdir(mode=0o700, exist_ok=True) backup_directory = backup_root / f"1780400000-{candidate[:12]}-0123456789abcdef" backup_directory.mkdir(mode=0o700) compose_digest = "e" * 64 manifest = { "schema": "itworx.mcp-hub-backup/v1", "repository": controller.MCP_HUB_REPOSITORY, "revision": candidate, "backupDirectory": str(backup_directory), "composeSha256": compose_digest, "configRestoreVerified": True, "restore": {"status": "verified"}, "databases": {name: {} for name in controller.MCP_HUB_DATABASES}, } manifest_bytes = json.dumps(manifest, sort_keys=True).encode("utf-8") manifest_path = backup_directory / "manifest.json" manifest_path.write_bytes(manifest_bytes) manifest_path.chmod(0o600) receipt_payload = { "schema": "itworx.mcp-hub-backup-receipt/v1", "repository": controller.MCP_HUB_REPOSITORY, "revision": candidate, "backupDirectory": str(backup_directory), "manifestSha256": hashlib.sha256(manifest_bytes).hexdigest(), "composeSha256": compose_digest, "restoreVerified": True, "createdAtEpoch": 1780400000, } receipt_path = receipt_root / f"backup-{candidate[:12]}.json" receipt_path.write_text(json.dumps(receipt_payload, sort_keys=True), encoding="utf-8") receipt_path.chmod(0o600) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "_root_owned", lambda _path: True) monkeypatch.setattr(controller, "_mcp_hub_private_directory", lambda *_args, **_kwargs: None) monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda *_args, **_kwargs: None) return {**receipt_payload, "receiptPath": str(receipt_path)} def _record_test_observed_baseline(controller, monkeypatch, tmp_path): project, candidate, model, _backup, runtime, _images, state_path, prior = _observed_live_fixture( controller, monkeypatch, tmp_path ) project["_mcp_hub_rendered_model"] = model original_state = state_path.read_bytes() monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) backup = _trusted_backup_receipt_fixture(controller, monkeypatch, tmp_path, candidate) (tmp_path / "mcp-hub-topology-migrations").mkdir(mode=0o700) verified_runtime, facts, _model = controller.mcp_hub_verify_observed_baseline( project, prior, model ) evidence = controller.mcp_hub_record_observed_baseline( project, prior, candidate, backup, verified_runtime, facts ) project["_mcp_hub_observed_baseline"] = evidence return project, candidate, model, backup, runtime, state_path, prior, evidence, original_state def test_observed_baseline_is_separate_evidence_and_preserves_unknown_controller_state( controller, monkeypatch, tmp_path ): ( _project, candidate, _model, backup, runtime, state_path, prior, evidence, original_state ) = _record_test_observed_baseline(controller, monkeypatch, tmp_path) assert state_path.read_bytes() == original_state assert evidence["schema"] == controller.MCP_HUB_OBSERVED_BASELINE_SCHEMA evidence_path = Path(evidence["evidencePath"]) record = json.loads(evidence_path.read_text(encoding="utf-8")) assert record["claim"] == "observed_mixed_runtime_facts_only" assert record["historicalStateSha256"] == hashlib.sha256(original_state).hexdigest() assert record["historicalStatus"] == "failed" assert record["historicalLiveStatus"] == "unknown" assert record["backupManifestSha256"] == backup["manifestSha256"] assert record["backupRestoreVerified"] is True assert record["postgresVolumeBaseline"] == [ { "type": "volume", "source": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, "rw": True, "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:00Z", }, { "type": "volume", "source": f"/var/lib/docker/volumes/{'f' * 64}/_data", "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, "rw": True, "name": "f" * 64, "driver": "local", "options": {}, "created_at": "2020-01-01T10:00:01Z", }, ] assert "live_status" not in record and "status" not in record assert record["candidateRevision"] == candidate assert set(record["containers"]) == set(controller.MCP_HUB_CONTAINERS) for name in controller.MCP_HUB_LEGACY_CONTAINERS: live = runtime[name] historical = prior["live_artifacts"]["containers"][name] assert ( live["Id"], live["Image"], live["Config"]["Image"], ) != ( historical["container_id"], historical["image_id"], historical["image_ref"], ) owned_revisions = { item["image_revision"] for item in record["containers"].values() if item.get("image_source") == controller.MCP_HUB_OCI_IMAGE_SOURCE } assert len(owned_revisions) > 1 assert all( isinstance(item, dict) and "image_tree_label_present" in item for item in record["containers"].values() ) assert evidence_path.is_relative_to(tmp_path / "mcp-hub-topology-migrations") assert hashlib.sha256(evidence_path.read_bytes()).hexdigest() == evidence["evidenceSha256"] if os.name != "nt": assert stat.S_IMODE(evidence_path.stat().st_mode) == 0o600 @pytest.mark.parametrize("service", ("postgres", "ingestion-clamd")) def test_reviewed_security_transition_preserves_the_exact_live_baseline( controller, monkeypatch, tmp_path, service ): project, _candidate, model, _backup, runtime, _images, state_path, prior = ( _observed_live_fixture(controller, monkeypatch, tmp_path) ) original_state = state_path.read_bytes() name = f"itworx-mcp-hub-{service}-1" observed_ref = runtime[name]["Config"]["Image"] security_ref = f"gitea.example.com/jens/itworx-mcp-hub/{service}:security-fixture@sha256:{'e' * 64}" bindings = dict(controller.MCP_HUB_APPROVED_EXTERNAL_IMAGE_BINDINGS) bindings[service] += ((security_ref, "fixture:security", security_ref),) monkeypatch.setattr(controller, "MCP_HUB_APPROVED_EXTERNAL_IMAGE_BINDINGS", bindings) transitions = dict(getattr(controller, "MCP_HUB_APPROVED_OBSERVED_EXTERNAL_TRANSITIONS", {})) transitions[service] = {security_ref: (observed_ref, security_ref)} monkeypatch.setattr(controller, "MCP_HUB_APPROVED_OBSERVED_EXTERNAL_TRANSITIONS", transitions, raising=False) model["services"][service]["image"] = security_ref verified, facts, observed_model = controller.mcp_hub_verify_observed_baseline(project, prior, model) assert verified[name]["Config"]["Image"] == observed_ref assert facts[name]["image_ref"] == observed_ref assert observed_model["services"][service]["image"] == observed_ref assert state_path.read_bytes() == original_state assert controller._mcp_hub_observed_external_ref_allowed(service, security_ref, observed_ref) assert not controller._mcp_hub_observed_external_ref_allowed(service, observed_ref, security_ref) assert not controller._mcp_hub_observed_external_ref_allowed(service, security_ref, observed_ref + "-unexpected") assert not controller._mcp_hub_observed_external_ref_allowed(service, security_ref[:-1] + "0", observed_ref) def test_backup_receipt_validator_rejects_an_unpersisted_boolean_claim( controller, monkeypatch, tmp_path ): candidate = "b" * 40 _trusted_backup_receipt_fixture(controller, monkeypatch, tmp_path, candidate) fake = { "restoreVerified": True, "repository": controller.MCP_HUB_REPOSITORY, "revision": candidate, "manifestSha256": "d" * 64, "composeSha256": "e" * 64, } with pytest.raises(controller.DeployError, match="exact controller-verified capture"): controller._mcp_hub_validate_backup_receipt(fake, candidate) def test_observed_baseline_rejects_missing_per_image_tree_resolution_without_state_write( controller, monkeypatch, tmp_path ): project, candidate, model, backup, runtime, _images, state_path, prior = _observed_live_fixture( controller, monkeypatch, tmp_path ) original_state = state_path.read_bytes() monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) project["_mcp_hub_rendered_model"] = model _verified_runtime, facts, _model = controller.mcp_hub_verify_observed_baseline( project, prior, model ) facts.pop(controller.MCP_HUB_CONTAINERS[0]) with pytest.raises(controller.DeployError, match="mixed runtime changed"): controller.mcp_hub_record_observed_baseline( project, prior, candidate, backup, runtime, facts ) assert state_path.read_bytes() == original_state @pytest.mark.parametrize("service_name", ["postgres", "redis"]) def test_observed_baseline_accepts_only_the_exact_reviewed_external_transition( controller, monkeypatch, tmp_path, service_name ): project, candidate, model, _backup, _runtime, _images, state_path, prior = ( _observed_live_fixture(controller, monkeypatch, tmp_path) ) original_state = state_path.read_bytes() model["services"][service_name]["image"] = f"{service_name}:latest@sha256:{'f' * 64}" with pytest.raises(controller.DeployError, match="explicitly reviewed legacy-to-candidate transition"): controller.mcp_hub_verify_observed_baseline(project, prior, model) assert state_path.read_bytes() == original_state def test_observed_baseline_requires_historical_state_artifacts_to_match_the_receipt( controller, monkeypatch, tmp_path ): project, _candidate, model, _backup, _runtime, _images, state_path, prior = ( _observed_live_fixture(controller, monkeypatch, tmp_path) ) web = "itworx-mcp-hub-web-1" prior["live_artifacts"]["containers"][web]["container_id"] = "f" * 64 state_path.write_text(json.dumps(prior), encoding="utf-8") state_path.chmod(0o600) original_state = state_path.read_bytes() with pytest.raises(controller.DeployError, match="historical state artifacts differ"): controller.mcp_hub_verify_observed_baseline(project, prior, model) assert state_path.read_bytes() == original_state @pytest.mark.parametrize( ("drift", "message"), [ ("no-health", "identity or health"), ("wrong-service", "identity or health"), ("mount-drift", "mount set"), ], ) def test_observed_baseline_rejects_health_label_and_mount_drift_before_recording( controller, monkeypatch, tmp_path, drift, message ): project, candidate, model, _backup, runtime, _images, state_path, prior = _observed_live_fixture( controller, monkeypatch, tmp_path ) original_state = state_path.read_bytes() target = "itworx-mcp-hub-web-1" if drift == "no-health": runtime[target]["State"]["Health"]["Status"] = "none" elif drift == "wrong-service": runtime[target]["Config"]["Labels"]["com.docker.compose.service"] = "worker" else: runtime[target]["Mounts"] = [{ "Type": "bind", "Source": "/srv/example/appdata/unexpected", "Destination": "/opt/unexpected", "RW": True, }] monkeypatch.setattr( controller, "mcp_hub_attested_live_mounts", lambda *_args, **_kwargs: (_ for _ in ()).throw( controller.DeployError("MCP Hub live mount set differs from its rendered Compose model") ), ) with pytest.raises(controller.DeployError, match=message): controller.mcp_hub_verify_observed_baseline(project, prior, model) assert state_path.read_bytes() == original_state assert not (tmp_path / "mcp-hub-topology-migrations").exists() def test_hub_backup_anchor_is_below_controller_state_not_appdata(controller): assert controller.MCP_HUB_BACKUP_ROOT == controller.STATE_ROOT / "mcp-hub-backups" assert not controller.MCP_HUB_BACKUP_ROOT.is_relative_to(controller.MCP_HUB_APPDATA_ROOT) def test_legacy_state_migration_cannot_promote_or_rewrite_the_unknown_baseline( controller, monkeypatch, tmp_path ): project, candidate, model, backup, runtime, state_path, prior = _legacy_migration_fixture( controller, monkeypatch, tmp_path ) prior["status"] = "failed" prior["live_status"] = "unknown" state_path.write_text(json.dumps(prior), encoding="utf-8") state_path.chmod(0o600) original_state = state_path.read_bytes() with pytest.raises(controller.DeployError, match="cannot be promoted"): controller.migrate_mcp_hub_legacy_state( project, controller.MCP_HUB_REPOSITORY, candidate, model, backup, runtime ) assert state_path.read_bytes() == original_state assert not (tmp_path / "mcp-hub-topology-migrations").exists() @pytest.mark.skipif(os.name == "nt", reason="migration intent durability requires POSIX directory fsync") def test_migration_intent_is_durable_and_bound_to_observed_evidence_and_backup( controller, monkeypatch, tmp_path ): project, candidate, _model, backup, _runtime, state_path, _prior, evidence, original_state = ( _record_test_observed_baseline(controller, monkeypatch, tmp_path) ) attempt_id = controller._mcp_hub_begin_migration_intent(project, candidate, backup) intent_path = tmp_path / "mcp-hub-topology-migrations" / "active-intent.json" intent_bytes = intent_path.read_bytes() intent = json.loads(intent_bytes) assert attempt_id == intent["attemptId"] assert intent["status"] == "started" assert intent["candidateRevision"] == candidate assert intent["backupManifestSha256"] == backup["manifestSha256"] assert intent["observedBaselinePath"] == evidence["evidencePath"] assert intent["observedBaselineSha256"] == evidence["evidenceSha256"] assert intent["baselineStateSha256"] == hashlib.sha256(original_state).hexdigest() assert stat.S_IMODE(intent_path.stat().st_mode) == 0o600 with pytest.raises(controller.DeployError, match="retry and image-only rollback are blocked"): controller._mcp_hub_assert_no_unresolved_migration_intent(project) with pytest.raises(controller.DeployError, match="retry and image-only rollback are blocked"): controller._mcp_hub_begin_migration_intent(project, candidate, backup) assert intent_path.read_bytes() == intent_bytes state = json.loads(state_path.read_text(encoding="utf-8")) state["status"] = "success" state["revision"] = candidate state["live_revision"] = candidate state["live_status"] = "healthy" state["live_legacy_provenance"] = False state["live_artifacts"] = { "containers": {name: {"container_id": name} for name in controller.MCP_HUB_CONTAINERS}, "topology_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, } state["last_attempt"] = { "revision": candidate, "status": "success", "metadata": { "mcpHubMigrationAttemptId": attempt_id, "mcpHubBackupReceipt": backup, }, } state_path.write_text(json.dumps(state), encoding="utf-8") state_path.chmod(0o600) controller._mcp_hub_assert_no_unresolved_migration_intent(project) completed = json.loads(intent_path.read_text(encoding="utf-8")) assert completed["status"] == "complete" assert completed["completedStateSha256"] == hashlib.sha256(state_path.read_bytes()).hexdigest() @pytest.mark.skipif(os.name == "nt", reason="migration intent durability requires POSIX directory fsync") def test_migration_intent_update_uses_compare_and_swap(controller, monkeypatch, tmp_path): project, candidate, _model, backup, _runtime, _state_path, _prior, _evidence, _original_state = ( _record_test_observed_baseline(controller, monkeypatch, tmp_path) ) controller._mcp_hub_begin_migration_intent(project, candidate, backup) path = tmp_path / "mcp-hub-topology-migrations" / "active-intent.json" intent, original = controller._mcp_hub_read_migration_intent() changed = {**intent, "startedAtEpoch": intent["startedAtEpoch"] + 1} path.write_text(json.dumps(changed), encoding="utf-8") path.chmod(0o600) with pytest.raises(controller.DeployError, match="changed before its compare-and-swap"): controller._mcp_hub_replace_migration_intent( path, json.dumps(intent).encode("utf-8"), expected_bytes=original, ) def test_retired_healthy_legacy_snapshot_path_is_not_available(controller): assert not hasattr(controller, "mcp_hub_migration_snapshot_state") def test_legacy_state_migration_rejects_promotion_even_when_receipt_identity_drifts( controller, monkeypatch, tmp_path ): project, candidate, model, backup, runtime, _state_path, prior = _legacy_migration_fixture( controller, monkeypatch, tmp_path ) changed = dict(prior) changed_artifacts = {**prior["live_artifacts"], "containers": dict(prior["live_artifacts"]["containers"])} changed_item = dict(changed_artifacts["containers"][controller.MCP_HUB_LEGACY_CONTAINERS[0]]) changed_item["container_id"] = "f" * 64 changed_artifacts["containers"][controller.MCP_HUB_LEGACY_CONTAINERS[0]] = changed_item changed["live_artifacts"] = changed_artifacts _state_path.write_text(json.dumps(changed), encoding="utf-8") _state_path.chmod(0o600) original_state = _state_path.read_bytes() with pytest.raises(controller.DeployError, match="cannot be promoted"): controller.migrate_mcp_hub_legacy_state( project, controller.MCP_HUB_REPOSITORY, candidate, model, backup, runtime, ) assert _state_path.read_bytes() == original_state def test_rollback_source_must_render_same_fixed_topology_and_live_image_refs( controller, monkeypatch, tmp_path ): project, candidate, model, _backup, runtime, _state_path, prior = _legacy_migration_fixture( controller, monkeypatch, tmp_path ) project["_repository_path"] = str(tmp_path / "hub.git") project["_deployment_metadata"] = {"revision": candidate} staged_prior = tmp_path / "trusted-releases" / str(prior["live_revision"]) staged_calls = [] monkeypatch.setattr( controller, "_mcp_hub_stage_protected_release_source", lambda _project, _repo, revision, *, protected_head: ( staged_calls.append((revision, protected_head)) or staged_prior ), ) for name, container in runtime.items(): service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") container["Config"] = {"Image": model["services"][service_name]["image"]} monkeypatch.setattr( controller, "extract_revision", lambda *_args: pytest.fail("rollback must use the trusted staged release tree"), ) def render_rollback(rollback_project, _revision, _repository): assert Path(rollback_project["source_dir"]) == staged_prior return model monkeypatch.setattr(controller, "mcp_hub_compose_model", render_rollback) monkeypatch.setattr( controller, "verify_mcp_hub_live_topology", lambda *_args: runtime ) monkeypatch.setattr(controller, "verify_mcp_hub_live_images", lambda *_args: None) candidate_model = _valid_model(controller) candidate_model["services"]["control-api"]["image"] = "registry.example/control-api:new-reviewed-tag" rollback_model, verified_runtime = controller.mcp_hub_verify_rollback_source( project, prior, candidate_model ) assert rollback_model is model assert verified_runtime is runtime assert staged_calls == [(prior["live_revision"], candidate)] assert project["_mcp_hub_allow_legacy_live_mounts"] is True changed_source = _valid_model(controller) changed_source["services"]["postgres"]["image"] = "postgres:unexpected" monkeypatch.setattr(controller, "mcp_hub_compose_model", lambda *_args: changed_source) with pytest.raises(controller.DeployError, match="exact running image references"): controller.mcp_hub_verify_rollback_source(project, prior, model) monkeypatch.setattr( controller, "mcp_hub_compose_model", lambda *_args: (_ for _ in ()).throw( controller.DeployError("MCP Hub rendered Compose validation failed") ), ) with pytest.raises(controller.DeployError, match="rendered Compose validation failed"): controller.mcp_hub_verify_rollback_source(project, prior, model) def test_legacy_snapshot_rollback_uses_trusted_prior_tree_and_rejects_legacy_after_up( controller, monkeypatch, tmp_path ): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) candidate = "c" * 40 prior = "a" * 40 build = { "id": prior[:12], "repository": controller.MCP_HUB_REPOSITORY, "revision": prior, "tree": "d" * 40, "treeDigest": "e" * 64, "builtAt": "2026-09-30T10:00:00.000Z", } project["_repository_path"] = str(tmp_path / "hub.git") project["_deployment_metadata"] = {"revision": candidate} trusted_prior = tmp_path / "trusted-releases" / prior model = _valid_model(controller) containers = [] by_name = {} for index, name in enumerate(controller.MCP_HUB_CONTAINERS): service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") item = { "name": name, "container_id": f"{index + 1:064x}", "image_id": "sha256:" + f"{index + 1:064x}", "image_ref": model["services"][service_name]["image"], "mounts": [], } containers.append(item) by_name[name] = item snapshot = { "topology_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "revision": prior, "legacy_provenance": True, "receipt": {"build": build}, "containers": containers, } staged = [] rendered = [] started = [] observed = [] monkeypatch.setattr(controller, "repository_path_from_project", lambda _project: tmp_path / "hub.git") monkeypatch.setattr( controller, "_mcp_hub_stage_protected_release_source", lambda rollback_project, _repo, revision, *, protected_head: ( staged.append((revision, protected_head)) or trusted_prior ), ) monkeypatch.setattr(controller, "compose_project_container_names", lambda _name: set(controller.MCP_HUB_CONTAINERS)) def render(rollback_project, revision, repository): rendered.append((Path(rollback_project["source_dir"]), revision, repository)) return model monkeypatch.setattr(controller, "mcp_hub_compose_model", render) monkeypatch.setattr(controller, "_mcp_hub_prepare_rollback_image", lambda *_args: None) monkeypatch.setattr( controller, "inspect_container", lambda name: {"Id": by_name[name]["container_id"], "Image": by_name[name]["image_id"]}, ) def attest(_model, _service, _project_name, _container, *, project, **_kwargs): assert project.get("_mcp_hub_allow_legacy_live_mounts") is True return [] monkeypatch.setattr(controller, "mcp_hub_attested_live_mounts", attest) monkeypatch.setattr( controller, "mcp_hub_up_services", lambda rollback_project, revision, repository, services: started.append( (Path(rollback_project["source_dir"]), revision, repository, services) ), ) def verify_topology(rollback_project, _model): assert "_mcp_hub_allow_legacy_live_mounts" not in rollback_project assert Path(rollback_project["source_dir"]) == trusted_prior observed.append("trusted-runtime-verified") return {} monkeypatch.setattr(controller, "verify_mcp_hub_live_topology", verify_topology) verified_receipt = {"repository": controller.MCP_HUB_REPOSITORY, "revision": prior} monkeypatch.setattr( controller, "verify_deployment", lambda rollback_project, metadata, *, expected_image_ids: ( observed.append((Path(rollback_project["source_dir"]), metadata, len(expected_image_ids))) or verified_receipt ), ) controller.restore_mcp_hub_snapshot( project, snapshot, controller.MCP_HUB_REPOSITORY, candidate ) assert staged == [(prior, candidate)] assert rendered == [(trusted_prior, prior, controller.MCP_HUB_REPOSITORY)] assert len(started) == 4 assert all(item[0] == trusted_prior and item[1] == prior for item in started) assert observed[0] == "trusted-runtime-verified" assert project["_rollback_receipt"] is verified_receipt assert project["_rollback_restored_revision"] == prior def test_hub_deploy_runs_the_fixed_migration_and_startup_stages(controller, monkeypatch, tmp_path): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) project["_repository_path"] = str(tmp_path / "hub.git") revision = "a" * 40 model = _valid_model(controller) commands = [] stages = [] migration_attempts = [] events = [] postgres_volume_baseline = [{"destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME}] project["_mcp_hub_observed_postgres_volume_baseline"] = postgres_volume_baseline monkeypatch.setattr(controller, "mcp_hub_require_docker_capacity", lambda: None) monkeypatch.setattr( controller, "mcp_hub_stage_release_source", lambda *_args: tmp_path / "release", ) monkeypatch.setattr(controller, "mcp_hub_compose_model", lambda *_args: model) monkeypatch.setattr(controller, "compose_command", lambda _project: (tmp_path, ["docker", "compose", "--project-name", controller.MCP_HUB_PROJECT])) monkeypatch.setattr(controller, "deployment_environment", lambda *_args: {}) monkeypatch.setattr(controller, "run", lambda command, **_kwargs: commands.append(command)) monkeypatch.setattr( controller, "mcp_hub_require_candidate_images", lambda *_args: events.append("candidate-verified") or {name: "sha256:" + "a" * 64 for name in controller.MCP_HUB_CONTAINERS}, ) monkeypatch.setattr( controller, "_mcp_hub_begin_migration_intent", lambda _project, _revision, _backup: ( migration_attempts.append("intent") or events.append("intent") or "1" * 32 ), ) monkeypatch.setattr( controller, "prepare_compose_migration", lambda _project: pytest.fail("fixed Hub deploy must not rename or adopt containers"), ) monkeypatch.setattr( controller, "mcp_hub_up_services", lambda _p, _r, _repo, names: stages.append(names) or events.append("up"), ) monkeypatch.setattr( controller, "_mcp_hub_require_observed_postgres_volume_continuity", lambda _project: events.append(("volume-continuity", len(stages))), ) monkeypatch.setattr(controller, "verify_mcp_hub_database_revision", lambda *_args: "abc123") monkeypatch.setattr( controller, "verify_mcp_hub_live_topology", lambda *_args, **_kwargs: {} ) monkeypatch.setattr( controller.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0), ) controller.deploy_mcp_hub_staged(project, revision, controller.MCP_HUB_REPOSITORY) assert not any("build" in command for command in commands) migration = next(command for command in commands if "SERVICE_NAME=database-migrate" in command) assert migration[-8:] == [ "run", "--rm", "--no-deps", "--pull", "never", "-e", "SERVICE_NAME=database-migrate", "control-api" ] assert project["_mcp_hub_migration_started"] is True assert project["_mcp_hub_migration_attempt_id"] == "1" * 32 assert project["_mcp_hub_production_mutation_started"] is True assert project["_legacy_rollbacks"] == [] assert migration_attempts == ["intent"] assert events == [ "candidate-verified", ("volume-continuity", 0), "intent", "up", ("volume-continuity", 1), "candidate-verified", "up", "up", "up", "candidate-verified", ("volume-continuity", 4), ] assert stages == [ controller.MCP_HUB_DATABASE_SERVICES, controller.MCP_HUB_CONTEXTFORGE_SERVICES, controller.MCP_HUB_APPLICATION_SERVICES, controller.MCP_HUB_INTAKE_SERVICES, ] assert "down" not in [part for command in commands for part in command] def test_hub_source_epoch_is_derived_from_the_exact_git_commit(controller, monkeypatch, tmp_path): commands = [] revision = "a" * 40 monkeypatch.setattr(controller, "output", lambda command: commands.append(command) or "1790888863") assert controller.mcp_hub_source_date_epoch(tmp_path / "hub.git", revision) == "1790888863" assert commands == [["git", f"--git-dir={tmp_path / 'hub.git'}", "show", "-s", "--format=%ct", revision]] @pytest.mark.parametrize("value", ["", "-1", "0", "1;touch /tmp/unsafe", "1\n2", "019", "true"]) def test_hub_source_epoch_rejects_noncanonical_git_output(controller, monkeypatch, tmp_path, value): monkeypatch.setattr(controller, "output", lambda _command: value) with pytest.raises(controller.DeployError, match="source epoch"): controller.mcp_hub_source_date_epoch(tmp_path / "hub.git", "a" * 40) def test_normal_hub_upgrade_requires_volume_baseline_before_migration(controller): project = {"_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, "_mcp_hub_rendered_model": _valid_model(controller)} with pytest.raises(controller.DeployError, match="volume baseline"): controller._mcp_hub_require_observed_postgres_volume_continuity(project) def _patch_release_volume_paths(controller, monkeypatch, tmp_path): releases = tmp_path / "protected-volume" / "releases" monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases) monkeypatch.setattr( controller, "_mcp_hub_release_directory", lambda revision: releases / revision, ) def require_release_anchor(*, create=False): if create: releases.mkdir(mode=0o700, parents=True, exist_ok=True) if not releases.is_dir() or releases.is_symlink(): raise controller.DeployError("MCP Hub protected release anchor is unavailable") return releases monkeypatch.setattr(controller, "_mcp_hub_require_trusted_release_anchor", require_release_anchor) return releases @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="migration-intent path-swap recovery requires root-owned POSIX fixtures", ) def test_hub_migration_intent_cas_removes_new_intent_from_swapped_directory( controller, monkeypatch, tmp_path ): shared = tmp_path / "shared" shared.mkdir(mode=0o777) shared.chmod(0o777) controller_parent = shared / "controller" controller_parent.mkdir(mode=0o700) state_root = controller_parent / "state" state_root.mkdir(mode=0o700) attacker = tmp_path / "attacker-migrations" attacker.mkdir(mode=0o700) attacker_intent = attacker / "active-intent.json" attacker_intent.write_bytes(b"attacker-intent") attacker_intent.chmod(0o600) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) path = controller._mcp_hub_migration_intent_path(create=True) migrations_path = path.parent pinned = controller._MCP_HUB_RELEASE_STORE.PinnedDirectory.open( migrations_path, check_ancestors=False ) helper = controller._MCP_HUB_RELEASE_STORE original_atomic_replace = helper.atomic_replace_at def swap_after_replace(*args, **kwargs): def swap(): migrations_path.rename(state_root / "migrations.saved") migrations_path.symlink_to(attacker, target_is_directory=True) kwargs["after_replace"] = swap return original_atomic_replace(*args, **kwargs) monkeypatch.setattr(helper, "atomic_replace_at", swap_after_replace) try: with pytest.raises(controller.DeployError, match="durable compare-and-swap"): controller._mcp_hub_replace_migration_intent( path, b"candidate-intent", expected_bytes=None ) with pytest.raises(FileNotFoundError): helper.read_at(pinned.fd, path.name, max_bytes=128) assert attacker_intent.read_bytes() == b"attacker-intent" finally: pinned.close() def test_hub_reconciler_requires_the_exact_four_file_release_bind_inventory(controller): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) revision = "d" * 40 project["_mcp_hub_required_release_revision"] = revision store = controller._MCP_HUB_RELEASE_STORE model = _valid_model(controller) for service, suffix, target, _kind in store.RELEASE_BIND_CONTRACT: model["services"][service]["volumes"].append( { "type": "bind", "source": f"{store.logical_release_path(revision)}/{suffix}", "target": target, "read_only": True, "mode": "ro", } ) assert controller.validate_mcp_hub_compose_model( project, model, require_catalog_mounts=True, catalog_source="logical", ) is model host_model = store.translate_release_mounts(model, revision) assert controller.validate_mcp_hub_compose_model( project, host_model, require_catalog_mounts=True, catalog_source="host", ) is host_model missing = copy.deepcopy(model) missing["services"]["redis"]["volumes"].pop(0) with pytest.raises(controller.DeployError, match="in-release bind inventory"): controller.validate_mcp_hub_compose_model( project, missing, require_catalog_mounts=True, catalog_source="logical", ) writable = copy.deepcopy(model) writable["services"]["contextforge"]["volumes"][0]["read_only"] = False with pytest.raises(controller.DeployError, match="in-release bind inventory"): controller.validate_mcp_hub_compose_model( project, writable, require_catalog_mounts=True, catalog_source="logical", ) broad = copy.deepcopy(model) broad["services"]["control-api"]["volumes"][0]["source"] = store.logical_release_path(revision) with pytest.raises(controller.DeployError, match="in-release bind inventory"): controller.validate_mcp_hub_compose_model( project, broad, require_catalog_mounts=True, catalog_source="logical", ) def test_hub_runtime_rejects_controller_state_mount_and_host_capabilities(controller): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) model = _valid_model(controller) model["services"]["control-api"]["volumes"] = [{ "type": "bind", "source": controller.STATE_ROOT.as_posix(), "target": "/controller-state", "read_only": True, }] with pytest.raises(controller.DeployError, match="only a protected catalog leaf read-only"): controller.validate_mcp_hub_compose_model(project, model) model["services"]["control-api"]["volumes"] = [] model["services"]["control-api"]["privileged"] = True with pytest.raises(controller.DeployError, match="host filesystem or remount capabilities"): controller.validate_mcp_hub_compose_model(project, model) model["services"]["control-api"].pop("privileged") model["services"]["control-api"]["cap_add"] = ["SYS_ADMIN"] with pytest.raises(controller.DeployError, match="host filesystem or remount capabilities"): controller.validate_mcp_hub_compose_model(project, model) @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="state path-swap recovery requires root-owned POSIX fixtures", ) def test_hub_state_cas_restores_original_file_before_path_readback(controller, monkeypatch, tmp_path): shared = tmp_path / "shared" shared.mkdir(mode=0o777) shared.chmod(0o777) controller_parent = shared / "controller" controller_parent.mkdir(mode=0o700) state_root = controller_parent / "state" state_root.mkdir(mode=0o700) state_path = state_root / f"{controller.MCP_HUB_PROJECT}.json" state_path.write_bytes(b"baseline-state") state_path.chmod(0o600) attacker = tmp_path / "attacker-state" attacker.mkdir(mode=0o700) attacker_state = attacker / state_path.name attacker_state.write_bytes(b"attacker-state") attacker_state.chmod(0o600) pinned = controller._MCP_HUB_RELEASE_STORE.PinnedDirectory.open( state_root, check_ancestors=False ) monkeypatch.setattr(controller, "STATE_ROOT", state_root) monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) helper = controller._MCP_HUB_RELEASE_STORE original_atomic_replace = helper.atomic_replace_at def swap_after_replace(*args, **kwargs): def swap(): state_root.rename(controller_parent / "state.saved") state_root.symlink_to(attacker, target_is_directory=True) kwargs["after_replace"] = swap return original_atomic_replace(*args, **kwargs) monkeypatch.setattr(helper, "atomic_replace_at", swap_after_replace) try: with pytest.raises(controller.DeployError, match="durable compare-and-swap"): controller._mcp_hub_write_controller_state_bytes( b"candidate-state", expected_bytes=b"baseline-state" ) assert helper.read_at(pinned.fd, state_path.name, max_bytes=128) == b"baseline-state" assert attacker_state.read_bytes() == b"attacker-state" finally: pinned.close() def test_projects_registry_uses_fixed_config_alias_only_for_protected_install(controller): assert controller.CONFIG_PATH == controller.ROOT / "projects.json" assert controller.controller_config_root(Path("/opt/gitea-deploy")) == Path( "/controller-hub-releases/controller-config" ) assert controller._CONTROLLER_PATHS.controller_readonly_config_root( Path("/opt/gitea-deploy") ) == Path("/controller-config") @pytest.mark.skipif(os.name == "nt", reason="POSIX ACL inspection is unavailable on Windows") def test_state_trust_fails_closed_when_acl_metadata_cannot_be_inspected(controller, monkeypatch): def unavailable(*_args, **_kwargs): raise OSError("filesystem ACL metadata is unavailable") monkeypatch.setattr(controller.os, "listxattr", unavailable) with pytest.raises(controller.DeployError, match="cannot inspect filesystem ACLs"): controller._mcp_hub_require_no_posix_acl(Path("/tmp")) def test_hub_full_sha_is_required_before_compose_preparation(controller, monkeypatch, tmp_path): target = _legacy_target(controller) monkeypatch.setattr(controller, "load_projects", lambda: {controller.MCP_HUB_REPOSITORY: target}) monkeypatch.setattr(controller, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr( controller, "output", lambda *_args, **_kwargs: pytest.fail("invalid revision reached Git or Compose preparation"), ) monkeypatch.setattr( controller, "deployment_environment", lambda *_args, **_kwargs: pytest.fail("invalid revision reached Compose environment setup"), ) monkeypatch.setattr( controller, "mcp_hub_load_release_candidate", lambda *_args, **_kwargs: pytest.fail("invalid revision reached candidate preparation"), ) monkeypatch.setattr( sys, "argv", ["deploy.py", "deploy", controller.MCP_HUB_REPOSITORY, "a" * 12], ) with pytest.raises(controller.DeployError, match="full lowercase hexadecimal Git SHA"): controller.main()