diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index ece1b6e..991c00f 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `f4a918d9dba3663faa1f675dcc335c9ec951f30c`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `2e53f628d284213b90ca2f9c5e121671f5933574`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index 7c575d0..ad032f1 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "f4a918d9dba3663faa1f675dcc335c9ec951f30c", + "sourceRevision": "2e53f628d284213b90ca2f9c5e121671f5933574", "files": [ { "path": ".dockerignore", @@ -509,8 +509,8 @@ }, { "path": "backend/app/services/deployment_control_plane.py", - "sha256": "bfea241b036111180b42259af770847f15e396284261cb29000e1db4f637867c", - "bytes": 28812 + "sha256": "4443d9a1581cf24c1cc3eccaaa1e0e67527228e28a873924ab0c3591fd67da99", + "bytes": 25070 }, { "path": "backend/app/services/deployment_target_registry.py", @@ -589,8 +589,8 @@ }, { "path": "backend/app/services/mcp_hub_deployment_reconciliation.py", - "sha256": "d45921d486b7debba7dc7d68a274beb2103c20fc5b2968f9380b5c22dfea6d95", - "bytes": 31417 + "sha256": "2367cf52e39091bd941b19e12aeccf72f65842cdf5ec74e5eb17bd0f636f48bf", + "bytes": 44919 }, { "path": "backend/app/services/mcp_operations.py", @@ -869,13 +869,13 @@ }, { "path": "backend/tests/test_deploy_broker.py", - "sha256": "7a59aa36564fb056def3453324f376bb0580fdeee99a32d4a3180a22fd16bebc", - "bytes": 305380 + "sha256": "6bc23cb5883961316acae07daf12ec261e1c21e229bf5ee8ebc56ff245f6a34d", + "bytes": 309532 }, { "path": "backend/tests/test_deployment_auto_rollback.py", - "sha256": "07e10b748bbfea5277b9b0ca9b2ab38edf90d76a5fe976560e5384ef1e8c90c8", - "bytes": 13355 + "sha256": "e8eb4fd4be632fcb25e85778adf4174492771c41b7cde0aae6f3746641efcc8a", + "bytes": 5966 }, { "path": "backend/tests/test_deployment_contract.py", @@ -982,21 +982,31 @@ "sha256": "d704cd4ba299ce2fc51d6f4a8a1b26ecdc2dea0e734e63dc6a65e1ba4c3ed37f", "bytes": 692 }, + { + "path": "backend/tests/test_mcp_hub_candidate_preparation.py", + "sha256": "aa5525214e034464ac3586252ba5bad09f59c96c861cb9449a3dc86ae7a91c74", + "bytes": 22525 + }, { "path": "backend/tests/test_mcp_hub_deployment_reconciliation.py", - "sha256": "0400d0b0ca31a62d869166b9368fa0c018c0e44edba98387d12633976988da29", - "bytes": 14176 + "sha256": "476d02f8c5261e55788b16f8ee490728291b2379395890bbedc3a9be56ae5f58", + "bytes": 28475 }, { "path": "backend/tests/test_mcp_hub_governed_topology.py", - "sha256": "f521c7d843f7d5fe9a5ac3d68bde1f072f38991d5e30e8f9bea72b77061c79f8", - "bytes": 106551 + "sha256": "117c07dfb34e9ab0fac5675dd51585906b33d7bf5ea0a059d2d8f6eb4d2897a7", + "bytes": 118901 }, { "path": "backend/tests/test_mcp_hub_policy_reconciliation.py", "sha256": "50cf920443570240331d8851195bdd3140eeb93397b938c7f78b35749f6b7399", "bytes": 5041 }, + { + "path": "backend/tests/test_mcp_hub_release_candidates.py", + "sha256": "f2add61e18641e4a5075948dc0b3b0e36b2b4bfe389d608078a7c5cf506fad54", + "bytes": 34190 + }, { "path": "backend/tests/test_mcp_hub_release_store.py", "sha256": "91dbd0da6a131f76a04c51f93e31a77f4acd057f788f4b1a6fce75d37c23525e", @@ -1344,12 +1354,12 @@ }, { "path": "pnpm-lock.yaml", - "sha256": "f699c9ed2c5649b0d41f0e01420ef377161b08ea163c58fd00c5d09cf696fe0d", + "sha256": "ca7ed34cc053954acb98b3a704d23f954f4e06eafddfbaf6c27a38ad72245e33", "bytes": 215991 }, { "path": "pnpm-workspace.yaml", - "sha256": "bd8c09cbbfdf628ec0c5cce6ceee452d6c8b75373f61da28f347b157fb96a7d4", + "sha256": "2dcca254705ad46c71c774714739640b640c11014db4a49171986e5308b30882", "bytes": 9120 }, { @@ -1972,6 +1982,11 @@ "sha256": "b11ff524bb105035e9320691ca80bdb4e8b616d7c28b1f0d0827b13c0f3914f5", "bytes": 681 }, + { + "path": "src/lib/buildDependencySecurity.test.ts", + "sha256": "a9e011262388ed6d11a173297819166cf0676cc8d68efff9872e0493c05e1ec7", + "bytes": 1369 + }, { "path": "src/lib/clipboard.ts", "sha256": "f85b6cfecaa4814d5c6519ba8d42fe0ebb63d8513eff9ab442c4ea286c3bd4a6", diff --git a/backend/app/services/deployment_control_plane.py b/backend/app/services/deployment_control_plane.py index da58df1..5a2faed 100644 --- a/backend/app/services/deployment_control_plane.py +++ b/backend/app/services/deployment_control_plane.py @@ -33,8 +33,7 @@ _TERMINAL = {"preflight_failed", "succeeded", "failed", "rolled_back", "rollback _BROKER_EVIDENCE_FIELDS = {"request", "repository", "revision", "status"} _OIDC_EVIDENCE_FIELDS = {"success", "status", "applicationId", "providerId", "clientSecretCreated"} _PROXY_EVIDENCE_FIELDS = {"success", "status", "hostId", "hostname"} -_AUTO_ROLLBACK_REPOSITORY = "jens/itworx_mcp_hub" -_AUTO_ROLLBACK_APPLICATION_ID = "itworx-mcp-hub" +_CANONICAL_HUB_REPOSITORY = "jens/itworx_mcp_hub" class DeploymentControlError(ValueError): @@ -312,123 +311,23 @@ class DeploymentControlPlane: **dict(job.evidence_json or {}), "postApplyVerification": {"status": "failed", "detail": message}, } - if ( - plan.repository.casefold() != _AUTO_ROLLBACK_REPOSITORY - or plan.application_id != _AUTO_ROLLBACK_APPLICATION_ID - ): - job.status = "failed" - job.stage = "external_integration_failed" - job.result_summary = message - job.completed_at = utc_now() - return - self._attempt_auto_rollback(session, job, plan, reason=message) - - def _attempt_auto_rollback( - self, - session: Session, - job: DeploymentJob, - plan: DeploymentPlanV1, - *, - reason: str, - ) -> None: - """Restore only the controller-retained prior receipt after verification failure.""" - policy = evaluate_policy( - session, - action="deployment.rollback", - target=plan.application_id, - source="manual", - ) - exact_policies = session.exec( - select(Policy).where( - (Policy.action == "deployment.rollback") - & (Policy.target == plan.application_id) - ) - ).all() - exact_policy = exact_policies[0] if len(exact_policies) == 1 else None - policy_is_exact_and_strict = ( - exact_policy is not None - and exact_policy.manual == "allowed" - and exact_policy.voice == "blocked" - and exact_policy.risk == "high" - and policy.get("policy_id") == exact_policy.id - ) - if policy.get("decision") != "allowed" or not policy_is_exact_and_strict: + job.status = "failed" + job.stage = "external_integration_failed" + if plan.repository.casefold() == _CANONICAL_HUB_REPOSITORY: job.evidence_json = { **dict(job.evidence_json or {}), - "autoRollback": {"status": "blocked", "reason": "exact-target-policy"}, + "recovery": { + "status": "manual_required", + "reason": "hub_data_aware_recovery_required", + }, } - job.status = "failed" - job.stage = "external_integration_failed" - job.result_summary = reason - job.completed_at = utc_now() - return - - secret = get_secret(session, plan.broker_secret_ref) - if not secret: - job.evidence_json = { - **dict(job.evidence_json or {}), - "autoRollback": {"status": "blocked", "reason": "controller-secret"}, - } - job.status = "failed" - job.stage = "external_integration_failed" - job.result_summary = reason - job.completed_at = utc_now() - return - - rollback_key = f"auto-rollback:{job.id}" - rollback = self.broker.rollback(plan, secret=secret, idempotency_key=rollback_key) - rollback_status = str(rollback.get("status") or "failed") - rollback_request = str(rollback.get("request") or "") or None - rollback_evidence = { - key: rollback[key] - for key in _BROKER_EVIDENCE_FIELDS - if key in rollback - } - rollback_evidence["status"] = rollback_status - job.evidence_json = { - **dict(job.evidence_json or {}), - "autoRollback": rollback_evidence, - } - if rollback_request: - job.broker_request_id = rollback_request - - if rollback_status == "rolled_back": - job.status = "rolled_back" - job.stage = "auto_rollback_verified" job.result_summary = ( - "Post-apply verification failed; the controller restored the prior verified deployment." + f"{message} Automatic image-only rollback is disabled for MCP Hub; " + "data-aware recovery is required." ) - job.completed_at = utc_now() - elif rollback_status in {"accepted", "already_queued", "running"}: - if not rollback_request: - job.status = "failed" - job.stage = "auto_rollback_failed" - job.evidence_json = { - **dict(job.evidence_json or {}), - "autoRollback": { - **rollback_evidence, - "reason": "missing-controller-request", - }, - } - job.result_summary = ( - "Post-apply verification failed and automatic rollback " - "returned no pollable controller request." - ) - job.completed_at = utc_now() - return - job.status = "rolling_back" - job.stage = "auto_rollback_queued" - job.result_summary = ( - "Post-apply verification failed; automatic rollback is in progress." - ) - job.completed_at = None else: - job.status = "failed" - job.stage = "auto_rollback_failed" - job.result_summary = ( - "Post-apply verification failed and the controller did not verify rollback." - ) - job.completed_at = utc_now() + job.result_summary = message + job.completed_at = utc_now() def execute( self, @@ -438,6 +337,13 @@ class DeploymentControlPlane: idempotency_key: str, operation: Literal["preflight", "apply", "rollback"], ) -> dict[str, Any]: + if ( + operation in {"apply", "rollback"} + and plan.repository.casefold() == _CANONICAL_HUB_REPOSITORY + ): + raise DeploymentControlError( + "MCP Hub apply and rollback require the signed protected push workflow" + ) job = self.create_job(session, plan=plan, idempotency_key=idempotency_key, operation=operation) if job.status != "queued": return self.serialize_job(job, replay=True) diff --git a/backend/app/services/mcp_hub_deployment_reconciliation.py b/backend/app/services/mcp_hub_deployment_reconciliation.py index 9845657..82a6c35 100644 --- a/backend/app/services/mcp_hub_deployment_reconciliation.py +++ b/backend/app/services/mcp_hub_deployment_reconciliation.py @@ -12,21 +12,21 @@ _REPOSITORY = "jens/itworx_mcp_hub" _COMPOSE_PROJECT = "itworx-mcp-hub" _TOPOLOGY_CONTRACT = "itworx.mcp-hub-topology/17-v1" -# One-time fail-closed re-attestation for the existing MCP Hub runtime. The -# script never builds, restarts, recreates, stops or deploys a container. It -# repairs only controller state after proving the exact target, Git ancestry, -# complete Compose topology and every currently running image identity. +# Read-only re-attestation for an already-successful modern MCP Hub receipt. +# Unknown, failed, legacy and mixed-topology states require the guarded deploy +# path. Reconciliation never repairs or writes controller state. _RECONCILE_SCRIPT = r''' import fcntl +import errno +import hashlib import json import os from pathlib import Path import re -import shutil import stat import subprocess import sys -import time +from datetime import datetime, timezone repository, expected_compose_project = sys.argv[1:3] registry_path = Path("/controller-config/projects.json") @@ -80,7 +80,9 @@ if ( or project.get("slug") != expected_compose_project or project.get("compose_project") != expected_compose_project or not isinstance(expected_containers, list) - or set(expected_containers) != expected_fixed + or any(not isinstance(item, str) for item in expected_containers) + or len(expected_containers) != len(set(expected_containers)) + or frozenset(expected_containers) not in {frozenset(expected_fixed), frozenset(expected_runtime)} ): raise SystemExit(23) expected_containers = sorted(expected_runtime) @@ -89,8 +91,6 @@ state_path = state_root / f"{expected_compose_project}.json" project_lock_path = state_root / "locks" / f"{expected_compose_project}.lock" owner, name = repository.split("/", 1) bare_repo = Path("/gitea-repositories") / owner / f"{name}.git" -canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories") -canonical_bare_repo = canonical_repository_root / owner / f"{name}.git" if not bare_repo.is_dir() or bare_repo.is_symlink(): raise SystemExit(24) @@ -106,44 +106,6 @@ def git_run(*args): ) -def canonical_git_run(*args): - return subprocess.run( - ["git", f"--git-dir={canonical_bare_repo}", *args], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - timeout=10, - check=False, - ) - - -def canonical_image_ancestry_is_proven(recorded_revision, image_revision, expected_main): - if not canonical_bare_repo.is_dir() or canonical_bare_repo.is_symlink(): - return False - resolved_recorded = canonical_git_run("rev-parse", f"{recorded_revision}^{{commit}}") - resolved_image = canonical_git_run("rev-parse", f"{image_revision}^{{commit}}") - resolved_main = canonical_git_run("rev-parse", "refs/heads/main") - if not ( - resolved_recorded.returncode == 0 - and resolved_recorded.stdout.strip() == recorded_revision - and resolved_image.returncode == 0 - and resolved_image.stdout.strip() == image_revision - and resolved_main.returncode == 0 - and resolved_main.stdout.strip() == expected_main - ): - return False - return ( - canonical_git_run( - "merge-base", "--is-ancestor", recorded_revision, image_revision - ).returncode - == 0 - and canonical_git_run( - "merge-base", "--is-ancestor", image_revision, expected_main - ).returncode - == 0 - ) - - main = git_run("rev-parse", "refs/heads/main") main_revision = main.stdout.strip() if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None: @@ -219,13 +181,20 @@ def main_declares_sessionless_dataplane_image(container_name, image_ref): def main_declares_contextforge_compat_image(container_name, image_ref): if container_name != "itworx-mcp-hub-contextforge-1": return False + candidate_reference = ( + re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None + and image_ref == f"itworx-mcp-hub/contextforge-schema:{recorded_live}" + ) + # The caller already binds recorded_live to the receipt and protected-main + # ancestry. Read a candidate's build contract from that exact source tree. + proof_revision = recorded_live if candidate_reference else main_revision manifest = git_run( "show", - f"{main_revision}:infrastructure/images/contextforge/compat-image.env", + f"{proof_revision}:infrastructure/images/contextforge/compat-image.env", ) builder = git_run( "show", - f"{main_revision}:scripts/build_contextforge_compat_image.sh", + f"{proof_revision}:scripts/build_contextforge_compat_image.sh", ) if manifest.returncode != 0 or builder.returncode != 0: return False @@ -257,7 +226,7 @@ def main_declares_contextforge_compat_image(container_name, image_ref): or re.fullmatch(r"[0-9a-f]{40}", upstream_commit) is None or re.fullmatch(r"[0-9a-f]{12}", alembic_head) is None or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", mcp_sdk) is None - or local_image != image_ref + or (local_image != image_ref and not candidate_reference) or local_image != f"itworx-mcp-hub/contextforge-schema:{alembic_head}" ): return False @@ -269,18 +238,370 @@ def main_declares_contextforge_compat_image(container_name, image_ref): 'alembic -c mcpgateway/alembic.ini heads', 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"', ) + if candidate_reference: + required_builder_contract += ( + '--release-candidate', + 'CANDIDATE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}"', + 'docker tag "$STAGED_IMAGE" "$CANDIDATE_IMAGE"', + ) return all(fragment in builder.stdout for fragment in required_builder_contract) -state_root.mkdir(parents=True, exist_ok=True) -project_lock_path.parent.mkdir(parents=True, exist_ok=True) +def live_attested_mounts(container): + raw_mounts = container.get("Mounts") + if not isinstance(raw_mounts, list): + raise SystemExit(52) + observed = [] + for raw in raw_mounts: + if not isinstance(raw, dict): + raise SystemExit(52) + kind = str(raw.get("Type") or "") + destination = str(raw.get("Destination") or "") + if kind not in {"bind", "volume", "tmpfs"} or not destination: + raise SystemExit(52) + source = str(raw.get("Source") or "") if kind in {"bind", "volume"} else "" + name = str(raw.get("Name") or "") if kind == "volume" else "" + mount = { + "type": kind, + "source": os.path.normpath(source) if source else "", + "destination": destination, + "rw": raw.get("RW") is True, + "name": name, + } + if kind == "volume": + if not name: + raise SystemExit(52) + volume = subprocess.run( + ["docker", "volume", "inspect", name], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=10, + check=False, + ) + if volume.returncode != 0: + raise SystemExit(52) + try: + volume_items = json.loads(volume.stdout) + volume_item = volume_items[0] + options = volume_item.get("Options") or {} + mountpoint = os.path.normpath(str(volume_item.get("Mountpoint") or "")) + created_at = volume_item.get("CreatedAt") + created = datetime.fromisoformat(str(created_at).replace("Z", "+00:00")) + except Exception: + raise SystemExit(52) + if ( + not isinstance(options, dict) + or str(volume_item.get("Name") or "") != name + or mountpoint != mount["source"] + or str(volume_item.get("Driver") or "") != str(raw.get("Driver") or "") + or not isinstance(created_at, str) + or created.tzinfo is None + or created.utcoffset() is None + or created.astimezone(timezone.utc) > datetime.now(timezone.utc) + ): + raise SystemExit(52) + mount["driver"] = str(volume_item.get("Driver") or "") + mount["options"] = {str(key): str(value) for key, value in options.items()} + mount["created_at"] = created_at + observed.append(mount) + return sorted(observed, key=lambda item: str(item["destination"])) + + +def fixed_image_reference(service_name, image_ref): + if service_name == "dataplane-router" and image_ref == "itworx-mcp-hub/dataplane:sessionless-final": + return main_declares_sessionless_dataplane_image( + "itworx-mcp-hub-dataplane-router-1", image_ref + ) + owned = { + "connector-appops": "gitea.example.com/jens/itworx-mcp-hub/connector-appops", + "connector-gitea": "gitea.example.com/jens/itworx-mcp-hub/connector-gitea", + "connector-ingestion": "gitea.example.com/jens/itworx-mcp-hub/connector-ingestion", + "connector-knowledge": "gitea.example.com/jens/itworx-mcp-hub/connector-knowledge", + "connector-mobilityops": "gitea.example.com/jens/itworx-mcp-hub/connector-mobilityops", + "connector-projectbrain": "gitea.example.com/jens/itworx-mcp-hub/connector-projectbrain", + "connector-stitch": "gitea.example.com/jens/itworx-mcp-hub/connector-stitch", + "connector-unraid": "gitea.example.com/jens/itworx-mcp-hub/connector-unraid", + "control-api": "gitea.example.com/jens/itworx-mcp-hub/control-api", + "dataplane-router": "gitea.example.com/jens/itworx-mcp-hub/dataplane", + "docker-read-broker": "gitea.example.com/jens/itworx-mcp-hub/docker-read-broker", + "web": "gitea.example.com/jens/itworx-mcp-hub/web", + "worker": "gitea.example.com/jens/itworx-mcp-hub/worker", + } + if service_name in owned: + return ( + re.fullmatch(r"sha256:[0-9a-f]{64}", image_ref) is not None + or ( + image_ref.startswith(owned[service_name] + "@sha256:") + and re.fullmatch(r".+@sha256:[0-9a-f]{64}", image_ref) is not None + ) + ) + if service_name == "postgres": + return image_ref == "postgres:18.4@sha256:3a82e1f56c8f0f5616a11103ac3d47e632c3938698946a7ad26da0df1334744a" or ( + image_ref == "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04" + and main_declares_external_image(image_ref) + ) + if service_name == "ingestion-clamd": + return image_ref == "clamav/clamav:1.4.6@sha256:a5f03c12a79dbe9f6d8a527b6bb1ea053fa8dd061d3738a26897f055ee2d9303" + if service_name == "redis": + return image_ref == "redis:8.10.0@sha256:c29e49ab2f85760a3827b53882e6dd9f5c6c3f0bb7d724e07bb31cbf275a5236" or ( + image_ref == "redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48" + and main_declares_external_image(image_ref) + ) + if service_name == "contextforge": + return main_declares_contextforge_compat_image( + "itworx-mcp-hub-contextforge-1", image_ref + ) + return False + + +def image_digest_matches(image, image_ref): + repository, separator, digest = image_ref.partition("@") + if not separator or re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None: + return False + first, slash, last = repository.partition("/") + if slash: + if ":" in last: + last = last.rsplit(":", 1)[0] + repository = first + slash + last + elif ":" in repository: + repository = repository.rsplit(":", 1)[0] + aliases = {repository} + first_component = repository.split("/", 1)[0] + is_registry = "." in first_component or ":" in first_component or first_component == "localhost" + if not is_registry: + if "/" in repository: + aliases.add("docker.io/" + repository) + else: + aliases.add("docker.io/library/" + repository) + if repository.startswith("docker.io/"): + aliases.add(repository.removeprefix("docker.io/")) + digests = image.get("RepoDigests", []) + return isinstance(digests, list) and any( + "@" in str(item) + and str(item).split("@", 1)[0] in aliases + and str(item).split("@", 1)[1] == digest + for item in digests + ) + + +def receipt_bound_image_reference(launch_image, image_id, recorded, receipt_item): + if not isinstance(recorded, dict) or not isinstance(receipt_item, dict): + raise SystemExit(52) + image_ref = receipt_item.get("image_ref") + if ( + re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None + or not isinstance(image_ref, str) + or not image_ref + or recorded.get("image_ref") != image_ref + or recorded.get("image_id") != image_id + or receipt_item.get("image_id") != image_id + or launch_image not in {image_ref, image_id} + or (image_ref.startswith("sha256:") and image_ref != image_id) + ): + raise SystemExit(52) + return image_ref + + +def trusted_private_path(path, expected_mode, *, directory): + try: + info = path.lstat() + except OSError: + return False + if ( + path.is_symlink() + or (not stat.S_ISDIR(info.st_mode) if directory else not stat.S_ISREG(info.st_mode)) + or info.st_uid != 0 + or (stat.S_IMODE(info.st_mode) != expected_mode if directory else stat.S_IMODE(info.st_mode) & 0o077 != 0) + ): + return False + try: + attributes = os.listxattr(path, follow_symlinks=False) + except OSError as exc: + if exc.errno in {errno.ENOTSUP, getattr(errno, "EOPNOTSUPP", errno.ENOTSUP)}: + attributes = [] + else: + return False + return not any( + item in {"system.posix_acl_access", "system.posix_acl_default"} + for item in attributes + ) + + +def validate_migration_intent_state( + state_root, state_bytes, repository, recorded_live, recorded_status, + recorded_live_status, last_attempt, attempt_status, attempt_revision, +): + attempt_metadata = dict(last_attempt.get("metadata") or {}) + migration_attempt_id = str(attempt_metadata.get("mcpHubMigrationAttemptId") or "") + migration_root = state_root / "mcp-hub-topology-migrations" + migration_path = migration_root / "active-intent.json" + if migration_root.is_symlink(): + raise SystemExit(56) + if migration_root.exists() and not trusted_private_path(migration_root, 0o700, directory=True): + raise SystemExit(56) + if migration_path.is_symlink(): + raise SystemExit(56) + if migration_path.exists(): + if ( + not trusted_private_path(migration_path, 0o600, directory=False) + or stat.S_IMODE(migration_path.lstat().st_mode) != 0o600 + ): + raise SystemExit(56) + try: + intent_bytes = migration_path.read_bytes() + intent = json.loads(intent_bytes.decode("utf-8")) + except Exception: + raise SystemExit(56) + common_intent_fields = { + "schema", "status", "attemptId", "repository", "candidateRevision", + "priorLiveRevision", "backupManifestSha256", "backupComposeSha256", + "baselineStateSha256", "startedAtEpoch", + } + observed_intent_fields = {"observedBaselinePath", "observedBaselineSha256"} + observed_fields_present = set(intent) & observed_intent_fields if isinstance(intent, dict) else set() + if observed_fields_present not in (set(), observed_intent_fields): + raise SystemExit(56) + required_intent_fields = common_intent_fields | observed_fields_present + if isinstance(intent, dict) and intent.get("status") == "complete": + required_intent_fields |= {"completedAtEpoch", "completedStateSha256"} + if ( + not isinstance(intent, dict) + or intent.get("status") != "complete" + or set(intent) != required_intent_fields + or intent.get("schema") != "itworx.mcp-hub-migration-intent/v1" + or intent.get("repository") != repository + or re.fullmatch(r"[0-9a-f]{32}", str(intent.get("attemptId") or "")) is None + or intent.get("attemptId") != migration_attempt_id + or intent.get("candidateRevision") != recorded_live + or re.fullmatch(r"[0-9a-f]{40}", str(intent.get("priorLiveRevision") or "")) is None + or any( + re.fullmatch(r"[0-9a-f]{64}", str(intent.get(field) or "")) is None + for field in ("backupManifestSha256", "backupComposeSha256", "baselineStateSha256", "completedStateSha256") + ) + or type(intent.get("startedAtEpoch")) is not int + or type(intent.get("completedAtEpoch")) is not int + or intent.get("completedAtEpoch") < intent.get("startedAtEpoch") + or intent.get("completedStateSha256") != hashlib.sha256(state_bytes).hexdigest() + or recorded_status != "success" + or recorded_live_status != "healthy" + or attempt_status != "success" + or attempt_revision != recorded_live + or not isinstance(attempt_metadata.get("mcpHubBackupReceipt"), dict) + or set(attempt_metadata.get("mcpHubBackupReceipt", {})) != { + "schema", "repository", "revision", "backupDirectory", "manifestSha256", + "composeSha256", "restoreVerified", "createdAtEpoch", "receiptPath", + } + or attempt_metadata.get("mcpHubBackupReceipt", {}).get("schema") != "itworx.mcp-hub-backup-receipt/v1" + or attempt_metadata.get("mcpHubBackupReceipt", {}).get("repository") != repository + or attempt_metadata.get("mcpHubBackupReceipt", {}).get("revision") != recorded_live + or attempt_metadata.get("mcpHubBackupReceipt", {}).get("restoreVerified") is not True + or type(attempt_metadata.get("mcpHubBackupReceipt", {}).get("createdAtEpoch")) is not int + or re.fullmatch(r"[0-9a-f]{64}", str(attempt_metadata.get("mcpHubBackupReceipt", {}).get("manifestSha256") or "")) is None + or re.fullmatch(r"[0-9a-f]{64}", str(attempt_metadata.get("mcpHubBackupReceipt", {}).get("composeSha256") or "")) is None + or attempt_metadata.get("mcpHubBackupReceipt", {}).get("manifestSha256") != intent.get("backupManifestSha256") + or attempt_metadata.get("mcpHubBackupReceipt", {}).get("composeSha256") != intent.get("backupComposeSha256") + ): + raise SystemExit(56) + backup_receipt = attempt_metadata.get("mcpHubBackupReceipt", {}) + receipt_root = state_root / "mcp-hub-backup-receipts" + backup_path = Path(str(backup_receipt.get("receiptPath") or "")) + if ( + backup_path.parent != receipt_root + or re.fullmatch( + rf"[0-9]+-{recorded_live[:12]}-[0-9a-f]{{16}}\.json", + backup_path.name, + ) is None + or backup_path.is_symlink() + or not trusted_private_path(receipt_root, 0o700, directory=True) + or not trusted_private_path(backup_path, 0o600, directory=False) + or stat.S_IMODE(backup_path.lstat().st_mode) != 0o600 + ): + raise SystemExit(56) + try: + stored_backup_receipt = json.loads(backup_path.read_text(encoding="utf-8")) + except Exception: + raise SystemExit(56) + if stored_backup_receipt != { + key: value for key, value in backup_receipt.items() if key != "receiptPath" + }: + raise SystemExit(56) + backup_root = state_root / "mcp-hub-backups" + backup_directory = Path(str(backup_receipt.get("backupDirectory") or "")) + manifest_path = backup_directory / "manifest.json" + if ( + backup_directory.parent != backup_root + or re.fullmatch( + rf"[0-9]+-{recorded_live[:12]}-[0-9a-f]{{16}}", + backup_directory.name, + ) is None + or not trusted_private_path(backup_root, 0o700, directory=True) + or not trusted_private_path(backup_directory, 0o700, directory=True) + or not trusted_private_path(manifest_path, 0o600, directory=False) + or stat.S_IMODE(manifest_path.lstat().st_mode) != 0o600 + ): + raise SystemExit(56) + try: + manifest_bytes = manifest_path.read_bytes() + manifest = json.loads(manifest_bytes.decode("utf-8")) + except Exception: + raise SystemExit(56) + if ( + hashlib.sha256(manifest_bytes).hexdigest() != intent.get("backupManifestSha256") + or not isinstance(manifest, dict) + or manifest.get("schema") != "itworx.mcp-hub-backup/v1" + or manifest.get("repository") != repository + or manifest.get("revision") != recorded_live + or manifest.get("backupDirectory") != str(backup_directory) + or manifest.get("composeSha256") != intent.get("backupComposeSha256") + or manifest.get("configRestoreVerified") is not True + or dict(manifest.get("restore") or {}).get("status") != "verified" + or set(dict(manifest.get("databases") or {})) != {"itworx_mcp_hub", "contextforge"} + ): + raise SystemExit(56) + if observed_fields_present: + observed_path = Path(str(intent.get("observedBaselinePath") or "")) + if ( + observed_path.parent != migration_root + or re.fullmatch( + rf"observed-baseline-{recorded_live[:12]}-[0-9a-f]{{20}}\.json", + observed_path.name, + ) is None + or observed_path.is_symlink() + or not trusted_private_path(observed_path, 0o600, directory=False) + or stat.S_IMODE(observed_path.lstat().st_mode) != 0o600 + ): + raise SystemExit(56) + try: + observed_bytes = observed_path.read_bytes() + observed = json.loads(observed_bytes.decode("utf-8")) + except Exception: + raise SystemExit(56) + if ( + hashlib.sha256(observed_bytes).hexdigest() != intent.get("observedBaselineSha256") + or not isinstance(observed, dict) + or observed.get("schema") != "itworx.mcp-hub-observed-baseline/v1" + or observed.get("claim") != "observed_mixed_runtime_facts_only" + or observed.get("candidateRevision") != recorded_live + or observed.get("historicalStateSha256") != intent.get("baselineStateSha256") + or observed.get("backupManifestSha256") != intent.get("backupManifestSha256") + or observed.get("backupComposeSha256") != intent.get("backupComposeSha256") + or observed.get("backupRestoreVerified") is not True + or not isinstance(observed.get("postgresVolumeBaseline"), list) + or len(observed.get("postgresVolumeBaseline", [])) != 2 + ): + raise SystemExit(56) + elif migration_attempt_id: + raise SystemExit(56) + + try: - project_lock = project_lock_path.open("a+", encoding="utf-8") - global_lock = global_lock_path.open("a+", encoding="utf-8") + project_lock_fd = os.open(project_lock_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + global_lock_fd = os.open(global_lock_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) except Exception: raise SystemExit(29) -with project_lock: +with os.fdopen(project_lock_fd, "rb") as project_lock, os.fdopen(global_lock_fd, "rb") as global_lock: try: fcntl.flock(project_lock, fcntl.LOCK_EX) except Exception: @@ -298,10 +619,17 @@ with project_lock: if current_projects.get(repository) != project: raise SystemExit(37) - if state_path.is_symlink() or not state_path.is_file(): + if ( + not trusted_private_path(state_root, 0o700, directory=True) + or not trusted_private_path(state_root / "locks", 0o700, directory=True) + or not trusted_private_path(project_lock_path, 0o600, directory=False) + or not trusted_private_path(global_lock_path, 0o600, directory=False) + or not trusted_private_path(state_path, 0o600, directory=False) + ): raise SystemExit(42) try: - state = json.loads(state_path.read_text(encoding="utf-8")) + state_bytes = state_path.read_bytes() + state = json.loads(state_bytes.decode("utf-8")) except Exception: raise SystemExit(42) if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository: @@ -313,9 +641,12 @@ with project_lock: last_attempt = dict(state.get("last_attempt") or {}) attempt_revision = str(last_attempt.get("revision") or "") attempt_status = str(last_attempt.get("status") or "") - last_attempt_message = str(last_attempt.get("message") or "") receipt = state.get("live_receipt") recorded_artifacts = state.get("live_artifacts") + validate_migration_intent_state( + state_root, state_bytes, repository, recorded_live, recorded_status, + recorded_live_status, last_attempt, attempt_status, attempt_revision, + ) registered_live_containers = ( recorded_artifacts.get("containers") @@ -326,86 +657,73 @@ with project_lock: print("mcp-hub-topology-migration-required", flush=True) raise SystemExit(56) + if set(expected_containers) != expected_runtime: + raise SystemExit(56) + + receipt_payload = receipt if isinstance(receipt, dict) else {} + receipt_tree = str(receipt_payload.get("tree") or "") + receipt_tree_digest = str(receipt_payload.get("treeDigest") or "") + raw_receipt_build = receipt_payload.get("build") + receipt_build = raw_receipt_build if isinstance(raw_receipt_build, dict) else {} + expected_build = { + "id": recorded_live[:12], + "repository": repository, + "revision": recorded_live, + "tree": receipt_tree, + "treeDigest": receipt_tree_digest, + "builtAt": str(receipt_build.get("builtAt") or ""), + } + receipt_identity_valid = ( + re.fullmatch(r"[0-9a-f]{40}", receipt_tree) is not None + and receipt_tree_digest == hashlib.sha256( + f"git-tree:{receipt_tree}".encode("utf-8") + ).hexdigest() + and receipt_build == expected_build + and re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", + expected_build["builtAt"], + ) is not None + and isinstance(recorded_artifacts, dict) + and set(recorded_artifacts) + == {"containers", "compose_sha256", "topology_contract"} + and re.fullmatch( + r"[0-9a-f]{64}", str(recorded_artifacts.get("compose_sha256") or "") + ) is not None + ) + already_reconciled = ( recorded_status == "success" and recorded_live_status == "healthy" + and state.get("revision") == recorded_live + and state.get("live_legacy_provenance") is False and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None and isinstance(recorded_artifacts, dict) and state.get("compose_project") == expected_compose_project + and state.get("slug") == expected_compose_project + and attempt_status == "success" + and attempt_revision == recorded_live and isinstance(registered_live_containers, dict) and set(registered_live_containers) == expected_runtime and recorded_artifacts.get("topology_contract") == topology_contract - and ( - state.get("live_legacy_provenance") is True - or ( - isinstance(receipt, dict) - and receipt.get("schema") == "gitea-deploy-receipt.v1" - and receipt.get("repository") == repository - and receipt.get("revision") == recorded_live - ) - ) + and state.get("artifacts") == recorded_artifacts + and isinstance(receipt, dict) + and set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"} + and receipt.get("schema") == "gitea-deploy-receipt.v1" + and receipt.get("repository") == repository + and receipt.get("revision") == recorded_live + and receipt_identity_valid + and isinstance(receipt.get("containers"), dict) + and set(dict(receipt.get("containers") or {})) == expected_runtime + and receipt.get("live") is None ) - if not already_reconciled and ( - recorded_status != "failed" - or recorded_live_status not in {"healthy", "unknown"} - or state.get("live_legacy_provenance") is not False - or attempt_status != "failed" - or re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None - or re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is None - or attempt_revision == recorded_live - or not isinstance(receipt, dict) - or receipt.get("schema") != "gitea-deploy-receipt.v1" - or receipt.get("repository") != repository - or receipt.get("revision") != recorded_live - or not isinstance(recorded_artifacts, dict) - ): - raise SystemExit(52) - - # deploy.py deliberately downgrades the prior live status to unknown - # when capture_compose_rollback() fails before production mutation. - # Accept that state only when the durable attempt message is one of the - # bounded snapshot-stage failure classes and no rollback/source-restore - # error was recorded. The runtime is still re-proven container by - # container below before any controller state is changed. - snapshot_failure_prefixes = ( - "Refusing to mutate a Compose project with unrecorded containers:", - "Deployment state claims a live revision but all production containers are missing", - "Cannot capture a complete rollback snapshot; containers are missing:", - "Existing production containers require a valid healthy live deployment state", - "Container ", - "Existing deployment state is not bound to its live revision", - "Recorded deployment receipt ", - "Existing deployment no longer matches its verified live receipt", - "Expected containers are not healthy/running:", - "Compose project container set does not match the allowlisted runtime:", - "Containers are linked to the wrong Compose project:", - "Running container image IDs do not match the verified artifacts:", - "Immutable image provenance does not match the requested deployment:", - ) - if ( - not already_reconciled - and recorded_live_status == "unknown" - and ( - not last_attempt_message.startswith(snapshot_failure_prefixes) - or "; rollback error:" in last_attempt_message - or "; source rollback error:" in last_attempt_message - ) - ): - raise SystemExit(55) + if not already_reconciled: + raise SystemExit(56 if recorded_live_status in {"unknown", "healthy"} else 52) resolved_live = git_run("rev-parse", f"{recorded_live}^{{commit}}") if resolved_live.returncode != 0 or resolved_live.stdout.strip() != recorded_live: raise SystemExit(25) if git_run("merge-base", "--is-ancestor", recorded_live, main_revision).returncode != 0: raise SystemExit(52) - if not already_reconciled: - resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}") - if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision: - raise SystemExit(25) - if git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision).returncode != 0: - raise SystemExit(52) - if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0: - raise SystemExit(52) listed = subprocess.run( [ @@ -428,11 +746,10 @@ with project_lock: raise SystemExit(47) old_containers = recorded_artifacts.get("containers") + receipt_containers = dict(receipt.get("containers") or {}) if not isinstance(old_containers, dict) or set(old_containers) != set(expected_containers): raise SystemExit(52) - container_artifacts = {} - drift_found = False for name in expected_containers: inspected = subprocess.run( ["docker", "inspect", name], @@ -450,17 +767,31 @@ with project_lock: raise SystemExit(44) runtime_state = dict(container.get("State") or {}) health = str(dict(runtime_state.get("Health") or {}).get("Status") or "none") - labels = dict(dict(container.get("Config") or {}).get("Labels") or {}) config = dict(container.get("Config") or {}) + labels = dict(config.get("Labels") or {}) + name_service = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") image_id = str(container.get("Image") or "") - image_ref = str(config.get("Image") or "") - if not runtime_state.get("Running") or health in {"starting", "unhealthy"}: + launch_image = str(config.get("Image") or "") + container_id = str(container.get("Id") or "") + if runtime_state.get("Running") is not True or health != "healthy": raise SystemExit(45) - if labels.get("com.docker.compose.project") != expected_compose_project: + if ( + labels.get("com.docker.compose.project") != expected_compose_project + or labels.get("com.docker.compose.service") != name_service + ): raise SystemExit(47) - if re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None or not image_ref: + if ( + re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None + or not launch_image + or not container_id + ): raise SystemExit(46) + recorded = old_containers.get(name) + receipt_item = receipt_containers.get(name) + image_ref = receipt_bound_image_reference( + launch_image, image_id, recorded, receipt_item + ) image_result = subprocess.run( ["docker", "image", "inspect", image_id], text=True, @@ -476,149 +807,80 @@ with project_lock: except Exception: raise SystemExit(50) image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {}) - image_source = str(image_labels.get("org.opencontainers.image.source") or "") - image_revision = str(image_labels.get("org.opencontainers.image.revision") or "") - - recorded = old_containers.get(name) - if not isinstance(recorded, dict): - raise SystemExit(52) - recorded_image_id = str(recorded.get("image_id") or "") - recorded_container_id = str(recorded.get("container_id") or "") - if re.fullmatch(r"sha256:[0-9a-f]{64}", recorded_image_id) is None: - raise SystemExit(52) - - image_changed = image_id != recorded_image_id - if image_changed: - contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref) - sessionless_dataplane_image = ( - not already_reconciled - and main_declares_sessionless_dataplane_image(name, image_ref) - ) - if contextforge_compat_image or sessionless_dataplane_image: - pass - elif image_source == owned_image_source: - if re.fullmatch(r"[0-9a-f]{40}", image_revision) is None: - raise SystemExit(51) - resolved_image = git_run("rev-parse", f"{image_revision}^{{commit}}") - mounted_recorded_to_image = git_run("merge-base", "--is-ancestor", recorded_live, image_revision) - mounted_image_to_main = git_run("merge-base", "--is-ancestor", image_revision, main_revision) - mounted_ancestry_proven = ( - resolved_image.returncode == 0 - and resolved_image.stdout.strip() == image_revision - and mounted_recorded_to_image.returncode == 0 - and mounted_image_to_main.returncode == 0 - ) - if not mounted_ancestry_proven and not canonical_image_ancestry_is_proven( - recorded_live, image_revision, main_revision - ): - raise SystemExit(51) - elif not main_declares_external_image(image_ref): - raise SystemExit(51) - - container_id = str(container.get("Id") or "") - if not container_id: - raise SystemExit(46) - if image_id != recorded_image_id or container_id != recorded_container_id: - drift_found = True - - container_artifacts[name] = { + artifact = { "container_id": container_id, "image_id": image_id, "image_ref": image_ref, "container_revision": str(labels.get("org.opencontainers.image.revision") or ""), - "image_revision": image_revision, - "image_source": image_source, + "image_revision": str(image_labels.get("org.opencontainers.image.revision") or ""), + "image_source": str(image_labels.get("org.opencontainers.image.source") or ""), "image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""), "image_version": str(image_labels.get("org.opencontainers.image.version") or ""), "image_created": str(image_labels.get("org.opencontainers.image.created") or ""), } - - if already_reconciled: - if drift_found: + if str(image.get("Id") or "") != image_id: + raise SystemExit(46) + if ( + set(recorded) != set(artifact) | {"mounts_json"} + or set(receipt_item) != set(artifact) | {"mounts_sha256"} + ): raise SystemExit(52) - print("already-reconciled", flush=True) - raise SystemExit(0) - if not drift_found: - raise SystemExit(52) + for key, value in artifact.items(): + if recorded.get(key) != value or receipt_item.get(key) != value: + raise SystemExit(52) - artifacts = { - "containers": container_artifacts, - "compose_sha256": "", - "topology_contract": topology_contract, - } - now = int(time.time()) - payload = { - "schema_version": 3, - "repository": repository, - "slug": expected_compose_project, - "compose_project": expected_compose_project, - "revision": recorded_live, - "live_revision": recorded_live, - "live_status": "healthy", - "live_legacy_provenance": True, - "live_artifacts": artifacts, - "live_receipt": {}, - "rollback_snapshot": {}, - "status": "success", - "message": "Verified mixed MCP Hub runtime reconciled as a legacy provenance baseline", - "updated_at_epoch": now, - "artifacts": artifacts, - "last_attempt": { - "revision": recorded_live, - "status": "success", - "message": "Verified MCP Hub runtime state reconciliation", - "updated_at_epoch": now, - "metadata": {"reconciled_failed_revision": attempt_revision}, - }, - "previous_revision": recorded_live, - "rollback_restored_revision": "", - } - - backup = state_path.with_name(f"{state_path.name}.mcp-hub-reconcile-backup-{now}-{os.getpid()}") - temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.mcp-hub-reconcile.tmp") - try: - mode = stat.S_IMODE(state_path.stat().st_mode) - shutil.copy2(state_path, backup) - with temporary.open("x", encoding="utf-8") as handle: - handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") - handle.flush() - os.fsync(handle.fileno()) - os.chmod(temporary, mode) - os.replace(temporary, state_path) - directory_fd = os.open(str(state_root), os.O_RDONLY) + raw_mounts = recorded.get("mounts_json") + if not isinstance(raw_mounts, str): + raise SystemExit(52) try: - os.fsync(directory_fd) - finally: - os.close(directory_fd) - except Exception: - try: - temporary.unlink(missing_ok=True) + recorded_mounts = json.loads(raw_mounts) except Exception: - pass - raise SystemExit(53) + raise SystemExit(52) + mount_bytes = json.dumps( + recorded_mounts, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + if ( + not isinstance(recorded_mounts, list) + or hashlib.sha256(mount_bytes).hexdigest() != receipt_item.get("mounts_sha256") + or live_attested_mounts(container) != recorded_mounts + ): + raise SystemExit(52) + + if not fixed_image_reference(name_service, image_ref): + raise SystemExit(51) + owned_service = name_service in { + "connector-appops", "connector-gitea", "connector-ingestion", + "connector-knowledge", "connector-mobilityops", "connector-projectbrain", + "connector-stitch", "connector-unraid", "control-api", "dataplane-router", + "docker-read-broker", "web", "worker", + } + if owned_service and image_ref != "itworx-mcp-hub/dataplane:sessionless-final": + if ( + artifact["image_source"] != owned_image_source + or artifact["image_revision"] != recorded_live + or artifact["image_tree"] != receipt_tree + or artifact["image_version"] != recorded_live[:12] + or artifact["image_created"] != expected_build["builtAt"] + ): + raise SystemExit(51) + if "@sha256:" in image_ref and not image_digest_matches(image, image_ref): + raise SystemExit(51) try: - verify = json.loads(state_path.read_text(encoding="utf-8")) + if state_path.read_bytes() != state_bytes: + raise SystemExit(52) + current_projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: - raise SystemExit(54) - verified_containers = dict(dict(verify.get("live_artifacts") or {}).get("containers") or {}) - if not ( - verify.get("repository") == repository - and verify.get("live_revision") == recorded_live - and verify.get("live_status") == "healthy" - and verify.get("live_legacy_provenance") is True - and verify.get("status") == "success" - and set(verified_containers) == set(expected_containers) - and all(str(dict(verified_containers[name]).get("image_id") or "") == str(container_artifacts[name]["image_id"]) for name in expected_containers) - ): - raise SystemExit(54) + raise SystemExit(37) + if current_projects.get(repository) != project: + raise SystemExit(37) -print("reconciled", flush=True) +print("already-reconciled", flush=True) ''' class McpHubDeploymentReconciliation: - """Re-attest only the exact proven MCP Hub state drift, without touching runtime.""" + """Read-only verify only an existing, exact modern MCP Hub success receipt.""" def __init__(self, portainer: PortainerConnector | None = None): self.portainer = portainer or PortainerConnector() @@ -706,11 +968,8 @@ class McpHubDeploymentReconciliation: 47: "live MCP Hub Compose project differs from the controller target", 50: "a live MCP Hub image cannot be inspected", 51: "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain", - 52: "controller state drift is not the exact known recoverable MCP Hub state", - 53: "reconciled MCP Hub deployment state could not be written durably", - 54: "reconciled MCP Hub deployment state did not verify after write", - 55: "unknown MCP Hub live status was not produced by a proven recoverable snapshot-stage failure", - 56: "MCP Hub 15-to-17 topology migration requires its guarded protected-main deployment", + 52: "controller state or live facts do not match the exact modern MCP Hub receipt", + 56: "MCP Hub legacy, unknown, mixed, or unresolved migration state requires guarded deployment or data-aware recovery", } if exit_code != 0: if exit_code == 56: diff --git a/backend/tests/test_deploy_broker.py b/backend/tests/test_deploy_broker.py index 4871382..58b74da 100644 --- a/backend/tests/test_deploy_broker.py +++ b/backend/tests/test_deploy_broker.py @@ -5718,6 +5718,131 @@ def test_appops_request_is_idempotent_and_runs_only_fixed_deploy(status_module, assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", "jens/repo-a", "a" * 40]] +@pytest.mark.parametrize("operation", ["apply", "rollback"]) +def test_generic_appops_hub_request_is_rejected_before_inbox_creation( + status_module, tmp_path, monkeypatch, operation +): + inbox = tmp_path / "appops-inbox" + monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) + + status, request_id = status_module._enqueue_appops_request( + repository="Jens/ITWorx_MCP_Hub", + revision="a" * 40, + operation=operation, + idempotency_key=f"appops-hub-{operation}-0001", + plan={}, + ) + + assert (status, request_id) == ("invalid", "") + assert not inbox.exists() + + +@pytest.mark.parametrize("operation", ["apply", "rollback"]) +@pytest.mark.parametrize( + ("prior_status", "expected_status"), + [("pending", "failed"), ("running", "outcome_unknown")], +) +def test_direct_appops_hub_record_is_denied_before_project_or_deploy_access( + status_module, tmp_path, monkeypatch, operation, prior_status, expected_status +): + inbox = tmp_path / "appops-inbox" + inbox.mkdir() + request_id = "a" * 64 + marker = inbox / f"{request_id}.json" + marker.write_text( + json.dumps( + { + "schema": "appops-deployment-request.v1", + "request": request_id, + "repository": "jens/itworx_mcp_hub", + "revision": "b" * 40, + "operation": operation, + "plan_sha256": "c" * 64, + "status": prior_status, + "received_at": 1, + "updated_at": 1, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + status_module, + "_load_projects", + lambda: pytest.fail("generic Hub records must not inspect deployment projects"), + ) + monkeypatch.setattr( + status_module.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("generic Hub records must not launch deploy.py"), + ) + + status_module._process_appops_record(marker) + + record = json.loads(marker.read_text(encoding="utf-8")) + assert record["status"] == expected_status + assert record["reason"] == "canonical_hub_push_required" + + +def test_hub_protected_push_worker_remains_the_fixed_deploy_path( + status_module, tmp_path, monkeypatch +): + repository = "jens/itworx_mcp_hub" + inbox = tmp_path / "webhook-inbox" + projects = tmp_path / "projects.json" + project = { + **_webhook_project(repository), + "slug": "itworx-mcp-hub", + "containers": ["itworx-mcp-hub"], + } + projects.write_text(json.dumps({repository: project}), encoding="utf-8") + monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) + monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) + + revision = "d" * 40 + raw_body = json.dumps( + _push_payload(revision, repository=repository), separators=(",", ":") + ).encode() + _, request_id = status_module._enqueue_webhook_delivery( + raw_body, + "145440ab-b7b1-4c74-ae84-996d8d48167e", + 77, + "refs/heads/main", + repository, + revision, + status_module._validation_policy(repository, project), + ) + marker = inbox / f"{request_id}.json" + calls = [] + + class Result: + returncode = 0 + + monkeypatch.setattr( + status_module.subprocess, + "run", + lambda arguments, **_kwargs: calls.append(arguments) or Result(), + ) + monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) + monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"ok": True}) + monkeypatch.setattr( + status_module, + "_fetch_gitea_validation", + lambda *_args: ( + "pass", + [{ + "context": "Managed validation / full (push)", + "status": "success", + "updated_at": "2026-09-14T00:00:00Z", + }], + ), + ) + + status_module._process_webhook_record(marker) + + assert json.loads(marker.read_text(encoding="utf-8"))["status"] == "deployed" + assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", repository, revision]] + + def test_durable_rollback_snapshot_is_receipt_bound_and_not_caller_selected(deploy_module, tmp_path, monkeypatch): previous_revision = "a" * 40 current_revision = "b" * 40 diff --git a/backend/tests/test_deployment_auto_rollback.py b/backend/tests/test_deployment_auto_rollback.py index 28ddead..f640703 100644 --- a/backend/tests/test_deployment_auto_rollback.py +++ b/backend/tests/test_deployment_auto_rollback.py @@ -1,10 +1,14 @@ from __future__ import annotations -from sqlmodel import Session, SQLModel, create_engine +import pytest +from sqlmodel import Session, SQLModel, create_engine, select -from backend.app.db.models import Policy +from backend.app.db.models import DeploymentJob, Policy from backend.app.schemas.deployment import DeploymentPlanV1 -from backend.app.services.deployment_control_plane import DeploymentControlPlane +from backend.app.services.deployment_control_plane import ( + DeploymentControlError, + DeploymentControlPlane, +) from backend.app.services.secret_store import set_secret @@ -31,19 +35,22 @@ def _plan() -> DeploymentPlanV1: class _Broker: - def __init__(self, rollback_status: str = "rolled_back") -> None: - self.rollback_status = rollback_status + def __init__(self) -> None: + self.preflight_calls = 0 + self.apply_calls: list[str] = [] self.rollback_calls: list[str] = [] def preflight(self, plan, *, secret): + self.preflight_calls += 1 return {"status": "preflight_ok"} def apply(self, plan, *, secret, idempotency_key): + self.apply_calls.append(idempotency_key) return {"status": "deployed", "request": "a" * 64} def rollback(self, plan, *, secret, idempotency_key): self.rollback_calls.append(idempotency_key) - return {"status": self.rollback_status, "request": "b" * 64} + return {"status": "rolled_back", "request": "b" * 64} def status(self, request_id, *, secret): return {"status": "rolled_back", "request": request_id} @@ -103,11 +110,10 @@ def _plane(broker: _Broker) -> DeploymentControlPlane: ) -def _configure(session: Session, *, rollback_policy: bool = True) -> None: +def _configure(session: Session) -> None: set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32) session.add(_policy("deployment.apply")) - if rollback_policy: - session.add(_policy("deployment.rollback")) + session.add(_policy("deployment.rollback")) session.commit() @@ -117,284 +123,75 @@ def test_default_compose_file_survives_durable_plan_roundtrip(): assert restored.topology.compose_file == ".itworx/deployment.yml" -def test_post_apply_verification_failure_auto_rolls_back_with_exact_policy(tmp_path): - broker = _Broker("rolled_back") +@pytest.mark.parametrize("operation", ["apply", "rollback"]) +def test_hub_typed_mutation_is_rejected_before_job_or_broker_queue( + tmp_path, operation +): + broker = _Broker() + plane = _plane(broker) + with _session(tmp_path) as session: _configure(session) - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0001", - operation="apply", - ) + with pytest.raises(DeploymentControlError, match="signed protected push"): + plane.execute( + session, + plan=_plan(), + idempotency_key=f"mcp-hub-{operation}-typed-0001", + operation=operation, + ) - assert result["status"] == "rolled_back" - assert result["stage"] == "auto_rollback_verified" - assert result["evidence"]["postApplyVerification"]["status"] == "failed" - assert result["evidence"]["autoRollback"]["status"] == "rolled_back" - assert broker.rollback_calls == [f"auto-rollback:{result['id']}"] + assert session.exec(select(DeploymentJob)).all() == [] - -def test_post_apply_verification_failure_does_not_rollback_without_exact_policy(tmp_path): - broker = _Broker("rolled_back") - with _session(tmp_path) as session: - _configure(session, rollback_policy=False) - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0002", - operation="apply", - ) - - assert result["status"] == "failed" - assert result["stage"] == "external_integration_failed" - assert result["evidence"]["autoRollback"]["status"] == "blocked" + assert broker.preflight_calls == 0 + assert broker.apply_calls == [] assert broker.rollback_calls == [] -def test_post_apply_verification_failure_keeps_failure_when_controller_rejects_rollback(tmp_path): - broker = _Broker("rejected") - with _session(tmp_path) as session: - _configure(session) - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0003", - operation="apply", - ) - - assert result["status"] == "failed" - assert result["stage"] == "auto_rollback_failed" - assert result["evidence"]["postApplyVerification"]["status"] == "failed" - assert result["evidence"]["autoRollback"]["status"] == "rejected" - assert len(broker.rollback_calls) == 1 - - -def test_queued_auto_rollback_is_polled_and_finishes_as_auto_rollback(tmp_path): - broker = _Broker("accepted") +def test_hub_read_only_preflight_remains_available(tmp_path): + broker = _Broker() plane = _plane(broker) - with _session(tmp_path) as session: - _configure(session) - initial = plane.execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0004", - operation="apply", - ) - assert initial["status"] == "rolling_back" - assert initial["stage"] == "auto_rollback_queued" - finished = plane.refresh(session, initial["id"]) - - assert finished["status"] == "rolled_back" - assert finished["stage"] == "auto_rollback_verified" - assert finished["evidence"]["autoRollback"]["status"] == "rolled_back" - - -def test_successful_post_apply_verification_never_invokes_auto_rollback(tmp_path): - class _HealthyNpm(_FailingNpm): - def ensure_managed_proxy_host(self, **kwargs): - return {"success": True, "status": "created", "hostId": 42} - - broker = _Broker("rolled_back") - plane = DeploymentControlPlane( - broker=broker, - portainer=_Portainer(), - npm=_HealthyNpm(), - authentik=_Authentik(), - unraid=_Unraid(), - ) with _session(tmp_path) as session: _configure(session) result = plane.execute( session, plan=_plan(), - idempotency_key="mcp-hub-apply-0005", - operation="apply", + idempotency_key="mcp-hub-preflight-0001", + operation="preflight", ) - assert result["status"] == "succeeded" - assert result["stage"] == "controller_and_edge_verified" + assert result["status"] == "preflight_succeeded" + assert broker.preflight_calls == 1 + assert broker.apply_calls == [] assert broker.rollback_calls == [] -def test_auto_rollback_is_scoped_to_exact_mcp_hub_target(tmp_path): - broker = _Broker("rolled_back") - plan = _plan().model_copy(update={"repository": "jens/other-repository"}) - with _session(tmp_path) as session: - _configure(session) - result = _plane(broker).execute( - session, - plan=plan, - idempotency_key="mcp-hub-apply-0006", - operation="apply", - ) - assert result["status"] == "failed" - assert result["stage"] == "external_integration_failed" - assert result["evidence"]["postApplyVerification"]["status"] == "failed" - assert "autoRollback" not in result["evidence"] - assert broker.rollback_calls == [] - -def test_auto_rollback_rejects_same_repository_with_different_application(tmp_path): - broker = _Broker("rolled_back") - plan = _plan().model_copy( - update={ - "application_id": "itworx-mcp-hub-copy", - "topology": _plan().topology.model_copy( - update={ - "compose_project": "itworx-mcp-hub-copy", - "containers": ("itworx-mcp-hub-copy",), - } - ), - } - ) - with _session(tmp_path) as session: - set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32) - session.add( - Policy( - id="allow-copy-apply", - action="deployment.apply", - target="itworx-mcp-hub-copy", - voice="blocked", - manual="allowed", - risk="high", - reason="Test-only exact target.", - ) - ) - session.add( - Policy( - id="allow-copy-rollback", - action="deployment.rollback", - target="itworx-mcp-hub-copy", - voice="blocked", - manual="allowed", - risk="high", - reason="Test-only exact target.", - ) - ) - session.commit() - result = _plane(broker).execute( - session, - plan=plan, - idempotency_key="mcp-hub-apply-0007", - operation="apply", - ) - - assert result["status"] == "failed" - assert result["stage"] == "external_integration_failed" - assert result["evidence"]["postApplyVerification"]["status"] == "failed" - assert "autoRollback" not in result["evidence"] - assert broker.rollback_calls == [] - -def test_queued_auto_rollback_rejects_deployed_status_as_rollback_proof(tmp_path): - class _AmbiguousRollbackBroker(_Broker): - def status(self, request_id, *, secret): - return {"status": "deployed", "request": request_id} - - broker = _AmbiguousRollbackBroker("accepted") +def test_hub_post_apply_failure_records_data_aware_recovery_without_rollback(tmp_path): + broker = _Broker() plane = _plane(broker) + plan = _plan() + job = DeploymentJob( + id="mcp-hub-existing-apply", + application_id=plan.application_id, + repository=plan.repository, + revision=plan.revision, + plan_sha256="a" * 64, + idempotency_sha256="b" * 64, + status="applying", + plan_json=plan.model_dump(mode="json"), + evidence_json={"brokerStatus": "deployed"}, + ) + with _session(tmp_path) as session: _configure(session) - initial = plane.execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0008", - operation="apply", - ) - assert initial["status"] == "rolling_back" + plane._complete_apply_job(session, job, plan) - finished = plane.refresh(session, initial["id"]) - - assert finished["status"] == "failed" - assert finished["stage"] == "auto_rollback_failed" - assert finished["evidence"]["autoRollback"]["status"] == "accepted" - assert "deployed state" in finished["result"] - -def test_manual_rollback_requires_rolled_back_terminal_proof(tmp_path): - broker = _Broker("deployed") - with _session(tmp_path) as session: - _configure(session) - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-rollback-0009", - operation="rollback", - ) - - assert result["status"] == "rollback_failed" - assert result["stage"] == "controller_rejected" - - -def test_apply_rejects_rolled_back_terminal_status(tmp_path): - class _RolledBackApplyBroker(_Broker): - def apply(self, plan, *, secret, idempotency_key): - return {"status": "rolled_back", "request": "c" * 64} - - broker = _RolledBackApplyBroker() - with _session(tmp_path) as session: - _configure(session) - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0010", - operation="apply", - ) - - assert result["status"] == "failed" - assert result["stage"] == "controller_rejected" - assert broker.rollback_calls == [] - -def test_auto_rollback_refuses_duplicate_exact_rollback_policies(tmp_path): - broker = _Broker("rolled_back") - with _session(tmp_path) as session: - _configure(session) - session.add( - Policy( - id="duplicate-rollback-policy", - action="deployment.rollback", - target="itworx-mcp-hub", - voice="blocked", - manual="allowed", - risk="high", - reason="Duplicate exact rollback policy.", - ) - ) - session.commit() - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0011", - operation="apply", - ) - - assert result["status"] == "failed" - assert result["stage"] == "external_integration_failed" - assert result["evidence"]["autoRollback"] == { - "status": "blocked", - "reason": "exact-target-policy", + assert job.status == "failed" + assert job.stage == "external_integration_failed" + assert job.evidence_json["postApplyVerification"]["status"] == "failed" + assert job.evidence_json["recovery"] == { + "status": "manual_required", + "reason": "hub_data_aware_recovery_required", } + assert "Automatic image-only rollback is disabled" in job.result_summary assert broker.rollback_calls == [] - -def test_auto_rollback_rejects_unpollable_accepted_response(tmp_path): - class _MissingRequestBroker(_Broker): - def rollback(self, plan, *, secret, idempotency_key): - self.rollback_calls.append(idempotency_key) - return {"status": "accepted"} - - broker = _MissingRequestBroker() - with _session(tmp_path) as session: - _configure(session) - result = _plane(broker).execute( - session, - plan=_plan(), - idempotency_key="mcp-hub-apply-0012", - operation="apply", - ) - - assert result["status"] == "failed" - assert result["stage"] == "auto_rollback_failed" - assert result["evidence"]["autoRollback"] == { - "status": "accepted", - "reason": "missing-controller-request", - } - assert len(broker.rollback_calls) == 1 diff --git a/backend/tests/test_mcp_hub_candidate_preparation.py b/backend/tests/test_mcp_hub_candidate_preparation.py new file mode 100644 index 0000000..b4f8585 --- /dev/null +++ b/backend/tests/test_mcp_hub_candidate_preparation.py @@ -0,0 +1,556 @@ +from __future__ import annotations + +import hashlib +import importlib +import json +import os +import subprocess +import sys +import types +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +BROKER_DIR = ROOT / "ops" / "gitea-deploy-broker" + + +@pytest.fixture +def modules(monkeypatch): + if "fcntl" not in sys.modules: + shim = types.ModuleType("fcntl") + shim.LOCK_EX, shim.LOCK_UN, shim.flock = 2, 8, lambda *_args: None + monkeypatch.setitem(sys.modules, "fcntl", shim) + monkeypatch.syspath_prepend(str(BROKER_DIR)) + sys.modules.pop("deploy", None) + sys.modules.pop("prepare_mcp_hub_candidate", None) + controller = importlib.import_module("deploy") + utility = importlib.import_module("prepare_mcp_hub_candidate") + monkeypatch.setattr(utility, "_is_host_root", lambda: True) + yield controller, utility + sys.modules.pop("prepare_mcp_hub_candidate", None) + sys.modules.pop("deploy", None) + + +@pytest.fixture +def harness(modules, tmp_path, monkeypatch): + controller, utility = modules + revision = "a" * 40 + epoch = "1790888863" + now = ( + datetime.now(timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z") + ) + metadata = { + "id": revision[:12], + "repository": controller.MCP_HUB_REPOSITORY, + "revision": revision, + "tree": "b" * 40, + "treeDigest": hashlib.sha256(("git-tree:" + "b" * 40).encode()).hexdigest(), + "builtAt": now, + } + repo = tmp_path / "hub.git" + project = { + "branch": "main", + "slug": controller.MCP_HUB_PROJECT, + "compose_project": controller.MCP_HUB_PROJECT, + "containers": list(controller.MCP_HUB_CONTAINERS), + "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + "_repository_path": str(repo), + "repo_name": "itworx_mcp_hub", + } + state = types.SimpleNamespace( + source=tmp_path / "releases" / revision, + compose_marker="baseline", + images={}, + mutations=[], + inspect_calls=[], + ) + state.source.mkdir(parents=True) + (state.source / "scripts").mkdir() + draft = state.source / "scripts" / "draft_cve_exceptions.py" + draft.write_text( + "import argparse, json\n" + "from datetime import date, timedelta\n" + "from pathlib import Path\n" + "p=argparse.ArgumentParser()\n" + "p.add_argument('--scan-summary', required=True)\n" + "p.add_argument('--image-digests', required=True)\n" + "p.add_argument('--evidence-reference', required=True)\n" + "p.add_argument('--output', required=True)\n" + "a=p.parse_args()\n" + "scan=json.loads(Path(a.scan_summary).read_text())\n" + "digest=json.loads(Path(a.image_digests).read_text())['images']\n" + "exceptions=[]\n" + "for row in scan['unresolvedCriticalOrHigh']:\n" + " refs=set(row['affectedImages'])\n" + " affected={s:v['imageId'] for s,v in digest.items() if v['tag'] in refs}\n" + " for cve in row['cves']:\n" + " exceptions.append({'id':row['package']+'-'+cve,'package':row['package'],'cve':cve,\n" + " 'installedVersion':row['installedVersion'],'severity':row['severity'],\n" + " 'approvalStatus':'pending','riskOwner':'','approver':'','affectedImageDigests':affected,\n" + " 'evidenceReference':a.evidence_reference})\n" + "Path(a.output).write_text(json.dumps({'schemaVersion':'1.0','exceptions':exceptions}))\n" + "", + encoding="utf-8", + ) + raw_project = dict(project) + monkeypatch.setattr(controller, "STATE_ROOT", tmp_path) + monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", tmp_path / "releases") + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda _path: None) + monkeypatch.setattr(controller, "_root_owned", lambda _path: True) + monkeypatch.setattr( + controller, + "load_projects", + lambda: {controller.MCP_HUB_REPOSITORY: raw_project}, + ) + monkeypatch.setattr( + controller, "effective_mcp_hub_project", lambda repository, value: dict(value) + ) + monkeypatch.setattr( + controller, "repository_path", lambda _repository, _project: repo + ) + monkeypatch.setattr(controller, "output", lambda _args: revision) + monkeypatch.setattr( + controller, + "validate_commit", + lambda _repo, branch, commit: commit if branch == "main" else "", + ) + monkeypatch.setattr( + controller, "deployment_metadata", lambda *_args, **_kwargs: dict(metadata) + ) + monkeypatch.setattr(controller, "mcp_hub_source_date_epoch", lambda *_args: epoch) + + def stage(_project, _repo, commit): + assert commit == revision + return state.source + + monkeypatch.setattr(controller, "mcp_hub_stage_release_source", stage) + monkeypatch.setattr( + controller, "exact_source_matches_revision", lambda *_args: True + ) + monkeypatch.setattr( + controller, "_mcp_hub_verify_release_tree_metadata", lambda *_args: None + ) + monkeypatch.setattr(controller, "mcp_hub_validate_secret_files", lambda _model: 21) + + def rendered(project_arg, commit, repository): + assert commit == revision + assert repository == controller.MCP_HUB_REPOSITORY + services = {} + for index, service in enumerate(controller.MCP_HUB_SERVICES, 1): + if service in controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES: + local = controller.MCP_HUB_LOCAL_BUILD_IMAGE_REPOSITORIES.get(service) + reference = ( + f"{local}:{revision[:12]}" + if local + else f"{controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES[service]}@sha256:{index:064x}" + ) + elif service == "contextforge": + reference = f"itworx-mcp-hub/contextforge-schema:{revision}" + elif service == "ingestion-clamd": + reference = controller.MCP_HUB_INTAKE_CLAMAV_IMAGE + elif service == "postgres": + reference = controller.MCP_HUB_POSTGRES_IMAGE + elif service == "redis": + reference = controller.MCP_HUB_REDIS_IMAGE + else: + reference = f"fixture/{service}@sha256:{index:064x}" + services[service] = {"image": reference} + return {"services": services, "testMarker": state.compose_marker} + + monkeypatch.setattr(controller, "mcp_hub_compose_model", rendered) + + def inspect(reference): + state.inspect_calls.append(reference) + return state.images.get(reference) + + monkeypatch.setattr(controller, "inspect_image", inspect) + monkeypatch.setattr( + controller, "run", lambda *_args, **_kwargs: state.mutations.append("run") + ) + monkeypatch.setattr(utility, "_is_host_root", lambda: True) + + plan = utility.plan_candidate(controller.MCP_HUB_REPOSITORY, revision) + candidate_dir = tmp_path / "mcp-hub-release-candidates" / revision + plan_doc = json.loads((candidate_dir / "build.json").read_text()) + model = rendered(project, revision, controller.MCP_HUB_REPOSITORY) + labels = { + "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, + "org.opencontainers.image.revision": revision, + "org.opencontainers.image.source-tree": metadata["tree"], + "org.opencontainers.image.version": metadata["id"], + "org.opencontainers.image.created": metadata["builtAt"], + "org.opencontainers.image.source-date-epoch": epoch, + } + for index, service in enumerate(controller.MCP_HUB_SERVICES, 1): + reference = model["services"][service]["image"] + image_id = "sha256:" + f"{index:064x}" + config = {"User": "10001:10001"} + if ( + service in controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES + or service == "contextforge" + ): + config["Labels"] = dict(labels) + state.images[reference] = {"Id": image_id, "Config": config} + + def write_input(name: str, value: object) -> Path: + path = candidate_dir / name + data = ( + json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n" + ).encode() + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + path.chmod(0o600) + return path + + for index, service in enumerate(controller.MCP_HUB_SERVICES, 1): + image_id = "sha256:" + f"{index:064x}" + vuln = { + "SchemaVersion": 2, + "Trivy": {"Version": "0.73.0"}, + "CreatedAt": now, + "Metadata": { + "ImageID": image_id, + "ImageConfig": {"architecture": "amd64", "os": "linux"}, + }, + "Results": [ + { + "Class": "os-pkgs", + "Packages": [{"Name": "fixture", "Version": "1"}], + "Vulnerabilities": [], + } + ], + } + secret = { + "ImageID": image_id, + "CreatedAt": now, + "TrivyVersion": "0.73.0", + "FindingCount": 0, + "SensitiveFindingDetailsPersisted": False, + "Results": [{"FindingCount": 0}], + } + write_input(f"vulnerability-{service}.json", vuln) + write_input(f"secret-{service}.json", secret) + write_input("trivy-db-metadata.json", {"UpdatedAt": now, "DownloadedAt": now}) + + # The reports must match the IDs in the final inspect inventory. Keep a + # convenient writer for intentional drift cases in each test. + monkeypatch.setattr(utility, "_test_write_input", write_input, raising=False) + yield types.SimpleNamespace( + controller=controller, + utility=utility, + revision=revision, + epoch=epoch, + metadata=metadata, + project=project, + state=state, + candidate_dir=candidate_dir, + model=model, + write_input=write_input, + now=now, + plan=plan, + plan_doc=plan_doc, + ) + + +def test_plan_freezes_protected_metadata_without_candidate_or_approval(harness): + h = harness + document = json.loads((h.candidate_dir / "build.json").read_text()) + assert document["schema"] == "itworx.mcp-hub-build-plan/v1" + assert document["repository"] == h.controller.MCP_HUB_REPOSITORY + assert document["revision"] == h.revision + assert document["build"] == h.metadata + assert document["sourceDateEpoch"] == h.epoch + assert len(document["composeSha256"]) == 64 + assert not (h.candidate_dir / "candidate.json").exists() + assert not (h.candidate_dir / "approval.json").exists() + assert h.state.mutations == [] + + +def test_wrong_repository_and_nonroot_fail_before_controller_access( + modules, monkeypatch +): + controller, utility = modules + monkeypatch.setattr(utility, "_is_host_root", lambda: True) + monkeypatch.setattr( + controller, "load_projects", lambda: pytest.fail("must reject repo first") + ) + with pytest.raises(controller.DeployError, match="fixed MCP Hub repository"): + utility.plan_candidate("someone/else", "a" * 40) + monkeypatch.setattr(utility, "_is_host_root", lambda: False) + with pytest.raises(controller.DeployError, match="root"): + utility.plan_candidate(controller.MCP_HUB_REPOSITORY, "a" * 40) + + +def test_non_exact_main_head_is_denied_before_staging(modules, monkeypatch, tmp_path): + controller, utility = modules + monkeypatch.setattr(utility, "_is_host_root", lambda: True) + project = { + "branch": "main", + "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + } + monkeypatch.setattr( + controller, "load_projects", lambda: {controller.MCP_HUB_REPOSITORY: project} + ) + monkeypatch.setattr( + controller, "effective_mcp_hub_project", lambda _repo, value: value + ) + monkeypatch.setattr( + controller, "repository_path", lambda *_args: tmp_path / "hub.git" + ) + monkeypatch.setattr(controller, "output", lambda _args: "c" * 40) + monkeypatch.setattr( + controller, + "mcp_hub_stage_release_source", + lambda *_args: pytest.fail("must not stage"), + ) + with pytest.raises(controller.DeployError, match="exact protected main head"): + utility.plan_candidate(controller.MCP_HUB_REPOSITORY, "a" * 40) + + +def test_capture_emits_candidate_and_pending_only_envelope_without_apply_calls( + harness, monkeypatch +): + h = harness + calls = [] + original_run = h.utility.subprocess.run + + def observe_draft(command, **kwargs): + calls.append((command, kwargs)) + return original_run(command, **kwargs) + + monkeypatch.setattr(h.utility.subprocess, "run", observe_draft) + result = h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + candidate = json.loads((h.candidate_dir / "candidate.json").read_text()) + exceptions = json.loads((h.candidate_dir / "exceptions.json").read_text()) + digests = json.loads((h.candidate_dir / "image-digests.json").read_text()) + assert result["status"] == "pending_review" + assert set(candidate["images"]) == set(h.controller.MCP_HUB_SERVICES) + assert digests["images"]["web"]["tag"] == candidate["images"]["web"]["imageRef"] + assert digests["images"]["web"]["imageId"] == candidate["images"]["web"]["imageId"] + candidate_bytes = (h.candidate_dir / "candidate.json").read_bytes() + exceptions_bytes = (h.candidate_dir / "exceptions.json").read_bytes() + assert result["candidateSha256"] == hashlib.sha256(candidate_bytes).hexdigest() + assert result["exceptionsSha256"] == hashlib.sha256(exceptions_bytes).hexdigest() + assert not (h.candidate_dir / "approval.json").exists() + assert all(item["approvalStatus"] == "pending" for item in exceptions["exceptions"]) + assert h.state.mutations == [] + assert len(h.state.inspect_calls) == 17 + assert len(calls) == 1 + assert ( + Path(calls[0][0][1]) == h.state.source / "scripts" / "draft_cve_exceptions.py" + ) + assert calls[0][1]["stdout"] is subprocess.DEVNULL + assert calls[0][1]["stderr"] is subprocess.DEVNULL + assert set(calls[0][1]["env"]) == { + "PATH", + "PYTHONDONTWRITEBYTECODE", + "PYTHONIOENCODING", + } + + +@pytest.mark.parametrize("mutation", ["label", "root"]) +def test_capture_rejects_contextforge_provenance_drift_without_candidate( + harness, mutation +): + h = harness + reference = h.model["services"]["contextforge"]["image"] + config = h.state.images[reference]["Config"] + if mutation == "label": + config["Labels"]["org.opencontainers.image.source-tree"] = "wrong" + else: + config["User"] = "0:10001" + with pytest.raises(h.controller.DeployError, match="provenance|non-root runtime"): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert not (h.candidate_dir / "candidate.json").exists() + assert not (h.candidate_dir / "approval.json").exists() + assert h.state.mutations == [] + + +def test_pending_high_findings_are_drafted_from_the_exact_staged_source(harness): + h = harness + report_path = h.candidate_dir / "vulnerability-web.json" + report = json.loads(report_path.read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "libfixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": "1.2.3", + "Severity": "HIGH", + } + ] + report_path.unlink() + h.write_input(report_path.name, report) + result = h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + candidate = json.loads((h.candidate_dir / "candidate.json").read_text()) + summary = json.loads((h.candidate_dir / "scan-summary.json").read_text()) + exceptions = json.loads((h.candidate_dir / "exceptions.json").read_text())[ + "exceptions" + ] + row = summary["unresolvedCriticalOrHigh"][0] + assert row["affectedImages"] == [candidate["images"]["web"]["imageRef"]] + assert exceptions[0]["approvalStatus"] == "pending" + assert exceptions[0]["package"] == "libfixture" + assert exceptions[0]["cve"] == "CVE-2026-12345" + assert exceptions[0]["affectedImageDigests"] == { + "web": candidate["images"]["web"]["imageId"] + } + assert result["status"] == "pending_review" + assert not h.state.mutations + + +def test_findings_for_same_cve_at_distinct_versions_remain_distinct(harness): + h = harness + for service, version in (("web", "1.2.3"), ("worker", "4.5.6")): + path = h.candidate_dir / f"vulnerability-{service}.json" + report = json.loads(path.read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "libfixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": version, + "Severity": "HIGH", + } + ] + path.unlink() + h.write_input(path.name, report) + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + summary = json.loads((h.candidate_dir / "scan-summary.json").read_text()) + assert {row["installedVersion"] for row in summary["unresolvedCriticalOrHigh"]} == { + "1.2.3", + "4.5.6", + } + + +def test_same_package_cve_version_with_conflicting_severity_fails_closed(harness): + h = harness + for service, severity in (("web", "HIGH"), ("worker", "CRITICAL")): + path = h.candidate_dir / f"vulnerability-{service}.json" + report = json.loads(path.read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "libfixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": "1.2.3", + "Severity": severity, + } + ] + path.unlink() + h.write_input(path.name, report) + with pytest.raises(h.controller.DeployError, match="conflicting scan severities"): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert not (h.candidate_dir / "candidate.json").exists() + + +def test_draft_for_all_images_is_rejected_when_finding_scope_is_narrower(harness): + h = harness + path = h.candidate_dir / "vulnerability-web.json" + report = json.loads(path.read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "libfixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": "1.2.3", + "Severity": "HIGH", + } + ] + path.unlink() + h.write_input(path.name, report) + draft_script = h.state.source / "scripts" / "draft_cve_exceptions.py" + draft_script.write_text( + draft_script.read_text().replace("if v['tag'] in refs", "if True"), + encoding="utf-8", + ) + with pytest.raises(h.controller.DeployError, match="pending-only document"): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert not (h.candidate_dir / "candidate.json").exists() + + +def test_missing_scan_report_fails_without_publishing_candidate(harness): + h = harness + (h.candidate_dir / "secret-web.json").unlink() + with pytest.raises(h.controller.DeployError): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert not (h.candidate_dir / "candidate.json").exists() + assert not (h.candidate_dir / "approval.json").exists() + assert h.state.mutations == [] + + +def test_frozen_compose_drift_and_image_identity_drift_fail_closed(harness): + h = harness + h.state.compose_marker = "changed" + with pytest.raises(h.controller.DeployError, match="Compose"): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + h.state.compose_marker = "baseline" + web_ref = h.model["services"]["web"]["image"] + h.state.images[web_ref]["Id"] = "sha256:" + "f" * 64 + with pytest.raises(h.controller.DeployError): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert not (h.candidate_dir / "candidate.json").exists() + + +def test_fixable_critical_or_high_finding_is_not_captured(harness): + h = harness + path = h.candidate_dir / "vulnerability-web.json" + report = json.loads(path.read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "libfixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": "1.2.3", + "FixedVersion": "1.2.4", + "Severity": "CRITICAL", + } + ] + path.unlink() + h.write_input(path.name, report) + with pytest.raises(h.controller.DeployError, match="fixable"): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert not (h.candidate_dir / "candidate.json").exists() + + +def test_capture_never_overwrites_existing_records(harness): + h = harness + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + original = (h.candidate_dir / "candidate.json").read_bytes() + with pytest.raises(h.controller.DeployError): + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + assert (h.candidate_dir / "candidate.json").read_bytes() == original + assert h.state.mutations == [] + + +def test_capture_preserves_full_supported_trivy_database_metadata(harness): + h = harness + path = h.candidate_dir / "trivy-db-metadata.json" + path.unlink() + metadata = { + "UpdatedAt": h.now, + "DownloadedAt": h.now, + "Version": 2, + "NextUpdate": h.now, + } + h.write_input(path.name, metadata) + h.utility.capture_candidate(h.controller.MCP_HUB_REPOSITORY, h.revision) + candidate = json.loads((h.candidate_dir / "candidate.json").read_text()) + assert ( + candidate["databaseMetadataSha256"] + == hashlib.sha256(path.read_bytes()).hexdigest() + ) + assert json.loads(path.read_text()) == metadata + + +def test_cli_requires_explicit_fixed_target_and_subcommand(modules, monkeypatch): + _controller, utility = modules + monkeypatch.setattr(utility, "_is_host_root", lambda: False) + with pytest.raises(SystemExit): + utility.main([]) + monkeypatch.setattr(utility, "_is_host_root", lambda: True) + assert ( + utility.main(["plan", "--repository", "other/repo", "--revision", "a" * 40]) + == 2 + ) diff --git a/backend/tests/test_mcp_hub_deployment_reconciliation.py b/backend/tests/test_mcp_hub_deployment_reconciliation.py index 6dcad69..16ccb49 100644 --- a/backend/tests/test_mcp_hub_deployment_reconciliation.py +++ b/backend/tests/test_mcp_hub_deployment_reconciliation.py @@ -1,5 +1,16 @@ +import ast +import hashlib +import json +import os +import re +import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path from types import SimpleNamespace +import pytest + from backend.app.services.mcp_hub_deployment_reconciliation import ( McpHubDeploymentReconciliation, ) @@ -43,259 +54,581 @@ class FakePortainer: raise AssertionError(path) -def test_reconciliation_is_fixed_to_mcp_hub_and_reconciles_only_verified_failed_forward_state(): +def test_reconciliation_is_fixed_and_read_only_for_mcp_hub(): fake = FakePortainer() result = McpHubDeploymentReconciliation(fake).run() - assert result == { "status": "reconciled", "repository": "jens/itworx_mcp_hub", "composeProject": "itworx-mcp-hub", } - create = fake.calls[0] - command = create[2]["json"]["Cmd"] + command = fake.calls[0][2]["json"]["Cmd"] assert command[:2] == ["python3", "-c"] assert command[3:] == ["jens/itworx_mcp_hub", "itworx-mcp-hub"] - assert create[2]["json"]["AttachStdout"] is False - assert create[2]["json"]["AttachStderr"] is False - script = command[2] assert "/controller-config/projects.json" in script assert "/controller-hub-releases/controller-config/projects.json" in script - assert "os.path.samefile(registry_path, host_registry_path)" in script assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script - assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script - assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script - assert 'project.get("branch") != "main"' in script - assert 'project.get("compose_project") != expected_compose_project' in script - assert 'recorded_status != "failed"' in script - assert 'recorded_live_status not in {"healthy", "unknown"}' in script - assert 'attempt_status != "failed"' in script - assert 'attempt_revision == recorded_live' in script - assert 'git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)' in script - assert 'git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)' in script + assert "os.path.samefile(registry_path, host_registry_path)" in script + assert 'os.open(project_lock_path, os.O_RDONLY' in script + assert 'os.open(global_lock_path, os.O_RDONLY' in script + assert 'state_root.mkdir' not in script + assert 'project_lock_path.parent.mkdir' not in script + assert 'os.replace(' not in script + assert 'shutil.copy2(' not in script + assert 'state_path.write_text(' not in script + assert '"status": "success"' not in script + assert '"live_legacy_provenance": True' not in script -def test_unknown_live_status_is_accepted_only_for_a_proven_snapshot_stage_failure_without_rollback_error(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert 'last_attempt_message = str(last_attempt.get("message") or "")' in script - assert "snapshot_failure_prefixes = (" in script - assert 'recorded_live_status == "unknown"' in script - assert 'last_attempt_message.startswith(snapshot_failure_prefixes)' in script - assert '"; rollback error:" in last_attempt_message' in script - assert '"; source rollback error:" in last_attempt_message' in script - assert "raise SystemExit(55)" in script - - -def test_reconciliation_proves_complete_live_compose_topology_and_preserves_actual_image_ids(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert 'expected_containers = project.get("containers")' in script - assert "set(project_names) != set(expected_containers)" in script - assert 'labels.get("com.docker.compose.project") != expected_compose_project' in script - assert 'not runtime_state.get("Running")' in script - assert 'health in {"starting", "unhealthy"}' in script - assert 'image_id = str(container.get("Image") or "")' in script - assert '"containers": container_artifacts' in script - assert '"compose_sha256": ""' in script - assert '"topology_contract": topology_contract' in script - assert '"live_legacy_provenance": True' in script - assert '"live_artifacts": artifacts' in script - assert '"live_revision": recorded_live' in script - - -def test_reconciliation_recognizes_only_the_fixed_mcp_hub_oci_source_as_owned(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert 'owned_image_source = "https://gitea.example.com/jens/itworx-mcp-hub"' in script - assert "if image_source == owned_image_source:" in script - assert 'git_run("merge-base", "--is-ancestor", recorded_live, image_revision)' in script - assert 'git_run("merge-base", "--is-ancestor", image_revision, main_revision)' in script - assert 'image_source.startswith("https://gitea.example.com/")' not in script - assert 'image_source == repository' not in script - assert '"live_revision": main_revision' not in script - assert '"live_revision": attempt_revision' not in script - - -def test_reconciliation_requires_the_controller_17_service_topology_migration(): +def test_unknown_or_legacy_state_requires_the_guarded_deployment_path(): result = McpHubDeploymentReconciliation(FakePortainer(exit_code=56)).run() assert result == { "status": "topology_migration_required", "repository": "jens/itworx_mcp_hub", "composeProject": "itworx-mcp-hub", - "reason": "MCP Hub 15-to-17 topology migration requires its guarded protected-main deployment", + "reason": "MCP Hub legacy, unknown, mixed, or unresolved migration state requires guarded deployment or data-aware recovery", } fake = FakePortainer() McpHubDeploymentReconciliation(fake).run() script = fake.calls[0][2]["json"]["Cmd"][2] - assert 'topology_contract = "itworx.mcp-hub-topology/17-v1"' in script + assert 'set(registered_live_containers) == expected_fixed' in script + assert 'raise SystemExit(56 if recorded_live_status in {"unknown", "healthy"} else 52)' in script + assert 'recorded_status == "success"' in script + assert 'recorded_live_status == "healthy"' in script + assert 'state.get("live_legacy_provenance") is False' in script assert 'recorded_artifacts.get("topology_contract") == topology_contract' in script - assert '"topology_contract": topology_contract' in script + assert 'receipt.get("schema") == "gitea-deploy-receipt.v1"' in script + assert 'receipt.get("live") is None' in script -def test_owned_image_ancestry_can_fail_closed_to_the_canonical_gitea_store(): +def test_modern_success_requires_a_complete_exact_receipt_and_protected_ancestry(): fake = FakePortainer() McpHubDeploymentReconciliation(fake).run() script = fake.calls[0][2]["json"]["Cmd"][2] - assert 'canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")' in script - assert 'canonical_bare_repo = canonical_repository_root / owner / f"{name}.git"' in script - assert "def canonical_git_run(*args):" in script - assert "def canonical_image_ancestry_is_proven(" in script - assert 'resolved_recorded = canonical_git_run("rev-parse", f"{recorded_revision}^{{commit}}")' in script - assert 'resolved_image = canonical_git_run("rev-parse", f"{image_revision}^{{commit}}")' in script - assert 'resolved_main = canonical_git_run("rev-parse", "refs/heads/main")' in script - assert 'resolved_main.stdout.strip() == expected_main' in script - assert '"merge-base", "--is-ancestor", recorded_revision, image_revision' in script - assert '"merge-base", "--is-ancestor", image_revision, expected_main' in script - assert "if not mounted_ancestry_proven and not canonical_image_ancestry_is_proven(" in script - assert "canonical_bare_repo.is_symlink()" in script - - -def test_unchanged_recorded_image_ids_do_not_reinterpret_historical_oci_revision_labels(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - change_probe = "image_changed = image_id != recorded_image_id" - provenance_gate = "if image_changed:" - compat_probe = "contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref)" - dataplane_probe = "main_declares_sessionless_dataplane_image(name, image_ref)" - owned_probe = "elif image_source == owned_image_source:" - external_probe = "elif not main_declares_external_image(image_ref):" - - assert change_probe in script - assert provenance_gate in script - assert compat_probe in script - assert dataplane_probe in script - assert "not already_reconciled" in script - assert owned_probe in script - assert external_probe in script - assert script.index(change_probe) < script.index(provenance_gate) - assert script.index(provenance_gate) < script.index(compat_probe) - assert script.index(compat_probe) < script.index(dataplane_probe) - assert script.index(dataplane_probe) < script.index(owned_probe) - assert script.index(owned_probe) < script.index(external_probe) - assert "if image_id != recorded_image_id or container_id != recorded_container_id:" in script - - -def test_external_image_drift_requires_an_immutable_digest_declared_in_reviewed_main_compose(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert "def main_declares_external_image(image_ref):" in script - assert "@sha256:" in script - assert 'git_run("show", f"{main_revision}:{compose_path}")' in script - assert '"infrastructure/compose/docker-compose.yml"' in script - assert '"infrastructure/compose/docker-compose.prod.yml"' in script - assert 'line.strip() in {f"image: {image_ref}", f\'image: "{image_ref}"\', f"image: \'{image_ref}\'"}' in script - assert "main_declares_external_image(image_ref)" in script - assert "elif not main_declares_external_image(image_ref):" in script - - -def test_contextforge_compat_image_drift_requires_exact_reviewed_main_contract(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert 'container_name != "itworx-mcp-hub-contextforge-1"' in script - assert 'f"{main_revision}:infrastructure/images/contextforge/compat-image.env"' in script - assert 'f"{main_revision}:scripts/build_contextforge_compat_image.sh"' in script - assert '"CONTEXTFORGE_UPSTREAM_COMMIT"' in script - assert '"CONTEXTFORGE_ALEMBIC_HEAD"' in script - assert '"CONTEXTFORGE_MCP_SDK"' in script - assert '"CONTEXTFORGE_LOCAL_IMAGE"' in script - assert 're.fullmatch(r"[0-9a-f]{40}", upstream_commit)' in script - assert 're.fullmatch(r"[0-9a-f]{12}", alembic_head)' in script - assert 'local_image != f"itworx-mcp-hub/contextforge-schema:{alembic_head}"' in script - assert 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"' in script - assert 'git -C "$SRC" checkout --detach FETCH_HEAD' in script - assert 'MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"' in script - assert 'alembic -c mcpgateway/alembic.ini heads' in script - assert 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"' in script - assert 'image_ref.startswith("itworx-mcp-hub/contextforge-schema:")' not in script - - - compat_probe = "contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref)" - owned_probe = "elif image_source == owned_image_source:" - assert compat_probe in script - assert "if contextforge_compat_image or sessionless_dataplane_image:" in script - assert owned_probe in script - assert script.index(compat_probe) < script.index(owned_probe) - - -def test_sessionless_dataplane_baseline_is_one_time_and_commit_bound(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert 'container_name != "itworx-mcp-hub-dataplane-router-1"' in script - assert 'image_ref != "itworx-mcp-hub/dataplane:sessionless-final"' in script - assert 'sessionless_commit = "b519a8c4cf17cd9df00a575acfe096838aab7789"' in script - assert 'f"{sessionless_commit}:services/dataplane_router/router.py"' in script - assert 'f"{sessionless_commit}:tests/test_dataplane_session_headers.py"' in script - assert 'f"{main_revision}:evidence/mcp-session-boundary-20260916/SUMMARY.md"' in script - assert 'MCP_TRANSPORT_STATE_HEADERS = frozenset({b"mcp-session-id"})' in script - assert 'del response.headers["mcp-session-id"]' in script - assert '"2025-11-25"' in script - assert '"2026-07-28"' in script - assert '"permanently enforce stateless public transport"' in script - assert ( - 'git_run("merge-base", "--is-ancestor", sessionless_commit, main_revision)' - in script - ) - assert "not already_reconciled" in script - - -def test_already_reconciled_state_is_reverified_before_idempotent_success(): - fake = FakePortainer() - McpHubDeploymentReconciliation(fake).run() - script = fake.calls[0][2]["json"]["Cmd"][2] - - assert "already_reconciled = (" in script - assert script.index("already_reconciled = (") < script.index("project_names =") - assert script.index("container_artifacts = {}") < script.index("if already_reconciled:") + assert 'set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"}' in script + assert 'receipt_identity_valid' in script + assert 'receipt.get("revision") == recorded_live' in script + assert 'git_run("merge-base", "--is-ancestor", recorded_live, main_revision)' in script + assert 'set(dict(receipt.get("containers") or {})) == expected_runtime' in script assert 'print("already-reconciled", flush=True)' in script - assert "if drift_found:" in script + assert 'print("reconciled", flush=True)' not in script + assert 'migration_path = migration_root / "active-intent.json"' in script + assert 'intent.get("status") != "complete"' in script + assert 'elif migration_attempt_id:' in script + assert 'intent.get("completedStateSha256") != hashlib.sha256(state_bytes).hexdigest()' in script + assert 'observed.get("historicalStateSha256") != intent.get("baselineStateSha256")' in script + assert 'rf"[0-9]+-{recorded_live[:12]}-[0-9a-f]{{16}}"' in script -def test_reconciliation_refuses_unknown_runtime_topology(): - result = McpHubDeploymentReconciliation(FakePortainer(exit_code=47)).run() - assert result == { - "status": "validation_failed", - "reason": "live MCP Hub Compose project differs from the controller target", +def test_reconciliation_intent_guard_rejects_unknown_and_accepts_exact_complete_intent(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + parsed = ast.parse(script) + function = next( + node for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name == "validate_migration_intent_state" + ) + namespace = { + "Path": Path, + "hashlib": hashlib, + "json": json, + "os": os, + "re": re, + "stat": SimpleNamespace(S_IMODE=lambda _mode: 0o600), + "trusted_private_path": lambda *_args, **_kwargs: True, + } + exec(compile(ast.Module(body=[function], type_ignores=[]), "intent-test", "exec"), namespace) + validate = namespace["validate_migration_intent_state"] + repository = "jens/itworx_mcp_hub" + candidate = "b" * 40 + migration_attempt = "a" * 32 + state_bytes = b"exact-success-state-bytes" + + for raw in (b'{"status":"started"}', b"not-json"): + with tempfile.TemporaryDirectory() as temporary: + state_root = Path(temporary) + migration_root = state_root / "mcp-hub-topology-migrations" + migration_root.mkdir() + (migration_root / "active-intent.json").write_bytes(raw) + last_attempt = {"metadata": {"mcpHubMigrationAttemptId": migration_attempt}} + with pytest.raises(SystemExit) as exc: + validate( + state_root, state_bytes, repository, candidate, "success", "healthy", + last_attempt, "success", candidate, + ) + assert exc.value.code == 56 + + with tempfile.TemporaryDirectory() as temporary: + last_attempt = {"metadata": {"mcpHubMigrationAttemptId": migration_attempt}} + with pytest.raises(SystemExit) as exc: + validate( + Path(temporary), state_bytes, repository, candidate, "success", "healthy", + last_attempt, "success", candidate, + ) + assert exc.value.code == 56 + + with tempfile.TemporaryDirectory() as temporary: + state_root = Path(temporary) + migration_root = state_root / "mcp-hub-topology-migrations" + migration_root.mkdir() + intent = { + "schema": "itworx.mcp-hub-migration-intent/v1", + "status": "complete", + "attemptId": migration_attempt, + "repository": repository, + "candidateRevision": candidate, + "priorLiveRevision": "c" * 40, + "backupManifestSha256": "d" * 64, + "backupComposeSha256": "e" * 64, + "baselineStateSha256": "f" * 64, + "startedAtEpoch": 10, + "completedAtEpoch": 11, + "completedStateSha256": "0" * 64, + } + active_intent = migration_root / "active-intent.json" + active_intent.write_text(json.dumps(intent), encoding="utf-8") + last_attempt = { + "metadata": { + "mcpHubMigrationAttemptId": migration_attempt, + "mcpHubBackupReceipt": { + "repository": repository, + "revision": candidate, + "restoreVerified": True, + "manifestSha256": "d" * 64, + "composeSha256": "e" * 64, + }, + } + } + with pytest.raises(SystemExit) as exc: + validate( + state_root, state_bytes, repository, candidate, "success", "healthy", + last_attempt, "success", candidate, + ) + assert exc.value.code == 56 + + with tempfile.TemporaryDirectory() as temporary: + state_root = Path(temporary) + migration_root = state_root / "mcp-hub-topology-migrations" + receipt_root = state_root / "mcp-hub-backup-receipts" + backup_root = state_root / "mcp-hub-backups" + migration_root.mkdir() + receipt_root.mkdir() + backup_root.mkdir() + backup_directory = backup_root / f"1780400000-{candidate[:12]}-0123456789abcdef" + backup_directory.mkdir() + compose_digest = "e" * 64 + manifest = { + "schema": "itworx.mcp-hub-backup/v1", + "repository": repository, + "revision": candidate, + "backupDirectory": str(backup_directory), + "composeSha256": compose_digest, + "configRestoreVerified": True, + "restore": {"status": "verified"}, + "databases": {"itworx_mcp_hub": {}, "contextforge": {}}, + } + manifest_path = backup_directory / "manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + manifest_digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest() + receipt_path = receipt_root / f"1780400000-{candidate[:12]}-0123456789abcdef.json" + backup_receipt = { + "schema": "itworx.mcp-hub-backup-receipt/v1", + "repository": repository, + "revision": candidate, + "backupDirectory": str(backup_directory), + "manifestSha256": manifest_digest, + "composeSha256": compose_digest, + "restoreVerified": True, + "createdAtEpoch": 1780400000, + "receiptPath": str(receipt_path), + } + receipt_path.write_text( + json.dumps({key: value for key, value in backup_receipt.items() if key != "receiptPath"}), + encoding="utf-8", + ) + observed_path = migration_root / f"observed-baseline-{candidate[:12]}-0123456789abcdef0123.json" + observed = { + "schema": "itworx.mcp-hub-observed-baseline/v1", + "claim": "observed_mixed_runtime_facts_only", + "candidateRevision": candidate, + "historicalStateSha256": "f" * 64, + "backupManifestSha256": manifest_digest, + "backupComposeSha256": compose_digest, + "backupRestoreVerified": True, + "postgresVolumeBaseline": [{}, {}], + } + observed_path.write_text(json.dumps(observed), encoding="utf-8") + observed_digest = hashlib.sha256(observed_path.read_bytes()).hexdigest() + successful_state_sha = hashlib.sha256(state_bytes).hexdigest() + intent = { + "schema": "itworx.mcp-hub-migration-intent/v1", + "status": "complete", + "attemptId": migration_attempt, + "repository": repository, + "candidateRevision": candidate, + "priorLiveRevision": "c" * 40, + "backupManifestSha256": manifest_digest, + "backupComposeSha256": compose_digest, + "baselineStateSha256": "f" * 64, + "startedAtEpoch": 10, + "completedAtEpoch": 11, + "completedStateSha256": successful_state_sha, + "observedBaselinePath": str(observed_path), + "observedBaselineSha256": observed_digest, + } + (migration_root / "active-intent.json").write_text(json.dumps(intent), encoding="utf-8") + last_attempt = { + "metadata": { + "mcpHubMigrationAttemptId": migration_attempt, + "mcpHubBackupReceipt": backup_receipt, + } + } + + validate( + state_root, state_bytes, repository, candidate, "success", "healthy", + last_attempt, "success", candidate, + ) + + +def test_container_reverification_requires_exact_health_service_label_ids_and_mounts(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + + assert 'runtime_state.get("Running") is not True or health != "healthy"' in script + assert 'labels.get("com.docker.compose.project") != expected_compose_project' in script + assert 'labels.get("com.docker.compose.service") != name_service' in script + assert 'if recorded.get(key) != value or receipt_item.get(key) != value:' in script + assert 'live_attested_mounts(container) != recorded_mounts' in script + assert 'receipt_item.get("mounts_sha256")' in script + assert 'def live_attested_mounts(container):' in script + assert 'docker", "volume", "inspect", name' in script + assert 'mount["created_at"] = created_at' in script + assert 'created.astimezone(timezone.utc) > datetime.now(timezone.utc)' in script + + +def test_reconciliation_distinguishes_recreated_postgres_volume_by_created_at(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + parsed = ast.parse(script) + function = next( + node for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name == "live_attested_mounts" + ) + volume = { + "Name": "hub-postgres", + "Driver": "local", + "Mountpoint": "/var/lib/docker/volumes/hub-postgres/_data", + "Options": {}, + "CreatedAt": "2020-01-01T10:00:00Z", + } + namespace = { + "json": json, + "os": os, + "subprocess": SimpleNamespace( + PIPE=subprocess.PIPE, + DEVNULL=subprocess.DEVNULL, + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, stdout=json.dumps([volume]) + ) + ), + "datetime": datetime, + "timezone": timezone, + } + exec(compile(ast.Module(body=[function], type_ignores=[]), "mount-test", "exec"), namespace) + attest = namespace["live_attested_mounts"] + container = { + "Mounts": [{ + "Type": "volume", "Name": "hub-postgres", + "Source": "/var/lib/docker/volumes/hub-postgres/_data", + "Destination": "/var/lib/postgresql/data", "Driver": "local", "RW": True, + }] } + original = attest(container) + assert original[0]["created_at"] == "2020-01-01T10:00:00Z" + volume["CreatedAt"] = "2021-01-01T10:00:00Z" + recreated = attest(container) + assert recreated[0]["name"] == original[0]["name"] + assert recreated[0]["source"] == original[0]["source"] + assert recreated[0]["created_at"] != original[0]["created_at"] -def test_reconciliation_refuses_unproven_runtime_revision_drift(): - result = McpHubDeploymentReconciliation(FakePortainer(exit_code=51)).run() - assert result == { - "status": "validation_failed", - "reason": "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain", + +def test_each_service_uses_its_fixed_image_repository_or_reviewed_external_contract(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + + assert 'def fixed_image_reference(service_name, image_ref):' in script + assert '"dataplane-router": "gitea.example.com/jens/itworx-mcp-hub/dataplane"' in script + assert '"docker-read-broker": "gitea.example.com/jens/itworx-mcp-hub/docker-read-broker"' in script + assert 'artifact["image_source"] != owned_image_source' in script + assert 'artifact["image_revision"] != recorded_live' in script + assert 'main_declares_external_image(image_ref)' in script + assert 'main_declares_contextforge_compat_image(' in script + assert 'def image_digest_matches(image, image_ref):' in script + assert 'aliases.add("docker.io/library/" + repository)' in script + + +def test_external_candidate_pins_are_exact_and_declared_by_protected_main(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + parsed = ast.parse(script) + function = next( + node for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name == "fixed_image_reference" + ) + declared = { + "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04", + "redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48", } - - -def test_reconciliation_refuses_state_that_is_not_the_known_failed_forward_shape(): - result = McpHubDeploymentReconciliation(FakePortainer(exit_code=52)).run() - assert result == { - "status": "validation_failed", - "reason": "controller state drift is not the exact known recoverable MCP Hub state", + namespace = { + "re": re, + "main_declares_external_image": lambda image_ref: image_ref in declared, + "main_declares_sessionless_dataplane_image": lambda *_args: False, + "main_declares_contextforge_compat_image": lambda *_args: False, } + exec(compile(ast.Module(body=[function], type_ignores=[]), "fixed-image-test", "exec"), namespace) + fixed = namespace["fixed_image_reference"] + + assert fixed("postgres", "postgres:18.4@sha256:3a82e1f56c8f0f5616a11103ac3d47e632c3938698946a7ad26da0df1334744a") + assert fixed( + "postgres", + "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04", + ) + assert fixed("redis", "redis:8.10.0@sha256:c29e49ab2f85760a3827b53882e6dd9f5c6c3f0bb7d724e07bb31cbf275a5236") + assert fixed("redis", "redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48") + assert not fixed("postgres", "postgres:18.6@sha256:" + "a" * 64) + assert not fixed("redis", "redis:8.10.3@sha256:" + "b" * 64) + for service in ("web", "control-api", "dataplane-router", "connector-ingestion"): + assert fixed(service, "sha256:" + "a" * 64) + for service in ("postgres", "redis", "contextforge", "ingestion-clamd", "foreign"): + assert not fixed(service, "sha256:" + "a" * 64) + + namespace["main_declares_external_image"] = lambda _image_ref: False + assert not fixed("postgres", "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04") + assert not fixed("redis", "redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48") -def test_reconciliation_refuses_unknown_state_without_snapshot_stage_proof(): - result = McpHubDeploymentReconciliation(FakePortainer(exit_code=55)).run() - assert result == { - "status": "validation_failed", - "reason": "unknown MCP Hub live status was not produced by a proven recoverable snapshot-stage failure", +@pytest.fixture +def contextforge_reconciliation_proof(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + parsed = ast.parse(fake.calls[0][2]["json"]["Cmd"][2]) + functions = [ + node for node in parsed.body + if isinstance(node, ast.FunctionDef) + and node.name in { + "main_declares_contextforge_compat_image", "fixed_image_reference", + "receipt_bound_image_reference", + } + ] + revision = "a" * 40 + main_revision = "b" * 40 + manifest = ( + f"CONTEXTFORGE_UPSTREAM_COMMIT={'c' * 40}\n" + "CONTEXTFORGE_ALEMBIC_HEAD=123456789abc\n" + "CONTEXTFORGE_MCP_SDK=1.2.3\n" + "CONTEXTFORGE_LOCAL_IMAGE=itworx-mcp-hub/contextforge-schema:123456789abc\n" + ) + builder = ( + 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"\n' + 'git -C "$SRC" checkout --detach FETCH_HEAD\n' + 'MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"\n' + 'alembic -c mcpgateway/alembic.ini heads\n' + 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"\n' + '--release-candidate\n' + 'CANDIDATE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}"\n' + 'docker tag "$STAGED_IMAGE" "$CANDIDATE_IMAGE"\n' + ) + files = { + f"{commit}:infrastructure/images/contextforge/compat-image.env": manifest + for commit in (revision, main_revision) } + files.update({ + f"{commit}:scripts/build_contextforge_compat_image.sh": builder + for commit in (revision, main_revision) + }) + + def git_run(*args): + assert args[0] == "show" + return SimpleNamespace(returncode=0 if args[1] in files else 1, + stdout=files.get(args[1], "")) + + namespace = {"re": re, "git_run": git_run, "main_revision": main_revision, + "recorded_live": revision} + exec( # noqa: S102 -- Execute only extracted repository-owned verifier functions. + compile(ast.Module(body=functions, type_ignores=[]), "contextforge-proof", "exec"), namespace + ) + return SimpleNamespace(namespace=namespace, files=files, revision=revision) + + +def test_contextforge_reconciliation_accepts_exact_receipt_revision(contextforge_reconciliation_proof): + proof = contextforge_reconciliation_proof + namespace = proof.namespace + fixed = namespace["fixed_image_reference"] + reference = f"itworx-mcp-hub/contextforge-schema:{proof.revision}" + image_id = "sha256:" + "d" * 64 + record = {"image_id": image_id, "image_ref": reference} + bound = namespace["receipt_bound_image_reference"](image_id, image_id, record, dict(record)) + assert fixed("contextforge", bound) + assert fixed("contextforge", "itworx-mcp-hub/contextforge-schema:123456789abc") + # A later main may remove the candidate builder; the recorded receipt's + # ancestor revision must supply the source proof for this immutable image. + proof.files[f"{'b' * 40}:scripts/build_contextforge_compat_image.sh"] = "" + assert fixed("contextforge", bound) + + +@pytest.mark.parametrize("mutation", [ + "wrong_revision", "short_revision", "mutable_tag", "wrong_service", + "missing_candidate_flag", "missing_candidate_assignment", "missing_candidate_tag", + "missing_fetch", "bad_manifest", "missing_recorded_source", +]) +def test_contextforge_reconciliation_rejects_unbound_candidates(contextforge_reconciliation_proof, mutation): + proof = contextforge_reconciliation_proof + reference = f"itworx-mcp-hub/contextforge-schema:{proof.revision}" + service = "contextforge" + builder_path = f"{proof.revision}:scripts/build_contextforge_compat_image.sh" + fragments = { + "missing_candidate_flag": '--release-candidate', + "missing_candidate_assignment": 'CANDIDATE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}"', + "missing_candidate_tag": 'docker tag "$STAGED_IMAGE" "$CANDIDATE_IMAGE"', + "missing_fetch": 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"', + } + if mutation == "wrong_revision": + reference = f"itworx-mcp-hub/contextforge-schema:{'b' * 40}" + elif mutation == "short_revision": + reference = f"itworx-mcp-hub/contextforge-schema:{proof.revision[:12]}" + elif mutation == "mutable_tag": + reference = "itworx-mcp-hub/contextforge-schema:latest" + elif mutation == "wrong_service": + service = "foreign" + elif mutation in fragments: + proof.files[builder_path] = proof.files[builder_path].replace(fragments[mutation], "") + elif mutation == "bad_manifest": + path = f"{proof.revision}:infrastructure/images/contextforge/compat-image.env" + proof.files[path] = proof.files[path].replace("CONTEXTFORGE_ALEMBIC_HEAD=123456789abc", "CONTEXTFORGE_ALEMBIC_HEAD=invalid") + else: + proof.files.pop(builder_path) + assert not proof.namespace["fixed_image_reference"](service, reference) + + +def test_digest_attestation_normalizes_tagged_docker_hub_repo_digests(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + parsed = ast.parse(script) + function = next( + node for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name == "image_digest_matches" + ) + namespace = {"re": re} + exec(compile(ast.Module(body=[function], type_ignores=[]), "reconcile-test", "exec"), namespace) + matches = namespace["image_digest_matches"] + digest = "sha256:" + "a" * 64 + + assert matches({"RepoDigests": [f"postgres@{digest}"]}, f"postgres:18.6@{digest}") + assert matches({"RepoDigests": [f"docker.io/library/redis@{digest}"]}, f"redis:8.10.2@{digest}") + assert matches({"RepoDigests": [f"clamav/clamav@{digest}"]}, f"clamav/clamav:1.4.6@{digest}") + assert matches( + {"RepoDigests": [f"registry.example:5000/team/web@{digest}"]}, + f"registry.example:5000/team/web:release@{digest}", + ) + assert not matches( + {"RepoDigests": [f"redis@sha256:{'b' * 64}"]}, f"redis:8.10.2@{digest}" + ) + assert not matches({"RepoDigests": [f"foreign/redis@{digest}"]}, f"redis:8.10.2@{digest}") + + +def test_id_launched_image_uses_exact_state_and_receipt_reference(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + parsed = ast.parse(script) + function = next( + node for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name == "receipt_bound_image_reference" + ) + namespace = {"re": re} + exec( # noqa: S102 + compile(ast.Module(body=[function], type_ignores=[]), "receipt-image-test", "exec"), + namespace, + ) + bound_reference = namespace["receipt_bound_image_reference"] + image_id = "sha256:" + "a" * 64 + image_ref = ( + "postgres:18.6@sha256:" + "0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04" + ) + recorded = {"image_id": image_id, "image_ref": image_ref} + receipt_item = dict(recorded) + + assert bound_reference(image_id, image_id, recorded, receipt_item) == image_ref + assert bound_reference(image_ref, image_id, recorded, receipt_item) == image_ref + assert 'image_ref = receipt_bound_image_reference(' in script + assert script.index('image_ref = receipt_bound_image_reference(') < script.index( + 'if not fixed_image_reference(name_service, image_ref):' + ) + assert 'image_digest_matches(image, image_ref)' in script + + for launch_image, live_id, state, receipt in ( + ("sha256:" + "c" * 64, image_id, recorded, receipt_item), + ( + image_id, image_id, + {**recorded, "image_id": "sha256:" + "c" * 64}, receipt_item, + ), + ( + image_id, image_id, + recorded, {**receipt_item, "image_id": "sha256:" + "c" * 64}, + ), + ( + image_id, image_id, + {**recorded, "image_ref": "redis:8.10.2@sha256:" + "b" * 64}, + receipt_item, + ), + (image_id, image_id, recorded, {**receipt_item, "image_ref": ""}), + (image_id, "arbitrary-image-id", recorded, receipt_item), + (image_id, image_id, + {**recorded, "image_ref": "sha256:" + "b" * 64}, + {**receipt_item, "image_ref": "sha256:" + "b" * 64}), + ): + with pytest.raises(SystemExit) as exc: + bound_reference(launch_image, live_id, state, receipt) + assert exc.value.code == 52 + + +def test_reconciliation_reports_invalid_state_and_live_evidence_fail_closed(): + expected = { + 42: ("validation_failed", "MCP Hub deployment state file is unavailable or invalid"), + 45: ("validation_failed", "a live MCP Hub container is not running and healthy"), + 47: ("validation_failed", "live MCP Hub Compose project differs from the controller target"), + 51: ("validation_failed", "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain"), + 52: ("validation_failed", "controller state or live facts do not match the exact modern MCP Hub receipt"), + 56: ("topology_migration_required", "MCP Hub legacy, unknown, mixed, or unresolved migration state requires guarded deployment or data-aware recovery"), + } + for exit_code, (status, reason) in expected.items(): + result = McpHubDeploymentReconciliation(FakePortainer(exit_code=exit_code)).run() + assert result["status"] == status + assert result["reason"] == reason + + +def test_reconciliation_only_allows_fixed_17_topology_and_rejects_registry_drift(): + fake = FakePortainer() + McpHubDeploymentReconciliation(fake).run() + script = fake.calls[0][2]["json"]["Cmd"][2] + + assert 'project.get("branch") != "main"' in script + assert 'project.get("slug") != expected_compose_project' in script + assert 'project.get("compose_project") != expected_compose_project' in script + assert 'frozenset(expected_containers) not in {frozenset(expected_fixed), frozenset(expected_runtime)}' in script + assert 'set(expected_containers) != expected_runtime' in script + assert 'set(project_names) != set(expected_containers)' in script + assert '"itworx-mcp-hub-connector-ingestion-1"' in script + assert '"itworx-mcp-hub-ingestion-clamd-1"' in script diff --git a/backend/tests/test_mcp_hub_governed_topology.py b/backend/tests/test_mcp_hub_governed_topology.py index 750b9e3..66a26ca 100644 --- a/backend/tests/test_mcp_hub_governed_topology.py +++ b/backend/tests/test_mcp_hub_governed_topology.py @@ -1,10 +1,9 @@ from __future__ import annotations - -import copy import hashlib import importlib.util import io import json +import copy import os import stat import subprocess @@ -13,13 +12,9 @@ import tarfile import types from datetime import datetime, timedelta, timezone from pathlib import Path - import pytest - ROOT = Path(__file__).resolve().parents[2] CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" / "deploy.py" - - def _load_controller(): if "fcntl" not in sys.modules: fcntl = types.ModuleType("fcntl") @@ -33,12 +28,10 @@ def _load_controller(): spec.loader.exec_module(module) return module - @pytest.fixture def controller(): return _load_controller() - def _legacy_target(controller): return { "branch": "main", @@ -55,80 +48,6 @@ def _legacy_target(controller): "source_dir": (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix(), } - -def test_projects_registry_uses_fixed_config_alias_only_for_protected_install(controller): - assert controller.CONFIG_PATH == controller.ROOT / "projects.json" - assert controller.controller_config_root(Path("/opt/gitea-deploy")) == Path( - "/controller-hub-releases/controller-config" - ) - assert controller._CONTROLLER_PATHS.controller_readonly_config_root( - Path("/opt/gitea-deploy") - ) == Path("/controller-config") - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX ACL inspection is unavailable on Windows") -def test_state_trust_fails_closed_when_acl_metadata_cannot_be_inspected(controller, monkeypatch): - def unavailable(*_args, **_kwargs): - raise OSError("filesystem ACL metadata is unavailable") - - monkeypatch.setattr(controller.os, "listxattr", unavailable) - with pytest.raises(controller.DeployError, match="cannot inspect filesystem ACLs"): - controller._mcp_hub_require_no_posix_acl(Path("/tmp")) - - -@pytest.mark.skipif(os.name == "nt", reason="backup ownership contract requires POSIX ownership and ACL metadata") -def test_hub_backup_child_is_created_with_fixed_parent_owner_and_private_mode( - controller, tmp_path, monkeypatch -): - appdata_root = tmp_path / "hub-appdata" - backup_root = appdata_root / "backups" - appdata_root.mkdir(mode=0o700) - backup_root.mkdir(mode=0o700) - os.chown(appdata_root, 99, 100) - os.chown(backup_root, 99, 100) - monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata_root) - monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) - - attempt = backup_root / "attempt" - controller._mcp_hub_private_directory(attempt, create=True, root=backup_root) - - metadata = attempt.stat() - assert (metadata.st_uid, metadata.st_gid) == (99, 100) - assert stat.S_IMODE(metadata.st_mode) == 0o700 - controller._mcp_hub_private_directory(attempt, root=backup_root) - - -@pytest.mark.skipif(os.name == "nt", reason="backup root contract requires POSIX mode and ACL metadata") -def test_hub_backup_root_rejects_group_readable_root(controller, tmp_path, monkeypatch): - appdata_root = tmp_path / "hub-appdata" - backup_root = appdata_root / "backups" - appdata_root.mkdir(mode=0o700) - backup_root.mkdir(mode=0o750) - monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", appdata_root) - monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) - - with pytest.raises(controller.DeployError, match="fixed private owner and mode"): - controller._mcp_hub_private_directory(backup_root, root=backup_root) - - -@pytest.mark.skipif(os.name == "nt", reason="release receipt storage requires POSIX mode and ACL metadata") -def test_hub_backup_receipt_directory_is_guarded_by_its_release_volume_parent( - controller, tmp_path, monkeypatch -): - volume_root = tmp_path / "controller-hub-releases" - state_root = volume_root / "state" - receipt_root = state_root / "backup-receipts" - receipt_root.mkdir(mode=0o700, parents=True) - volume_root.chmod(0o700) - state_root.chmod(0o700) - monkeypatch.setattr(controller, "MCP_HUB_RELEASE_STORE_STATE_ROOT", state_root) - monkeypatch.setattr(controller, "MCP_HUB_BACKUP_RECEIPT_ROOT", receipt_root) - - controller._mcp_hub_private_directory(receipt_root, root=state_root) - with pytest.raises(controller.DeployError, match="escapes its fixed parent"): - controller._mcp_hub_private_directory(receipt_root, root=controller.STATE_ROOT) - - def _valid_model(controller): services = {} for service_name in controller.MCP_HUB_SERVICES: @@ -136,6 +55,8 @@ def _valid_model(controller): image = "itworx-mcp-hub/contextforge-schema:0123456789ab" elif service_name == "postgres": image = controller.MCP_HUB_POSTGRES_IMAGE + elif service_name == "redis": + image = controller.MCP_HUB_REDIS_IMAGE elif service_name == "ingestion-clamd": image = controller.MCP_HUB_INTAKE_CLAMAV_IMAGE else: @@ -152,26 +73,56 @@ def _valid_model(controller): services[service_name] = service return {"services": services, "volumes": {}, "secrets": {}} - -def _patch_release_volume_paths(controller, monkeypatch, tmp_path): - releases = tmp_path / "protected-volume" / "releases" - monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases) - monkeypatch.setattr( - controller, - "_mcp_hub_release_directory", - lambda revision: releases / revision, +def _observed_candidate_model(controller): + model = _valid_model(controller) + for service_name, local_repository in controller.MCP_HUB_LOCAL_BUILD_IMAGE_REPOSITORIES.items(): + model["services"][service_name]["image"] = f"{local_repository}:bbbbbbbbbbbb" + model["services"]["connector-ingestion"]["image"] = ( + f"{controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES['connector-ingestion']}@sha256:{'b' * 64}" ) + return model - def require_release_anchor(*, create=False): - if create: - releases.mkdir(mode=0o700, parents=True, exist_ok=True) - if not releases.is_dir() or releases.is_symlink(): - raise controller.DeployError("MCP Hub protected release anchor is unavailable") - return releases - - monkeypatch.setattr(controller, "_mcp_hub_require_trusted_release_anchor", require_release_anchor) - return releases +def test_contextforge_candidate_model_accepts_only_the_frozen_full_revision(controller): + revision = "a" * 40 + project = { + "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + "_deployment_metadata": {"revision": revision}, + } + model = _valid_model(controller) + model["services"]["contextforge"]["image"] = ( + f"itworx-mcp-hub/contextforge-schema:{revision}" + ) + assert controller.validate_mcp_hub_compose_model(project, model) is model + model["services"]["contextforge"]["image"] = ( + f"itworx-mcp-hub/contextforge-schema:{'b' * 40}" + ) + with pytest.raises(controller.DeployError, match="ContextForge"): + controller.validate_mcp_hub_compose_model(project, model) +@pytest.mark.parametrize("mutation", ["revision", "tree", "epoch", "root", "tag"]) +def test_frozen_contextforge_provenance_rejects_changed_build_fields(controller, mutation): + revision = "a" * 40 + build = {"tree": "b" * 40, "id": revision[:12], "builtAt": "2026-10-02T00:00:00.000Z"} + epoch = "1790888863" + reference = f"itworx-mcp-hub/contextforge-schema:{revision}" + image = {"Config": {"User": "10001:10001", "Labels": { + "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, + "org.opencontainers.image.revision": revision, + "org.opencontainers.image.source-tree": build["tree"], + "org.opencontainers.image.version": build["id"], + "org.opencontainers.image.created": build["builtAt"], + "org.opencontainers.image.source-date-epoch": epoch, + }}} + controller._mcp_hub_verify_frozen_image_provenance("contextforge", reference, image, revision, build, epoch) + if mutation == "root": + image["Config"]["User"] = "root:10001" + elif mutation == "tag": + reference = "itworx-mcp-hub/contextforge-schema:12d4a0c7789c" + else: + key = {"revision": "revision", "tree": "source-tree", "epoch": "source-date-epoch"}[mutation] + image["Config"]["Labels"][f"org.opencontainers.image.{key}"] = "changed" + with pytest.raises(controller.DeployError, match="provenance|ContextForge"): + controller._mcp_hub_verify_frozen_image_provenance("contextforge", reference, image, revision, build, epoch) def test_exact_legacy_target_derives_only_the_fixed_17_service_contract(controller): target = _legacy_target(controller) @@ -185,6 +136,90 @@ def test_exact_legacy_target_derives_only_the_fixed_17_service_contract(controll assert target["containers"] == list(controller.MCP_HUB_LEGACY_CONTAINERS) assert controller.effective_mcp_hub_project("jens/other", target) is target +def test_postgres_candidate_pin_matches_the_reviewed_release_evidence(controller): + expected = ( + "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04" + ) + assert controller.MCP_HUB_POSTGRES_IMAGE == expected + assert controller.MCP_HUB_OBSERVED_EXTERNAL_IMAGE_TRANSITIONS["postgres"] == ( + controller.MCP_HUB_POSTGRES_LEGACY_IMAGE, + expected, + ) + +def test_installed_legacy_registry_shape_is_adapted_only_in_memory(controller): + target = { + "branch": "main", + "slug": controller.MCP_HUB_PROJECT, + "compose_project": controller.MCP_HUB_PROJECT, + "containers": list(controller.MCP_HUB_LEGACY_CONTAINERS), + "source_dir": "/srv/example/appdata/itworx-mcp-hub/deployment/repository", + "compose_dir": "infrastructure/compose", + "compose_files": [ + "docker-compose.yml", + "docker-compose.prod.yml", + "docker-compose.autodeploy.yml", + ], + "exact_source": None, + "verify_oci_revision": None, + "manual": False, + "deploy_script": "", + "archive_deploy_script": "", + "live_probe": None, + "managed_plan_version": "", + "clean_untracked_paths": None, + "env_source": "/srv/example/appdata/itworx-mcp-hub/deploy.env", + } + original = copy.deepcopy(target) + + result = controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, target) + + assert result["source_dir"] == (controller.EXACT_SOURCE_ROOT / "itworx-mcp-hub").as_posix() + assert result["exact_source"] is True + assert result["compose_files"] == list(controller.MCP_HUB_COMPOSE_FILES) + assert set(result["containers"]) == set(controller.MCP_HUB_CONTAINERS) + assert target == original + + for field, value in ( + ("source_dir", "/srv/example/appdata/other/repository"), + ("compose_files", ["docker-compose.yml", "unexpected.yml"]), + ("exact_source", True), + ("verify_oci_revision", False), + ("env_source", "/tmp/untrusted.env"), + ): + changed = {**original, field: value} + with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): + controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, changed) + +def test_hub_verified_build_date_cannot_be_overridden_by_target_environment( + controller, monkeypatch +): + project = controller.effective_mcp_hub_project( + controller.MCP_HUB_REPOSITORY, _legacy_target(controller) + ) + project["environment"] = { + "APPOPS_BUILD_DATE": "untrusted-override", + "APPOPS_SOURCE_TREE": "untrusted-tree", + "APPOPS_BUILD_ID": "untrusted-id", + "SOURCE_DATE_EPOCH": "untrusted-epoch", + } + project["_mcp_hub_source_date_epoch"] = "1790888863" + metadata = { + "repository": controller.MCP_HUB_REPOSITORY, + "revision": "a" * 40, + "builtAt": "2026-10-01T18:00:00.000Z", + "tree": "b" * 40, + "id": "a" * 12, + } + monkeypatch.setenv("APPOPS_BUILD_DATE", "process-override") + + environment = controller.deployment_environment( + project, "a" * 40, controller.MCP_HUB_REPOSITORY, metadata + ) + + assert environment["APPOPS_BUILD_DATE"] == metadata["builtAt"] + assert environment["APPOPS_SOURCE_TREE"] == metadata["tree"] + assert environment["APPOPS_BUILD_ID"] == metadata["id"] + assert environment["SOURCE_DATE_EPOCH"] == "1790888863" def test_hub_compose_command_preserves_candidate_build_metadata_and_fixed_intake_profile( controller, monkeypatch, tmp_path @@ -233,238 +268,6 @@ def test_hub_compose_command_preserves_candidate_build_metadata_and_fixed_intake assert command[-2:] == ["--file", str(overlay)] -def _built_hub_candidate(controller, tmp_path, revision="a" * 40): - project = controller.effective_mcp_hub_project( - controller.MCP_HUB_REPOSITORY, _legacy_target(controller) - ) - project["source_dir"] = str(tmp_path) - project["compose_dir"] = "infrastructure/compose" - project["_deployment_metadata"] = { - "id": revision[:12], - "repository": controller.MCP_HUB_REPOSITORY, - "revision": revision, - "tree": "b" * 40, - "treeDigest": "c" * 64, - "builtAt": "2026-10-02T10:00:00.000Z", - } - model = _valid_model(controller) - image_ids = {} - labels = { - "org.opencontainers.image.revision": revision, - "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.source-tree": project["_deployment_metadata"]["tree"], - "org.opencontainers.image.version": revision[:12], - "org.opencontainers.image.created": project["_deployment_metadata"]["builtAt"], - } - for index, service_name in enumerate(controller.MCP_HUB_APPLICATION_SERVICES, 1): - service = model["services"][service_name] - service["build"] = {"context": "."} - service["image"] = f"itworx-mcp-hub/{service_name}:{revision[:12]}" - image_ids[service["image"]] = "sha256:" + f"{index:064x}" - return project, model, image_ids, labels - - -def test_hub_source_builds_are_inspected_then_started_by_immutable_ids( - controller, monkeypatch, tmp_path -): - project, model, image_ids, labels = _built_hub_candidate(controller, tmp_path) - for filename in controller.MCP_HUB_COMPOSE_FILES: - (tmp_path / "infrastructure" / "compose" / filename).parent.mkdir( - parents=True, exist_ok=True - ) - (tmp_path / "infrastructure" / "compose" / filename).write_text( - "services: {}\n", encoding="utf-8" - ) - catalog_override = tmp_path / "catalog.yaml" - catalog_override.write_text("services: {}\n", encoding="utf-8") - inspections = dict(image_ids) - monkeypatch.setattr( - controller, - "inspect_image", - lambda reference: { - "Id": inspections.get(reference, reference), - "Config": {"Labels": labels}, - }, - ) - monkeypatch.setattr( - controller, - "_mcp_hub_compose_override", - lambda *_args, **_kwargs: catalog_override, - ) - monkeypatch.setattr(controller, "deployment_environment", lambda *_args: {}) - monkeypatch.setattr(controller, "mcp_hub_wait_services", lambda *_args, **_kwargs: None) - launched = [] - - def record_compose(command, **_kwargs): - pin_index = next( - index - for index, argument in enumerate(command[:-1]) - if argument == "--file" and Path(command[index + 1]).name == "image-pins.json" - ) - pin_path = Path(command[pin_index + 1]) - override = json.loads(pin_path.read_text(encoding="ascii")) - launched.append((command, override)) - return None - - monkeypatch.setattr(controller, "run", record_compose) - controller._mcp_hub_pin_built_image_ids(project, model) - expected_pins = { - name: {"image": project["_mcp_hub_rendered_model"]["services"][name]["image"]} - for name in sorted(controller.MCP_HUB_APPLICATION_SERVICES) - } - - # Simulate mutable tags being retargeted after inspection. Compose must use - # only the image IDs captured from the successful build inspection. - for index, tag in enumerate(image_ids): - inspections[tag] = "sha256:" + f"{index + 101:064x}" - controller.mcp_hub_up_services( - project, - project["_deployment_metadata"]["revision"], - controller.MCP_HUB_REPOSITORY, - controller.MCP_HUB_APPLICATION_SERVICES, - ) - - assert len(launched) == 1 - command, override = launched[0] - assert command[command.index("up") + 1:command.index("up") + 6] == [ - "--detach", "--no-build", "--pull", "never", "control-api" - ] - assert override == {"services": expected_pins} - assert all( - project["_mcp_hub_rendered_model"]["services"][name]["image"] - == image_ids[f"itworx-mcp-hub/{name}:aaaaaaaaaaaa"] - for name in controller.MCP_HUB_APPLICATION_SERVICES - ) - - -def test_hub_source_build_rejects_wrong_image_revision_before_pinning( - controller, monkeypatch, tmp_path -): - project, model, image_ids, labels = _built_hub_candidate(controller, tmp_path) - wrong_labels = {**labels, "org.opencontainers.image.revision": "d" * 40} - monkeypatch.setattr( - controller, - "inspect_image", - lambda reference: { - "Id": image_ids[reference], - "Config": {"Labels": wrong_labels}, - }, - ) - - with pytest.raises(controller.DeployError, match="not bound to the candidate source"): - controller._mcp_hub_pin_built_image_ids(project, model) - assert "_mcp_hub_rendered_model" not in project - - -def test_hub_recorded_source_builds_repin_the_exact_receipt_image_ids(controller): - model = _valid_model(controller) - containers = {} - expected = {} - for index, service_name in enumerate(controller.MCP_HUB_APPLICATION_SERVICES, 1): - model["services"][service_name]["build"] = {"context": "."} - image_id = "sha256:" + f"{index:064x}" - container_name = f"itworx-mcp-hub-{service_name}-1" - containers[container_name] = {"image_id": image_id, "image_ref": image_id} - expected[service_name] = image_id - - pinned = controller._mcp_hub_pin_recorded_build_images(model, containers) - - assert pinned == expected - assert all( - model["services"][name]["image"] == image_id - for name, image_id in expected.items() - ) - containers["itworx-mcp-hub-control-api-1"]["image_ref"] = "mutable:tag" - with pytest.raises(controller.DeployError, match="not pinned to its image ID"): - controller._mcp_hub_pin_recorded_build_images(model, containers) - - -def test_hub_runtime_rejects_controller_state_mount_and_host_capabilities(controller): - project = controller.effective_mcp_hub_project( - controller.MCP_HUB_REPOSITORY, _legacy_target(controller) - ) - model = _valid_model(controller) - model["services"]["control-api"]["volumes"] = [{ - "type": "bind", - "source": controller.STATE_ROOT.as_posix(), - "target": "/controller-state", - "read_only": True, - }] - with pytest.raises(controller.DeployError, match="only a protected catalog leaf read-only"): - controller.validate_mcp_hub_compose_model(project, model) - - model["services"]["control-api"]["volumes"] = [] - model["services"]["control-api"]["privileged"] = True - with pytest.raises(controller.DeployError, match="host filesystem or remount capabilities"): - controller.validate_mcp_hub_compose_model(project, model) - model["services"]["control-api"].pop("privileged") - model["services"]["control-api"]["cap_add"] = ["SYS_ADMIN"] - with pytest.raises(controller.DeployError, match="host filesystem or remount capabilities"): - controller.validate_mcp_hub_compose_model(project, model) - - -def test_hub_reconciler_requires_the_exact_four_file_release_bind_inventory(controller): - project = controller.effective_mcp_hub_project( - controller.MCP_HUB_REPOSITORY, _legacy_target(controller) - ) - revision = "d" * 40 - project["_mcp_hub_required_release_revision"] = revision - store = controller._MCP_HUB_RELEASE_STORE - model = _valid_model(controller) - for service, suffix, target, _kind in store.RELEASE_BIND_CONTRACT: - model["services"][service]["volumes"].append( - { - "type": "bind", - "source": f"{store.logical_release_path(revision)}/{suffix}", - "target": target, - "read_only": True, - "mode": "ro", - } - ) - - assert controller.validate_mcp_hub_compose_model( - project, - model, - require_catalog_mounts=True, - catalog_source="logical", - ) is model - host_model = store.translate_release_mounts(model, revision) - assert controller.validate_mcp_hub_compose_model( - project, - host_model, - require_catalog_mounts=True, - catalog_source="host", - ) is host_model - - missing = copy.deepcopy(model) - missing["services"]["redis"]["volumes"].pop(0) - with pytest.raises(controller.DeployError, match="in-release bind inventory"): - controller.validate_mcp_hub_compose_model( - project, - missing, - require_catalog_mounts=True, - catalog_source="logical", - ) - writable = copy.deepcopy(model) - writable["services"]["contextforge"]["volumes"][0]["read_only"] = False - with pytest.raises(controller.DeployError, match="in-release bind inventory"): - controller.validate_mcp_hub_compose_model( - project, - writable, - require_catalog_mounts=True, - catalog_source="logical", - ) - broad = copy.deepcopy(model) - broad["services"]["control-api"]["volumes"][0]["source"] = store.logical_release_path(revision) - with pytest.raises(controller.DeployError, match="in-release bind inventory"): - controller.validate_mcp_hub_compose_model( - project, - broad, - require_catalog_mounts=True, - catalog_source="logical", - ) - - @pytest.mark.skipif(os.name == "nt", reason="release publication requires POSIX ownership and directory fsync") def test_hub_release_source_stage_is_immutable_and_idempotent( controller, monkeypatch, tmp_path @@ -511,7 +314,6 @@ def test_hub_release_source_stage_is_immutable_and_idempotent( with pytest.raises(controller.DeployError, match="differs from its protected Git tree"): controller.mcp_hub_stage_release_source(project, tmp_path / "hub.git", revision) - @pytest.mark.skipif(os.name == "nt", reason="release publication requires POSIX directory fsync") def test_hub_release_stage_syncs_tree_before_rename_and_parent_after( controller, monkeypatch, tmp_path @@ -569,7 +371,6 @@ def test_hub_release_stage_syncs_tree_before_rename_and_parent_after( assert any(event == ("directory", "catalog") for event in events[:rename_index]) assert events.index(("release-parent", "")) > rename_index - @pytest.mark.skipif(os.name == "nt", reason="release publication requires POSIX ownership and directory fsync") def test_prior_release_can_only_be_staged_from_protected_main_ancestry( controller, monkeypatch, tmp_path @@ -613,7 +414,6 @@ def test_prior_release_can_only_be_staged_from_protected_main_ancestry( project, tmp_path / "hub.git", "c" * 40, protected_head=protected_head ) - @pytest.mark.skipif(os.name == "nt", reason="release metadata checks require POSIX mode bits") def test_hub_release_anchor_and_tree_reject_writable_metadata( controller, monkeypatch, tmp_path @@ -642,7 +442,6 @@ def test_hub_release_anchor_and_tree_reject_writable_metadata( with pytest.raises(controller.DeployError, match="writable or not canonical"): controller._mcp_hub_verify_release_tree_metadata(release) - def test_hub_rollback_mount_contract_ignores_only_release_generation_sha(controller): old = [{ "type": "bind", @@ -663,7 +462,6 @@ def test_hub_rollback_mount_contract_ignores_only_release_generation_sha(control candidate[0]["source"] = "/srv/example/appdata/other/worker.py" assert not controller._mcp_hub_mount_contract_equal(old, candidate) - def test_legacy_release_source_is_only_a_canonical_captured_path_identity( controller, monkeypatch, tmp_path ): @@ -676,7 +474,6 @@ def test_legacy_release_source_is_only_a_canonical_captured_path_identity( assert controller._mcp_hub_legacy_release_source(legacy + "/../catalog") is None assert controller._mcp_hub_release_source(legacy) is None - def test_live_mount_release_generation_is_bound_to_each_service_image(controller, monkeypatch, tmp_path): image_revision = "a" * 40 candidate_revision = "c" * 40 @@ -736,7 +533,6 @@ def test_live_mount_release_generation_is_bound_to_each_service_image(controller model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) - def test_legacy_live_release_mount_is_rebased_to_protected_tree_only_in_transition( controller, monkeypatch, tmp_path ): @@ -816,7 +612,6 @@ def test_legacy_live_release_mount_is_rebased_to_protected_tree_only_in_transiti model, "control-api", controller.MCP_HUB_PROJECT, container, project=project ) - @pytest.mark.parametrize( "field,value", [ @@ -838,7 +633,6 @@ def test_hub_topology_derivation_rejects_any_target_outside_the_fixed_legacy_con with pytest.raises(controller.DeployError, match="exact legacy 15-container contract"): controller.effective_mcp_hub_project(controller.MCP_HUB_REPOSITORY, target) - def test_rendered_model_requires_the_exact_17_services_and_intake_safety_gates(controller): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) @@ -869,7 +663,6 @@ def test_rendered_model_requires_the_exact_17_services_and_intake_safety_gates(c with pytest.raises(controller.DeployError, match="PostgreSQL image"): controller.validate_mcp_hub_compose_model(project, mutable_database) - def test_live_enabled_connector_flags_must_match_all_17_verified_services( controller, monkeypatch ): @@ -924,7 +717,6 @@ def test_live_enabled_connector_flags_must_match_all_17_verified_services( {"environment": {"INGESTION_CONNECTOR_ENABLED": "false"}}, ) - def test_backup_inputs_match_the_secret_free_live_contract(controller): recent_status = { "schemaVersion": "1", @@ -950,7 +742,6 @@ def test_backup_inputs_match_the_secret_free_live_contract(controller): with pytest.raises(controller.DeployError, match="revocation state is invalid"): controller._mcp_hub_validate_revocations(json.dumps(malformed_revocations).encode()) - def test_registered_target_archive_projection_accepts_only_the_broker_schema(controller): registrations = { "schemaVersion": "1.0", @@ -979,7 +770,6 @@ def test_registered_target_archive_projection_accepts_only_the_broker_schema(con with pytest.raises(controller.DeployError, match="registration schema"): controller._mcp_hub_validate_registered_targets(json.dumps(with_unknown_scope).encode()) - def test_backup_mount_inventory_includes_exact_registry_source(controller, monkeypatch, tmp_path): config_root = tmp_path / "appdata" / "config" config_root.mkdir(parents=True) @@ -1021,7 +811,6 @@ def test_backup_mount_inventory_includes_exact_registry_source(controller, monke with pytest.raises(controller.DeployError, match="registration mount"): controller.mcp_hub_fixed_backup_inputs(model) - def test_all_secret_file_mounts_are_fixed_private_metadata_only_inputs( controller, monkeypatch, tmp_path ): @@ -1087,7 +876,6 @@ def test_all_secret_file_mounts_are_fixed_private_metadata_only_inputs( metadata, 1001, 1000, platform_name="posix" ) - def test_database_backup_capacity_uses_dump_and_restore_size_headroom(controller, monkeypatch): sizes = {"itworx_mcp_hub": 1024**3, "contextforge": 2 * 1024**3} required_before = ( @@ -1112,7 +900,6 @@ def test_database_backup_capacity_uses_dump_and_restore_size_headroom(controller ) controller._mcp_hub_require_backup_space(sizes, after_dumps=True) - def test_database_size_query_is_limited_to_two_fixed_database_names(controller, monkeypatch): captured = {} @@ -1134,7 +921,6 @@ def test_database_size_query_is_limited_to_two_fixed_database_names(controller, assert "pg_database_size" in query assert captured["timeout"] == 30 - def test_failed_backup_attempt_removes_only_its_unique_private_directory( controller, monkeypatch, tmp_path ): @@ -1142,12 +928,7 @@ def test_failed_backup_attempt_removes_only_its_unique_private_directory( backup_root.mkdir(mode=0o700) monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", tmp_path) - private_directory = controller._mcp_hub_private_directory - monkeypatch.setattr( - controller, - "_mcp_hub_private_directory", - lambda path, **kwargs: private_directory(path, root=backup_root, **kwargs), - ) + monkeypatch.setattr(controller, "_mcp_hub_private_directory", lambda *_args, **_kwargs: None) monkeypatch.setattr(controller, "mcp_hub_validate_secret_files", lambda _model: 21) monkeypatch.setattr( controller, @@ -1186,7 +967,6 @@ def test_failed_backup_attempt_removes_only_its_unique_private_directory( assert list(backup_root.iterdir()) == [] - def test_rollback_keeps_digest_refs_immutable_and_only_tags_local_refs(controller, monkeypatch): image_id = "sha256:" + "a" * 64 digest = "sha256:" + "b" * 64 @@ -1206,6 +986,29 @@ def test_rollback_keeps_digest_refs_immutable_and_only_tags_local_refs(controlle controller._mcp_hub_prepare_rollback_image(image_id, "itworx-mcp-hub/service:old") assert tagged == [["docker", "image", "tag", image_id, "itworx-mcp-hub/service:old"]] +def test_digest_proof_normalizes_docker_hub_aliases_without_losing_registry_ports( + controller, +): + digest = "sha256:" + "a" * 64 + assert controller._mcp_hub_digest_ref_proven( + {"RepoDigests": [f"docker.io/library/postgres@{digest}"]}, + f"postgres:18.6@{digest}", + ) + assert controller._mcp_hub_digest_ref_proven( + {"RepoDigests": [f"redis@{digest}"]}, + f"redis:8.10.2@{digest}", + ) + assert controller._mcp_hub_digest_ref_proven( + {"RepoDigests": [f"registry.example:5000/team/service@{digest}"]}, + f"registry.example:5000/team/service:release@{digest}", + ) + assert not controller._mcp_hub_digest_ref_proven( + {"RepoDigests": [f"foreign/postgres@{digest}"]}, + f"postgres:18.6@{digest}", + ) + assert not controller._mcp_hub_digest_ref_proven( + {"RepoDigests": ["postgres@sha256:bad"]}, "postgres:18.6@sha256:bad" + ) def test_connector_ingestion_requires_both_digest_and_hub_source_ancestry(controller, monkeypatch): image_id = "sha256:" + "a" * 64 @@ -1256,7 +1059,6 @@ def test_connector_ingestion_requires_both_digest_and_hub_source_ancestry(contro ) assert result["image_revision"] == revision - def test_owned_image_revision_may_be_older_than_the_mixed_controller_receipt( controller, monkeypatch ): @@ -1303,254 +1105,6 @@ def test_owned_image_revision_may_be_older_than_the_mixed_controller_receipt( assert proven["image_revision"] == older_service_revision assert ancestor_checks == [(older_service_revision, candidate_revision)] - -@pytest.mark.parametrize( - ("container_name", "image_ref"), - [ - ("itworx-mcp-hub-contextforge-1", "itworx-mcp-hub/contextforge-schema:abc123456789"), - ("itworx-mcp-hub-dataplane-router-1", "itworx-mcp-hub/dataplane:sessionless-final"), - ], -) -def test_compatibility_tags_do_not_prove_candidate_image_bytes( - controller, monkeypatch, container_name, image_ref -): - image_id = "sha256:" + "a" * 64 - image = { - "Id": image_id, - "RepoDigests": [], - "Config": {"Labels": { - "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.revision": "b" * 40, - }}, - } - container = { - "Id": "c" * 64, - "Image": image_id, - "Config": {"Image": image_ref, "Labels": {}}, - } - monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) - monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) - monkeypatch.setattr(controller, "_mcp_hub_git_output", lambda *_args: "") - - with pytest.raises(controller.DeployError, match="provenance"): - controller._mcp_hub_live_image_proven( - {"_repository_path": "/fixed/hub.git"}, - container_name, - container, - image_ref, - "d" * 40, - "e" * 40, - require_current_provenance=True, - ) - - -def test_contextforge_candidate_requires_protected_digest_and_hub_image_provenance( - controller, monkeypatch -): - image_id = "sha256:" + "a" * 64 - digest = "sha256:" + "b" * 64 - image_ref = f"itworx-mcp-hub/contextforge-schema:abc123456789@{digest}" - candidate = "e" * 40 - image_revision = "d" * 40 - manifest = "\n".join( - ( - "CONTEXTFORGE_UPSTREAM_COMMIT=" + "1" * 40, - "CONTEXTFORGE_ALEMBIC_HEAD=abc123456789", - "CONTEXTFORGE_MCP_SDK=1.2.3", - "CONTEXTFORGE_LOCAL_IMAGE=itworx-mcp-hub/contextforge-schema:abc123456789", - f"CONTEXTFORGE_IMAGE_DIGEST={digest}", - ) - ) - builder = "\n".join( - ( - 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"', - 'git -C "$SRC" checkout --detach FETCH_HEAD', - "alembic -c mcpgateway/alembic.ini heads", - 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"', - ) - ) - image = { - "Id": image_id, - "RepoDigests": [f"itworx-mcp-hub/contextforge-schema@{digest}"], - "Config": {"Labels": { - "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.revision": image_revision, - }}, - } - monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) - monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) - monkeypatch.setattr( - controller, - "_mcp_hub_git_output", - lambda _project, _operation, path: manifest if path.endswith("compat-image.env") else builder, - ) - project = {"_repository_path": "/fixed/hub.git"} - - assert controller._mcp_hub_contextforge_compat_proven( - project, candidate, image_ref, image_id - ) - - for labels, image_reference, contents in ( - ( - {"org.opencontainers.image.source": "https://untrusted.example/hub", - "org.opencontainers.image.revision": image_revision}, - image_ref, - manifest, - ), - ( - {"org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.revision": "invalid"}, - image_ref, - manifest, - ), - ( - {"org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.revision": image_revision}, - image_ref, - manifest.replace(f"CONTEXTFORGE_IMAGE_DIGEST={digest}", ""), - ), - ( - {"org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.revision": image_revision}, - image_ref, - manifest.replace(digest, "sha256:" + "c" * 64), - ), - ): - image["Config"]["Labels"] = labels - monkeypatch.setattr( - controller, - "_mcp_hub_git_output", - lambda _project, _operation, path, contents=contents: ( - contents if path.endswith("compat-image.env") else builder - ), - ) - assert not controller._mcp_hub_contextforge_compat_proven( - project, candidate, image_reference, image_id - ) - - image["Config"]["Labels"] = { - "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.revision": image_revision, - } - monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: False) - monkeypatch.setattr( - controller, - "_mcp_hub_git_output", - lambda _project, _operation, path: manifest if path.endswith("compat-image.env") else builder, - ) - assert not controller._mcp_hub_contextforge_compat_proven( - project, candidate, image_ref, image_id - ) - - -@pytest.mark.parametrize( - ("image_source", "image_revision", "ancestor"), - [ - ("https://gitea.example.com/jens/itworx-mcp-hub", "c" * 40, True), - ("https://untrusted.example/hub", "c" * 40, True), - ("https://gitea.example.com/jens/itworx-mcp-hub", "invalid", True), - ("https://gitea.example.com/jens/itworx-mcp-hub", "a" * 40, False), - ], -) -def test_dataplane_candidate_requires_exact_source_and_revision_with_the_fix( - controller, monkeypatch, image_source, image_revision, ancestor -): - image_id = "sha256:" + "a" * 64 - digest = "sha256:" + "b" * 64 - image_ref = f"itworx-mcp-hub/dataplane:sessionless-final@{digest}" - candidate = "d" * 40 - required_commit = "b519a8c4cf17cd9df00a575acfe096838aab7789" - image = { - "Id": image_id, - "RepoDigests": [f"itworx-mcp-hub/dataplane@{digest}"], - "Config": {"Labels": { - "org.opencontainers.image.source": image_source, - "org.opencontainers.image.revision": image_revision, - }}, - } - monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) - checks = [] - - def is_ancestor(_project, base, head): - checks.append((base, head)) - return ancestor and ( - (base == required_commit and head == image_revision) - or (base == image_revision and head == candidate) - ) - - monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", is_ancestor) - monkeypatch.setattr( - controller, - "_mcp_hub_git_output", - lambda _project, _operation, path: ( - 'MCP_TRANSPORT_STATE_HEADERS = frozenset({b"mcp-session-id"})\n' - 'del response.headers["mcp-session-id"]\n2025-11-25 2026-07-28 mcp-session-id' - if path.endswith("router.py") or path.endswith("test_dataplane_session_headers.py") - else f"{required_commit} itworx-mcp-hub/dataplane:sessionless-final permanently enforce stateless public transport" - ), - ) - - proven = controller._mcp_hub_dataplane_compat_proven( - {"_repository_path": "/fixed/hub.git"}, - candidate, - "itworx-mcp-hub-dataplane-router-1", - image_ref, - image_id, - ) - assert proven is (image_source == controller.MCP_HUB_OCI_IMAGE_SOURCE - and image_revision != "invalid" and ancestor) - if proven: - assert checks == [ - (required_commit, image_revision), - (image_revision, candidate), - ] - - -def test_unchanged_compatibility_image_requires_exact_legacy_receipt_identity( - controller, monkeypatch -): - image_id = "sha256:" + "a" * 64 - image_ref = "itworx-mcp-hub/dataplane:sessionless-final" - container_id = "c" * 64 - image = {"Id": image_id, "RepoDigests": [], "Config": {"Labels": {}}} - container = { - "Id": container_id, - "Image": image_id, - "Config": {"Image": image_ref, "Labels": {}}, - } - monkeypatch.setattr(controller, "inspect_image", lambda _image_id: image) - receipt_identity = { - "container_id": container_id, - "image_id": image_id, - "image_ref": image_ref, - } - - observed = controller._mcp_hub_live_image_proven( - {"_mcp_hub_legacy_receipt": True}, - "itworx-mcp-hub-dataplane-router-1", - container, - image_ref, - "b" * 40, - "c" * 40, - require_current_provenance=False, - prior_receipt_identity=receipt_identity, - ) - assert observed["image_id"] == image_id - - changed = {**container, "Id": "d" * 64} - with pytest.raises(controller.DeployError, match="exact verified receipt"): - controller._mcp_hub_live_image_proven( - {"_mcp_hub_legacy_receipt": True}, - "itworx-mcp-hub-dataplane-router-1", - changed, - image_ref, - "b" * 40, - "c" * 40, - require_current_provenance=False, - prior_receipt_identity=receipt_identity, - ) - - def test_catalog_archiver_accepts_only_bounded_regular_catalog_members(controller, monkeypatch): stream = io.BytesIO() with tarfile.open(fileobj=stream, mode="w:") as archive: @@ -1579,7 +1133,6 @@ def test_catalog_archiver_accepts_only_bounded_regular_catalog_members(controlle with pytest.raises(controller.DeployError, match="unsafe member"): controller._mcp_hub_git_catalog_archive({"slug": "itworx-mcp-hub"}, "a" * 40) - def test_backup_catalog_archives_bind_exact_mount_and_active_image_revisions( controller, monkeypatch, tmp_path ): @@ -1587,7 +1140,7 @@ def test_backup_catalog_archives_bind_exact_mount_and_active_image_revisions( monkeypatch.setattr(controller, "MCP_HUB_APPDATA_ROOT", legacy_appdata_root) source_dir = tmp_path / "source" candidate_revision = "b" * 40 - candidate_catalog = f"{controller._MCP_HUB_RELEASE_STORE.logical_release_path(candidate_revision)}/catalog" + candidate_catalog = (controller.MCP_HUB_RELEASES_ROOT / candidate_revision / "catalog").as_posix() model = {"services": {}, "volumes": {}, "secrets": {}} runtime = {} image_facts = {} @@ -1654,7 +1207,6 @@ def test_backup_catalog_archives_bind_exact_mount_and_active_image_revisions( tmp_path / "negative-backup", ) - def test_pg_restore_streams_private_dump_over_stdin_without_mounting_backup( controller, monkeypatch, tmp_path ): @@ -1684,7 +1236,6 @@ def test_pg_restore_streams_private_dump_over_stdin_without_mounting_backup( "--dbname=contextforge", "-", ] - def test_private_backup_artifacts_publish_atomically_without_overwrite( controller, tmp_path ): @@ -1699,16 +1250,20 @@ def test_private_backup_artifacts_publish_atomically_without_overwrite( assert artifact.read_bytes() == b"first" assert list(tmp_path.glob("*.tmp")) == [] - +@pytest.mark.parametrize( + "image_ref_name", + ["MCP_HUB_POSTGRES_IMAGE", "MCP_HUB_POSTGRES_LEGACY_IMAGE"], +) def test_live_mount_inventory_includes_volume_identity_source_and_read_only_state( - controller, monkeypatch + controller, monkeypatch, image_ref_name ): + image_ref = getattr(controller, image_ref_name) config_source = (Path("/srv/example/appdata/itworx-mcp-hub/config") if sys.platform != "win32" else Path("C:/appdata/itworx-mcp-hub/config")) model = { "services": { "postgres": { - "image": controller.MCP_HUB_POSTGRES_IMAGE, + "image": image_ref, "environment": {"PGDATA": controller.MCP_HUB_POSTGRES_PGDATA}, "volumes": [ {"type": "volume", "source": "pgdata", "target": "/var/lib/postgresql/data", "read_only": False}, @@ -1727,7 +1282,7 @@ def test_live_mount_inventory_includes_volume_identity_source_and_read_only_stat image_id = "sha256:" + "a" * 64 live = { "Image": image_id, - "Config": {"Image": controller.MCP_HUB_POSTGRES_IMAGE}, + "Config": {"Image": image_ref}, "Mounts": [ {"Type": "volume", "Name": "itworx-mcp-hub-postgres", "Source": mountpoint, "Driver": "local", "Destination": "/var/lib/postgresql/data", "RW": True}, {"Type": "volume", "Name": anon_name, "Source": anon_mountpoint, "Driver": "local", "Destination": "/var/lib/postgresql", "RW": True}, @@ -1739,13 +1294,14 @@ def test_live_mount_inventory_includes_volume_identity_source_and_read_only_stat "itworx-mcp-hub-postgres": mountpoint, anon_name: anon_mountpoint, } + created_at = "2020-01-01T10:00:00Z" monkeypatch.setattr( controller, "_mcp_hub_docker", lambda _kind, _operation, name, **_kwargs: subprocess.CompletedProcess( [], 0, stdout=( - f'[{json.dumps({"Name": name, "Driver": "local", "Mountpoint": volume_data[name], "Options": {}})}]' + f'[{json.dumps({"Name": name, "Driver": "local", "Mountpoint": volume_data[name], "Options": {}, "CreatedAt": created_at})}]' ).encode("utf-8"), ), ) @@ -1754,7 +1310,7 @@ def test_live_mount_inventory_includes_volume_identity_source_and_read_only_stat "inspect_image", lambda image: { "Id": image_id, - "RepoDigests": [controller.MCP_HUB_POSTGRES_IMAGE.split("@", 1)[0].split(":", 1)[0] + "@" + controller.MCP_HUB_POSTGRES_IMAGE.split("@", 1)[1]], + "RepoDigests": [controller._mcp_hub_image_repository(image_ref) + "@" + image_ref.split("@", 1)[1]], "Config": {"Volumes": {"/var/lib/postgresql": {}}}, }, ) @@ -1774,6 +1330,7 @@ def test_live_mount_inventory_includes_volume_identity_source_and_read_only_stat "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, + "created_at": created_at, } anonymous_volume = next(item for item in attested if item["destination"] == "/var/lib/postgresql") assert anonymous_volume == { @@ -1784,6 +1341,7 @@ def test_live_mount_inventory_includes_volume_identity_source_and_read_only_stat "name": anon_name, "driver": "local", "options": {}, + "created_at": created_at, } assert next(item for item in attested if item["destination"] == "/etc/itworx")["rw"] is False live["Mounts"][0]["Name"] = "unexpected-postgres-volume" @@ -1792,7 +1350,6 @@ def test_live_mount_inventory_includes_volume_identity_source_and_read_only_stat model, "postgres", controller.MCP_HUB_PROJECT, live ) - @pytest.mark.parametrize( "mutate,match", [ @@ -1829,6 +1386,7 @@ def test_postgres_anonymous_volume_is_fixed_to_the_pinned_image_and_path( "Driver": "local", "Mountpoint": volume_data[name], "Options": {}, + "CreatedAt": "2020-01-01T10:00:00Z", } return subprocess.CompletedProcess([], 0, stdout=json.dumps([payload]).encode("utf-8")) @@ -1867,7 +1425,6 @@ def test_postgres_anonymous_volume_is_fixed_to_the_pinned_image_and_path( model, "postgres", controller.MCP_HUB_PROJECT, container ) - def test_postgres_staged_deploy_must_preserve_both_volume_identities(controller): data_volume = { "type": "volume", @@ -1877,6 +1434,7 @@ def test_postgres_staged_deploy_must_preserve_both_volume_identities(controller) "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, + "created_at": "2020-01-01T10:00:00Z", } anonymous_name = "f" * 64 anonymous_volume = { @@ -1887,6 +1445,7 @@ def test_postgres_staged_deploy_must_preserve_both_volume_identities(controller) "name": anonymous_name, "driver": "local", "options": {}, + "created_at": "2020-01-01T10:00:01Z", } snapshot = { "containers": [{ @@ -1909,6 +1468,153 @@ def test_postgres_staged_deploy_must_preserve_both_volume_identities(controller) with pytest.raises(controller.DeployError, match="changed a PostgreSQL data volume identity"): controller._mcp_hub_require_postgres_volumes_preserved(snapshot, changed) + changed = json.loads(json.dumps(runtime)) + changed[controller.MCP_HUB_POSTGRES_CONTAINER]["_appops_attested_mounts"][0][ + "created_at" + ] = "2021-01-01T09:00:00Z" + with pytest.raises(controller.DeployError, match="changed a PostgreSQL data volume identity"): + controller._mcp_hub_require_postgres_volumes_preserved(snapshot, changed) + +@pytest.mark.parametrize( + "created_at", + [None, "not-a-timestamp", "2020-01-01T10:00:00", "2999-01-01T00:00:00Z"], +) +def test_docker_volume_metadata_requires_valid_nonfuture_creation_time( + controller, monkeypatch, created_at +): + payload = { + "Name": "itworx-mcp-hub-postgres", + "Driver": "local", + "Mountpoint": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", + "Options": {}, + } + if created_at is not None: + payload["CreatedAt"] = created_at + monkeypatch.setattr( + controller, + "_mcp_hub_docker", + lambda *_args, **_kwargs: subprocess.CompletedProcess( + [], 0, stdout=json.dumps([payload]).encode("utf-8") + ), + ) + + with pytest.raises(controller.DeployError, match="creation timestamp"): + controller._mcp_hub_volume_metadata("itworx-mcp-hub-postgres", {}) + +@pytest.mark.parametrize("baseline_kind", ["observed", "normal_upgrade"]) +def test_observed_postgres_volume_continuity_rechecks_created_at_independently( + controller, monkeypatch, baseline_kind +): + baseline = [ + { + "type": "volume", "source": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", + "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, "rw": True, + "name": "itworx-mcp-hub-postgres", "driver": "local", "options": {}, + "created_at": "2020-01-01T10:00:00Z", + }, + { + "type": "volume", "source": f"/var/lib/docker/volumes/{'f' * 64}/_data", + "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, "rw": True, + "name": "f" * 64, "driver": "local", "options": {}, + "created_at": "2020-01-01T10:00:01Z", + }, + ] + current = json.loads(json.dumps(baseline)) + project = { + "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + "_mcp_hub_observed_postgres_volume_baseline": baseline, + "_mcp_hub_rendered_model": _valid_model(controller), + "compose_project": controller.MCP_HUB_PROJECT, + } + if baseline_kind == "normal_upgrade": + project.pop("_mcp_hub_observed_postgres_volume_baseline") + project["_compose_rollback"] = { + "containers": [{"name": controller.MCP_HUB_POSTGRES_CONTAINER, "mounts": baseline}] + } + postgres = { + "Id": "a" * 64, + "Image": "sha256:" + "b" * 64, + "Config": {"Image": controller.MCP_HUB_POSTGRES_IMAGE, "Labels": { + "com.docker.compose.project": controller.MCP_HUB_PROJECT, + "com.docker.compose.service": "postgres", + }}, + "State": {"Running": True, "Health": {"Status": "healthy"}}, + } + monkeypatch.setattr(controller, "inspect_container", lambda _name: postgres) + monkeypatch.setattr( + controller, "mcp_hub_attested_live_mounts", lambda *_args, **_kwargs: current + ) + + controller._mcp_hub_require_observed_postgres_volume_continuity(project) + + # Docker can recreate a named volume with the same name and mountpoint. + # Creation-time evidence must still detect that replacement. + current[0]["created_at"] = "2021-01-01T10:00:00Z" + with pytest.raises(controller.DeployError, match="PostgreSQL volume identity changed"): + controller._mcp_hub_require_observed_postgres_volume_continuity(project) + +@pytest.mark.parametrize("intent_status", ["started", "complete", None]) +def test_explicit_hub_rollback_is_blocked_before_repository_or_runtime_actions( + controller, monkeypatch, intent_status +): + target = _legacy_target(controller) + monkeypatch.setattr( + controller, + "load_projects", + lambda: {controller.MCP_HUB_REPOSITORY: target}, + ) + intent = ( + ({"status": intent_status}, b"private-intent") + if intent_status is not None + else None + ) + monkeypatch.setattr(controller, "_mcp_hub_read_migration_intent", lambda: intent) + monkeypatch.setattr( + controller, "repository_path", + lambda *_args: pytest.fail("rollback guard must precede repository/source actions"), + ) + monkeypatch.setattr( + sys, "argv", + ["deploy.py", "rollback", controller.MCP_HUB_REPOSITORY, "a" * 40], + ) + + with pytest.raises(controller.DeployError, match="image-only rollback is unsupported"): + controller.main() + +def test_explicit_rollback_guard_does_not_block_unrelated_targets(controller, monkeypatch): + class ReachedRepositoryResolution(Exception): + pass + + target = {"slug": "unraid-appops-gateway", "branch": "main", "containers": []} + monkeypatch.setattr( + controller, "load_projects", lambda: {"jens/unraid-appops-gateway": target} + ) + monkeypatch.setattr( + controller, "repository_path", + lambda *_args: (_ for _ in ()).throw(ReachedRepositoryResolution()), + ) + monkeypatch.setattr( + sys, "argv", + ["deploy.py", "rollback", "jens/unraid-appops-gateway", "a" * 40], + ) + + with pytest.raises(ReachedRepositoryResolution): + controller.main() + +def test_started_migration_marker_requires_manual_recovery_without_rereading_intent( + controller, monkeypatch +): + project = { + "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + "_mcp_hub_migration_started": True, + } + monkeypatch.setattr( + controller, + "_mcp_hub_read_migration_intent", + lambda: pytest.fail("durable pre-apply marker already requires manual recovery"), + ) + + assert controller._mcp_hub_intent_requires_manual_recovery(project) is True def test_restore_probe_is_offline_and_uses_only_temporary_database_storage( controller, monkeypatch, tmp_path @@ -1963,7 +1669,6 @@ def test_restore_probe_is_offline_and_uses_only_temporary_database_storage( assert all(identifier == "a" * 64 and source == backup for identifier, _, source in restored_dumps) assert [call[0][0] for call in calls].count("rm") == 1 - def _legacy_migration_fixture(controller, monkeypatch, tmp_path): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) @@ -2038,7 +1743,6 @@ def _legacy_migration_fixture(controller, monkeypatch, tmp_path): state_path.chmod(0o600) monkeypatch.setattr(controller, "STATE_ROOT", tmp_path) monkeypatch.setattr(controller, "_root_owned", lambda _path: True) - monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) monkeypatch.setattr(controller, "validate_recorded_receipt", lambda *_args: {}) monkeypatch.setattr(controller, "_mcp_hub_git_output", lambda *_args: candidate) monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) @@ -2061,59 +1765,430 @@ def _legacy_migration_fixture(controller, monkeypatch, tmp_path): } return project, candidate, model, backup_receipt, runtime, state_path, prior +def _baseline_image_facts(controller, runtime): + facts = {} + for name, container in runtime.items(): + service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") + image_source = ( + controller.MCP_HUB_OCI_IMAGE_SOURCE + if service_name not in {"postgres", "redis", "contextforge", "ingestion-clamd"} + else "" + ) + image_revision = "a" * 40 if image_source else "" + proof_kind = ( + "owned_git" + if image_source + else "contextforge_compat" + if service_name == "contextforge" + else "external_contract" + ) + facts[name] = { + "container_id": str(container["Id"]), + "image_id": str(container["Image"]), + "image_ref": str(container["Config"]["Image"]), + "container_revision": "", + "image_revision": image_revision, + "image_source": image_source, + "image_tree": "", + "image_tree_label_present": False, + "image_git_tree": "b" * 40 if image_source else "", + "proof_kind": proof_kind, + "mounts": list(container["_appops_attested_mounts"]), + } + return facts -def test_legacy_state_migration_is_backup_bound_and_writes_a_complete_17_service_baseline( +def _observed_live_fixture(controller, monkeypatch, tmp_path): + project, candidate, _model, backup, runtime, state_path, prior = _legacy_migration_fixture( + controller, monkeypatch, tmp_path + ) + model = _observed_candidate_model(controller) + prior["status"] = "failed" + prior["live_status"] = "unknown" + state_path.write_text(json.dumps(prior), encoding="utf-8") + state_path.chmod(0o600) + project["_deployment_metadata"] = {"revision": candidate} + project["_repository_path"] = str(tmp_path / "hub.git") + monkeypatch.setattr(controller, "_mcp_hub_git_output", lambda _project, _cmd, *args: ( + candidate if args == ("refs/heads/main",) else "c" * 40 + )) + monkeypatch.setattr(controller, "_mcp_hub_is_ancestor", lambda *_args: True) + monkeypatch.setattr(controller, "compose_project_container_names", lambda _project: set(runtime)) + monkeypatch.setattr(controller, "inspect_container", lambda name: runtime.get(name)) + image_inventory = {} + for index, name in enumerate(controller.MCP_HUB_CONTAINERS): + service_name = name.removeprefix("itworx-mcp-hub-").removesuffix("-1") + container = runtime[name] + container["Id"] = f"{index + 101:064x}" + container["Image"] = "sha256:" + f"{index + 101:064x}" + container["State"] = {"Running": True, "Health": {"Status": "healthy"}} + container["Config"]["Labels"] = { + "com.docker.compose.project": controller.MCP_HUB_PROJECT, + "com.docker.compose.service": service_name, + } + container["Config"]["Env"] = ( + ["INGESTION_CONNECTOR_ENABLED=true"] + if service_name in {"control-api", "worker"} + else [] + ) + if service_name == "postgres": + container["Config"]["Env"].append( + f"PGDATA={controller.MCP_HUB_POSTGRES_PGDATA}" + ) + if service_name == "contextforge": + container["Config"]["Env"].append("MCPGATEWAY_SKIP_MIGRATIONS=true") + image_revision = ("a" if index % 2 == 0 else "d") * 40 + image_labels = {} + if service_name not in {"postgres", "redis", "contextforge", "ingestion-clamd"}: + image_labels = { + "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, + "org.opencontainers.image.revision": image_revision, + } + if service_name in controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES: + image_repository = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES[service_name] + image_ref = f"{image_repository}@sha256:{index + 1:064x}" + elif service_name == "postgres": + image_ref = controller.MCP_HUB_POSTGRES_LEGACY_IMAGE + elif service_name == "redis": + image_ref = controller.MCP_HUB_REDIS_LEGACY_IMAGE + else: + image_ref = str(model["services"][service_name]["image"]) + container["Config"]["Image"] = image_ref + if service_name == "postgres": + data_source = "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data" + anonymous_name = "f" * 64 + container["_test_attested_mounts"] = [ + { + "type": "volume", + "source": data_source, + "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, + "rw": True, + "name": "itworx-mcp-hub-postgres", + "driver": "local", + "options": {}, + "created_at": "2020-01-01T10:00:00Z", + }, + { + "type": "volume", + "source": f"/var/lib/docker/volumes/{anonymous_name}/_data", + "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, + "rw": True, + "name": anonymous_name, + "driver": "local", + "options": {}, + "created_at": "2020-01-01T10:00:01Z", + }, + ] + repo_digests = [] + if "@" in image_ref: + repo_digests = [ + f"{controller._mcp_hub_image_repository(image_ref)}@{image_ref.split('@', 1)[1]}" + ] + image_inventory[str(container["Image"])] = { + "Id": str(container["Image"]), + "RepoDigests": repo_digests, + "Config": {"Labels": image_labels}, + } + monkeypatch.setattr(controller, "inspect_image", lambda image_id: image_inventory.get(image_id)) + def observed_live_image_proof(_project, _name, container, image_ref, *_args, **_kwargs): + image = image_inventory[str(container["Image"])] + labels = dict(image["Config"]["Labels"]) + return { + "container_id": str(container["Id"]), + "image_id": str(container["Image"]), + "image_ref": image_ref, + "container_revision": str( + dict(container.get("Config") or {}).get("Labels", {}).get( + "org.opencontainers.image.revision", "" + ) + ), + "image_revision": str(labels.get("org.opencontainers.image.revision") or ""), + "image_source": str(labels.get("org.opencontainers.image.source") or ""), + "image_tree": str(labels.get("org.opencontainers.image.source-tree") or ""), + "image_version": str(labels.get("org.opencontainers.image.version") or ""), + "image_created": str(labels.get("org.opencontainers.image.created") or ""), + } + + monkeypatch.setattr(controller, "_mcp_hub_live_image_proven", observed_live_image_proof) + monkeypatch.setattr(controller, "_mcp_hub_contextforge_compat_proven", lambda *_args: True) + monkeypatch.setattr(controller, "_mcp_hub_dataplane_compat_proven", lambda *_args: True) + monkeypatch.setattr( + controller, + "mcp_hub_attested_live_mounts", + lambda _model, _service, _project, container, **_kwargs: list( + container.get("_test_attested_mounts", []) + ), + ) + return project, candidate, model, backup, runtime, image_inventory, state_path, prior + +def _trusted_backup_receipt_fixture(controller, monkeypatch, tmp_path, candidate): + state_root = tmp_path + backup_root = state_root / "mcp-hub-backups" + backup_root.mkdir(mode=0o700, exist_ok=True) + receipt_root = state_root / "mcp-hub-backup-receipts" + receipt_root.mkdir(mode=0o700, exist_ok=True) + backup_directory = backup_root / f"1780400000-{candidate[:12]}-0123456789abcdef" + backup_directory.mkdir(mode=0o700) + compose_digest = "e" * 64 + manifest = { + "schema": "itworx.mcp-hub-backup/v1", + "repository": controller.MCP_HUB_REPOSITORY, + "revision": candidate, + "backupDirectory": str(backup_directory), + "composeSha256": compose_digest, + "configRestoreVerified": True, + "restore": {"status": "verified"}, + "databases": {name: {} for name in controller.MCP_HUB_DATABASES}, + } + manifest_bytes = json.dumps(manifest, sort_keys=True).encode("utf-8") + manifest_path = backup_directory / "manifest.json" + manifest_path.write_bytes(manifest_bytes) + manifest_path.chmod(0o600) + receipt_payload = { + "schema": "itworx.mcp-hub-backup-receipt/v1", + "repository": controller.MCP_HUB_REPOSITORY, + "revision": candidate, + "backupDirectory": str(backup_directory), + "manifestSha256": hashlib.sha256(manifest_bytes).hexdigest(), + "composeSha256": compose_digest, + "restoreVerified": True, + "createdAtEpoch": 1780400000, + } + receipt_path = receipt_root / f"backup-{candidate[:12]}.json" + receipt_path.write_text(json.dumps(receipt_payload, sort_keys=True), encoding="utf-8") + receipt_path.chmod(0o600) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "MCP_HUB_BACKUP_ROOT", backup_root) + monkeypatch.setattr(controller, "_root_owned", lambda _path: True) + monkeypatch.setattr(controller, "_mcp_hub_private_directory", lambda *_args, **_kwargs: None) + monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda *_args, **_kwargs: None) + return {**receipt_payload, "receiptPath": str(receipt_path)} + +def _record_test_observed_baseline(controller, monkeypatch, tmp_path): + project, candidate, model, _backup, runtime, _images, state_path, prior = _observed_live_fixture( + controller, monkeypatch, tmp_path + ) + project["_mcp_hub_rendered_model"] = model + original_state = state_path.read_bytes() + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + backup = _trusted_backup_receipt_fixture(controller, monkeypatch, tmp_path, candidate) + (tmp_path / "mcp-hub-topology-migrations").mkdir(mode=0o700) + verified_runtime, facts, _model = controller.mcp_hub_verify_observed_baseline( + project, prior, model + ) + evidence = controller.mcp_hub_record_observed_baseline( + project, prior, candidate, backup, verified_runtime, facts + ) + project["_mcp_hub_observed_baseline"] = evidence + return project, candidate, model, backup, runtime, state_path, prior, evidence, original_state + +def test_observed_baseline_is_separate_evidence_and_preserves_unknown_controller_state( + controller, monkeypatch, tmp_path +): + ( + _project, candidate, _model, backup, runtime, state_path, prior, evidence, original_state + ) = _record_test_observed_baseline(controller, monkeypatch, tmp_path) + + assert state_path.read_bytes() == original_state + assert evidence["schema"] == controller.MCP_HUB_OBSERVED_BASELINE_SCHEMA + evidence_path = Path(evidence["evidencePath"]) + record = json.loads(evidence_path.read_text(encoding="utf-8")) + assert record["claim"] == "observed_mixed_runtime_facts_only" + assert record["historicalStateSha256"] == hashlib.sha256(original_state).hexdigest() + assert record["historicalStatus"] == "failed" + assert record["historicalLiveStatus"] == "unknown" + assert record["backupManifestSha256"] == backup["manifestSha256"] + assert record["backupRestoreVerified"] is True + assert record["postgresVolumeBaseline"] == [ + { + "type": "volume", + "source": "/var/lib/docker/volumes/itworx-mcp-hub-postgres/_data", + "destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME, + "rw": True, + "name": "itworx-mcp-hub-postgres", + "driver": "local", + "options": {}, + "created_at": "2020-01-01T10:00:00Z", + }, + { + "type": "volume", + "source": f"/var/lib/docker/volumes/{'f' * 64}/_data", + "destination": controller.MCP_HUB_POSTGRES_ANONYMOUS_VOLUME, + "rw": True, + "name": "f" * 64, + "driver": "local", + "options": {}, + "created_at": "2020-01-01T10:00:01Z", + }, + ] + assert "live_status" not in record and "status" not in record + assert record["candidateRevision"] == candidate + assert set(record["containers"]) == set(controller.MCP_HUB_CONTAINERS) + for name in controller.MCP_HUB_LEGACY_CONTAINERS: + live = runtime[name] + historical = prior["live_artifacts"]["containers"][name] + assert ( + live["Id"], + live["Image"], + live["Config"]["Image"], + ) != ( + historical["container_id"], + historical["image_id"], + historical["image_ref"], + ) + owned_revisions = { + item["image_revision"] + for item in record["containers"].values() + if item.get("image_source") == controller.MCP_HUB_OCI_IMAGE_SOURCE + } + assert len(owned_revisions) > 1 + assert all( + isinstance(item, dict) and "image_tree_label_present" in item + for item in record["containers"].values() + ) + assert evidence_path.is_relative_to(tmp_path / "mcp-hub-topology-migrations") + assert hashlib.sha256(evidence_path.read_bytes()).hexdigest() == evidence["evidenceSha256"] + if os.name != "nt": + assert stat.S_IMODE(evidence_path.stat().st_mode) == 0o600 + +def test_backup_receipt_validator_rejects_an_unpersisted_boolean_claim( + controller, monkeypatch, tmp_path +): + candidate = "b" * 40 + _trusted_backup_receipt_fixture(controller, monkeypatch, tmp_path, candidate) + fake = { + "restoreVerified": True, + "repository": controller.MCP_HUB_REPOSITORY, + "revision": candidate, + "manifestSha256": "d" * 64, + "composeSha256": "e" * 64, + } + with pytest.raises(controller.DeployError, match="exact controller-verified capture"): + controller._mcp_hub_validate_backup_receipt(fake, candidate) + +def test_observed_baseline_rejects_missing_per_image_tree_resolution_without_state_write( + controller, monkeypatch, tmp_path +): + project, candidate, model, backup, runtime, _images, state_path, prior = _observed_live_fixture( + controller, monkeypatch, tmp_path + ) + original_state = state_path.read_bytes() + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + project["_mcp_hub_rendered_model"] = model + _verified_runtime, facts, _model = controller.mcp_hub_verify_observed_baseline( + project, prior, model + ) + facts.pop(controller.MCP_HUB_CONTAINERS[0]) + + with pytest.raises(controller.DeployError, match="mixed runtime changed"): + controller.mcp_hub_record_observed_baseline( + project, prior, candidate, backup, runtime, facts + ) + + assert state_path.read_bytes() == original_state + +@pytest.mark.parametrize("service_name", ["postgres", "redis"]) +def test_observed_baseline_accepts_only_the_exact_reviewed_external_transition( + controller, monkeypatch, tmp_path, service_name +): + project, candidate, model, _backup, _runtime, _images, state_path, prior = ( + _observed_live_fixture(controller, monkeypatch, tmp_path) + ) + original_state = state_path.read_bytes() + model["services"][service_name]["image"] = f"{service_name}:latest@sha256:{'f' * 64}" + + with pytest.raises(controller.DeployError, match="explicitly reviewed legacy-to-candidate transition"): + controller.mcp_hub_verify_observed_baseline(project, prior, model) + + assert state_path.read_bytes() == original_state + +def test_observed_baseline_requires_historical_state_artifacts_to_match_the_receipt( + controller, monkeypatch, tmp_path +): + project, _candidate, model, _backup, _runtime, _images, state_path, prior = ( + _observed_live_fixture(controller, monkeypatch, tmp_path) + ) + web = "itworx-mcp-hub-web-1" + prior["live_artifacts"]["containers"][web]["container_id"] = "f" * 64 + state_path.write_text(json.dumps(prior), encoding="utf-8") + state_path.chmod(0o600) + original_state = state_path.read_bytes() + + with pytest.raises(controller.DeployError, match="historical state artifacts differ"): + controller.mcp_hub_verify_observed_baseline(project, prior, model) + + assert state_path.read_bytes() == original_state + +@pytest.mark.parametrize( + ("drift", "message"), + [ + ("no-health", "identity or health"), + ("wrong-service", "identity or health"), + ("mount-drift", "mount set"), + ], +) +def test_observed_baseline_rejects_health_label_and_mount_drift_before_recording( + controller, monkeypatch, tmp_path, drift, message +): + project, candidate, model, _backup, runtime, _images, state_path, prior = _observed_live_fixture( + controller, monkeypatch, tmp_path + ) + original_state = state_path.read_bytes() + target = "itworx-mcp-hub-web-1" + if drift == "no-health": + runtime[target]["State"]["Health"]["Status"] = "none" + elif drift == "wrong-service": + runtime[target]["Config"]["Labels"]["com.docker.compose.service"] = "worker" + else: + runtime[target]["Mounts"] = [{ + "Type": "bind", + "Source": "/srv/example/appdata/unexpected", + "Destination": "/opt/unexpected", + "RW": True, + }] + monkeypatch.setattr( + controller, + "mcp_hub_attested_live_mounts", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + controller.DeployError("MCP Hub live mount set differs from its rendered Compose model") + ), + ) + + with pytest.raises(controller.DeployError, match=message): + controller.mcp_hub_verify_observed_baseline(project, prior, model) + + assert state_path.read_bytes() == original_state + assert not (tmp_path / "mcp-hub-topology-migrations").exists() + +def test_hub_backup_anchor_is_below_controller_state_not_appdata(controller): + assert controller.MCP_HUB_BACKUP_ROOT == controller.STATE_ROOT / "mcp-hub-backups" + assert not controller.MCP_HUB_BACKUP_ROOT.is_relative_to(controller.MCP_HUB_APPDATA_ROOT) + +def test_legacy_state_migration_cannot_promote_or_rewrite_the_unknown_baseline( controller, monkeypatch, tmp_path ): project, candidate, model, backup, runtime, state_path, prior = _legacy_migration_fixture( controller, monkeypatch, tmp_path ) - if os.name == "nt": - real_open = os.open - sync_marker = tmp_path / "directory-fsync-marker" + prior["status"] = "failed" + prior["live_status"] = "unknown" + state_path.write_text(json.dumps(prior), encoding="utf-8") + state_path.chmod(0o600) + original_state = state_path.read_bytes() - def open_directory_marker(path, flags, *args, **kwargs): - if Path(path) == tmp_path: - return real_open(sync_marker, os.O_CREAT | os.O_RDWR, 0o600) - return real_open(path, flags, *args, **kwargs) - - monkeypatch.setattr(controller.os, "open", open_directory_marker) - - result = controller.migrate_mcp_hub_legacy_state( - project, - controller.MCP_HUB_REPOSITORY, - candidate, - model, - backup, - runtime, - ) - - assert set(result["live_artifacts"]["containers"]) == set(controller.MCP_HUB_CONTAINERS) - assert result["live_artifacts"]["topology_contract"] == controller.MCP_HUB_TOPOLOGY_CONTRACT - assert result["live_legacy_provenance"] is True - assert result["live_receipt"] == prior["live_receipt"] - assert result["rollback_snapshot"] == {} - migration_backups = list((tmp_path / "mcp-hub-topology-migrations").glob("*.json")) - assert len(migration_backups) == 1 - assert json.loads(migration_backups[0].read_text(encoding="utf-8")) == prior - assert json.loads(state_path.read_text(encoding="utf-8"))["live_artifacts"]["topology_contract"] == controller.MCP_HUB_TOPOLOGY_CONTRACT + with pytest.raises(controller.DeployError, match="cannot be promoted"): + controller.migrate_mcp_hub_legacy_state( + project, controller.MCP_HUB_REPOSITORY, candidate, model, backup, runtime + ) + assert state_path.read_bytes() == original_state + assert not (tmp_path / "mcp-hub-topology-migrations").exists() @pytest.mark.skipif(os.name == "nt", reason="migration intent durability requires POSIX directory fsync") -def test_migration_intent_is_durable_attempt_bound_and_blocks_restart_retry( +def test_migration_intent_is_durable_and_bound_to_observed_evidence_and_backup( controller, monkeypatch, tmp_path ): - project, candidate, model, backup, runtime, state_path, _prior = _legacy_migration_fixture( - controller, monkeypatch, tmp_path - ) - monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) - controller.migrate_mcp_hub_legacy_state( - project, - controller.MCP_HUB_REPOSITORY, - candidate, - model, - backup, - runtime, + project, candidate, _model, backup, _runtime, state_path, _prior, evidence, original_state = ( + _record_test_observed_baseline(controller, monkeypatch, tmp_path) ) attempt_id = controller._mcp_hub_begin_migration_intent(project, candidate, backup) @@ -2124,25 +2199,27 @@ def test_migration_intent_is_durable_attempt_bound_and_blocks_restart_retry( assert intent["status"] == "started" assert intent["candidateRevision"] == candidate assert intent["backupManifestSha256"] == backup["manifestSha256"] - assert intent["baselineStateSha256"] == hashlib.sha256(state_path.read_bytes()).hexdigest() + assert intent["observedBaselinePath"] == evidence["evidencePath"] + assert intent["observedBaselineSha256"] == evidence["evidenceSha256"] + assert intent["baselineStateSha256"] == hashlib.sha256(original_state).hexdigest() assert stat.S_IMODE(intent_path.stat().st_mode) == 0o600 - # A fresh controller process sees the durable marker and refuses to start - # another migration while the preceding outcome is unknown. with pytest.raises(controller.DeployError, match="retry and image-only rollback are blocked"): - controller._mcp_hub_assert_no_unresolved_migration_intent( - {"_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT} - ) + controller._mcp_hub_assert_no_unresolved_migration_intent(project) with pytest.raises(controller.DeployError, match="retry and image-only rollback are blocked"): controller._mcp_hub_begin_migration_intent(project, candidate, backup) assert intent_path.read_bytes() == intent_bytes - # The only restart reconciliation that clears a started intent is a - # durable success state containing the same attempt and backup identity. state = json.loads(state_path.read_text(encoding="utf-8")) + state["status"] = "success" state["revision"] = candidate state["live_revision"] = candidate state["live_status"] = "healthy" + state["live_legacy_provenance"] = False + state["live_artifacts"] = { + "containers": {name: {"container_id": name} for name in controller.MCP_HUB_CONTAINERS}, + "topology_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + } state["last_attempt"] = { "revision": candidate, "status": "success", @@ -2158,20 +2235,10 @@ def test_migration_intent_is_durable_attempt_bound_and_blocks_restart_retry( assert completed["status"] == "complete" assert completed["completedStateSha256"] == hashlib.sha256(state_path.read_bytes()).hexdigest() - @pytest.mark.skipif(os.name == "nt", reason="migration intent durability requires POSIX directory fsync") def test_migration_intent_update_uses_compare_and_swap(controller, monkeypatch, tmp_path): - project, candidate, model, backup, runtime, _state_path, _prior = _legacy_migration_fixture( - controller, monkeypatch, tmp_path - ) - monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) - controller.migrate_mcp_hub_legacy_state( - project, - controller.MCP_HUB_REPOSITORY, - candidate, - model, - backup, - runtime, + project, candidate, _model, backup, _runtime, _state_path, _prior, _evidence, _original_state = ( + _record_test_observed_baseline(controller, monkeypatch, tmp_path) ) controller._mcp_hub_begin_migration_intent(project, candidate, backup) path = tmp_path / "mcp-hub-topology-migrations" / "active-intent.json" @@ -2187,129 +2254,10 @@ def test_migration_intent_update_uses_compare_and_swap(controller, monkeypatch, expected_bytes=original, ) +def test_retired_healthy_legacy_snapshot_path_is_not_available(controller): + assert not hasattr(controller, "mcp_hub_migration_snapshot_state") -@pytest.mark.skipif( - os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, - reason="state path-swap recovery requires root-owned POSIX fixtures", -) -def test_hub_state_cas_restores_original_file_before_path_readback(controller, monkeypatch, tmp_path): - shared = tmp_path / "shared" - shared.mkdir(mode=0o777) - shared.chmod(0o777) - controller_parent = shared / "controller" - controller_parent.mkdir(mode=0o700) - state_root = controller_parent / "state" - state_root.mkdir(mode=0o700) - state_path = state_root / f"{controller.MCP_HUB_PROJECT}.json" - state_path.write_bytes(b"baseline-state") - state_path.chmod(0o600) - attacker = tmp_path / "attacker-state" - attacker.mkdir(mode=0o700) - attacker_state = attacker / state_path.name - attacker_state.write_bytes(b"attacker-state") - attacker_state.chmod(0o600) - pinned = controller._MCP_HUB_RELEASE_STORE.PinnedDirectory.open( - state_root, check_ancestors=False - ) - monkeypatch.setattr(controller, "STATE_ROOT", state_root) - monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) - helper = controller._MCP_HUB_RELEASE_STORE - original_atomic_replace = helper.atomic_replace_at - - def swap_after_replace(*args, **kwargs): - def swap(): - state_root.rename(controller_parent / "state.saved") - state_root.symlink_to(attacker, target_is_directory=True) - - kwargs["after_replace"] = swap - return original_atomic_replace(*args, **kwargs) - - monkeypatch.setattr(helper, "atomic_replace_at", swap_after_replace) - try: - with pytest.raises(controller.DeployError, match="durable compare-and-swap"): - controller._mcp_hub_write_controller_state_bytes( - b"candidate-state", expected_bytes=b"baseline-state" - ) - assert helper.read_at(pinned.fd, state_path.name, max_bytes=128) == b"baseline-state" - assert attacker_state.read_bytes() == b"attacker-state" - finally: - pinned.close() - - -@pytest.mark.skipif( - os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, - reason="migration-intent path-swap recovery requires root-owned POSIX fixtures", -) -def test_hub_migration_intent_cas_removes_new_intent_from_swapped_directory( - controller, monkeypatch, tmp_path -): - shared = tmp_path / "shared" - shared.mkdir(mode=0o777) - shared.chmod(0o777) - controller_parent = shared / "controller" - controller_parent.mkdir(mode=0o700) - state_root = controller_parent / "state" - state_root.mkdir(mode=0o700) - attacker = tmp_path / "attacker-migrations" - attacker.mkdir(mode=0o700) - attacker_intent = attacker / "active-intent.json" - attacker_intent.write_bytes(b"attacker-intent") - attacker_intent.chmod(0o600) - monkeypatch.setattr(controller, "STATE_ROOT", state_root) - monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) - path = controller._mcp_hub_migration_intent_path(create=True) - migrations_path = path.parent - pinned = controller._MCP_HUB_RELEASE_STORE.PinnedDirectory.open( - migrations_path, check_ancestors=False - ) - helper = controller._MCP_HUB_RELEASE_STORE - original_atomic_replace = helper.atomic_replace_at - - def swap_after_replace(*args, **kwargs): - def swap(): - migrations_path.rename(state_root / "migrations.saved") - migrations_path.symlink_to(attacker, target_is_directory=True) - - kwargs["after_replace"] = swap - return original_atomic_replace(*args, **kwargs) - - monkeypatch.setattr(helper, "atomic_replace_at", swap_after_replace) - try: - with pytest.raises(controller.DeployError, match="durable compare-and-swap"): - controller._mcp_hub_replace_migration_intent( - path, b"candidate-intent", expected_bytes=None - ) - with pytest.raises(FileNotFoundError): - helper.read_at(pinned.fd, path.name, max_bytes=128) - assert attacker_intent.read_bytes() == b"attacker-intent" - finally: - pinned.close() - - -def test_migration_snapshot_attests_all_17_services_without_rewriting_legacy_state( - controller, monkeypatch, tmp_path -): - project, _candidate, model, _backup, runtime, _state_path, prior = _legacy_migration_fixture( - controller, monkeypatch, tmp_path - ) - project["source_dir"] = str(tmp_path) - project["compose_dir"] = "compose" - compose_dir = tmp_path / "compose" - compose_dir.mkdir() - for filename in controller.MCP_HUB_COMPOSE_FILES: - (compose_dir / filename).write_text(filename, encoding="utf-8") - - snapshot_state = controller.mcp_hub_migration_snapshot_state( - project, prior, runtime, model - ) - - assert set(snapshot_state["live_artifacts"]["containers"]) == set(controller.MCP_HUB_CONTAINERS) - assert snapshot_state["live_legacy_provenance"] is True - assert snapshot_state["live_receipt"] == prior["live_receipt"] - assert prior["live_artifacts"]["containers"].keys() == set(controller.MCP_HUB_LEGACY_CONTAINERS) - - -def test_legacy_state_migration_rejects_receipt_container_identity_drift( +def test_legacy_state_migration_rejects_promotion_even_when_receipt_identity_drifts( controller, monkeypatch, tmp_path ): project, candidate, model, backup, runtime, _state_path, prior = _legacy_migration_fixture( @@ -2323,8 +2271,9 @@ def test_legacy_state_migration_rejects_receipt_container_identity_drift( changed["live_artifacts"] = changed_artifacts _state_path.write_text(json.dumps(changed), encoding="utf-8") _state_path.chmod(0o600) + original_state = _state_path.read_bytes() - with pytest.raises(controller.DeployError, match="legacy artifact does not match"): + with pytest.raises(controller.DeployError, match="cannot be promoted"): controller.migrate_mcp_hub_legacy_state( project, controller.MCP_HUB_REPOSITORY, @@ -2333,7 +2282,7 @@ def test_legacy_state_migration_rejects_receipt_container_identity_drift( backup, runtime, ) - + assert _state_path.read_bytes() == original_state def test_rollback_source_must_render_same_fixed_topology_and_live_image_refs( controller, monkeypatch, tmp_path @@ -2398,7 +2347,6 @@ def test_rollback_source_must_render_same_fixed_topology_and_live_image_refs( with pytest.raises(controller.DeployError, match="rendered Compose validation failed"): controller.mcp_hub_verify_rollback_source(project, prior, model) - def test_legacy_snapshot_rollback_uses_trusted_prior_tree_and_rejects_legacy_after_up( controller, monkeypatch, tmp_path ): @@ -2507,49 +2455,19 @@ def test_legacy_snapshot_rollback_uses_trusted_prior_tree_and_rejects_legacy_aft assert project["_rollback_receipt"] is verified_receipt assert project["_rollback_restored_revision"] == prior - def test_hub_deploy_runs_the_fixed_migration_and_startup_stages(controller, monkeypatch, tmp_path): project = controller.effective_mcp_hub_project( controller.MCP_HUB_REPOSITORY, _legacy_target(controller) ) project["_repository_path"] = str(tmp_path / "hub.git") - project["source_dir"] = str(tmp_path) - project["compose_dir"] = "infrastructure/compose" - compose_dir = tmp_path / "infrastructure" / "compose" - compose_dir.mkdir(parents=True) - for filename in controller.MCP_HUB_COMPOSE_FILES: - (compose_dir / filename).write_text("services: {}\n", encoding="utf-8") - catalog_override = tmp_path / "catalog.yaml" - catalog_override.write_text("services: {}\n", encoding="utf-8") revision = "a" * 40 model = _valid_model(controller) - build = { - "id": revision[:12], - "repository": controller.MCP_HUB_REPOSITORY, - "revision": revision, - "tree": "b" * 40, - "treeDigest": "c" * 64, - "builtAt": "2026-10-02T10:00:00.000Z", - } - project["_deployment_metadata"] = build - image_ids = {} - for index, service_name in enumerate(controller.MCP_HUB_APPLICATION_SERVICES, 1): - model["services"][service_name]["build"] = {"context": "."} - model["services"][service_name]["image"] = f"itworx-mcp-hub/{service_name}:{revision[:12]}" - image_ids[model["services"][service_name]["image"]] = "sha256:" + f"{index:064x}" - image_labels = { - "org.opencontainers.image.revision": revision, - "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, - "org.opencontainers.image.source-tree": build["tree"], - "org.opencontainers.image.version": build["id"], - "org.opencontainers.image.created": build["builtAt"], - } commands = [] stages = [] migration_attempts = [] events = [] - verified_models = [] - migration_pins = [] + postgres_volume_baseline = [{"destination": controller.MCP_HUB_POSTGRES_DATA_VOLUME}] + project["_mcp_hub_observed_postgres_volume_baseline"] = postgres_volume_baseline monkeypatch.setattr(controller, "mcp_hub_require_docker_capacity", lambda: None) monkeypatch.setattr( controller, @@ -2557,34 +2475,13 @@ def test_hub_deploy_runs_the_fixed_migration_and_startup_stages(controller, monk lambda *_args: tmp_path / "release", ) monkeypatch.setattr(controller, "mcp_hub_compose_model", lambda *_args: model) - monkeypatch.setattr( - controller, - "inspect_image", - lambda reference: { - "Id": image_ids[reference], - "Config": {"Labels": image_labels}, - }, - ) - monkeypatch.setattr( - controller, - "_mcp_hub_compose_override", - lambda *_args, **_kwargs: catalog_override, - ) + monkeypatch.setattr(controller, "compose_command", lambda _project: (tmp_path, ["docker", "compose", "--project-name", controller.MCP_HUB_PROJECT])) monkeypatch.setattr(controller, "deployment_environment", lambda *_args: {}) - - def record_command(command, **_kwargs): - commands.append(command) - if "SERVICE_NAME=database-migrate" in command: - pin_index = next( - index - for index, argument in enumerate(command[:-1]) - if argument == "--file" and Path(command[index + 1]).name == "image-pins.json" - ) - migration_pins.append( - json.loads(Path(command[pin_index + 1]).read_text(encoding="ascii")) - ) - - monkeypatch.setattr(controller, "run", record_command) + monkeypatch.setattr(controller, "run", lambda command, **_kwargs: commands.append(command)) + monkeypatch.setattr( + controller, "mcp_hub_require_candidate_images", + lambda *_args: events.append("candidate-verified") or {name: "sha256:" + "a" * 64 for name in controller.MCP_HUB_CONTAINERS}, + ) monkeypatch.setattr( controller, "_mcp_hub_begin_migration_intent", @@ -2595,18 +2492,21 @@ def test_hub_deploy_runs_the_fixed_migration_and_startup_stages(controller, monk monkeypatch.setattr( controller, "prepare_compose_migration", - lambda _project: events.append("prepare") or [], + lambda _project: pytest.fail("fixed Hub deploy must not rename or adopt containers"), ) monkeypatch.setattr( controller, "mcp_hub_up_services", lambda _p, _r, _repo, names: stages.append(names) or events.append("up"), ) - monkeypatch.setattr(controller, "verify_mcp_hub_database_revision", lambda *_args: "abc123") monkeypatch.setattr( controller, - "verify_mcp_hub_live_topology", - lambda _project, verified_model, **_kwargs: verified_models.append(verified_model) or {}, + "_mcp_hub_require_observed_postgres_volume_continuity", + lambda _project: events.append(("volume-continuity", len(stages))), + ) + monkeypatch.setattr(controller, "verify_mcp_hub_database_revision", lambda *_args: "abc123") + monkeypatch.setattr( + controller, "verify_mcp_hub_live_topology", lambda *_args, **_kwargs: {} ) monkeypatch.setattr( controller.subprocess, @@ -2616,30 +2516,29 @@ def test_hub_deploy_runs_the_fixed_migration_and_startup_stages(controller, monk controller.deploy_mcp_hub_staged(project, revision, controller.MCP_HUB_REPOSITORY) - assert commands[0][-1] == "build" + assert not any("build" in command for command in commands) migration = next(command for command in commands if "SERVICE_NAME=database-migrate" in command) assert migration[-8:] == [ "run", "--rm", "--no-deps", "--pull", "never", "-e", "SERVICE_NAME=database-migrate", "control-api" ] - assert all( - project["_mcp_hub_rendered_model"]["services"][service_name]["image"] == image_id - for service_name, image_id in zip(controller.MCP_HUB_APPLICATION_SERVICES, image_ids.values()) - ) - assert "--build" not in migration - assert migration_pins == [ - { - "services": { - name: {"image": project["_mcp_hub_rendered_model"]["services"][name]["image"]} - for name in sorted(controller.MCP_HUB_APPLICATION_SERVICES) - } - } - ] - assert verified_models == [project["_mcp_hub_rendered_model"]] assert project["_mcp_hub_migration_started"] is True assert project["_mcp_hub_migration_attempt_id"] == "1" * 32 + assert project["_mcp_hub_production_mutation_started"] is True + assert project["_legacy_rollbacks"] == [] assert migration_attempts == ["intent"] - assert events[:2] == ["intent", "prepare"] - assert events[2:] == ["up"] * 4 + assert events == [ + "candidate-verified", + ("volume-continuity", 0), + "intent", + "up", + ("volume-continuity", 1), + "candidate-verified", + "up", + "up", + "up", + "candidate-verified", + ("volume-continuity", 4), + ] assert stages == [ controller.MCP_HUB_DATABASE_SERVICES, controller.MCP_HUB_CONTEXTFORGE_SERVICES, @@ -2647,3 +2546,269 @@ def test_hub_deploy_runs_the_fixed_migration_and_startup_stages(controller, monk controller.MCP_HUB_INTAKE_SERVICES, ] assert "down" not in [part for command in commands for part in command] + +def test_hub_source_epoch_is_derived_from_the_exact_git_commit(controller, monkeypatch, tmp_path): + commands = [] + revision = "a" * 40 + monkeypatch.setattr(controller, "output", lambda command: commands.append(command) or "1790888863") + assert controller.mcp_hub_source_date_epoch(tmp_path / "hub.git", revision) == "1790888863" + assert commands == [["git", f"--git-dir={tmp_path / 'hub.git'}", "show", "-s", "--format=%ct", revision]] + +@pytest.mark.parametrize("value", ["", "-1", "0", "1;touch /tmp/unsafe", "1\n2", "019", "true"]) +def test_hub_source_epoch_rejects_noncanonical_git_output(controller, monkeypatch, tmp_path, value): + monkeypatch.setattr(controller, "output", lambda _command: value) + with pytest.raises(controller.DeployError, match="source epoch"): + controller.mcp_hub_source_date_epoch(tmp_path / "hub.git", "a" * 40) + +def test_normal_hub_upgrade_requires_volume_baseline_before_migration(controller): + project = {"_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + "_mcp_hub_rendered_model": _valid_model(controller)} + with pytest.raises(controller.DeployError, match="volume baseline"): + controller._mcp_hub_require_observed_postgres_volume_continuity(project) + +def _patch_release_volume_paths(controller, monkeypatch, tmp_path): + releases = tmp_path / "protected-volume" / "releases" + monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases) + monkeypatch.setattr( + controller, + "_mcp_hub_release_directory", + lambda revision: releases / revision, + ) + + def require_release_anchor(*, create=False): + if create: + releases.mkdir(mode=0o700, parents=True, exist_ok=True) + if not releases.is_dir() or releases.is_symlink(): + raise controller.DeployError("MCP Hub protected release anchor is unavailable") + return releases + + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_release_anchor", require_release_anchor) + return releases + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="migration-intent path-swap recovery requires root-owned POSIX fixtures", +) +def test_hub_migration_intent_cas_removes_new_intent_from_swapped_directory( + controller, monkeypatch, tmp_path +): + shared = tmp_path / "shared" + shared.mkdir(mode=0o777) + shared.chmod(0o777) + controller_parent = shared / "controller" + controller_parent.mkdir(mode=0o700) + state_root = controller_parent / "state" + state_root.mkdir(mode=0o700) + attacker = tmp_path / "attacker-migrations" + attacker.mkdir(mode=0o700) + attacker_intent = attacker / "active-intent.json" + attacker_intent.write_bytes(b"attacker-intent") + attacker_intent.chmod(0o600) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + path = controller._mcp_hub_migration_intent_path(create=True) + migrations_path = path.parent + pinned = controller._MCP_HUB_RELEASE_STORE.PinnedDirectory.open( + migrations_path, check_ancestors=False + ) + helper = controller._MCP_HUB_RELEASE_STORE + original_atomic_replace = helper.atomic_replace_at + + def swap_after_replace(*args, **kwargs): + def swap(): + migrations_path.rename(state_root / "migrations.saved") + migrations_path.symlink_to(attacker, target_is_directory=True) + + kwargs["after_replace"] = swap + return original_atomic_replace(*args, **kwargs) + + monkeypatch.setattr(helper, "atomic_replace_at", swap_after_replace) + try: + with pytest.raises(controller.DeployError, match="durable compare-and-swap"): + controller._mcp_hub_replace_migration_intent( + path, b"candidate-intent", expected_bytes=None + ) + with pytest.raises(FileNotFoundError): + helper.read_at(pinned.fd, path.name, max_bytes=128) + assert attacker_intent.read_bytes() == b"attacker-intent" + finally: + pinned.close() + +def test_hub_reconciler_requires_the_exact_four_file_release_bind_inventory(controller): + project = controller.effective_mcp_hub_project( + controller.MCP_HUB_REPOSITORY, _legacy_target(controller) + ) + revision = "d" * 40 + project["_mcp_hub_required_release_revision"] = revision + store = controller._MCP_HUB_RELEASE_STORE + model = _valid_model(controller) + for service, suffix, target, _kind in store.RELEASE_BIND_CONTRACT: + model["services"][service]["volumes"].append( + { + "type": "bind", + "source": f"{store.logical_release_path(revision)}/{suffix}", + "target": target, + "read_only": True, + "mode": "ro", + } + ) + + assert controller.validate_mcp_hub_compose_model( + project, + model, + require_catalog_mounts=True, + catalog_source="logical", + ) is model + host_model = store.translate_release_mounts(model, revision) + assert controller.validate_mcp_hub_compose_model( + project, + host_model, + require_catalog_mounts=True, + catalog_source="host", + ) is host_model + + missing = copy.deepcopy(model) + missing["services"]["redis"]["volumes"].pop(0) + with pytest.raises(controller.DeployError, match="in-release bind inventory"): + controller.validate_mcp_hub_compose_model( + project, + missing, + require_catalog_mounts=True, + catalog_source="logical", + ) + writable = copy.deepcopy(model) + writable["services"]["contextforge"]["volumes"][0]["read_only"] = False + with pytest.raises(controller.DeployError, match="in-release bind inventory"): + controller.validate_mcp_hub_compose_model( + project, + writable, + require_catalog_mounts=True, + catalog_source="logical", + ) + broad = copy.deepcopy(model) + broad["services"]["control-api"]["volumes"][0]["source"] = store.logical_release_path(revision) + with pytest.raises(controller.DeployError, match="in-release bind inventory"): + controller.validate_mcp_hub_compose_model( + project, + broad, + require_catalog_mounts=True, + catalog_source="logical", + ) + +def test_hub_runtime_rejects_controller_state_mount_and_host_capabilities(controller): + project = controller.effective_mcp_hub_project( + controller.MCP_HUB_REPOSITORY, _legacy_target(controller) + ) + model = _valid_model(controller) + model["services"]["control-api"]["volumes"] = [{ + "type": "bind", + "source": controller.STATE_ROOT.as_posix(), + "target": "/controller-state", + "read_only": True, + }] + with pytest.raises(controller.DeployError, match="only a protected catalog leaf read-only"): + controller.validate_mcp_hub_compose_model(project, model) + + model["services"]["control-api"]["volumes"] = [] + model["services"]["control-api"]["privileged"] = True + with pytest.raises(controller.DeployError, match="host filesystem or remount capabilities"): + controller.validate_mcp_hub_compose_model(project, model) + model["services"]["control-api"].pop("privileged") + model["services"]["control-api"]["cap_add"] = ["SYS_ADMIN"] + with pytest.raises(controller.DeployError, match="host filesystem or remount capabilities"): + controller.validate_mcp_hub_compose_model(project, model) + +@pytest.mark.skipif( + os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, + reason="state path-swap recovery requires root-owned POSIX fixtures", +) +def test_hub_state_cas_restores_original_file_before_path_readback(controller, monkeypatch, tmp_path): + shared = tmp_path / "shared" + shared.mkdir(mode=0o777) + shared.chmod(0o777) + controller_parent = shared / "controller" + controller_parent.mkdir(mode=0o700) + state_root = controller_parent / "state" + state_root.mkdir(mode=0o700) + state_path = state_root / f"{controller.MCP_HUB_PROJECT}.json" + state_path.write_bytes(b"baseline-state") + state_path.chmod(0o600) + attacker = tmp_path / "attacker-state" + attacker.mkdir(mode=0o700) + attacker_state = attacker / state_path.name + attacker_state.write_bytes(b"attacker-state") + attacker_state.chmod(0o600) + pinned = controller._MCP_HUB_RELEASE_STORE.PinnedDirectory.open( + state_root, check_ancestors=False + ) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + helper = controller._MCP_HUB_RELEASE_STORE + original_atomic_replace = helper.atomic_replace_at + + def swap_after_replace(*args, **kwargs): + def swap(): + state_root.rename(controller_parent / "state.saved") + state_root.symlink_to(attacker, target_is_directory=True) + + kwargs["after_replace"] = swap + return original_atomic_replace(*args, **kwargs) + + monkeypatch.setattr(helper, "atomic_replace_at", swap_after_replace) + try: + with pytest.raises(controller.DeployError, match="durable compare-and-swap"): + controller._mcp_hub_write_controller_state_bytes( + b"candidate-state", expected_bytes=b"baseline-state" + ) + assert helper.read_at(pinned.fd, state_path.name, max_bytes=128) == b"baseline-state" + assert attacker_state.read_bytes() == b"attacker-state" + finally: + pinned.close() + +def test_projects_registry_uses_fixed_config_alias_only_for_protected_install(controller): + assert controller.CONFIG_PATH == controller.ROOT / "projects.json" + assert controller.controller_config_root(Path("/opt/gitea-deploy")) == Path( + "/controller-hub-releases/controller-config" + ) + assert controller._CONTROLLER_PATHS.controller_readonly_config_root( + Path("/opt/gitea-deploy") + ) == Path("/controller-config") + +@pytest.mark.skipif(os.name == "nt", reason="POSIX ACL inspection is unavailable on Windows") +def test_state_trust_fails_closed_when_acl_metadata_cannot_be_inspected(controller, monkeypatch): + def unavailable(*_args, **_kwargs): + raise OSError("filesystem ACL metadata is unavailable") + + monkeypatch.setattr(controller.os, "listxattr", unavailable) + with pytest.raises(controller.DeployError, match="cannot inspect filesystem ACLs"): + controller._mcp_hub_require_no_posix_acl(Path("/tmp")) + + + +def test_hub_full_sha_is_required_before_compose_preparation(controller, monkeypatch, tmp_path): + target = _legacy_target(controller) + monkeypatch.setattr(controller, "load_projects", lambda: {controller.MCP_HUB_REPOSITORY: target}) + monkeypatch.setattr(controller, "repository_path", lambda *_args: tmp_path) + monkeypatch.setattr( + controller, + "output", + lambda *_args, **_kwargs: pytest.fail("invalid revision reached Git or Compose preparation"), + ) + monkeypatch.setattr( + controller, + "deployment_environment", + lambda *_args, **_kwargs: pytest.fail("invalid revision reached Compose environment setup"), + ) + monkeypatch.setattr( + controller, + "mcp_hub_load_release_candidate", + lambda *_args, **_kwargs: pytest.fail("invalid revision reached candidate preparation"), + ) + monkeypatch.setattr( + sys, + "argv", + ["deploy.py", "deploy", controller.MCP_HUB_REPOSITORY, "a" * 12], + ) + + with pytest.raises(controller.DeployError, match="full lowercase hexadecimal Git SHA"): + controller.main() diff --git a/backend/tests/test_mcp_hub_release_candidates.py b/backend/tests/test_mcp_hub_release_candidates.py new file mode 100644 index 0000000..170f797 --- /dev/null +++ b/backend/tests/test_mcp_hub_release_candidates.py @@ -0,0 +1,926 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import sys +import types +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def controller(): + if "fcntl" not in sys.modules: + shim = types.ModuleType("fcntl") + shim.LOCK_EX, shim.LOCK_UN, shim.flock = 2, 8, lambda *_args: None + sys.modules["fcntl"] = shim + spec = importlib.util.spec_from_file_location( + "hub_candidate_controller", ROOT / "ops/gitea-deploy-broker/deploy.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def prepared(controller, tmp_path, monkeypatch): + monkeypatch.setattr(controller, "STATE_ROOT", tmp_path) + monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None) + monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda _path: None) + revision = "a" * 40 + directory = tmp_path / "mcp-hub-release-candidates" / revision + directory.mkdir(parents=True, mode=0o700) + directory.parent.chmod(0o700) + now = ( + datetime.now(timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z") + ) + metadata = { + "id": revision[:12], + "repository": controller.MCP_HUB_REPOSITORY, + "revision": revision, + "tree": "b" * 40, + "treeDigest": hashlib.sha256(("git-tree:" + "b" * 40).encode()).hexdigest(), + "builtAt": now, + } + project = { + "_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT, + "_repository_path": str(tmp_path / "hub.git"), + "_deployment_metadata": metadata, + "_mcp_hub_source_date_epoch": "1790888863", + } + monkeypatch.setattr( + controller, "deployment_metadata", lambda *_args, **_kwargs: metadata + ) + monkeypatch.setattr( + controller, "mcp_hub_source_date_epoch", lambda *_args: "1790888863" + ) + monkeypatch.setattr( + controller, "mcp_hub_stage_release_source", lambda *_args: tmp_path / "release" + ) + checks = [] + monkeypatch.setattr( + controller, + "_mcp_hub_check_candidate_exceptions", + lambda *_args: checks.append("gate"), + ) + images = {} + model = {"services": {}} + + def write(name, value): + data = ( + json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n" + ).encode() + path = directory / name + path.write_bytes(data) + path.chmod(0o600) + return hashlib.sha256(data).hexdigest() + + for index, service in enumerate(controller.MCP_HUB_SERVICES, 1): + image_id = "sha256:" + f"{index:064x}" + ref = f"fixture/{service}@sha256:" + f"{index:064x}" + if service == "contextforge": + ref = f"itworx-mcp-hub/contextforge-schema:{revision}" + model["services"][service] = {"image": ref} + vuln = { + "SchemaVersion": 2, + "Trivy": {"Version": "0.73.0"}, + "CreatedAt": now, + "Metadata": { + "ImageID": image_id, + "ImageConfig": {"architecture": "amd64", "os": "linux"}, + }, + "Results": [ + {"Class": "os-pkgs", "Packages": [{"Name": "fixture", "Version": "1"}]} + ], + } + secret = { + "ImageID": image_id, + "CreatedAt": now, + "TrivyVersion": "0.73.0", + "FindingCount": 0, + "SensitiveFindingDetailsPersisted": False, + "Results": [{"FindingCount": 0}], + } + images[service] = { + "imageRef": ref, + "imageId": image_id, + "vulnerabilitySha256": write(f"vulnerability-{service}.json", vuln), + "secretSha256": write(f"secret-{service}.json", secret), + } + candidate = { + "schema": "itworx.mcp-hub-release-candidate/v1", + "repository": controller.MCP_HUB_REPOSITORY, + "revision": revision, + "build": metadata, + "sourceDateEpoch": "1790888863", + "images": images, + "composeSha256": hashlib.sha256( + json.dumps(model, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest(), + "databaseMetadataSha256": write( + "trivy-db-metadata.json", {"UpdatedAt": now, "DownloadedAt": now} + ), + "summarySha256": write("scan-summary.json", {"unresolvedCriticalOrHigh": []}), + "digestsSha256": write( + "image-digests.json", + { + "images": { + s: {"imageId": v["imageId"], "tag": v["imageRef"]} + for s, v in images.items() + } + }, + ), + } + candidate_sha = write("candidate.json", candidate) + exceptions_sha = write("exceptions.json", {"exceptions": []}) + write( + "approval.json", + { + "schema": "itworx.mcp-hub-cve-approval/v1", + "candidateSha256": candidate_sha, + "exceptionsSha256": exceptions_sha, + }, + ) + labels = { + "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, + "org.opencontainers.image.revision": revision, + "org.opencontainers.image.source-tree": metadata["tree"], + "org.opencontainers.image.version": metadata["id"], + "org.opencontainers.image.created": metadata["builtAt"], + "org.opencontainers.image.source-date-epoch": "1790888863", + } + facts = { + v["imageRef"]: { + "Id": v["imageId"], + "Config": {"User": "10001:10001", "Labels": dict(labels)}, + } + for v in images.values() + } + monkeypatch.setattr(controller, "inspect_image", lambda ref: facts.get(ref)) + return types.SimpleNamespace( + project=project, + revision=revision, + directory=directory, + candidate=candidate, + model=model, + metadata=metadata, + write=write, + checks=checks, + facts=facts, + ) + + +def test_fresh_complete_candidate_freezes_metadata_and_checks_explicit_gate( + controller, prepared +): + result = controller.mcp_hub_load_release_candidate( + prepared.project, prepared.revision + ) + assert result["build"] == prepared.metadata + assert set(result["images"]) == set(controller.MCP_HUB_SERVICES) + assert prepared.checks == ["gate"] + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "missing", + "extra", + "revision", + "build", + "epoch", + "scan_hash", + "scan_identity", + "scan_empty", + "secret", + "stale", + "db_stale", + "approval", + "summary", + "digests", + ], +) +def test_candidate_drift_rejects_before_any_production_action( + controller, prepared, mutation +): + value = copy.deepcopy(prepared.candidate) + if mutation == "missing": + value["images"].pop("web") + elif mutation == "extra": + value["images"]["foreign"] = value["images"]["web"] + elif mutation == "revision": + value["revision"] = "c" * 40 + elif mutation == "build": + value["build"]["tree"] = "c" * 40 + elif mutation == "epoch": + value["sourceDateEpoch"] = "1" + elif mutation == "scan_hash": + value["images"]["web"]["vulnerabilitySha256"] = "c" * 64 + elif mutation in {"scan_identity", "scan_empty", "stale"}: + path = prepared.directory / "vulnerability-web.json" + report = json.loads(path.read_text()) + if mutation == "scan_identity": + report["Metadata"]["ImageID"] = "sha256:" + "c" * 64 + elif mutation == "scan_empty": + report["Results"] = [] + else: + report["CreatedAt"] = "2026-01-01T00:00:00Z" + value["images"]["web"]["vulnerabilitySha256"] = prepared.write( + path.name, report + ) + elif mutation == "secret": + report = json.loads((prepared.directory / "secret-web.json").read_text()) + report["FindingCount"] = 1 + value["images"]["web"]["secretSha256"] = prepared.write( + "secret-web.json", report + ) + elif mutation == "db_stale": + value["databaseMetadataSha256"] = prepared.write( + "trivy-db-metadata.json", + { + "UpdatedAt": "2026-01-01T00:00:00Z", + "DownloadedAt": "2026-01-01T00:00:00Z", + }, + ) + elif mutation == "summary": + value["summarySha256"] = prepared.write( + "scan-summary.json", {"unresolvedCriticalOrHigh": [{"package": "invented"}]} + ) + elif mutation == "digests": + value["digestsSha256"] = prepared.write("image-digests.json", {"images": {}}) + candidate_sha = prepared.write("candidate.json", value) + exceptions_sha = hashlib.sha256( + (prepared.directory / "exceptions.json").read_bytes() + ).hexdigest() + prepared.write( + "approval.json", + { + "schema": "itworx.mcp-hub-cve-approval/v1", + "candidateSha256": "c" * 64 if mutation == "approval" else candidate_sha, + "exceptionsSha256": exceptions_sha, + }, + ) + with pytest.raises(controller.DeployError): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + assert prepared.checks == [] + + +def test_pending_or_expired_human_exception_gate_cannot_be_substituted( + controller, prepared, monkeypatch +): + def denied(*_args): + raise controller.DeployError("CVE exception gate did not pass") + + monkeypatch.setattr(controller, "_mcp_hub_check_candidate_exceptions", denied) + with pytest.raises(controller.DeployError, match="CVE exception gate"): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + + +@pytest.mark.parametrize( + "service", ["web", "contextforge", "postgres", "connector-ingestion"] +) +def test_retagged_or_changed_candidate_image_is_rejected_before_apply( + controller, prepared, service +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.facts[prepared.candidate["images"][service]["imageRef"]]["Id"] = ( + "sha256:" + "f" * 64 + ) + with pytest.raises(controller.DeployError, match="image identity"): + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +def test_changed_rendered_model_is_rejected(controller, prepared): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.model["services"]["web"]["environment"] = {"unexpected": "change"} + with pytest.raises(controller.DeployError, match="Compose"): + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +def test_changed_approval_is_rejected_between_stages(controller, prepared): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.write("exceptions.json", {"exceptions": [{"approvalStatus": "pending"}]}) + with pytest.raises(controller.DeployError): + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +@pytest.mark.parametrize("service", ["web", "contextforge"]) +@pytest.mark.parametrize( + "label", + [ + "org.opencontainers.image.source", + "org.opencontainers.image.revision", + "org.opencontainers.image.source-tree", + "org.opencontainers.image.source-date-epoch", + "org.opencontainers.image.created", + "org.opencontainers.image.version", + ], +) +def test_frozen_image_provenance_is_checked_before_database_intent( + controller, prepared, service, label +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.facts[prepared.candidate["images"][service]["imageRef"]]["Config"][ + "Labels" + ][label] = "wrong" + with pytest.raises(controller.DeployError, match="provenance"): + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +@pytest.mark.parametrize("service", ["web", "contextforge"]) +@pytest.mark.parametrize("user", ["", "root", "root:10001", "0", "00:10001"]) +def test_frozen_image_root_user_is_rejected_before_database_intent( + controller, prepared, service, user +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.facts[prepared.candidate["images"][service]["imageRef"]]["Config"][ + "User" + ] = user + with pytest.raises(controller.DeployError, match="non-root runtime"): + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +def test_changed_raw_scan_is_rejected_between_stages(controller, prepared): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.write("vulnerability-contextforge.json", {"Results": []}) + with pytest.raises(controller.DeployError, match="hash"): + controller.mcp_hub_require_candidate_images( + prepared.project, prepared.model, prepared.revision + ) + + +@pytest.mark.parametrize("changed", [False, True]) +def test_staged_start_consumes_no_build_no_pull_and_checks_running_ids( + controller, prepared, monkeypatch, changed +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.project["_mcp_hub_rendered_model"] = prepared.model + commands = [] + monkeypatch.setattr( + controller, + "compose_command", + lambda _p: (prepared.directory, ["docker", "compose"]), + ) + monkeypatch.setattr(controller, "deployment_environment", lambda *_args: {}) + monkeypatch.setattr( + controller, "run", lambda command, **_kwargs: commands.append(command) + ) + monkeypatch.setattr(controller, "mcp_hub_wait_services", lambda *_args: None) + expected = prepared.candidate["images"]["postgres"]["imageId"] + monkeypatch.setattr( + controller, + "inspect_container", + lambda _name: {"Image": "sha256:" + "f" * 64 if changed else expected}, + ) + if changed: + with pytest.raises(controller.DeployError, match="exact scanned image"): + controller.mcp_hub_up_services( + prepared.project, + prepared.revision, + controller.MCP_HUB_REPOSITORY, + ("postgres",), + ) + else: + controller.mcp_hub_up_services( + prepared.project, + prepared.revision, + controller.MCP_HUB_REPOSITORY, + ("postgres",), + ) + assert commands == [ + [ + "docker", + "compose", + "up", + "--detach", + "--no-deps", + "--no-build", + "--pull", + "never", + "postgres", + ] + ] + + +@pytest.mark.parametrize("operation", ["start", "migrate"]) +def test_compose_operations_pin_all_scanned_ids_without_changing_source_model( + controller, prepared, monkeypatch, operation +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.project["_mcp_hub_rendered_model"] = prepared.model + original = copy.deepcopy(prepared.model) + observed = [] + + def compose(project): + path = Path(project["_mcp_hub_image_override_file"]) + assert path.stat().st_mode & 0o777 == 0o600 + observed.append((path, json.loads(path.read_text()))) + return prepared.directory, ["docker", "compose", "-f", str(path)] + + commands = [] + monkeypatch.setattr(controller, "compose_command", compose) + monkeypatch.setattr(controller, "deployment_environment", lambda *_args: {}) + monkeypatch.setattr(controller, "run", lambda cmd, **_kwargs: commands.append(cmd)) + monkeypatch.setattr(controller, "mcp_hub_wait_services", lambda *_args: None) + monkeypatch.setattr( + controller, + "inspect_container", + lambda _name: {"Image": prepared.candidate["images"]["postgres"]["imageId"]}, + ) + if operation == "start": + controller.mcp_hub_up_services( + prepared.project, + prepared.revision, + controller.MCP_HUB_REPOSITORY, + ("postgres",), + ) + else: + controller.mcp_hub_run_database_migration( + prepared.project, prepared.revision, controller.MCP_HUB_REPOSITORY + ) + assert commands[0][-3:] == [ + "-e", + "SERVICE_NAME=database-migrate", + "control-api", + ] + assert len(observed) == len(commands) == 1 + path, payload = observed[0] + assert payload == { + "services": { + service: {"image": record["imageId"]} + for service, record in prepared.candidate["images"].items() + } + } + assert "--no-deps" in commands[0] + assert commands[0][commands[0].index("--pull") + 1] == "never" + assert not path.exists() + assert "_mcp_hub_image_override_file" not in prepared.project + assert prepared.model == original + + +def test_migration_rechecks_candidate_before_running(controller, prepared, monkeypatch): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + prepared.project["_mcp_hub_rendered_model"] = prepared.model + prepared.write("vulnerability-control-api.json", {"Results": []}) + monkeypatch.setattr( + controller, "run", lambda *_args, **_kwargs: pytest.fail("must not run") + ) + with pytest.raises(controller.DeployError, match="hash"): + controller.mcp_hub_run_database_migration( + prepared.project, prepared.revision, controller.MCP_HUB_REPOSITORY + ) + + +def test_candidate_runtime_verifies_pinned_ids_after_migration( + controller, prepared, monkeypatch +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + project = prepared.project + project["compose_project"] = controller.MCP_HUB_PROJECT + project["slug"] = "itworx-mcp-hub" + project["source_dir"] = str(prepared.directory) + project["compose_files"] = [] + project["containers"] = list(controller.MCP_HUB_CONTAINERS) + project["_mcp_hub_rendered_model"] = prepared.model + project["_mcp_hub_candidate_image_ids"] = ( + controller.mcp_hub_require_candidate_images( + project, prepared.model, prepared.revision + ) + ) + for service in ("control-api", "worker"): + prepared.model["services"][service]["environment"] = { + "INGESTION_CONNECTOR_ENABLED": "true" + } + containers = {} + for service, record in prepared.candidate["images"].items(): + name = f"itworx-mcp-hub-{service}-1" + containers[name] = { + "Id": "c" * 64, + "Image": record["imageId"], + "Config": { + "Image": record["imageId"], + "Env": ( + ["INGESTION_CONNECTOR_ENABLED=true"] + if service in {"control-api", "worker"} + else [] + ), + "Labels": { + "com.docker.compose.project": controller.MCP_HUB_PROJECT, + "com.docker.compose.service": service, + }, + }, + "State": {"Running": True, "Health": {"Status": "healthy"}}, + } + monkeypatch.setattr( + controller, "compose_project_container_names", lambda _project: set(containers) + ) + monkeypatch.setattr( + controller, "inspect_container", lambda name: containers.get(name) + ) + monkeypatch.setattr( + controller, + "container_state", + lambda _name: (True, "healthy", controller.MCP_HUB_PROJECT), + ) + labels = { + "org.opencontainers.image.revision": prepared.revision, + "org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE, + "org.opencontainers.image.source-tree": prepared.metadata["tree"], + "org.opencontainers.image.version": prepared.metadata["id"], + "org.opencontainers.image.created": prepared.metadata["builtAt"], + } + image_ids_by_ref = { + record["imageRef"]: record["imageId"] + for record in prepared.candidate["images"].values() + } + monkeypatch.setattr( + controller, + "inspect_image", + lambda image_ref: { + "Id": image_ids_by_ref.get(image_ref, image_ref), + "Config": {"Labels": labels}, + "RepoDigests": [], + }, + ) + monkeypatch.setattr( + controller, "mcp_hub_attested_live_mounts", lambda *_args, **_kwargs: [] + ) + assert set(controller.verify_mcp_hub_live_topology(project, prepared.model)) == set( + containers + ) + assert set( + controller.verify_containers( + project, expected_image_ids=project["_mcp_hub_candidate_image_ids"] + ) + ) == set(containers) + for service in controller.MCP_HUB_APPLICATION_SERVICES: + prepared.model["services"][service]["build"] = {"context": "."} + manifest = ( + "CONTEXTFORGE_UPSTREAM_COMMIT=" + "1" * 40 + "\n" + "CONTEXTFORGE_ALEMBIC_HEAD=12d4a0c7789c\n" + "CONTEXTFORGE_MCP_SDK=1.30.0\n" + "CONTEXTFORGE_LOCAL_IMAGE=itworx-mcp-hub/contextforge-schema:12d4a0c7789c\n" + ) + builder = "\n".join( + [ + "--release-candidate", + 'CANDIDATE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}"', + 'docker tag "$STAGED_IMAGE" "$CANDIDATE_IMAGE"', + 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"', + 'git -C "$SRC" checkout --detach FETCH_HEAD', + "alembic -c mcpgateway/alembic.ini heads", + 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"', + ] + ) + monkeypatch.setattr( + controller, + "_mcp_hub_git_output", + lambda _project, _show, path: manifest + if path.endswith("compat-image.env") + else builder, + ) + monkeypatch.setattr( + controller, "_mcp_hub_dataplane_compat_proven", lambda *_args: True + ) + monkeypatch.setattr(controller, "_mcp_hub_digest_ref_proven", lambda *_args: True) + monkeypatch.setattr( + controller, + "_mcp_hub_live_image_proven", + lambda _project, name, item, source_ref, *_args, **_kwargs: { + "container_id": item["Id"], + "image_id": item["Image"], + "image_ref": source_ref, + "container_revision": "", + "image_revision": prepared.revision, + "image_source": controller.MCP_HUB_OCI_IMAGE_SOURCE, + "image_tree": prepared.metadata["tree"], + "image_version": prepared.metadata["id"], + "image_created": prepared.metadata["builtAt"], + }, + ) + receipt = controller.verify_deployment( + project, + prepared.metadata, + expected_image_ids=project["_mcp_hub_candidate_image_ids"], + ) + for service in controller.MCP_HUB_APPLICATION_SERVICES: + name = f"itworx-mcp-hub-{service}-1" + assert ( + receipt["containers"][name]["image_ref"] + == receipt["containers"][name]["image_id"] + ) + assert ( + controller.validate_recorded_receipt( + project, receipt, controller.MCP_HUB_REPOSITORY, prepared.revision + ) + == prepared.metadata + ) + state = {} + monkeypatch.setattr( + controller, + "_mcp_hub_read_controller_state_bytes", + lambda **_kwargs: state.get("bytes"), + ) + monkeypatch.setattr( + controller, + "_mcp_hub_write_controller_state_bytes", + lambda payload, **_kwargs: state.__setitem__("bytes", payload), + ) + controller.write_state( + controller.MCP_HUB_REPOSITORY, + project, + prepared.revision, + "success", + "verified", + live_receipt=receipt, + ) + saved = json.loads(state["bytes"]) + assert saved["live_status"] == "healthy" + for name in controller.MCP_HUB_CONTAINERS: + assert ( + saved["live_artifacts"]["containers"][name]["image_ref"] + == receipt["containers"][name]["image_ref"] + ) + project.pop("_mcp_hub_candidate_image_ids") + assert ( + controller.verified_current_receipt( + controller.MCP_HUB_REPOSITORY, + project, + prepared.revision, + prepared.metadata["tree"], + ) + == receipt + ) + assert "_mcp_hub_receipt_image_ids" not in project + project["_repository_name"] = controller.MCP_HUB_REPOSITORY + rollback_commands = [] + monkeypatch.setattr( + controller, "run", lambda command, **_kwargs: rollback_commands.append(command) + ) + snapshot = controller.capture_compose_rollback(project) + assert len(snapshot["containers"]) == len(rollback_commands) == 17 + for item in snapshot["containers"]: + assert item["image_ref"] == receipt["containers"][item["name"]]["image_ref"] + cf = prepared.candidate["images"]["contextforge"] + assert not controller._mcp_hub_contextforge_compat_proven( + project, prepared.revision, cf["imageRef"], "sha256:" + "f" * 64 + ) + containers["itworx-mcp-hub-web-1"]["Config"]["Image"] = "web:mutable" + with pytest.raises(controller.DeployError, match="identity or health"): + controller.verify_mcp_hub_live_topology(project, prepared.model) + + +@pytest.mark.parametrize("same_revision", [True, False]) +def test_same_revision_receipt_prepares_exact_source_before_verification( + controller, prepared, monkeypatch, same_revision +): + project = prepared.project + project.update({"source_dir": str(prepared.directory), "slug": "itworx-mcp-hub"}) + monkeypatch.setattr( + controller, + "read_state", + lambda _project: { + "live_revision": prepared.revision if same_revision else "b" * 40 + }, + ) + events = [] + monkeypatch.setattr( + controller, + "sync_source", + lambda *_args, **kwargs: events.append(("sync", kwargs["exact"])), + ) + monkeypatch.setattr( + controller, "copy_environment", lambda *_args: events.append("environment") + ) + monkeypatch.setattr( + controller, + "mcp_hub_compose_model", + lambda *_args: events.append("model") or prepared.model, + ) + controller.mcp_hub_prepare_current_receipt_model( + prepared.directory, project, controller.MCP_HUB_REPOSITORY, prepared.revision + ) + assert events == ([("sync", True), "environment", "model"] if same_revision else []) + assert (project.get("_mcp_hub_rendered_model") is prepared.model) is same_revision + + +@pytest.mark.parametrize("mutation", ["missing", "extra", "tag", "nonmapping"]) +def test_candidate_compose_override_rejects_malformed_ids( + controller, prepared, mutation +): + ids = { + f"itworx-mcp-hub-{service}-1": record["imageId"] + for service, record in prepared.candidate["images"].items() + } + if mutation == "missing": + ids.pop("itworx-mcp-hub-web-1") + elif mutation == "extra": + ids["foreign"] = "sha256:" + "a" * 64 + elif mutation == "tag": + ids["itworx-mcp-hub-web-1"] = "web:latest" + else: + ids = [] + prepared.project["_mcp_hub_rendered_model"] = prepared.model + prepared.project["_mcp_hub_candidate_image_ids"] = ids + with pytest.raises(controller.DeployError, match="candidate image IDs"): + with controller._mcp_hub_compose_image_override(prepared.project): + pytest.fail("must reject invalid image IDs") + + +def test_actual_high_findings_are_bound_to_current_service_tags(controller, prepared): + report = json.loads((prepared.directory / "vulnerability-web.json").read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "fixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": "1", + "Severity": "HIGH", + } + ] + prepared.candidate["images"]["web"]["vulnerabilitySha256"] = prepared.write( + "vulnerability-web.json", report + ) + summary = { + "unresolvedCriticalOrHigh": [ + { + "package": "fixture", + "cves": ["CVE-2026-12345"], + "installedVersion": "1", + "severity": "HIGH", + "affectedImages": [prepared.candidate["images"]["web"]["imageRef"]], + } + ] + } + prepared.candidate["summarySha256"] = prepared.write("scan-summary.json", summary) + candidate_sha = prepared.write("candidate.json", prepared.candidate) + prepared.write( + "approval.json", + { + "schema": "itworx.mcp-hub-cve-approval/v1", + "candidateSha256": candidate_sha, + "exceptionsSha256": hashlib.sha256( + (prepared.directory / "exceptions.json").read_bytes() + ).hexdigest(), + }, + ) + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + assert prepared.checks == ["gate"] + + +def test_fixable_high_findings_cannot_be_approved_around(controller, prepared): + report = json.loads((prepared.directory / "vulnerability-web.json").read_text()) + report["Results"][0]["Vulnerabilities"] = [ + { + "PkgName": "fixture", + "VulnerabilityID": "CVE-2026-12345", + "InstalledVersion": "1", + "FixedVersion": "2", + "Severity": "HIGH", + } + ] + prepared.candidate["images"]["web"]["vulnerabilitySha256"] = prepared.write( + "vulnerability-web.json", report + ) + candidate_sha = prepared.write("candidate.json", prepared.candidate) + prepared.write( + "approval.json", + { + "schema": "itworx.mcp-hub-cve-approval/v1", + "candidateSha256": candidate_sha, + "exceptionsSha256": hashlib.sha256( + (prepared.directory / "exceptions.json").read_bytes() + ).hexdigest(), + }, + ) + with pytest.raises(controller.DeployError, match="fixable critical/high"): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + assert prepared.checks == [] + + +def test_missing_normal_upgrade_volume_baseline_denies_before_intent_or_start( + controller, prepared, monkeypatch +): + controller.mcp_hub_load_release_candidate(prepared.project, prepared.revision) + monkeypatch.setattr(controller, "mcp_hub_require_docker_capacity", lambda: None) + monkeypatch.setattr( + controller, "mcp_hub_stage_release_source", lambda *_args: prepared.directory + ) + monkeypatch.setattr( + controller, "mcp_hub_compose_model", lambda *_args: prepared.model + ) + monkeypatch.setattr( + controller, + "compose_command", + lambda *_args: (prepared.directory, ["docker", "compose"]), + ) + monkeypatch.setattr(controller, "deployment_environment", lambda *_args: {}) + monkeypatch.setattr( + controller.subprocess, + "run", + lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0), + ) + monkeypatch.setattr( + controller, + "_mcp_hub_begin_migration_intent", + lambda *_args: pytest.fail( + "missing volume identity must precede durable intent" + ), + ) + monkeypatch.setattr( + controller, + "mcp_hub_up_services", + lambda *_args: pytest.fail( + "missing volume identity must precede Compose mutation" + ), + ) + with pytest.raises(controller.DeployError, match="volume baseline"): + controller.deploy_mcp_hub_staged( + prepared.project, prepared.revision, controller.MCP_HUB_REPOSITORY + ) + + +@pytest.mark.parametrize( + "mode, candidate_epoch", + [("deploy", "1790888863"), ("deploy", "1"), ("preflight", "1790888863")], +) +def test_main_hub_path_sets_validated_epoch_before_compose_render( + controller, prepared, monkeypatch, mode, candidate_epoch +): + class ReachedValidatedRender(Exception): + pass + + target = { + **prepared.project, + "branch": "main", + "slug": controller.MCP_HUB_PROJECT, + "source_dir": str(prepared.directory), + } + target["compose_project"] = controller.MCP_HUB_PROJECT + target.pop("_mcp_hub_source_date_epoch", None) + monkeypatch.setattr( + controller, "load_projects", lambda: {controller.MCP_HUB_REPOSITORY: target} + ) + monkeypatch.setattr( + controller, + "effective_mcp_hub_project", + lambda _repository, project: dict(project), + ) + monkeypatch.setattr( + controller, "repository_path", lambda *_args: prepared.directory + ) + monkeypatch.setattr(controller, "validate_commit", lambda *_args: prepared.revision) + candidate = copy.deepcopy(prepared.candidate) + candidate["sourceDateEpoch"] = candidate_epoch + monkeypatch.setattr( + controller, "mcp_hub_load_release_candidate", lambda *_args: candidate + ) + monkeypatch.setenv("SOURCE_DATE_EPOCH", "untrusted-environment-override") + + def render(_repo, project, revision): + if mode == "deploy" and candidate_epoch != "1790888863": + pytest.fail( + "mismatched candidate epoch must deny before Compose validation" + ) + assert project["_mcp_hub_source_date_epoch"] == "1790888863" + assert ( + controller.deployment_environment( + project, revision, controller.MCP_HUB_REPOSITORY + )["SOURCE_DATE_EPOCH"] + == "1790888863" + ) + raise ReachedValidatedRender() + + monkeypatch.setattr(controller, "validate_project_config", render) + monkeypatch.setattr( + sys, + "argv", + ["deploy.py", mode, controller.MCP_HUB_REPOSITORY, prepared.revision], + ) + if mode == "deploy" and candidate_epoch != "1790888863": + with pytest.raises(controller.DeployError, match="epoch"): + controller.main() + else: + with pytest.raises(ReachedValidatedRender): + controller.main() diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a8e688..8b49e29 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,7 +7,7 @@ settings: overrides: '@humanfs/node@>=0.16.6 <0.16.8': 0.16.8 brace-expansion@>=1.0.0 <1.1.21: 1.1.21 - brace-expansion@>=2.0.0 <2.0.3: 2.0.3 + brace-expansion@>=2.0.0 <2.1.6: 2.1.6 brace-expansion@>=5.0.0 <5.0.12: 5.0.12 browserslist@>=4.0.0 <4.28.7: 4.28.7 flatted@<3.4.2: 3.4.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5783155..c0e0d9f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -373,7 +373,7 @@ minimumReleaseAge: 1440 overrides: "@humanfs/node@>=0.16.6 <0.16.8": 0.16.8 "brace-expansion@>=1.0.0 <1.1.21": 1.1.21 - "brace-expansion@>=2.0.0 <2.0.3": 2.0.3 + "brace-expansion@>=2.0.0 <2.1.6": 2.1.6 "brace-expansion@>=5.0.0 <5.0.12": 5.0.12 "browserslist@>=4.0.0 <4.28.7": 4.28.7 "flatted@<3.4.2": 3.4.2 diff --git a/src/lib/buildDependencySecurity.test.ts b/src/lib/buildDependencySecurity.test.ts new file mode 100644 index 0000000..2940eeb --- /dev/null +++ b/src/lib/buildDependencySecurity.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { describe, it } from "node:test"; + +// Exercise the dependency actually used by Tailwind's globbing toolchain. +// A direct root import would miss the transitive version selected by pnpm. +let requireFromPackage = createRequire(import.meta.url); +for (const packageName of ["tailwindcss", "sucrase", "glob", "minimatch"]) { + requireFromPackage = createRequire( + requireFromPackage.resolve(`${packageName}/package.json`), + ); +} +const { expand } = requireFromPackage("brace-expansion") as { + expand: (pattern: string) => string[]; +}; + +describe("build dependency security", () => { + it("preserves ordinary brace expansion", () => { + assert.deepEqual(expand("assets/{one,two}.css"), [ + "assets/one.css", + "assets/two.css", + ]); + }); + + it("bounds deeply nested brace sets without overflowing the stack", () => { + const pattern = "{".repeat(3200) + "a,b" + "}".repeat(3200); + let result: string[] = []; + assert.doesNotThrow(() => { result = expand(pattern); }); + assert.ok(Array.isArray(result)); + assert.ok(result.length <= 2); + }); + + it("bounds deeply nested comma alternatives without overflowing the stack", () => { + const pattern = "{a,".repeat(4000) + "z" + "}".repeat(4000); + assert.doesNotThrow(() => expand(pattern)); + }); +});