from __future__ import annotations import time from typing import Any from urllib.parse import quote from ..connectors.portainer import PortainerConnector from ..core.redaction import redact_message _CONTROLLER = "gitea-deploy-control" _REPOSITORY = "jens/itworx_mcp_hub" _COMPOSE_PROJECT = "itworx-mcp-hub" # One-time fail-closed re-attestation for the existing MCP Hub runtime. The # script never builds, restarts, recreates, stops or deploys a container. It # repairs only controller state after proving the exact target, Git ancestry, # complete Compose topology and every currently running image identity. _RECONCILE_SCRIPT = r''' import fcntl import json import os from pathlib import Path import re import shutil import stat import subprocess import sys import time repository, expected_compose_project = sys.argv[1:3] registry_path = Path("/opt/gitea-deploy/projects.json") host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") owned_image_source = "https://gitea.example.com/jens/itworx-mcp-hub" if repository != "jens/itworx_mcp_hub" or expected_compose_project != "itworx-mcp-hub": raise SystemExit(40) if registry_path.is_symlink() or not registry_path.is_file(): raise SystemExit(20) try: if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path): raise SystemExit(36) except OSError: raise SystemExit(36) try: projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(31) project = projects.get(repository) if isinstance(projects, dict) else None if not isinstance(project, dict): raise SystemExit(22) expected_containers = project.get("containers") expected_fixed = { "itworx-mcp-hub-connector-appops-1", "itworx-mcp-hub-connector-gitea-1", "itworx-mcp-hub-connector-knowledge-1", "itworx-mcp-hub-connector-mobilityops-1", "itworx-mcp-hub-connector-projectbrain-1", "itworx-mcp-hub-connector-stitch-1", "itworx-mcp-hub-connector-unraid-1", "itworx-mcp-hub-contextforge-1", "itworx-mcp-hub-control-api-1", "itworx-mcp-hub-dataplane-router-1", "itworx-mcp-hub-docker-read-broker-1", "itworx-mcp-hub-postgres-1", "itworx-mcp-hub-redis-1", "itworx-mcp-hub-web-1", "itworx-mcp-hub-worker-1", } if ( project.get("branch") != "main" or project.get("slug") != expected_compose_project or project.get("compose_project") != expected_compose_project or not isinstance(expected_containers, list) or set(expected_containers) != expected_fixed ): raise SystemExit(23) state_path = state_root / f"{expected_compose_project}.json" project_lock_path = state_root / "locks" / f"{expected_compose_project}.lock" owner, name = repository.split("/", 1) bare_repo = Path("/gitea-repositories") / owner / f"{name}.git" canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories") canonical_bare_repo = canonical_repository_root / owner / f"{name}.git" if not bare_repo.is_dir() or bare_repo.is_symlink(): raise SystemExit(24) def git_run(*args): return subprocess.run( ["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) def canonical_git_run(*args): return subprocess.run( ["git", f"--git-dir={canonical_bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) def canonical_image_ancestry_is_proven(recorded_revision, image_revision, expected_main): if not canonical_bare_repo.is_dir() or canonical_bare_repo.is_symlink(): return False resolved_recorded = canonical_git_run("rev-parse", f"{recorded_revision}^{{commit}}") resolved_image = canonical_git_run("rev-parse", f"{image_revision}^{{commit}}") resolved_main = canonical_git_run("rev-parse", "refs/heads/main") if not ( resolved_recorded.returncode == 0 and resolved_recorded.stdout.strip() == recorded_revision and resolved_image.returncode == 0 and resolved_image.stdout.strip() == image_revision and resolved_main.returncode == 0 and resolved_main.stdout.strip() == expected_main ): return False return ( canonical_git_run( "merge-base", "--is-ancestor", recorded_revision, image_revision ).returncode == 0 and canonical_git_run( "merge-base", "--is-ancestor", image_revision, expected_main ).returncode == 0 ) main = git_run("rev-parse", "refs/heads/main") main_revision = main.stdout.strip() if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None: raise SystemExit(26) def main_declares_external_image(image_ref): if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{1,300}@sha256:[0-9a-f]{64}", image_ref) is None: return False for compose_path in ( "infrastructure/compose/docker-compose.yml", "infrastructure/compose/docker-compose.prod.yml", ): source = git_run("show", f"{main_revision}:{compose_path}") if source.returncode != 0: return False for line in source.stdout.splitlines(): if line.strip() in {f"image: {image_ref}", f'image: "{image_ref}"', f"image: '{image_ref}'"}: return True return False def main_declares_sessionless_dataplane_image(container_name, image_ref): if ( container_name != "itworx-mcp-hub-dataplane-router-1" or image_ref != "itworx-mcp-hub/dataplane:sessionless-final" ): return False sessionless_commit = "b519a8c4cf17cd9df00a575acfe096838aab7789" resolved = git_run("rev-parse", f"{sessionless_commit}^{{commit}}") if resolved.returncode != 0 or resolved.stdout.strip() != sessionless_commit: return False if git_run("merge-base", "--is-ancestor", sessionless_commit, main_revision).returncode != 0: return False router = git_run( "show", f"{sessionless_commit}:services/dataplane_router/router.py", ) tests = git_run( "show", f"{sessionless_commit}:tests/test_dataplane_session_headers.py", ) evidence = git_run( "show", f"{main_revision}:evidence/mcp-session-boundary-20260916/SUMMARY.md", ) if router.returncode != 0 or tests.returncode != 0 or evidence.returncode != 0: return False router_contract = ( 'MCP_TRANSPORT_STATE_HEADERS = frozenset({b"mcp-session-id"})', 'if "mcp-session-id" in response.headers:', 'del response.headers["mcp-session-id"]', ) test_contract = ( "2025-11-25", "2026-07-28", "mcp-session-id", ) evidence_contract = ( sessionless_commit, "itworx-mcp-hub/dataplane:sessionless-final", "permanently enforce stateless public transport", ) return ( all(fragment in router.stdout for fragment in router_contract) and all(fragment in tests.stdout for fragment in test_contract) and all(fragment in evidence.stdout for fragment in evidence_contract) ) def main_declares_contextforge_compat_image(container_name, image_ref): if container_name != "itworx-mcp-hub-contextforge-1": return False manifest = git_run( "show", f"{main_revision}:infrastructure/images/contextforge/compat-image.env", ) builder = git_run( "show", f"{main_revision}:scripts/build_contextforge_compat_image.sh", ) if manifest.returncode != 0 or builder.returncode != 0: return False values = {} for raw_line in manifest.stdout.splitlines(): line = raw_line.strip() if not line or line.startswith("#"): continue if "=" not in line: return False key, value = line.split("=", 1) if key in values: return False values[key] = value.strip().strip('"').strip("'") expected_keys = { "CONTEXTFORGE_UPSTREAM_COMMIT", "CONTEXTFORGE_ALEMBIC_HEAD", "CONTEXTFORGE_MCP_SDK", "CONTEXTFORGE_LOCAL_IMAGE", } upstream_commit = str(values.get("CONTEXTFORGE_UPSTREAM_COMMIT") or "") alembic_head = str(values.get("CONTEXTFORGE_ALEMBIC_HEAD") or "") mcp_sdk = str(values.get("CONTEXTFORGE_MCP_SDK") or "") local_image = str(values.get("CONTEXTFORGE_LOCAL_IMAGE") or "") if ( set(values) != expected_keys or re.fullmatch(r"[0-9a-f]{40}", upstream_commit) is None or re.fullmatch(r"[0-9a-f]{12}", alembic_head) is None or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", mcp_sdk) is None or local_image != image_ref or local_image != f"itworx-mcp-hub/contextforge-schema:{alembic_head}" ): return False required_builder_contract = ( 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"', 'git -C "$SRC" checkout --detach FETCH_HEAD', 'MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"', 'alembic -c mcpgateway/alembic.ini heads', 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"', ) return all(fragment in builder.stdout for fragment in required_builder_contract) state_root.mkdir(parents=True, exist_ok=True) project_lock_path.parent.mkdir(parents=True, exist_ok=True) try: project_lock = project_lock_path.open("a+", encoding="utf-8") global_lock = global_lock_path.open("a+", encoding="utf-8") except Exception: raise SystemExit(29) with project_lock: try: fcntl.flock(project_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(30) with global_lock: try: fcntl.flock(global_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(30) try: current_projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(31) if current_projects.get(repository) != project: raise SystemExit(37) if state_path.is_symlink() or not state_path.is_file(): raise SystemExit(42) try: state = json.loads(state_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(42) if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository: raise SystemExit(43) recorded_live = str(state.get("live_revision") or state.get("revision") or "") recorded_status = str(state.get("status") or "") recorded_live_status = str(state.get("live_status") or "") last_attempt = dict(state.get("last_attempt") or {}) attempt_revision = str(last_attempt.get("revision") or "") attempt_status = str(last_attempt.get("status") or "") last_attempt_message = str(last_attempt.get("message") or "") receipt = state.get("live_receipt") recorded_artifacts = state.get("live_artifacts") already_reconciled = ( state.get("live_legacy_provenance") is True and recorded_status == "success" and recorded_live_status == "healthy" and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None and isinstance(recorded_artifacts, dict) ) if not already_reconciled and ( recorded_status != "failed" or recorded_live_status not in {"healthy", "unknown"} or state.get("live_legacy_provenance") is not False or attempt_status != "failed" or re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None or re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is None or attempt_revision == recorded_live or not isinstance(receipt, dict) or receipt.get("schema") != "gitea-deploy-receipt.v1" or receipt.get("repository") != repository or receipt.get("revision") != recorded_live or not isinstance(recorded_artifacts, dict) ): raise SystemExit(52) # deploy.py deliberately downgrades the prior live status to unknown # when capture_compose_rollback() fails before production mutation. # Accept that state only when the durable attempt message is one of the # bounded snapshot-stage failure classes and no rollback/source-restore # error was recorded. The runtime is still re-proven container by # container below before any controller state is changed. snapshot_failure_prefixes = ( "Refusing to mutate a Compose project with unrecorded containers:", "Deployment state claims a live revision but all production containers are missing", "Cannot capture a complete rollback snapshot; containers are missing:", "Existing production containers require a valid healthy live deployment state", "Container ", "Existing deployment state is not bound to its live revision", "Recorded deployment receipt ", "Existing deployment no longer matches its verified live receipt", "Expected containers are not healthy/running:", "Compose project container set does not match the allowlisted runtime:", "Containers are linked to the wrong Compose project:", "Running container image IDs do not match the verified artifacts:", "Immutable image provenance does not match the requested deployment:", ) if ( not already_reconciled and recorded_live_status == "unknown" and ( not last_attempt_message.startswith(snapshot_failure_prefixes) or "; rollback error:" in last_attempt_message or "; source rollback error:" in last_attempt_message ) ): raise SystemExit(55) resolved_live = git_run("rev-parse", f"{recorded_live}^{{commit}}") if resolved_live.returncode != 0 or resolved_live.stdout.strip() != recorded_live: raise SystemExit(25) if git_run("merge-base", "--is-ancestor", recorded_live, main_revision).returncode != 0: raise SystemExit(52) if not already_reconciled: resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}") if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision: raise SystemExit(25) if git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision).returncode != 0: raise SystemExit(52) if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0: raise SystemExit(52) listed = subprocess.run( [ "docker", "ps", "--all", "--filter", f"label=com.docker.compose.project={expected_compose_project}", "--format", "{{.Names}}", ], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) project_names = [line.strip() for line in listed.stdout.splitlines() if line.strip()] if listed.returncode != 0 or set(project_names) != set(expected_containers): raise SystemExit(47) old_containers = recorded_artifacts.get("containers") if not isinstance(old_containers, dict) or set(old_containers) != set(expected_containers): raise SystemExit(52) container_artifacts = {} drift_found = False for name in expected_containers: inspected = subprocess.run( ["docker", "inspect", name], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) if inspected.returncode != 0: raise SystemExit(44) try: container = json.loads(inspected.stdout)[0] except Exception: raise SystemExit(44) runtime_state = dict(container.get("State") or {}) health = str(dict(runtime_state.get("Health") or {}).get("Status") or "none") labels = dict(dict(container.get("Config") or {}).get("Labels") or {}) config = dict(container.get("Config") or {}) image_id = str(container.get("Image") or "") image_ref = str(config.get("Image") or "") if not runtime_state.get("Running") or health in {"starting", "unhealthy"}: raise SystemExit(45) if labels.get("com.docker.compose.project") != expected_compose_project: raise SystemExit(47) if re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None or not image_ref: raise SystemExit(46) image_result = subprocess.run( ["docker", "image", "inspect", image_id], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) if image_result.returncode != 0: raise SystemExit(50) try: image = json.loads(image_result.stdout)[0] except Exception: raise SystemExit(50) image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {}) image_source = str(image_labels.get("org.opencontainers.image.source") or "") image_revision = str(image_labels.get("org.opencontainers.image.revision") or "") recorded = old_containers.get(name) if not isinstance(recorded, dict): raise SystemExit(52) recorded_image_id = str(recorded.get("image_id") or "") recorded_container_id = str(recorded.get("container_id") or "") if re.fullmatch(r"sha256:[0-9a-f]{64}", recorded_image_id) is None: raise SystemExit(52) image_changed = image_id != recorded_image_id if image_changed: contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref) sessionless_dataplane_image = ( not already_reconciled and main_declares_sessionless_dataplane_image(name, image_ref) ) if contextforge_compat_image or sessionless_dataplane_image: pass elif image_source == owned_image_source: if re.fullmatch(r"[0-9a-f]{40}", image_revision) is None: raise SystemExit(51) resolved_image = git_run("rev-parse", f"{image_revision}^{{commit}}") mounted_recorded_to_image = git_run("merge-base", "--is-ancestor", recorded_live, image_revision) mounted_image_to_main = git_run("merge-base", "--is-ancestor", image_revision, main_revision) mounted_ancestry_proven = ( resolved_image.returncode == 0 and resolved_image.stdout.strip() == image_revision and mounted_recorded_to_image.returncode == 0 and mounted_image_to_main.returncode == 0 ) if not mounted_ancestry_proven and not canonical_image_ancestry_is_proven( recorded_live, image_revision, main_revision ): raise SystemExit(51) elif not main_declares_external_image(image_ref): raise SystemExit(51) container_id = str(container.get("Id") or "") if not container_id: raise SystemExit(46) if image_id != recorded_image_id or container_id != recorded_container_id: drift_found = True container_artifacts[name] = { "container_id": container_id, "image_id": image_id, "image_ref": image_ref, "container_revision": str(labels.get("org.opencontainers.image.revision") or ""), "image_revision": image_revision, "image_source": image_source, "image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""), "image_version": str(image_labels.get("org.opencontainers.image.version") or ""), "image_created": str(image_labels.get("org.opencontainers.image.created") or ""), } if already_reconciled: if drift_found: raise SystemExit(52) print("already-reconciled", flush=True) raise SystemExit(0) if not drift_found: raise SystemExit(52) artifacts = {"containers": container_artifacts, "compose_sha256": ""} now = int(time.time()) payload = { "schema_version": 3, "repository": repository, "slug": expected_compose_project, "compose_project": expected_compose_project, "revision": recorded_live, "live_revision": recorded_live, "live_status": "healthy", "live_legacy_provenance": True, "live_artifacts": artifacts, "live_receipt": {}, "rollback_snapshot": {}, "status": "success", "message": "Verified mixed MCP Hub runtime reconciled as a legacy provenance baseline", "updated_at_epoch": now, "artifacts": artifacts, "last_attempt": { "revision": recorded_live, "status": "success", "message": "Verified MCP Hub runtime state reconciliation", "updated_at_epoch": now, "metadata": {"reconciled_failed_revision": attempt_revision}, }, "previous_revision": recorded_live, "rollback_restored_revision": "", } backup = state_path.with_name(f"{state_path.name}.mcp-hub-reconcile-backup-{now}-{os.getpid()}") temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.mcp-hub-reconcile.tmp") try: mode = stat.S_IMODE(state_path.stat().st_mode) shutil.copy2(state_path, backup) with temporary.open("x", encoding="utf-8") as handle: handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") handle.flush() os.fsync(handle.fileno()) os.chmod(temporary, mode) os.replace(temporary, state_path) directory_fd = os.open(str(state_root), os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except Exception: try: temporary.unlink(missing_ok=True) except Exception: pass raise SystemExit(53) try: verify = json.loads(state_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(54) verified_containers = dict(dict(verify.get("live_artifacts") or {}).get("containers") or {}) if not ( verify.get("repository") == repository and verify.get("live_revision") == recorded_live and verify.get("live_status") == "healthy" and verify.get("live_legacy_provenance") is True and verify.get("status") == "success" and set(verified_containers) == set(expected_containers) and all(str(dict(verified_containers[name]).get("image_id") or "") == str(container_artifacts[name]["image_id"]) for name in expected_containers) ): raise SystemExit(54) print("reconciled", flush=True) ''' class McpHubDeploymentReconciliation: """Re-attest only the exact proven MCP Hub state drift, without touching runtime.""" def __init__(self, portainer: PortainerConnector | None = None): self.portainer = portainer or PortainerConnector() def run(self) -> dict[str, Any]: status = self.portainer.get_container_status(_CONTROLLER) if status.get("status") != "ok": return { "status": "controller_unavailable", "reason": redact_message( str(status.get("reason") or status.get("status") or "unavailable") ), } controller = dict(status.get("container") or {}) if controller.get("name") != _CONTROLLER or controller.get("state") != "running": return { "status": "controller_unavailable", "reason": "deployment controller is not running", } container_id = str(controller.get("id") or "") config = self.portainer._config() if not config or not container_id: return { "status": "connector_unavailable", "reason": "Portainer deployment control is unavailable", } command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _COMPOSE_PROJECT] endpoint = quote(str(config.endpoint_id), safe="") safe_container = quote(container_id, safe="") try: created = self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={ "AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command, }, ).json() exec_id = str(created.get("Id") or "") if not exec_id: return {"status": "error", "reason": "MCP Hub state reconciliation could not start"} safe_exec = quote(exec_id, safe="") self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False}, ) deadline = time.monotonic() + 60 exit_code: int | None = None while time.monotonic() < deadline: inspected = self.portainer._request( "GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json" ).json() if not bool(inspected.get("Running")): raw_exit = inspected.get("ExitCode") exit_code = int(raw_exit) if raw_exit is not None else -1 break time.sleep(0.25) except (RuntimeError, ValueError, TypeError) as exc: return {"status": "error", "reason": redact_message(str(exc))} if exit_code is None: return {"status": "error", "reason": "MCP Hub state reconciliation timed out"} reasons = { 20: "deployment projects registry is unavailable or unsafe", 22: "MCP Hub deployment target is missing", 23: "MCP Hub deployment target differs from the bounded production topology", 24: "mounted MCP Hub bare repository is unavailable", 25: "MCP Hub deployment revisions cannot be resolved", 26: "MCP Hub main cannot be resolved", 29: "deployment lock files cannot be opened", 30: "deployment locks cannot be acquired", 31: "deployment projects registry cannot be read", 36: "deployment projects registry host alias is unavailable or mismatched", 37: "MCP Hub deployment target changed while reconciliation was waiting", 40: "fixed MCP Hub reconciliation contract was altered", 42: "MCP Hub deployment state file is unavailable or invalid", 43: "MCP Hub deployment state is not a modern controller state", 44: "a live MCP Hub container cannot be inspected", 45: "a live MCP Hub container is not running and healthy", 46: "a live MCP Hub container has an invalid immutable image identity", 47: "live MCP Hub Compose project differs from the controller target", 50: "a live MCP Hub image cannot be inspected", 51: "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain", 52: "controller state drift is not the exact known recoverable MCP Hub state", 53: "reconciled MCP Hub deployment state could not be written durably", 54: "reconciled MCP Hub deployment state did not verify after write", 55: "unknown MCP Hub live status was not produced by a proven recoverable snapshot-stage failure", } if exit_code != 0: return { "status": "validation_failed", "reason": reasons.get(exit_code, "MCP Hub deployment reconciliation failed closed"), } return { "status": "reconciled", "repository": _REPOSITORY, "composeProject": _COMPOSE_PROJECT, }