991 lines
44 KiB
Python
991 lines
44 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..core.redaction import redact_message
|
|
|
|
_CONTROLLER = "gitea-deploy-control"
|
|
_REPOSITORY = "jens/itworx_mcp_hub"
|
|
_COMPOSE_PROJECT = "itworx-mcp-hub"
|
|
_TOPOLOGY_CONTRACT = "itworx.mcp-hub-topology/17-v1"
|
|
|
|
# Read-only re-attestation for an already-successful modern MCP Hub receipt.
|
|
# Unknown, failed, legacy and mixed-topology states require the guarded deploy
|
|
# path. Reconciliation never repairs or writes controller state.
|
|
_RECONCILE_SCRIPT = r'''
|
|
import fcntl
|
|
import errno
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
repository, expected_compose_project = sys.argv[1:3]
|
|
registry_path = Path("/controller-config/projects.json")
|
|
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
|
|
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
|
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
|
owned_image_source = "https://gitea.example.com/jens/itworx-mcp-hub"
|
|
topology_contract = "itworx.mcp-hub-topology/17-v1"
|
|
|
|
if repository != "jens/itworx_mcp_hub" or expected_compose_project != "itworx-mcp-hub":
|
|
raise SystemExit(40)
|
|
if registry_path.is_symlink() or not registry_path.is_file():
|
|
raise SystemExit(20)
|
|
try:
|
|
if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path):
|
|
raise SystemExit(36)
|
|
except OSError:
|
|
raise SystemExit(36)
|
|
|
|
try:
|
|
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
project = projects.get(repository) if isinstance(projects, dict) else None
|
|
if not isinstance(project, dict):
|
|
raise SystemExit(22)
|
|
expected_containers = project.get("containers")
|
|
expected_fixed = {
|
|
"itworx-mcp-hub-connector-appops-1",
|
|
"itworx-mcp-hub-connector-gitea-1",
|
|
"itworx-mcp-hub-connector-knowledge-1",
|
|
"itworx-mcp-hub-connector-mobilityops-1",
|
|
"itworx-mcp-hub-connector-projectbrain-1",
|
|
"itworx-mcp-hub-connector-stitch-1",
|
|
"itworx-mcp-hub-connector-unraid-1",
|
|
"itworx-mcp-hub-contextforge-1",
|
|
"itworx-mcp-hub-control-api-1",
|
|
"itworx-mcp-hub-dataplane-router-1",
|
|
"itworx-mcp-hub-docker-read-broker-1",
|
|
"itworx-mcp-hub-postgres-1",
|
|
"itworx-mcp-hub-redis-1",
|
|
"itworx-mcp-hub-web-1",
|
|
"itworx-mcp-hub-worker-1",
|
|
}
|
|
expected_runtime = expected_fixed | {
|
|
"itworx-mcp-hub-connector-ingestion-1",
|
|
"itworx-mcp-hub-ingestion-clamd-1",
|
|
}
|
|
if (
|
|
project.get("branch") != "main"
|
|
or project.get("slug") != expected_compose_project
|
|
or project.get("compose_project") != expected_compose_project
|
|
or not isinstance(expected_containers, list)
|
|
or any(not isinstance(item, str) for item in expected_containers)
|
|
or len(expected_containers) != len(set(expected_containers))
|
|
or frozenset(expected_containers) not in {frozenset(expected_fixed), frozenset(expected_runtime)}
|
|
):
|
|
raise SystemExit(23)
|
|
expected_containers = sorted(expected_runtime)
|
|
|
|
state_path = state_root / f"{expected_compose_project}.json"
|
|
project_lock_path = state_root / "locks" / f"{expected_compose_project}.lock"
|
|
owner, name = repository.split("/", 1)
|
|
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
|
if not bare_repo.is_dir() or bare_repo.is_symlink():
|
|
raise SystemExit(24)
|
|
|
|
|
|
def git_run(*args):
|
|
return subprocess.run(
|
|
["git", f"--git-dir={bare_repo}", *args],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
|
|
|
|
main = git_run("rev-parse", "refs/heads/main")
|
|
main_revision = main.stdout.strip()
|
|
if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None:
|
|
raise SystemExit(26)
|
|
|
|
|
|
def main_declares_external_image(image_ref):
|
|
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/:@-]{1,300}@sha256:[0-9a-f]{64}", image_ref) is None:
|
|
return False
|
|
for compose_path in (
|
|
"infrastructure/compose/docker-compose.yml",
|
|
"infrastructure/compose/docker-compose.prod.yml",
|
|
):
|
|
source = git_run("show", f"{main_revision}:{compose_path}")
|
|
if source.returncode != 0:
|
|
return False
|
|
for line in source.stdout.splitlines():
|
|
if line.strip() in {f"image: {image_ref}", f'image: "{image_ref}"', f"image: '{image_ref}'"}:
|
|
return True
|
|
return False
|
|
|
|
|
|
def main_declares_sessionless_dataplane_image(container_name, image_ref):
|
|
if (
|
|
container_name != "itworx-mcp-hub-dataplane-router-1"
|
|
or image_ref != "itworx-mcp-hub/dataplane:sessionless-final"
|
|
):
|
|
return False
|
|
sessionless_commit = "b519a8c4cf17cd9df00a575acfe096838aab7789"
|
|
resolved = git_run("rev-parse", f"{sessionless_commit}^{{commit}}")
|
|
if resolved.returncode != 0 or resolved.stdout.strip() != sessionless_commit:
|
|
return False
|
|
if git_run("merge-base", "--is-ancestor", sessionless_commit, main_revision).returncode != 0:
|
|
return False
|
|
|
|
router = git_run(
|
|
"show",
|
|
f"{sessionless_commit}:services/dataplane_router/router.py",
|
|
)
|
|
tests = git_run(
|
|
"show",
|
|
f"{sessionless_commit}:tests/test_dataplane_session_headers.py",
|
|
)
|
|
evidence = git_run(
|
|
"show",
|
|
f"{main_revision}:evidence/mcp-session-boundary-20260916/SUMMARY.md",
|
|
)
|
|
if router.returncode != 0 or tests.returncode != 0 or evidence.returncode != 0:
|
|
return False
|
|
|
|
router_contract = (
|
|
'MCP_TRANSPORT_STATE_HEADERS = frozenset({b"mcp-session-id"})',
|
|
'if "mcp-session-id" in response.headers:',
|
|
'del response.headers["mcp-session-id"]',
|
|
)
|
|
test_contract = (
|
|
"2025-11-25",
|
|
"2026-07-28",
|
|
"mcp-session-id",
|
|
)
|
|
evidence_contract = (
|
|
sessionless_commit,
|
|
"itworx-mcp-hub/dataplane:sessionless-final",
|
|
"permanently enforce stateless public transport",
|
|
)
|
|
return (
|
|
all(fragment in router.stdout for fragment in router_contract)
|
|
and all(fragment in tests.stdout for fragment in test_contract)
|
|
and all(fragment in evidence.stdout for fragment in evidence_contract)
|
|
)
|
|
|
|
|
|
def main_declares_contextforge_compat_image(container_name, image_ref):
|
|
if container_name != "itworx-mcp-hub-contextforge-1":
|
|
return False
|
|
candidate_reference = (
|
|
re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None
|
|
and image_ref == f"itworx-mcp-hub/contextforge-schema:{recorded_live}"
|
|
)
|
|
# The caller already binds recorded_live to the receipt and protected-main
|
|
# ancestry. Read a candidate's build contract from that exact source tree.
|
|
proof_revision = recorded_live if candidate_reference else main_revision
|
|
manifest = git_run(
|
|
"show",
|
|
f"{proof_revision}:infrastructure/images/contextforge/compat-image.env",
|
|
)
|
|
builder = git_run(
|
|
"show",
|
|
f"{proof_revision}:scripts/build_contextforge_compat_image.sh",
|
|
)
|
|
if manifest.returncode != 0 or builder.returncode != 0:
|
|
return False
|
|
|
|
values = {}
|
|
for raw_line in manifest.stdout.splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if "=" not in line:
|
|
return False
|
|
key, value = line.split("=", 1)
|
|
if key in values:
|
|
return False
|
|
values[key] = value.strip().strip('"').strip("'")
|
|
|
|
expected_keys = {
|
|
"CONTEXTFORGE_UPSTREAM_COMMIT",
|
|
"CONTEXTFORGE_ALEMBIC_HEAD",
|
|
"CONTEXTFORGE_MCP_SDK",
|
|
"CONTEXTFORGE_LOCAL_IMAGE",
|
|
}
|
|
upstream_commit = str(values.get("CONTEXTFORGE_UPSTREAM_COMMIT") or "")
|
|
alembic_head = str(values.get("CONTEXTFORGE_ALEMBIC_HEAD") or "")
|
|
mcp_sdk = str(values.get("CONTEXTFORGE_MCP_SDK") or "")
|
|
local_image = str(values.get("CONTEXTFORGE_LOCAL_IMAGE") or "")
|
|
if (
|
|
set(values) != expected_keys
|
|
or re.fullmatch(r"[0-9a-f]{40}", upstream_commit) is None
|
|
or re.fullmatch(r"[0-9a-f]{12}", alembic_head) is None
|
|
or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", mcp_sdk) is None
|
|
or (local_image != image_ref and not candidate_reference)
|
|
or local_image != f"itworx-mcp-hub/contextforge-schema:{alembic_head}"
|
|
):
|
|
return False
|
|
|
|
required_builder_contract = (
|
|
'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"',
|
|
'git -C "$SRC" checkout --detach FETCH_HEAD',
|
|
'MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"',
|
|
'alembic -c mcpgateway/alembic.ini heads',
|
|
'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"',
|
|
)
|
|
if candidate_reference:
|
|
required_builder_contract += (
|
|
'--release-candidate',
|
|
'CANDIDATE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}"',
|
|
'docker tag "$STAGED_IMAGE" "$CANDIDATE_IMAGE"',
|
|
)
|
|
return all(fragment in builder.stdout for fragment in required_builder_contract)
|
|
|
|
|
|
def live_attested_mounts(container):
|
|
raw_mounts = container.get("Mounts")
|
|
if not isinstance(raw_mounts, list):
|
|
raise SystemExit(52)
|
|
observed = []
|
|
for raw in raw_mounts:
|
|
if not isinstance(raw, dict):
|
|
raise SystemExit(52)
|
|
kind = str(raw.get("Type") or "")
|
|
destination = str(raw.get("Destination") or "")
|
|
if kind not in {"bind", "volume", "tmpfs"} or not destination:
|
|
raise SystemExit(52)
|
|
source = str(raw.get("Source") or "") if kind in {"bind", "volume"} else ""
|
|
name = str(raw.get("Name") or "") if kind == "volume" else ""
|
|
mount = {
|
|
"type": kind,
|
|
"source": os.path.normpath(source) if source else "",
|
|
"destination": destination,
|
|
"rw": raw.get("RW") is True,
|
|
"name": name,
|
|
}
|
|
if kind == "volume":
|
|
if not name:
|
|
raise SystemExit(52)
|
|
volume = subprocess.run(
|
|
["docker", "volume", "inspect", name],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
if volume.returncode != 0:
|
|
raise SystemExit(52)
|
|
try:
|
|
volume_items = json.loads(volume.stdout)
|
|
volume_item = volume_items[0]
|
|
options = volume_item.get("Options") or {}
|
|
mountpoint = os.path.normpath(str(volume_item.get("Mountpoint") or ""))
|
|
created_at = volume_item.get("CreatedAt")
|
|
created = datetime.fromisoformat(str(created_at).replace("Z", "+00:00"))
|
|
except Exception:
|
|
raise SystemExit(52)
|
|
if (
|
|
not isinstance(options, dict)
|
|
or str(volume_item.get("Name") or "") != name
|
|
or mountpoint != mount["source"]
|
|
or str(volume_item.get("Driver") or "") != str(raw.get("Driver") or "")
|
|
or not isinstance(created_at, str)
|
|
or created.tzinfo is None
|
|
or created.utcoffset() is None
|
|
or created.astimezone(timezone.utc) > datetime.now(timezone.utc)
|
|
):
|
|
raise SystemExit(52)
|
|
mount["driver"] = str(volume_item.get("Driver") or "")
|
|
mount["options"] = {str(key): str(value) for key, value in options.items()}
|
|
mount["created_at"] = created_at
|
|
observed.append(mount)
|
|
return sorted(observed, key=lambda item: str(item["destination"]))
|
|
|
|
|
|
def fixed_image_reference(service_name, image_ref):
|
|
if service_name == "dataplane-router" and image_ref == "itworx-mcp-hub/dataplane:sessionless-final":
|
|
return main_declares_sessionless_dataplane_image(
|
|
"itworx-mcp-hub-dataplane-router-1", image_ref
|
|
)
|
|
owned = {
|
|
"connector-appops": "gitea.example.com/jens/itworx-mcp-hub/connector-appops",
|
|
"connector-gitea": "gitea.example.com/jens/itworx-mcp-hub/connector-gitea",
|
|
"connector-ingestion": "gitea.example.com/jens/itworx-mcp-hub/connector-ingestion",
|
|
"connector-knowledge": "gitea.example.com/jens/itworx-mcp-hub/connector-knowledge",
|
|
"connector-mobilityops": "gitea.example.com/jens/itworx-mcp-hub/connector-mobilityops",
|
|
"connector-projectbrain": "gitea.example.com/jens/itworx-mcp-hub/connector-projectbrain",
|
|
"connector-stitch": "gitea.example.com/jens/itworx-mcp-hub/connector-stitch",
|
|
"connector-unraid": "gitea.example.com/jens/itworx-mcp-hub/connector-unraid",
|
|
"control-api": "gitea.example.com/jens/itworx-mcp-hub/control-api",
|
|
"dataplane-router": "gitea.example.com/jens/itworx-mcp-hub/dataplane",
|
|
"docker-read-broker": "gitea.example.com/jens/itworx-mcp-hub/docker-read-broker",
|
|
"web": "gitea.example.com/jens/itworx-mcp-hub/web",
|
|
"worker": "gitea.example.com/jens/itworx-mcp-hub/worker",
|
|
}
|
|
if service_name in owned:
|
|
return (
|
|
re.fullmatch(r"sha256:[0-9a-f]{64}", image_ref) is not None
|
|
or (
|
|
image_ref.startswith(owned[service_name] + "@sha256:")
|
|
and re.fullmatch(r".+@sha256:[0-9a-f]{64}", image_ref) is not None
|
|
)
|
|
)
|
|
if service_name == "postgres":
|
|
return image_ref == "postgres:18.4@sha256:3a82e1f56c8f0f5616a11103ac3d47e632c3938698946a7ad26da0df1334744a" or (
|
|
image_ref == "postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04"
|
|
and main_declares_external_image(image_ref)
|
|
)
|
|
if service_name == "ingestion-clamd":
|
|
return image_ref == "clamav/clamav:1.4.6@sha256:a5f03c12a79dbe9f6d8a527b6bb1ea053fa8dd061d3738a26897f055ee2d9303"
|
|
if service_name == "redis":
|
|
return image_ref == "redis:8.10.0@sha256:c29e49ab2f85760a3827b53882e6dd9f5c6c3f0bb7d724e07bb31cbf275a5236" or (
|
|
image_ref == "redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48"
|
|
and main_declares_external_image(image_ref)
|
|
)
|
|
if service_name == "contextforge":
|
|
return main_declares_contextforge_compat_image(
|
|
"itworx-mcp-hub-contextforge-1", image_ref
|
|
)
|
|
return False
|
|
|
|
|
|
def image_digest_matches(image, image_ref):
|
|
repository, separator, digest = image_ref.partition("@")
|
|
if not separator or re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None:
|
|
return False
|
|
first, slash, last = repository.partition("/")
|
|
if slash:
|
|
if ":" in last:
|
|
last = last.rsplit(":", 1)[0]
|
|
repository = first + slash + last
|
|
elif ":" in repository:
|
|
repository = repository.rsplit(":", 1)[0]
|
|
aliases = {repository}
|
|
first_component = repository.split("/", 1)[0]
|
|
is_registry = "." in first_component or ":" in first_component or first_component == "localhost"
|
|
if not is_registry:
|
|
if "/" in repository:
|
|
aliases.add("docker.io/" + repository)
|
|
else:
|
|
aliases.add("docker.io/library/" + repository)
|
|
if repository.startswith("docker.io/"):
|
|
aliases.add(repository.removeprefix("docker.io/"))
|
|
digests = image.get("RepoDigests", [])
|
|
return isinstance(digests, list) and any(
|
|
"@" in str(item)
|
|
and str(item).split("@", 1)[0] in aliases
|
|
and str(item).split("@", 1)[1] == digest
|
|
for item in digests
|
|
)
|
|
|
|
|
|
def receipt_bound_image_reference(launch_image, image_id, recorded, receipt_item):
|
|
if not isinstance(recorded, dict) or not isinstance(receipt_item, dict):
|
|
raise SystemExit(52)
|
|
image_ref = receipt_item.get("image_ref")
|
|
if (
|
|
re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None
|
|
or not isinstance(image_ref, str)
|
|
or not image_ref
|
|
or recorded.get("image_ref") != image_ref
|
|
or recorded.get("image_id") != image_id
|
|
or receipt_item.get("image_id") != image_id
|
|
or launch_image not in {image_ref, image_id}
|
|
or (image_ref.startswith("sha256:") and image_ref != image_id)
|
|
):
|
|
raise SystemExit(52)
|
|
return image_ref
|
|
|
|
|
|
def trusted_private_path(path, expected_mode, *, directory):
|
|
try:
|
|
info = path.lstat()
|
|
except OSError:
|
|
return False
|
|
if (
|
|
path.is_symlink()
|
|
or (not stat.S_ISDIR(info.st_mode) if directory else not stat.S_ISREG(info.st_mode))
|
|
or info.st_uid != 0
|
|
or (stat.S_IMODE(info.st_mode) != expected_mode if directory else stat.S_IMODE(info.st_mode) & 0o077 != 0)
|
|
):
|
|
return False
|
|
try:
|
|
attributes = os.listxattr(path, follow_symlinks=False)
|
|
except OSError as exc:
|
|
if exc.errno in {errno.ENOTSUP, getattr(errno, "EOPNOTSUPP", errno.ENOTSUP)}:
|
|
attributes = []
|
|
else:
|
|
return False
|
|
return not any(
|
|
item in {"system.posix_acl_access", "system.posix_acl_default"}
|
|
for item in attributes
|
|
)
|
|
|
|
|
|
def validate_migration_intent_state(
|
|
state_root, state_bytes, repository, recorded_live, recorded_status,
|
|
recorded_live_status, last_attempt, attempt_status, attempt_revision,
|
|
):
|
|
attempt_metadata = dict(last_attempt.get("metadata") or {})
|
|
migration_attempt_id = str(attempt_metadata.get("mcpHubMigrationAttemptId") or "")
|
|
migration_root = state_root / "mcp-hub-topology-migrations"
|
|
migration_path = migration_root / "active-intent.json"
|
|
if migration_root.is_symlink():
|
|
raise SystemExit(56)
|
|
if migration_root.exists() and not trusted_private_path(migration_root, 0o700, directory=True):
|
|
raise SystemExit(56)
|
|
if migration_path.is_symlink():
|
|
raise SystemExit(56)
|
|
if migration_path.exists():
|
|
if (
|
|
not trusted_private_path(migration_path, 0o600, directory=False)
|
|
or stat.S_IMODE(migration_path.lstat().st_mode) != 0o600
|
|
):
|
|
raise SystemExit(56)
|
|
try:
|
|
intent_bytes = migration_path.read_bytes()
|
|
intent = json.loads(intent_bytes.decode("utf-8"))
|
|
except Exception:
|
|
raise SystemExit(56)
|
|
common_intent_fields = {
|
|
"schema", "status", "attemptId", "repository", "candidateRevision",
|
|
"priorLiveRevision", "backupManifestSha256", "backupComposeSha256",
|
|
"baselineStateSha256", "startedAtEpoch",
|
|
}
|
|
observed_intent_fields = {"observedBaselinePath", "observedBaselineSha256"}
|
|
observed_fields_present = set(intent) & observed_intent_fields if isinstance(intent, dict) else set()
|
|
if observed_fields_present not in (set(), observed_intent_fields):
|
|
raise SystemExit(56)
|
|
required_intent_fields = common_intent_fields | observed_fields_present
|
|
if isinstance(intent, dict) and intent.get("status") == "complete":
|
|
required_intent_fields |= {"completedAtEpoch", "completedStateSha256"}
|
|
if (
|
|
not isinstance(intent, dict)
|
|
or intent.get("status") != "complete"
|
|
or set(intent) != required_intent_fields
|
|
or intent.get("schema") != "itworx.mcp-hub-migration-intent/v1"
|
|
or intent.get("repository") != repository
|
|
or re.fullmatch(r"[0-9a-f]{32}", str(intent.get("attemptId") or "")) is None
|
|
or intent.get("attemptId") != migration_attempt_id
|
|
or intent.get("candidateRevision") != recorded_live
|
|
or re.fullmatch(r"[0-9a-f]{40}", str(intent.get("priorLiveRevision") or "")) is None
|
|
or any(
|
|
re.fullmatch(r"[0-9a-f]{64}", str(intent.get(field) or "")) is None
|
|
for field in ("backupManifestSha256", "backupComposeSha256", "baselineStateSha256", "completedStateSha256")
|
|
)
|
|
or type(intent.get("startedAtEpoch")) is not int
|
|
or type(intent.get("completedAtEpoch")) is not int
|
|
or intent.get("completedAtEpoch") < intent.get("startedAtEpoch")
|
|
or intent.get("completedStateSha256") != hashlib.sha256(state_bytes).hexdigest()
|
|
or recorded_status != "success"
|
|
or recorded_live_status != "healthy"
|
|
or attempt_status != "success"
|
|
or attempt_revision != recorded_live
|
|
or not isinstance(attempt_metadata.get("mcpHubBackupReceipt"), dict)
|
|
or set(attempt_metadata.get("mcpHubBackupReceipt", {})) != {
|
|
"schema", "repository", "revision", "backupDirectory", "manifestSha256",
|
|
"composeSha256", "restoreVerified", "createdAtEpoch", "receiptPath",
|
|
}
|
|
or attempt_metadata.get("mcpHubBackupReceipt", {}).get("schema") != "itworx.mcp-hub-backup-receipt/v1"
|
|
or attempt_metadata.get("mcpHubBackupReceipt", {}).get("repository") != repository
|
|
or attempt_metadata.get("mcpHubBackupReceipt", {}).get("revision") != recorded_live
|
|
or attempt_metadata.get("mcpHubBackupReceipt", {}).get("restoreVerified") is not True
|
|
or type(attempt_metadata.get("mcpHubBackupReceipt", {}).get("createdAtEpoch")) is not int
|
|
or re.fullmatch(r"[0-9a-f]{64}", str(attempt_metadata.get("mcpHubBackupReceipt", {}).get("manifestSha256") or "")) is None
|
|
or re.fullmatch(r"[0-9a-f]{64}", str(attempt_metadata.get("mcpHubBackupReceipt", {}).get("composeSha256") or "")) is None
|
|
or attempt_metadata.get("mcpHubBackupReceipt", {}).get("manifestSha256") != intent.get("backupManifestSha256")
|
|
or attempt_metadata.get("mcpHubBackupReceipt", {}).get("composeSha256") != intent.get("backupComposeSha256")
|
|
):
|
|
raise SystemExit(56)
|
|
backup_receipt = attempt_metadata.get("mcpHubBackupReceipt", {})
|
|
receipt_root = state_root / "mcp-hub-backup-receipts"
|
|
backup_path = Path(str(backup_receipt.get("receiptPath") or ""))
|
|
if (
|
|
backup_path.parent != receipt_root
|
|
or re.fullmatch(
|
|
rf"[0-9]+-{recorded_live[:12]}-[0-9a-f]{{16}}\.json",
|
|
backup_path.name,
|
|
) is None
|
|
or backup_path.is_symlink()
|
|
or not trusted_private_path(receipt_root, 0o700, directory=True)
|
|
or not trusted_private_path(backup_path, 0o600, directory=False)
|
|
or stat.S_IMODE(backup_path.lstat().st_mode) != 0o600
|
|
):
|
|
raise SystemExit(56)
|
|
try:
|
|
stored_backup_receipt = json.loads(backup_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(56)
|
|
if stored_backup_receipt != {
|
|
key: value for key, value in backup_receipt.items() if key != "receiptPath"
|
|
}:
|
|
raise SystemExit(56)
|
|
backup_root = state_root / "mcp-hub-backups"
|
|
backup_directory = Path(str(backup_receipt.get("backupDirectory") or ""))
|
|
manifest_path = backup_directory / "manifest.json"
|
|
if (
|
|
backup_directory.parent != backup_root
|
|
or re.fullmatch(
|
|
rf"[0-9]+-{recorded_live[:12]}-[0-9a-f]{{16}}",
|
|
backup_directory.name,
|
|
) is None
|
|
or not trusted_private_path(backup_root, 0o700, directory=True)
|
|
or not trusted_private_path(backup_directory, 0o700, directory=True)
|
|
or not trusted_private_path(manifest_path, 0o600, directory=False)
|
|
or stat.S_IMODE(manifest_path.lstat().st_mode) != 0o600
|
|
):
|
|
raise SystemExit(56)
|
|
try:
|
|
manifest_bytes = manifest_path.read_bytes()
|
|
manifest = json.loads(manifest_bytes.decode("utf-8"))
|
|
except Exception:
|
|
raise SystemExit(56)
|
|
if (
|
|
hashlib.sha256(manifest_bytes).hexdigest() != intent.get("backupManifestSha256")
|
|
or not isinstance(manifest, dict)
|
|
or manifest.get("schema") != "itworx.mcp-hub-backup/v1"
|
|
or manifest.get("repository") != repository
|
|
or manifest.get("revision") != recorded_live
|
|
or manifest.get("backupDirectory") != str(backup_directory)
|
|
or manifest.get("composeSha256") != intent.get("backupComposeSha256")
|
|
or manifest.get("configRestoreVerified") is not True
|
|
or dict(manifest.get("restore") or {}).get("status") != "verified"
|
|
or set(dict(manifest.get("databases") or {})) != {"itworx_mcp_hub", "contextforge"}
|
|
):
|
|
raise SystemExit(56)
|
|
if observed_fields_present:
|
|
observed_path = Path(str(intent.get("observedBaselinePath") or ""))
|
|
if (
|
|
observed_path.parent != migration_root
|
|
or re.fullmatch(
|
|
rf"observed-baseline-{recorded_live[:12]}-[0-9a-f]{{20}}\.json",
|
|
observed_path.name,
|
|
) is None
|
|
or observed_path.is_symlink()
|
|
or not trusted_private_path(observed_path, 0o600, directory=False)
|
|
or stat.S_IMODE(observed_path.lstat().st_mode) != 0o600
|
|
):
|
|
raise SystemExit(56)
|
|
try:
|
|
observed_bytes = observed_path.read_bytes()
|
|
observed = json.loads(observed_bytes.decode("utf-8"))
|
|
except Exception:
|
|
raise SystemExit(56)
|
|
if (
|
|
hashlib.sha256(observed_bytes).hexdigest() != intent.get("observedBaselineSha256")
|
|
or not isinstance(observed, dict)
|
|
or observed.get("schema") != "itworx.mcp-hub-observed-baseline/v1"
|
|
or observed.get("claim") != "observed_mixed_runtime_facts_only"
|
|
or observed.get("candidateRevision") != recorded_live
|
|
or observed.get("historicalStateSha256") != intent.get("baselineStateSha256")
|
|
or observed.get("backupManifestSha256") != intent.get("backupManifestSha256")
|
|
or observed.get("backupComposeSha256") != intent.get("backupComposeSha256")
|
|
or observed.get("backupRestoreVerified") is not True
|
|
or not isinstance(observed.get("postgresVolumeBaseline"), list)
|
|
or len(observed.get("postgresVolumeBaseline", [])) != 2
|
|
):
|
|
raise SystemExit(56)
|
|
elif migration_attempt_id:
|
|
raise SystemExit(56)
|
|
|
|
|
|
try:
|
|
project_lock_fd = os.open(project_lock_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
global_lock_fd = os.open(global_lock_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
|
except Exception:
|
|
raise SystemExit(29)
|
|
|
|
with os.fdopen(project_lock_fd, "rb") as project_lock, os.fdopen(global_lock_fd, "rb") as global_lock:
|
|
try:
|
|
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(30)
|
|
with global_lock:
|
|
try:
|
|
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(30)
|
|
|
|
try:
|
|
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
if current_projects.get(repository) != project:
|
|
raise SystemExit(37)
|
|
|
|
if (
|
|
not trusted_private_path(state_root, 0o700, directory=True)
|
|
or not trusted_private_path(state_root / "locks", 0o700, directory=True)
|
|
or not trusted_private_path(project_lock_path, 0o600, directory=False)
|
|
or not trusted_private_path(global_lock_path, 0o600, directory=False)
|
|
or not trusted_private_path(state_path, 0o600, directory=False)
|
|
):
|
|
raise SystemExit(42)
|
|
try:
|
|
state_bytes = state_path.read_bytes()
|
|
state = json.loads(state_bytes.decode("utf-8"))
|
|
except Exception:
|
|
raise SystemExit(42)
|
|
if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository:
|
|
raise SystemExit(43)
|
|
|
|
recorded_live = str(state.get("live_revision") or state.get("revision") or "")
|
|
recorded_status = str(state.get("status") or "")
|
|
recorded_live_status = str(state.get("live_status") or "")
|
|
last_attempt = dict(state.get("last_attempt") or {})
|
|
attempt_revision = str(last_attempt.get("revision") or "")
|
|
attempt_status = str(last_attempt.get("status") or "")
|
|
receipt = state.get("live_receipt")
|
|
recorded_artifacts = state.get("live_artifacts")
|
|
validate_migration_intent_state(
|
|
state_root, state_bytes, repository, recorded_live, recorded_status,
|
|
recorded_live_status, last_attempt, attempt_status, attempt_revision,
|
|
)
|
|
|
|
registered_live_containers = (
|
|
recorded_artifacts.get("containers")
|
|
if isinstance(recorded_artifacts, dict)
|
|
else None
|
|
)
|
|
if isinstance(registered_live_containers, dict) and set(registered_live_containers) == expected_fixed:
|
|
print("mcp-hub-topology-migration-required", flush=True)
|
|
raise SystemExit(56)
|
|
|
|
if set(expected_containers) != expected_runtime:
|
|
raise SystemExit(56)
|
|
|
|
receipt_payload = receipt if isinstance(receipt, dict) else {}
|
|
receipt_tree = str(receipt_payload.get("tree") or "")
|
|
receipt_tree_digest = str(receipt_payload.get("treeDigest") or "")
|
|
raw_receipt_build = receipt_payload.get("build")
|
|
receipt_build = raw_receipt_build if isinstance(raw_receipt_build, dict) else {}
|
|
expected_build = {
|
|
"id": recorded_live[:12],
|
|
"repository": repository,
|
|
"revision": recorded_live,
|
|
"tree": receipt_tree,
|
|
"treeDigest": receipt_tree_digest,
|
|
"builtAt": str(receipt_build.get("builtAt") or ""),
|
|
}
|
|
receipt_identity_valid = (
|
|
re.fullmatch(r"[0-9a-f]{40}", receipt_tree) is not None
|
|
and receipt_tree_digest == hashlib.sha256(
|
|
f"git-tree:{receipt_tree}".encode("utf-8")
|
|
).hexdigest()
|
|
and receipt_build == expected_build
|
|
and re.fullmatch(
|
|
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z",
|
|
expected_build["builtAt"],
|
|
) is not None
|
|
and isinstance(recorded_artifacts, dict)
|
|
and set(recorded_artifacts)
|
|
== {"containers", "compose_sha256", "topology_contract"}
|
|
and re.fullmatch(
|
|
r"[0-9a-f]{64}", str(recorded_artifacts.get("compose_sha256") or "")
|
|
) is not None
|
|
)
|
|
|
|
already_reconciled = (
|
|
recorded_status == "success"
|
|
and recorded_live_status == "healthy"
|
|
and state.get("revision") == recorded_live
|
|
and state.get("live_legacy_provenance") is False
|
|
and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None
|
|
and isinstance(recorded_artifacts, dict)
|
|
and state.get("compose_project") == expected_compose_project
|
|
and state.get("slug") == expected_compose_project
|
|
and attempt_status == "success"
|
|
and attempt_revision == recorded_live
|
|
and isinstance(registered_live_containers, dict)
|
|
and set(registered_live_containers) == expected_runtime
|
|
and recorded_artifacts.get("topology_contract") == topology_contract
|
|
and state.get("artifacts") == recorded_artifacts
|
|
and isinstance(receipt, dict)
|
|
and set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"}
|
|
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
|
and receipt.get("repository") == repository
|
|
and receipt.get("revision") == recorded_live
|
|
and receipt_identity_valid
|
|
and isinstance(receipt.get("containers"), dict)
|
|
and set(dict(receipt.get("containers") or {})) == expected_runtime
|
|
and receipt.get("live") is None
|
|
)
|
|
if not already_reconciled:
|
|
raise SystemExit(56 if recorded_live_status in {"unknown", "healthy"} else 52)
|
|
|
|
resolved_live = git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
|
if resolved_live.returncode != 0 or resolved_live.stdout.strip() != recorded_live:
|
|
raise SystemExit(25)
|
|
if git_run("merge-base", "--is-ancestor", recorded_live, main_revision).returncode != 0:
|
|
raise SystemExit(52)
|
|
|
|
listed = subprocess.run(
|
|
[
|
|
"docker",
|
|
"ps",
|
|
"--all",
|
|
"--filter",
|
|
f"label=com.docker.compose.project={expected_compose_project}",
|
|
"--format",
|
|
"{{.Names}}",
|
|
],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
project_names = [line.strip() for line in listed.stdout.splitlines() if line.strip()]
|
|
if listed.returncode != 0 or set(project_names) != set(expected_containers):
|
|
raise SystemExit(47)
|
|
|
|
old_containers = recorded_artifacts.get("containers")
|
|
receipt_containers = dict(receipt.get("containers") or {})
|
|
if not isinstance(old_containers, dict) or set(old_containers) != set(expected_containers):
|
|
raise SystemExit(52)
|
|
|
|
for name in expected_containers:
|
|
inspected = subprocess.run(
|
|
["docker", "inspect", name],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
if inspected.returncode != 0:
|
|
raise SystemExit(44)
|
|
try:
|
|
container = json.loads(inspected.stdout)[0]
|
|
except Exception:
|
|
raise SystemExit(44)
|
|
runtime_state = dict(container.get("State") or {})
|
|
health = str(dict(runtime_state.get("Health") or {}).get("Status") or "none")
|
|
config = dict(container.get("Config") or {})
|
|
labels = dict(config.get("Labels") or {})
|
|
name_service = name.removeprefix("itworx-mcp-hub-").removesuffix("-1")
|
|
image_id = str(container.get("Image") or "")
|
|
launch_image = str(config.get("Image") or "")
|
|
container_id = str(container.get("Id") or "")
|
|
if runtime_state.get("Running") is not True or health != "healthy":
|
|
raise SystemExit(45)
|
|
if (
|
|
labels.get("com.docker.compose.project") != expected_compose_project
|
|
or labels.get("com.docker.compose.service") != name_service
|
|
):
|
|
raise SystemExit(47)
|
|
if (
|
|
re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None
|
|
or not launch_image
|
|
or not container_id
|
|
):
|
|
raise SystemExit(46)
|
|
|
|
recorded = old_containers.get(name)
|
|
receipt_item = receipt_containers.get(name)
|
|
image_ref = receipt_bound_image_reference(
|
|
launch_image, image_id, recorded, receipt_item
|
|
)
|
|
image_result = subprocess.run(
|
|
["docker", "image", "inspect", image_id],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
if image_result.returncode != 0:
|
|
raise SystemExit(50)
|
|
try:
|
|
image = json.loads(image_result.stdout)[0]
|
|
except Exception:
|
|
raise SystemExit(50)
|
|
image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {})
|
|
artifact = {
|
|
"container_id": container_id,
|
|
"image_id": image_id,
|
|
"image_ref": image_ref,
|
|
"container_revision": str(labels.get("org.opencontainers.image.revision") or ""),
|
|
"image_revision": str(image_labels.get("org.opencontainers.image.revision") or ""),
|
|
"image_source": str(image_labels.get("org.opencontainers.image.source") or ""),
|
|
"image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""),
|
|
"image_version": str(image_labels.get("org.opencontainers.image.version") or ""),
|
|
"image_created": str(image_labels.get("org.opencontainers.image.created") or ""),
|
|
}
|
|
if str(image.get("Id") or "") != image_id:
|
|
raise SystemExit(46)
|
|
if (
|
|
set(recorded) != set(artifact) | {"mounts_json"}
|
|
or set(receipt_item) != set(artifact) | {"mounts_sha256"}
|
|
):
|
|
raise SystemExit(52)
|
|
for key, value in artifact.items():
|
|
if recorded.get(key) != value or receipt_item.get(key) != value:
|
|
raise SystemExit(52)
|
|
|
|
raw_mounts = recorded.get("mounts_json")
|
|
if not isinstance(raw_mounts, str):
|
|
raise SystemExit(52)
|
|
try:
|
|
recorded_mounts = json.loads(raw_mounts)
|
|
except Exception:
|
|
raise SystemExit(52)
|
|
mount_bytes = json.dumps(
|
|
recorded_mounts, sort_keys=True, separators=(",", ":")
|
|
).encode("utf-8")
|
|
if (
|
|
not isinstance(recorded_mounts, list)
|
|
or hashlib.sha256(mount_bytes).hexdigest() != receipt_item.get("mounts_sha256")
|
|
or live_attested_mounts(container) != recorded_mounts
|
|
):
|
|
raise SystemExit(52)
|
|
|
|
if not fixed_image_reference(name_service, image_ref):
|
|
raise SystemExit(51)
|
|
owned_service = name_service in {
|
|
"connector-appops", "connector-gitea", "connector-ingestion",
|
|
"connector-knowledge", "connector-mobilityops", "connector-projectbrain",
|
|
"connector-stitch", "connector-unraid", "control-api", "dataplane-router",
|
|
"docker-read-broker", "web", "worker",
|
|
}
|
|
if owned_service and image_ref != "itworx-mcp-hub/dataplane:sessionless-final":
|
|
if (
|
|
artifact["image_source"] != owned_image_source
|
|
or artifact["image_revision"] != recorded_live
|
|
or artifact["image_tree"] != receipt_tree
|
|
or artifact["image_version"] != recorded_live[:12]
|
|
or artifact["image_created"] != expected_build["builtAt"]
|
|
):
|
|
raise SystemExit(51)
|
|
if "@sha256:" in image_ref and not image_digest_matches(image, image_ref):
|
|
raise SystemExit(51)
|
|
|
|
try:
|
|
if state_path.read_bytes() != state_bytes:
|
|
raise SystemExit(52)
|
|
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(37)
|
|
if current_projects.get(repository) != project:
|
|
raise SystemExit(37)
|
|
|
|
print("already-reconciled", flush=True)
|
|
'''
|
|
|
|
|
|
class McpHubDeploymentReconciliation:
|
|
"""Read-only verify only an existing, exact modern MCP Hub success receipt."""
|
|
|
|
def __init__(self, portainer: PortainerConnector | None = None):
|
|
self.portainer = portainer or PortainerConnector()
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
status = self.portainer.get_container_status(_CONTROLLER)
|
|
if status.get("status") != "ok":
|
|
return {
|
|
"status": "controller_unavailable",
|
|
"reason": redact_message(
|
|
str(status.get("reason") or status.get("status") or "unavailable")
|
|
),
|
|
}
|
|
controller = dict(status.get("container") or {})
|
|
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
|
return {
|
|
"status": "controller_unavailable",
|
|
"reason": "deployment controller is not running",
|
|
}
|
|
container_id = str(controller.get("id") or "")
|
|
config = self.portainer._config()
|
|
if not config or not container_id:
|
|
return {
|
|
"status": "connector_unavailable",
|
|
"reason": "Portainer deployment control is unavailable",
|
|
}
|
|
|
|
command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _COMPOSE_PROJECT]
|
|
endpoint = quote(str(config.endpoint_id), safe="")
|
|
safe_container = quote(container_id, safe="")
|
|
try:
|
|
created = self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
|
json={
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"Cmd": command,
|
|
},
|
|
).json()
|
|
exec_id = str(created.get("Id") or "")
|
|
if not exec_id:
|
|
return {"status": "error", "reason": "MCP Hub state reconciliation could not start"}
|
|
safe_exec = quote(exec_id, safe="")
|
|
self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
|
json={"Detach": True, "Tty": False},
|
|
)
|
|
deadline = time.monotonic() + 60
|
|
exit_code: int | None = None
|
|
while time.monotonic() < deadline:
|
|
inspected = self.portainer._request(
|
|
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
|
).json()
|
|
if not bool(inspected.get("Running")):
|
|
raw_exit = inspected.get("ExitCode")
|
|
exit_code = int(raw_exit) if raw_exit is not None else -1
|
|
break
|
|
time.sleep(0.25)
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
return {"status": "error", "reason": redact_message(str(exc))}
|
|
|
|
if exit_code is None:
|
|
return {"status": "error", "reason": "MCP Hub state reconciliation timed out"}
|
|
reasons = {
|
|
20: "deployment projects registry is unavailable or unsafe",
|
|
22: "MCP Hub deployment target is missing",
|
|
23: "MCP Hub deployment target differs from the bounded production topology",
|
|
24: "mounted MCP Hub bare repository is unavailable",
|
|
25: "MCP Hub deployment revisions cannot be resolved",
|
|
26: "MCP Hub main cannot be resolved",
|
|
29: "deployment lock files cannot be opened",
|
|
30: "deployment locks cannot be acquired",
|
|
31: "deployment projects registry cannot be read",
|
|
36: "deployment projects registry host alias is unavailable or mismatched",
|
|
37: "MCP Hub deployment target changed while reconciliation was waiting",
|
|
40: "fixed MCP Hub reconciliation contract was altered",
|
|
42: "MCP Hub deployment state file is unavailable or invalid",
|
|
43: "MCP Hub deployment state is not a modern controller state",
|
|
44: "a live MCP Hub container cannot be inspected",
|
|
45: "a live MCP Hub container is not running and healthy",
|
|
46: "a live MCP Hub container has an invalid immutable image identity",
|
|
47: "live MCP Hub Compose project differs from the controller target",
|
|
50: "a live MCP Hub image cannot be inspected",
|
|
51: "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain",
|
|
52: "controller state or live facts do not match the exact modern MCP Hub receipt",
|
|
56: "MCP Hub legacy, unknown, mixed, or unresolved migration state requires guarded deployment or data-aware recovery",
|
|
}
|
|
if exit_code != 0:
|
|
if exit_code == 56:
|
|
return {
|
|
"status": "topology_migration_required",
|
|
"repository": _REPOSITORY,
|
|
"composeProject": _COMPOSE_PROJECT,
|
|
"reason": reasons[exit_code],
|
|
}
|
|
return {
|
|
"status": "validation_failed",
|
|
"reason": reasons.get(exit_code, "MCP Hub deployment reconciliation failed closed"),
|
|
}
|
|
return {
|
|
"status": "reconciled",
|
|
"repository": _REPOSITORY,
|
|
"composeProject": _COMPOSE_PROJECT,
|
|
}
|