From 570089b749ce576eee2482dada4c17509bf4e03c Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Sat, 3 Oct 2026 20:13:28 +0000 Subject: [PATCH] Publish curated ITWorx_MCP_Hub source from 5d65966d0393e868ed5df1d95a21412a4c587d78 --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 56 +- infrastructure/compose/docker-compose.yml | 4 +- scripts/create_manifest.py | 52 +- scripts/publish_mcp_hub_ingestion.py | 1186 +++++++++++++++++++ scripts/security_scan.py | 61 +- scripts/verify.py | 11 + tests/test_compose.py | 116 +- tests/test_database_candidate_provenance.py | 263 ++++ tests/test_mcp_hub_ingestion_publisher.py | 812 +++++++++++++ tests/test_mcp_hub_release_artifact.py | 364 ++++++ tests/test_mcp_hub_release_images.py | 531 +++++++++ tests/test_release_images_workflow.py | 63 + tests/test_wp160_security.py | 48 +- 14 files changed, 3491 insertions(+), 78 deletions(-) create mode 100644 scripts/publish_mcp_hub_ingestion.py create mode 100644 tests/test_database_candidate_provenance.py create mode 100644 tests/test_mcp_hub_ingestion_publisher.py create mode 100644 tests/test_mcp_hub_release_artifact.py create mode 100644 tests/test_mcp_hub_release_images.py create mode 100644 tests/test_release_images_workflow.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index ed24f59..fc5e02d 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `2aa63ba527a98b38b4d28f2a1bb85b1b1dbe7f6d`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `5d65966d0393e868ed5df1d95a21412a4c587d78`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index 624ed7e..102ba3c 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "2aa63ba527a98b38b4d28f2a1bb85b1b1dbe7f6d", + "sourceRevision": "5d65966d0393e868ed5df1d95a21412a4c587d78", "files": [ { "path": ".dockerignore", @@ -1999,8 +1999,8 @@ }, { "path": "infrastructure/compose/docker-compose.yml", - "sha256": "5071e83745b659bf920624a8a28dc86017ea225b900fe42cfce176bd75885fef", - "bytes": 27747 + "sha256": "7dbf120b195e491e48047a863dd353362d6933e60e0777af5e38d7d0b8f627fb", + "bytes": 27883 }, { "path": "infrastructure/database/README.md", @@ -2454,8 +2454,8 @@ }, { "path": "scripts/create_manifest.py", - "sha256": "560a0b8996292594935bed98f29a403887c813a8e72318f72797842ef560b6eb", - "bytes": 11451 + "sha256": "d44978e5a2a9cafb26506996a8095c05e1ff1501937f526d8b7c0e1e762e41ae", + "bytes": 10863 }, { "path": "scripts/draft_cve_exceptions.py", @@ -2522,6 +2522,11 @@ "sha256": "96b2c25d5f22838bb499f3000646f803f2af29fa99dc68b699f75ac9f3474a92", "bytes": 2515 }, + { + "path": "scripts/publish_mcp_hub_ingestion.py", + "sha256": "e0ddf54fcbf8609dbca1273b2a7bcea8845511412ec0f3e41554646283083e59", + "bytes": 45333 + }, { "path": "scripts/reconcile_gitea_public_surface.py", "sha256": "bd58744a6fe1e448845f96b037f459cf1f0d2fce034c91a3766b66ec4b2ea1b1", @@ -2549,8 +2554,8 @@ }, { "path": "scripts/security_scan.py", - "sha256": "07612f3e995d2a15dc0a72f650f27a4c25f9ad235ac0ded2124ce22f7bd24f87", - "bytes": 5833 + "sha256": "8a8b152e276f6ca9bb52d5d6791888b2777fd18c179b614bf069da67c115f393", + "bytes": 5570 }, { "path": "scripts/update_state.py", @@ -2564,8 +2569,8 @@ }, { "path": "scripts/verify.py", - "sha256": "da819db31ca891d8f9f2aaad42c02728b0dd8fa9bd2801a43f1fd0e947facca8", - "bytes": 14006 + "sha256": "2219c1db92ddb2d02d0c352ed12be7b43acc6c275dad9b9454206bab515b2a07", + "bytes": 14593 }, { "path": "scripts/verify_demo_token_flow.py", @@ -3109,8 +3114,8 @@ }, { "path": "tests/test_compose.py", - "sha256": "b0cf2be5b0b53aa6698a30568ad7999e113daa37bd831aa535d1866a0d373fc6", - "bytes": 18737 + "sha256": "f0a31081471405ff796c29de800a013c107f3c4a55970e05cf7d85ea09a69ae7", + "bytes": 24213 }, { "path": "tests/test_contextforge.py", @@ -3162,6 +3167,11 @@ "sha256": "84cc24fec155594bb5fc97a6966f2312b81d9b80dce22a2903c17de45797f006", "bytes": 3962 }, + { + "path": "tests/test_database_candidate_provenance.py", + "sha256": "1ddc6b769e6f3d8908d978b082ddb84734af9da0fcd13bbf98c89d1fd8a0b9ed", + "bytes": 13049 + }, { "path": "tests/test_dataplane_router.py", "sha256": "14a387851eacec3da9547f76e5294d212b32ad4b15903980c4e67104da4d0023", @@ -3327,6 +3337,21 @@ "sha256": "8e4d3e81862570d281259b6eabadbe9568d2d48e56c240b1fa1da6a73efd3eb4", "bytes": 721 }, + { + "path": "tests/test_mcp_hub_ingestion_publisher.py", + "sha256": "b6ec7c10fd6c62f9c079346f9af09e0b73153e7ca37d0ea750edff93676c3f8c", + "bytes": 36963 + }, + { + "path": "tests/test_mcp_hub_release_artifact.py", + "sha256": "9663cb159b2d430c0d561a23413ee3f5fe4fcc16822f39e83843cc271ae95fe1", + "bytes": 12742 + }, + { + "path": "tests/test_mcp_hub_release_images.py", + "sha256": "8507bffd7ea3b34a00ec6f52849f9106fae37f1623de0804bda0b475e776b803", + "bytes": 21009 + }, { "path": "tests/test_mcp_workflow_prompts.py", "sha256": "f411909eff398eb9cca4c17e83b9e248b305c2075df576b8bcf40b940289891b", @@ -3397,6 +3422,11 @@ "sha256": "f076ff9b84f8803c9e06e77980319e060aacebbd171fbaa4eb8fe5dab7f0197a", "bytes": 2556 }, + { + "path": "tests/test_release_images_workflow.py", + "sha256": "92e1e03d71aaab8ef2f31eb6643d42b9e6189b9ccca86f1f0b55425d841512c2", + "bytes": 2510 + }, { "path": "tests/test_stitch_connector.py", "sha256": "c34cd2dc20721052dc63a15dc763ecc9b4f1adaef1d6ec2d5189adff9161c269", @@ -3439,8 +3469,8 @@ }, { "path": "tests/test_wp160_security.py", - "sha256": "ba823df74a30aa4c18d17021ea848e328bbdf7649f0b3f77d4a1445ee88ae1ae", - "bytes": 5100 + "sha256": "410086ffcf45619c78ef82b6d3319d65734a8fb5fe679c21b4365e1cca2f28d0", + "bytes": 7295 }, { "path": "tests/test_wp170_release.py", diff --git a/infrastructure/compose/docker-compose.yml b/infrastructure/compose/docker-compose.yml index fb91c49..a269782 100644 --- a/infrastructure/compose/docker-compose.yml +++ b/infrastructure/compose/docker-compose.yml @@ -29,7 +29,7 @@ x-third-party: &third-party services: postgres: <<: *third-party - image: postgres:18.6@sha256:0377e72c5289ed2f98cf61b1a9c2db9eb9d300317fe14244492fbc94343b3d04 + image: gitea.example.com/jens/itworx-mcp-hub/postgres:wp400-gosu-v2-20261002-a94e0b4d2029@sha256:a60d81e06987671ae903757f1b71cdfbbff4d481b42610e4de2b7dcfa32aad73 entrypoint: ["sh", "/opt/itworx/postgres-entrypoint.sh"] command: ["postgres"] environment: @@ -53,7 +53,7 @@ services: redis: <<: *third-party - image: redis:8.10.2@sha256:7ef5b5cec96495a04ca7feff88a9492efeab8053fb284d24bdd73344c9245a48 + image: gitea.example.com/jens/itworx-mcp-hub/redis:wp400-remediation-20261002-4e18f61c@sha256:7d826b44f245ad32e8fe4bf40d42eae95855c3942c5f2e42784ee8fd3f9547a2 entrypoint: ["sh", "/opt/itworx/redis-entrypoint.sh"] environment: REDIS_AUTH_ENABLED: "${REDIS_AUTH_ENABLED:-false}" diff --git a/scripts/create_manifest.py b/scripts/create_manifest.py index 6b1c745..197218d 100644 --- a/scripts/create_manifest.py +++ b/scripts/create_manifest.py @@ -10,21 +10,19 @@ Every entry is hashed from `git archive` output at a chosen ref (default byte-for-byte (see the WP-220 evidence and ``.gitattributes`` for why that already caused a false-positive manifest failure once, in commit 95329ca). -2. Raw blob content is *also* not what a real archive extraction or checkout - produces for every file: Git always stores blobs LF-normalized - internally, but a file with an explicit `eol=crlf` `.gitattributes` - override (e.g. scripts/bootstrap.ps1, intentionally always-CRLF for - native Windows tooling) gets CRLF from `git archive`/`git checkout` - regardless. Hashing raw blobs would make the manifest describe bytes - nobody ever actually receives. Hashing `git archive` output instead makes - manifest generation and "verify a real archive extraction" - (`validate_pack.py --verify-tree`) definitionally the same operation. +2. The bytes received from Git depend on the path's .gitattributes rules. + Text files with an explicit eol rule are normalized in the object store + and converted on archive/checkout; paths marked -text retain their exact + source bytes in both places. Hashing raw blobs would miss the conversion + for text files, while hashing git archive output makes manifest generation + and "verify a real archive extraction" (validate_pack.py --verify-tree) + definitionally the same operation. -Both properties hold together because ``.gitattributes`` pins an explicit -`eol` for every tracked file type in this repository — no file is left to -`core.autocrlf` auto-detection ambiguity, so `git archive` output is fully -deterministic across platforms even though it is not identical to the raw -blob for CRLF-pinned files. +Both properties hold together because .gitattributes pins explicit behavior +for every tracked path: ordinary text files have a pinned eol, and the WP-400 +retained-provenance bundle is a scoped -text exact-byte exception. This avoids +core.autocrlf auto-detection ambiguity and keeps git archive output +deterministic across platforms. Only files tracked by Git at the chosen ref are ever considered — ignored, untracked, local-only, generated, cache, secret, editor, and agent-setting @@ -110,18 +108,13 @@ def batch_blob_contents(ref: str, paths: list[str], repo_root: Path = ROOT) -> d Deliberately sourced from `git archive`, not raw `git cat-file` blob content. Those two differ for any file with an explicit `eol=crlf` (or `eol=lf`) `.gitattributes` override — e.g. scripts/bootstrap.ps1, which - is intentionally always-CRLF for native Windows tooling. Git stores every - blob LF-normalized internally regardless of that attribute; the - attribute only governs the conversion `git archive` and `git checkout` - apply on the way *out*. Hashing raw blob bytes would make the manifest - describe content nobody ever actually receives from a real archive - extraction or checkout — hashing `git archive` output instead makes the - manifest and "verify a real archive extraction" (scripts/validate_pack.py - --verify-tree) definitionally the same operation, by construction. This - is still fully platform-independent: .gitattributes pins an explicit - `eol` for every tracked file type, so no file is left to `core.autocrlf` - auto-detection ambiguity (verified: a Windows checkout with - core.autocrlf=true and a Linux checkout produce byte-identical archives). + is intentionally always-CRLF for native Windows tooling. Text paths are + normalized in the object store and converted by archive/checkout; -text + paths such as the retained WP-400 provenance bundle keep their raw bytes. + Raw blobs omit archive-time conversion for text files, while hashing + git archive output matches the bytes consumers extract and check out. + This makes the manifest and "verify a real archive extraction" + (scripts/validate_pack.py --verify-tree) definitionally the same operation. """ if not paths: return {} @@ -202,9 +195,8 @@ def build_manifest(ref: str, repo_root: Path = ROOT) -> dict[str, object]: "contentDigest": compute_content_digest(files), "description": ( "Every entry below is hashed from `git archive` output at " - "this commit, not from checked-out filesystem bytes and not " - "from raw `git cat-file` blob content (those differ for " - "files with an explicit eol= override, e.g. " + "this commit, not from checked-out filesystem bytes or raw " + "blob content for text paths with an eol conversion (e.g. " "scripts/bootstrap.ps1). Regenerating this manifest from the " "same commit on any platform (Windows checkout, Linux " "checkout, fresh clone, or `git archive` extraction) must " @@ -224,7 +216,7 @@ def build_manifest(ref: str, repo_root: Path = ROOT) -> dict[str, object]: def main() -> int: parser = argparse.ArgumentParser( - description=("Create a deterministic Git-blob-based checksum manifest for the design pack.") + description="Create a deterministic Git-archive-based checksum manifest.", ) parser.add_argument("--output", default="PACK_MANIFEST.json") parser.add_argument( diff --git a/scripts/publish_mcp_hub_ingestion.py b/scripts/publish_mcp_hub_ingestion.py new file mode 100644 index 0000000..92799f2 --- /dev/null +++ b/scripts/publish_mcp_hub_ingestion.py @@ -0,0 +1,1186 @@ +#!/usr/bin/env python3 +"""Publish only the frozen MCP Hub connector-ingestion image by digest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import stat +import subprocess +import sys +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +FIXED_REPOSITORY = "jens/itworx_mcp_hub" +FIXED_IMAGE_REPOSITORY = "gitea.example.com/jens/itworx-mcp-hub/connector-ingestion" +FIXED_SOURCE_URI = "https://gitea.example.com/jens/itworx-mcp-hub" +FIXED_SERVICE = "connector-ingestion" +FIXED_CONTAINER = "gitea-deploy-control" +FIXED_DOCKER_BINARY = "/usr/bin/docker" +FIXED_DOCKER_CONTEXT = "default" +FIXED_DOCKER_SOCKET = "unix:///var/run/docker.sock" +FIXED_DOCKER_ROOT = "/var/lib/docker" +FIXED_DOCKER_HOST_NAME = "Tower" +FIXED_DOCKER_VERSION = "27.5.1" +FIXED_BUILDX_VERSION = "v0.20.1" +FIXED_BUILDER_NAME = "itworx-mcp-hub-ingestion" +FIXED_BUILDKIT_IMAGE = ( + "moby/buildkit:v0.33.0@sha256:6c2fa84a6b61ccd72899dde4239f8d5717f05f9a8ca6f3cad185fb1a95a94de3" +) +FIXED_BUILDKIT_CONTAINER = "buildx_buildkit_itworx-mcp-hub-ingestion0" +FIXED_BUILDKIT_CONFIG_PATH = Path("/root/.docker/buildx/buildkitd.default.toml") +FIXED_BUILDKIT_ENTRYPOINT = ["/usr/bin/buildkitd-entrypoint"] +FIXED_BUILDKIT_COMMAND = ["--allow-insecure-entitlement=network.host"] +FIXED_BUILDKIT_STATE_VOLUME = f"{FIXED_BUILDKIT_CONTAINER}_state" +FIXED_BUILDKIT_STATE_PATH = "/var/lib/buildkit" +FIXED_VOLUME_NAME = "itworx-controller-hub-releases" +FIXED_VOLUME_ROOT = Path("/var/lib/docker/volumes/itworx-controller-hub-releases/_data") +FIXED_RELEASES_ROOT = FIXED_VOLUME_ROOT / "releases" +FIXED_STATE_ROOT = Path("/mnt/cache/appdata/gitea-deploy-runners/state") +ATTESTATION_SCHEMA = "itworx.mcp-hub-ingestion-freeze-attestation/v1" +RECEIPT_SCHEMA = "itworx.mcp-hub-ingestion-publication/v1" +MAX_METADATA_BYTES = 1024 * 1024 +COMMAND_TIMEOUT_SECONDS = 30 +BUILDER_BOOTSTRAP_TIMEOUT_SECONDS = 300 +BUILD_TIMEOUT_SECONDS = 1800 + +OCI_LABELS = ( + ("org.opencontainers.image.source", FIXED_SOURCE_URI), + ("org.opencontainers.image.revision", "revision"), + ("org.opencontainers.image.source-tree", "tree"), + ("org.opencontainers.image.version", "id"), + ("org.opencontainers.image.created", "builtAt"), + ("org.opencontainers.image.source-date-epoch", "epoch"), +) +ALLOWED_MANIFEST_MEDIA_TYPES = { + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.oci.image.manifest.v1+json", +} +EXPECTED_VOLUME_LABELS = { + "itworx.managed-by": "deployment-controller", + "itworx.purpose": "mcp-hub-protected-release", +} +DOCKER_INFO_FORMAT = "{{.ID}}|{{.DockerRootDir}}|{{.ServerVersion}}|{{.Name}}" +IMAGE_INSPECT_FORMAT = "|".join( + [ + "{{.Id}}", + "{{.Config.User}}", + *(f'{{{{index .Config.Labels "{name}"}}}}' for name, _ in OCI_LABELS), + ] +) +PINNED_BUILDKIT_IMAGE_FIELDS = ( + "Id", + "Config.Env", + "Config.Entrypoint", + "Config.User", + "Config.WorkingDir", + "Config.Labels", +) +PINNED_BUILDKIT_IMAGE_FORMAT = "\n".join( + f"{{{{json .{field}}}}}" for field in PINNED_BUILDKIT_IMAGE_FIELDS +) +BUILDKIT_CONTAINER_FIELDS = ( + "Image", + "Config.Image", + "Config.Env", + "Config.Entrypoint", + "Config.Cmd", + "Config.User", + "Config.WorkingDir", + "Config.Labels", + "State.Running", + "HostConfig.Privileged", + "HostConfig.Init", + "HostConfig.RestartPolicy.Name", + "HostConfig.RestartPolicy.MaximumRetryCount", + "HostConfig.NetworkMode", + "HostConfig.UsernsMode", + "HostConfig.CgroupParent", + "HostConfig.CgroupnsMode", + "HostConfig.Mounts", + "Mounts", + "HostConfig.Binds", + "HostConfig.Devices", + "HostConfig.DeviceRequests", + "HostConfig.CapAdd", + "HostConfig.CapDrop", + "HostConfig.SecurityOpt", + "HostConfig.PortBindings", + "HostConfig.PublishAllPorts", + "HostConfig.ReadonlyRootfs", + "HostConfig.PidMode", + "HostConfig.IpcMode", + "HostConfig.UTSMode", + "HostConfig.AutoRemove", + "HostConfig.Runtime", + "HostConfig.Memory", + "HostConfig.MemorySwap", + "HostConfig.CpuShares", + "HostConfig.CpuQuota", + "HostConfig.CpuPeriod", + "HostConfig.CpusetCpus", + "HostConfig.CpusetMems", + "HostConfig.OomKillDisable", +) +BUILDKIT_CONTAINER_FORMAT = "\n".join( + f"{{{{json .{field}}}}}" for field in BUILDKIT_CONTAINER_FIELDS +) + + +class PublishError(RuntimeError): + """A fixed publication precondition or attestation failed.""" + + +@dataclass(frozen=True) +class FrozenSource: + repository: str + revision: str + build: dict[str, str] + source_date_epoch: str + freeze_sha256: str + controller_source_path: str + + +Runner = Callable[..., subprocess.CompletedProcess[str]] + + +_CONTROLLER_ATTESTATION_PROGRAM = r""" +import hashlib, json, sys +from pathlib import Path +sys.path.insert(0, "/opt/gitea-deploy") +import deploy as controller +import prepare_mcp_hub_candidate as candidate + +repository = "jens/itworx_mcp_hub" +revision = sys.argv[1] +project, repo = candidate._fixed_target(repository, revision) +record = candidate._load_freeze_record(revision) +build, epoch = candidate._validate_freeze_record(record, repository, revision, repo) +source = controller.MCP_HUB_RELEASES_ROOT / revision +if not controller.exact_source_matches_revision(repo, revision, source): + raise RuntimeError("staged source mismatch") +controller._mcp_hub_verify_release_tree_metadata(source) +controller._mcp_hub_require_trusted_state_root() +freeze_path = candidate._freeze_directory(revision, create=False) / candidate.FREEZE_FILENAME +raw = freeze_path.read_bytes() +if json.loads(raw) != record: + raise RuntimeError("freeze changed") +result = { + "schema": "itworx.mcp-hub-ingestion-freeze-attestation/v1", + "repository": repository, + "revision": revision, + "build": build, + "sourceDateEpoch": epoch, + "freezeSha256": hashlib.sha256(raw).hexdigest(), + "sourcePath": source.as_posix(), +} +print(json.dumps(result, sort_keys=True, separators=(",", ":"))) +""" + + +def _docker(*arguments: str) -> list[str]: + return [FIXED_DOCKER_BINARY, "--context", FIXED_DOCKER_CONTEXT, *arguments] + + +def _run( + runner: Runner, + command: list[str], + *, + cwd: Path | None = None, + timeout: int = COMMAND_TIMEOUT_SECONDS, + stage: str, +) -> str: + try: + result = runner(command, cwd=cwd, timeout=timeout) + except (OSError, subprocess.SubprocessError): + raise PublishError(f"{stage} command failed") from None + if result.returncode != 0 or not isinstance(result.stdout, str): + raise PublishError(f"{stage} command failed") + return result.stdout.strip() + + +def _docker_runner( + command: list[str], *, cwd: Path | None, timeout: int +) -> subprocess.CompletedProcess[str]: + return subprocess.run( # noqa: S603 -- command is fixed by this module; callers inject only the test runner. + command, + cwd=cwd, + timeout=timeout, + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + + +def _validate_revision(revision: str) -> None: + if re.fullmatch(r"[0-9a-f]{40}", revision) is None: + raise PublishError("revision must be a full lowercase Git SHA") + + +def _validate_digest(value: object) -> str: + digest = str(value or "") + if re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None: + raise PublishError("image digest is invalid") + return digest + + +def _validate_frozen_attestation(document: object, revision: str) -> FrozenSource: + _validate_revision(revision) + required = { + "schema", + "repository", + "revision", + "build", + "sourceDateEpoch", + "freezeSha256", + "sourcePath", + } + if not isinstance(document, dict) or set(document) != required: + raise PublishError("official source-freeze attestation has an invalid shape") + build = document.get("build") + build_fields = {"id", "repository", "revision", "tree", "treeDigest", "builtAt"} + if ( + document.get("schema") != ATTESTATION_SCHEMA + or document.get("repository") != FIXED_REPOSITORY + or document.get("revision") != revision + or not isinstance(build, dict) + or set(build) != build_fields + or any(not isinstance(value, str) for value in build.values()) + ): + raise PublishError("official source-freeze attestation does not match the fixed target") + tree = build["tree"] + epoch = document.get("sourceDateEpoch") + if re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", build["builtAt"]) is None: + raise PublishError("official source-freeze build time is invalid") from None + try: + datetime.strptime(build["builtAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + raise PublishError("official source-freeze build time is invalid") from None + if ( + build["id"] != revision[:12] + or build["repository"] != FIXED_REPOSITORY + or build["revision"] != revision + or re.fullmatch(r"[0-9a-f]{40}", tree) is None + or build["treeDigest"] != hashlib.sha256(f"git-tree:{tree}".encode("ascii")).hexdigest() + or not isinstance(epoch, str) + or re.fullmatch(r"[1-9][0-9]{0,11}", epoch) is None + or re.fullmatch(r"[0-9a-f]{64}", str(document.get("freezeSha256") or "")) is None + or document.get("sourcePath") != f"/controller-hub-releases/releases/{revision}" + ): + raise PublishError("official source-freeze metadata is invalid") + return FrozenSource( + repository=FIXED_REPOSITORY, + revision=revision, + build=dict(build), + source_date_epoch=epoch, + freeze_sha256=document["freezeSha256"], + controller_source_path=document["sourcePath"], + ) + + +def _attest_official_freeze(revision: str, runner: Runner) -> FrozenSource: + _validate_revision(revision) + command = _docker( + "exec", + "--user", + "0", + FIXED_CONTAINER, + "python3", + "-c", + _CONTROLLER_ATTESTATION_PROGRAM, + revision, + ) + raw = _run(runner, command, stage="official source-freeze attestation") + if len(raw) > 16 * 1024: + raise PublishError("official source-freeze attestation is oversized") + try: + document = json.loads(raw) + except (ValueError, TypeError): + raise PublishError("official source-freeze attestation is invalid") from None + return _validate_frozen_attestation(document, revision) + + +def _parse_engine_identity(value: str, *, source: str) -> tuple[str, str, str, str]: + parts = value.split("|") + if ( + len(parts) != 4 + or re.fullmatch(r"[A-Za-z0-9:_-]{16,128}", parts[0]) is None + or parts[1] != FIXED_DOCKER_ROOT + or parts[2] != FIXED_DOCKER_VERSION + or parts[3] != FIXED_DOCKER_HOST_NAME + ): + raise PublishError(f"{source} Docker engine identity is not the fixed Tower engine") + return parts[0], parts[1], parts[2], parts[3] + + +def _preflight_engine(runner: Runner) -> tuple[str, str]: + selected = _run(runner, [FIXED_DOCKER_BINARY, "context", "show"], stage="Docker context") + endpoint = _run( + runner, + [ + FIXED_DOCKER_BINARY, + "context", + "inspect", + FIXED_DOCKER_CONTEXT, + "--format", + '{{(index .Endpoints "docker").Host}}', + ], + stage="Docker endpoint inspection", + ) + if selected != FIXED_DOCKER_CONTEXT or endpoint != FIXED_DOCKER_SOCKET: + raise PublishError("Docker CLI is not bound to the fixed local Tower socket") + host = _parse_engine_identity( + _run( + runner, + _docker("info", "--format", DOCKER_INFO_FORMAT), + stage="host Docker identity inspection", + ), + source="host", + ) + controller = _parse_engine_identity( + _run( + runner, + _docker( + "exec", + "--user", + "0", + FIXED_CONTAINER, + "docker", + "--host", + FIXED_DOCKER_SOCKET, + "info", + "--format", + DOCKER_INFO_FORMAT, + ), + stage="controller Docker identity inspection", + ), + source="controller-socket", + ) + if host != controller: + raise PublishError("host and controller Docker socket identities differ") + inspected = _run( + runner, + _docker("inspect", "--format", "{{.Id}}|{{.State.Running}}", FIXED_CONTAINER), + stage="controller container inspection", + ).split("|") + if ( + len(inspected) != 2 + or re.fullmatch(r"[0-9a-f]{64}", inspected[0]) is None + or inspected[1] != "true" + ): + raise PublishError("fixed deployment controller is not running") + buildx_version = _run(runner, _docker("buildx", "version"), stage="Buildx availability") + if re.search(rf"(?:^|\s){re.escape(FIXED_BUILDX_VERSION)}(?:\s|$)", buildx_version) is None: + raise PublishError("Buildx version differs from the fixed Tower toolchain") + return host[0], inspected[0] + + +def _builder_rows(runner: Runner) -> set[tuple[str, str]]: + raw = _run( + runner, + _docker("buildx", "ls", "--format", "{{.Name}}|{{.DriverEndpoint}}"), + stage="buildx builder inventory", + ) + rows: set[tuple[str, str]] = set() + for line in raw.splitlines(): + parts = line.strip().split("|", 1) + if len(parts) != 2 or not parts[0] or not parts[1]: + raise PublishError("buildx builder inventory is invalid") + rows.add((parts[0], parts[1])) + return rows + + +def _json_inspection_record(raw: str, fields: tuple[str, ...], *, stage: str) -> dict[str, Any]: + lines = raw.splitlines() + if len(lines) != len(fields): + raise PublishError(f"{stage} returned an invalid fixed record") + try: + values = [json.loads(line) for line in lines] + except (ValueError, TypeError): + raise PublishError(f"{stage} returned an invalid fixed record") from None + return dict(zip(fields, values, strict=True)) + + +def _empty_host_config(value: Any) -> bool: + return value in (None, [], {}, "") + + +def _mount_extras_are_empty(mount: dict[str, Any], allowed: set[str]) -> bool: + return all(key in allowed or value in (None, "", False, [], {}) for key, value in mount.items()) + + +def _verify_dedicated_builder_container(runner: Runner, *, require_running: bool) -> None: + image_raw = _run( + runner, + _docker( + "image", + "inspect", + "--format", + PINNED_BUILDKIT_IMAGE_FORMAT, + FIXED_BUILDKIT_IMAGE, + ), + stage="pinned BuildKit image configuration inspection", + ) + image = _json_inspection_record( + image_raw, + PINNED_BUILDKIT_IMAGE_FIELDS, + stage="pinned BuildKit image configuration inspection", + ) + image_id = image["Id"] + image_env = image["Config.Env"] + image_entrypoint = image["Config.Entrypoint"] + image_user = image["Config.User"] + image_working_dir = image["Config.WorkingDir"] + image_labels = image["Config.Labels"] + if ( + not isinstance(image_id, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None + or image_entrypoint != FIXED_BUILDKIT_ENTRYPOINT + or not isinstance(image_env, list) + or not all(isinstance(item, str) for item in image_env) + or image_user is not None + and not isinstance(image_user, str) + or image_working_dir is not None + and not isinstance(image_working_dir, str) + or image_labels is not None + and not isinstance(image_labels, dict) + ): + raise PublishError("pinned BuildKit image configuration is outside its fixed contract") + + container_raw = _run( + runner, + _docker( + "inspect", + "--format", + BUILDKIT_CONTAINER_FORMAT, + FIXED_BUILDKIT_CONTAINER, + ), + stage="dedicated BuildKit container inspection", + ) + container = _json_inspection_record( + container_raw, + BUILDKIT_CONTAINER_FIELDS, + stage="dedicated BuildKit container inspection", + ) + config_matches_image = all( + container[container_field] == image[image_field] + for container_field, image_field in ( + ("Config.Env", "Config.Env"), + ("Config.Entrypoint", "Config.Entrypoint"), + ("Config.User", "Config.User"), + ("Config.WorkingDir", "Config.WorkingDir"), + ("Config.Labels", "Config.Labels"), + ) + ) + mounts = container["HostConfig.Mounts"] + actual_mounts = container["Mounts"] + expected_host_mount = { + "Type": "volume", + "Source": FIXED_BUILDKIT_STATE_VOLUME, + "Target": FIXED_BUILDKIT_STATE_PATH, + } + expected_actual_mount = { + "Type": "volume", + "Name": FIXED_BUILDKIT_STATE_VOLUME, + "Source": f"{FIXED_DOCKER_ROOT}/volumes/{FIXED_BUILDKIT_STATE_VOLUME}/_data", + "Destination": FIXED_BUILDKIT_STATE_PATH, + "Driver": "local", + "Mode": "", + "RW": True, + "Propagation": "", + } + mounts_match = ( + isinstance(mounts, list) + and len(mounts) == 1 + and isinstance(mounts[0], dict) + and all(mounts[0].get(key) == value for key, value in expected_host_mount.items()) + # Docker's Mount.ReadOnly is tagged json:",omitempty". Buildx leaves it + # unset for the writable state volume, so the daemon omits the key. + # Accept only omission or the explicit JSON boolean false; reject null, + # numbers, strings, and true rather than treating them as equivalent. + and ("ReadOnly" not in mounts[0] or mounts[0]["ReadOnly"] is False) + and _mount_extras_are_empty(mounts[0], set(expected_host_mount) | {"ReadOnly"}) + and isinstance(actual_mounts, list) + and len(actual_mounts) == 1 + and isinstance(actual_mounts[0], dict) + and all(actual_mounts[0].get(key) == value for key, value in expected_actual_mount.items()) + and _mount_extras_are_empty(actual_mounts[0], set(expected_actual_mount)) + ) + empty_config_fields = ( + "HostConfig.Binds", + "HostConfig.Devices", + "HostConfig.DeviceRequests", + "HostConfig.CapAdd", + "HostConfig.CapDrop", + "HostConfig.SecurityOpt", + "HostConfig.PortBindings", + ) + no_extra_host_config = all( + _empty_host_config(container[field]) for field in empty_config_fields + ) + zero_resource_limits = all( + container[field] in (None, 0, "") + for field in ( + "HostConfig.Memory", + "HostConfig.MemorySwap", + "HostConfig.CpuShares", + "HostConfig.CpuQuota", + "HostConfig.CpuPeriod", + "HostConfig.CpusetCpus", + "HostConfig.CpusetMems", + ) + ) + if ( + container["Image"] != image_id + or container["Config.Image"] != FIXED_BUILDKIT_IMAGE + or not config_matches_image + or container["Config.Cmd"] != FIXED_BUILDKIT_COMMAND + or not mounts_match + or container["HostConfig.Privileged"] is not True + or container["HostConfig.Init"] is not True + or container["HostConfig.RestartPolicy.Name"] != "unless-stopped" + or container["HostConfig.RestartPolicy.MaximumRetryCount"] != 0 + or container["HostConfig.NetworkMode"] != "default" + or container["HostConfig.UsernsMode"] not in (None, "") + or container["HostConfig.CgroupParent"] != "/docker/buildx" + or container["HostConfig.CgroupnsMode"] != "private" + or not no_extra_host_config + or container["HostConfig.PublishAllPorts"] is not False + or container["HostConfig.ReadonlyRootfs"] is not False + or any( + container[field] not in (None, "") + for field in ( + "HostConfig.PidMode", + "HostConfig.UTSMode", + ) + ) + or container["HostConfig.IpcMode"] != "private" + or container["HostConfig.Runtime"] != "runc" + or container["HostConfig.AutoRemove"] is not False + or not zero_resource_limits + or container["HostConfig.OomKillDisable"] not in (None, False) + or (require_running and container["State.Running"] is not True) + ): + raise PublishError("dedicated BuildKit container runtime configuration is unsafe") + + +def _verify_buildkit_default_config_absent() -> None: + path = FIXED_BUILDKIT_CONFIG_PATH + current = Path(path.anchor) + for part in path.parts[1:]: + current = current / part + try: + metadata = current.lstat() + except FileNotFoundError: + continue + except OSError: + raise PublishError("default BuildKit config path cannot be verified") from None + if stat.S_ISLNK(metadata.st_mode): + raise PublishError("default BuildKit config path contains a symlink") + try: + path.lstat() + except FileNotFoundError: + return + except OSError: + raise PublishError("default BuildKit config path cannot be verified") from None + raise PublishError("default BuildKit config would alter the pinned daemon command") + + +def _inspect_dedicated_builder( + runner: Runner, *, require_running: bool, validate_container: bool = False +) -> None: + rows = _builder_rows(runner) + expected_rows = { + (FIXED_BUILDER_NAME, "docker-container"), + (f"{FIXED_BUILDER_NAME}0", FIXED_DOCKER_SOCKET), + } + own_rows = {row for row in rows if row[0].startswith(FIXED_BUILDER_NAME)} + if own_rows != expected_rows: + raise PublishError("dedicated BuildKit builder does not match its fixed local contract") + details = _run( + runner, + _docker("buildx", "inspect", FIXED_BUILDER_NAME), + stage="dedicated builder inspection", + ) + names = re.findall(r"(?m)^Name:\s*(\S+)\s*$", details) + options = re.search(r"(?m)^Driver Options:\s*(.*?)\s*$", details) + if ( + names != [FIXED_BUILDER_NAME, f"{FIXED_BUILDER_NAME}0"] + or not re.search(r"(?m)^Driver:\s+docker-container\s*$", details) + or not re.search(rf"(?m)^Endpoint:\s+{re.escape(FIXED_DOCKER_SOCKET)}\s*$", details) + or options is None + or options.group(1) != f'image="{FIXED_BUILDKIT_IMAGE}"' + ): + raise PublishError("dedicated BuildKit builder is not bound to the fixed local engine") + if require_running: + version_lines = re.findall(r"(?m)^BuildKit(?: version)?:\s*(\S+)\s*$", details) + if version_lines != ["v0.33.0"] or not re.search(r"(?m)^Status:\s+running\s*$", details): + raise PublishError("dedicated BuildKit version differs from the pinned release") + if validate_container or require_running: + _verify_dedicated_builder_container(runner, require_running=require_running) + + +def _ensure_dedicated_builder(runner: Runner, frozen: FrozenSource) -> None: + checked_frozen = _validate_frozen_attestation( + { + "schema": ATTESTATION_SCHEMA, + "repository": frozen.repository, + "revision": frozen.revision, + "build": frozen.build, + "sourceDateEpoch": frozen.source_date_epoch, + "freezeSha256": frozen.freeze_sha256, + "sourcePath": frozen.controller_source_path, + }, + frozen.revision, + ) + if checked_frozen != frozen: + raise PublishError("a validated source freeze is required before builder setup") + _verify_buildkit_default_config_absent() + rows = _builder_rows(runner) + builder_registered = any( + name in {FIXED_BUILDER_NAME, f"{FIXED_BUILDER_NAME}0"} for name, _ in rows + ) + containers = _run( + runner, + _docker("container", "ls", "--all", "--format", "{{.Names}}"), + stage="dedicated builder container inventory", + ).splitlines() + has_reserved_container = FIXED_BUILDKIT_CONTAINER in containers + if not builder_registered and has_reserved_container: + raise PublishError("reserved BuildKit container name is occupied without its builder") + if builder_registered: + # Validate the configured pin before starting any existing builder. + _inspect_dedicated_builder( + runner, + require_running=False, + validate_container=has_reserved_container, + ) + else: + command = _docker( + "buildx", + "create", + "--name", + FIXED_BUILDER_NAME, + "--driver", + "docker-container", + "--driver-opt", + f"image={FIXED_BUILDKIT_IMAGE}", + FIXED_DOCKER_CONTEXT, + ) + _run(runner, command, stage="fixed dedicated BuildKit builder creation") + _inspect_dedicated_builder(runner, require_running=False) + _run( + runner, + _docker("buildx", "inspect", "--bootstrap", FIXED_BUILDER_NAME), + timeout=BUILDER_BOOTSTRAP_TIMEOUT_SECONDS, + stage="pinned dedicated BuildKit bootstrap", + ) + _inspect_dedicated_builder(runner, require_running=True, validate_container=True) + + +def _verify_release_volume(runner: Runner) -> None: + raw = _run( + runner, + _docker( + "volume", + "inspect", + "--format", + "{{json .}}", + FIXED_VOLUME_NAME, + ), + stage="fixed release-volume inspection", + ) + try: + document = json.loads(raw) + except (ValueError, TypeError): + raise PublishError("fixed release volume metadata is invalid") from None + if isinstance(document, list): + if len(document) != 1: + raise PublishError("fixed release volume metadata is ambiguous") + document = document[0] + if ( + not isinstance(document, dict) + or document.get("Name") != FIXED_VOLUME_NAME + or document.get("Driver") != "local" + or document.get("Scope") != "local" + or document.get("Mountpoint") != FIXED_VOLUME_ROOT.as_posix() + or document.get("Options") not in (None, {}) + or document.get("Labels") != EXPECTED_VOLUME_LABELS + ): + raise PublishError("fixed release volume differs from its reviewed contract") + + +def _require_no_acl(path: Path) -> None: + if os.name == "nt": + return + listxattr = getattr(os, "listxattr", None) + if not callable(listxattr): + raise PublishError("filesystem ACLs cannot be checked") + try: + names = listxattr(path, follow_symlinks=False) + except OSError: + raise PublishError("filesystem ACLs cannot be checked") from None + if any(name in {"system.posix_acl_access", "system.posix_acl_default"} for name in names): + raise PublishError("source path has an unreviewed POSIX ACL") + + +def _verify_local_release_root(path: Path, revision: str) -> None: + expected = FIXED_RELEASES_ROOT / revision + if path != expected or path.is_symlink(): + raise PublishError("staged source path is not the fixed release generation") + try: + info = path.lstat() + volume = FIXED_VOLUME_ROOT.lstat() + except OSError: + raise PublishError("staged source generation is unavailable") from None + releases_info = FIXED_RELEASES_ROOT.lstat() + if ( + path.is_symlink() + or not stat.S_ISDIR(info.st_mode) + or not stat.S_ISDIR(volume.st_mode) + or not stat.S_ISDIR(releases_info.st_mode) + or FIXED_VOLUME_ROOT.is_symlink() + or FIXED_RELEASES_ROOT.is_symlink() + ): + raise PublishError("staged source generation metadata is unsafe") + if os.name != "nt" and ( + info.st_uid != 0 + or stat.S_IMODE(info.st_mode) != 0o555 + or volume.st_uid != 0 + or stat.S_IMODE(volume.st_mode) & 0o022 + or releases_info.st_uid != 0 + or stat.S_IMODE(releases_info.st_mode) & 0o022 + ): + raise PublishError("staged source generation metadata is unsafe") + _require_no_acl(path) + _require_no_acl(FIXED_VOLUME_ROOT) + _require_no_acl(FIXED_RELEASES_ROOT) + _require_no_acl(path / "infrastructure") + dockerfile = path / "infrastructure" / "images" / "python-service" / "Dockerfile" + try: + dockerfile_info = dockerfile.lstat() + except OSError: + raise PublishError("fixed ingestion Dockerfile is unavailable") from None + if dockerfile.is_symlink() or not stat.S_ISREG(dockerfile_info.st_mode): + raise PublishError("fixed ingestion Dockerfile metadata is unsafe") + if os.name != "nt" and ( + dockerfile_info.st_uid != 0 + or stat.S_IMODE(dockerfile_info.st_mode) not in (0o444, 0o555) + or dockerfile_info.st_nlink != 1 + ): + raise PublishError("fixed ingestion Dockerfile metadata is unsafe") + _require_no_acl(dockerfile) + + +def _parse_build_metadata(path: Path) -> tuple[str, str]: + try: + info = path.lstat() + if ( + path.is_symlink() + or not stat.S_ISREG(info.st_mode) + or info.st_nlink != 1 + or (os.name != "nt" and info.st_uid != 0) + or info.st_size <= 0 + or info.st_size > MAX_METADATA_BYTES + ): + raise PublishError("build metadata file metadata is unsafe") + _require_no_acl(path) + metadata = json.loads(path.read_text(encoding="utf-8")) + except PublishError: + raise + except (OSError, ValueError, TypeError): + raise PublishError("build metadata file is invalid") from None + if not isinstance(metadata, dict): + raise PublishError("build metadata file is invalid") + return ( + _validate_digest(metadata.get("containerimage.digest")), + _validate_digest(metadata.get("containerimage.config.digest")), + ) + + +def _parse_manifest_descriptor(raw: str, expected_digest: str) -> None: + if len(raw) > 16 * 1024: + raise PublishError("registry manifest descriptor is oversized") + try: + descriptor = json.loads(raw) + except (ValueError, TypeError): + raise PublishError("registry manifest descriptor is invalid") from None + if not isinstance(descriptor, dict) or set(descriptor) != {"digest", "mediaType", "size"}: + raise PublishError("registry manifest descriptor has an invalid shape") + if ( + descriptor.get("digest") != expected_digest + or descriptor.get("mediaType") not in ALLOWED_MANIFEST_MEDIA_TYPES + or isinstance(descriptor.get("size"), bool) + or not isinstance(descriptor.get("size"), int) + or descriptor["size"] <= 0 + ): + raise PublishError("registry manifest digest does not match the frozen build") + + +def _validate_image_inspection(raw: str, frozen: FrozenSource, config_digest: str) -> None: + fields = raw.split("|") + if len(fields) != 2 + len(OCI_LABELS): + raise PublishError("registry image inspection has an invalid shape") + if fields[0] != config_digest or fields[1] != "10001:10001": + raise PublishError("registry image identity or non-root user does not match the build") + expected = [ + FIXED_SOURCE_URI, + frozen.revision, + frozen.build["tree"], + frozen.build["id"], + frozen.build["builtAt"], + frozen.source_date_epoch, + ] + if fields[2:] != expected: + raise PublishError("registry image provenance labels do not match the freeze") + + +def _safe_private_dir(path: Path, *, create: bool) -> None: + if os.name != "nt" and (not hasattr(os, "geteuid") or os.geteuid() != 0): + raise PublishError("private publication receipts require host root on POSIX") + fixed_parent = FIXED_STATE_ROOT.parent + for trusted in (fixed_parent, FIXED_STATE_ROOT): + try: + info = trusted.lstat() + except OSError: + raise PublishError("fixed controller state root is unavailable") from None + if ( + trusted.is_symlink() + or not stat.S_ISDIR(info.st_mode) + or (os.name != "nt" and (info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o022)) + ): + raise PublishError("fixed controller state root is unsafe") + _require_no_acl(trusted) + if not path.is_relative_to(FIXED_STATE_ROOT): + raise PublishError("publication receipt path is outside fixed controller state") + relative = path.relative_to(FIXED_STATE_ROOT) + current = FIXED_STATE_ROOT + for part in relative.parts: + current = current / part + if create: + try: + current.mkdir(mode=0o700) + except FileExistsError: + pass + except OSError: + raise PublishError("private publication directory could not be prepared") from None + elif not current.exists() and not current.is_symlink(): + raise PublishError("private publication directory is unavailable") + try: + info = current.lstat() + except OSError: + raise PublishError("private publication directory is unavailable") from None + if current.is_symlink() or not stat.S_ISDIR(info.st_mode): + raise PublishError("private publication directory metadata is unsafe") + if os.name != "nt" and (info.st_uid != 0 or stat.S_IMODE(info.st_mode) != 0o700): + raise PublishError("private publication directory metadata is unsafe") + _require_no_acl(current) + + +def _require_publication_slot(state_root: Path, revision: str) -> None: + if state_root != FIXED_STATE_ROOT: + raise PublishError("publication receipt root is not fixed") + _safe_private_dir(state_root, create=False) + current = state_root + for part in ("mcp-hub-ingestion-publications", revision): + current = current / part + if not current.exists() and not current.is_symlink(): + break + try: + info = current.lstat() + except OSError: + raise PublishError("publication receipt path is unsafe") from None + if current.is_symlink() or not stat.S_ISDIR(info.st_mode): + raise PublishError("publication receipt path is unsafe") + if os.name != "nt" and (info.st_uid != 0 or stat.S_IMODE(info.st_mode) != 0o700): + raise PublishError("publication receipt path is unsafe") + _require_no_acl(current) + receipt = state_root / "mcp-hub-ingestion-publications" / revision / "publication.json" + if receipt.exists() or receipt.is_symlink(): + raise PublishError("publication receipt already exists and is never overwritten") + + +def _write_receipt(path: Path, payload: dict[str, Any]) -> str: + _safe_private_dir(path.parent, create=True) + if path.exists() or path.is_symlink(): + raise PublishError("publication receipt already exists and is never overwritten") + receipt_hash = hashlib.sha256( + (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + ).hexdigest() + envelope = {"receipt": payload, "receiptSha256": receipt_hash} + serialized = (json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) + temporary = path.with_name(f".{path.name}.{os.urandom(8).hex()}.tmp") + created = False + try: + descriptor = os.open( + temporary, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + 0o600, + ) + created = True + try: + if os.name != "nt": + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb", closefd=False) as stream: + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + finally: + os.close(descriptor) + info = temporary.lstat() + if temporary.is_symlink() or not stat.S_ISREG(info.st_mode): + raise PublishError("private publication receipt temporary file is unsafe") + if os.name != "nt" and ( + info.st_uid != 0 or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o600 + ): + raise PublishError("private publication receipt temporary file is unsafe") + _require_no_acl(temporary) + os.link(temporary, path, follow_symlinks=False) + temporary.unlink() + created = False + if os.name != "nt": + directory_fd = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except PublishError: + raise + except FileExistsError: + raise PublishError("publication receipt already exists and is never overwritten") from None + except OSError: + raise PublishError("private publication receipt could not be written") from None + finally: + if created: + temporary.unlink(missing_ok=True) + return receipt_hash + + +def _publish_frozen_revision( + revision: str, + *, + frozen: FrozenSource, + docker_runner: Runner, + source_root: Path, + state_root: Path, + script_path: Path, + before_receipt: Callable[[], None] | None = None, +) -> dict[str, str]: + """Build and publish only the fixed service from an attested immutable freeze.""" + _validate_revision(revision) + if frozen.revision != revision or frozen.repository != FIXED_REPOSITORY: + raise PublishError("frozen source does not match the fixed target") + checked_frozen = _validate_frozen_attestation( + { + "schema": ATTESTATION_SCHEMA, + "repository": frozen.repository, + "revision": frozen.revision, + "build": frozen.build, + "sourceDateEpoch": frozen.source_date_epoch, + "freezeSha256": frozen.freeze_sha256, + "sourcePath": frozen.controller_source_path, + }, + revision, + ) + if checked_frozen != frozen: + raise PublishError("frozen source attestation is invalid") + expected_state_root = FIXED_STATE_ROOT if os.name != "nt" else state_root + if state_root != expected_state_root: + raise PublishError("publication receipt root is not the fixed controller state root") + _verify_local_release_root(source_root, revision) + expected_script = source_root / "scripts" / "publish_mcp_hub_ingestion.py" + if script_path.resolve() != expected_script.resolve() or script_path.is_symlink(): + raise PublishError("publisher script is not from the frozen source tree") + if not script_path.is_file(): + raise PublishError("publisher script is not present in the frozen source tree") + if os.name != "nt": + script_info = script_path.lstat() + if ( + script_path.is_symlink() + or not stat.S_ISREG(script_info.st_mode) + or script_info.st_uid != 0 + or stat.S_IMODE(script_info.st_mode) not in (0o444, 0o555) + or script_info.st_nlink != 1 + ): + raise PublishError("publisher script metadata is unsafe") + _require_no_acl(script_path) + _require_publication_slot(state_root, revision) + receipt_path = state_root / "mcp-hub-ingestion-publications" / revision / "publication.json" + + _inspect_dedicated_builder(docker_runner, require_running=True) + with tempfile.TemporaryDirectory(prefix="mcp-hub-ingestion-") as temporary: + metadata_path = Path(temporary) / "build-metadata.json" + output = ( + "type=image," + f"name={FIXED_IMAGE_REPOSITORY}," + "push=true,push-by-digest=true,name-canonical=true" + ) + command = _docker( + "buildx", + "build", + "--builder", + FIXED_BUILDER_NAME, + "--platform", + "linux/amd64", + "--pull", + "--provenance=false", + "--file", + "infrastructure/images/python-service/Dockerfile", + "--metadata-file", + str(metadata_path), + "--output", + output, + ) + for key, value in ( + ("SERVICE_NAME", FIXED_SERVICE), + ("GIT_REVISION", revision), + ("GIT_SOURCE_TREE", frozen.build["tree"]), + ("IMAGE_VERSION", frozen.build["id"]), + ("SOURCE_DATE_EPOCH", frozen.source_date_epoch), + ("BUILD_DATE", frozen.build["builtAt"]), + ): + command.extend(("--build-arg", f"{key}={value}")) + command.append(".") + _run( + docker_runner, + command, + cwd=source_root, + timeout=BUILD_TIMEOUT_SECONDS, + stage="fixed ingestion image build and digest publication", + ) + _inspect_dedicated_builder(docker_runner, require_running=True) + manifest_digest, config_digest = _parse_build_metadata(metadata_path) + image_ref = f"{FIXED_IMAGE_REPOSITORY}@{manifest_digest}" + descriptor = _run( + docker_runner, + _docker( + "buildx", + "imagetools", + "inspect", + "--format", + "{{json .Manifest}}", + image_ref, + ), + stage="registry manifest readback", + ) + _parse_manifest_descriptor(descriptor, manifest_digest) + _run( + docker_runner, + _docker("pull", "--quiet", image_ref), + timeout=BUILD_TIMEOUT_SECONDS, + stage="immutable registry image pull", + ) + inspected = _run( + docker_runner, + _docker("image", "inspect", "--format", IMAGE_INSPECT_FORMAT, image_ref), + stage="immutable image provenance inspection", + ) + _validate_image_inspection(inspected, frozen, config_digest) + + receipt = { + "schema": RECEIPT_SCHEMA, + "repository": FIXED_REPOSITORY, + "service": FIXED_SERVICE, + "revision": revision, + "build": dict(frozen.build), + "sourceDateEpoch": frozen.source_date_epoch, + "sourceFreezeSha256": frozen.freeze_sha256, + "imageRepository": FIXED_IMAGE_REPOSITORY, + "imageRef": image_ref, + "manifestDigest": manifest_digest, + "imageId": config_digest, + "platform": "linux/amd64", + "registryReadbackVerified": True, + } + if before_receipt is not None: + before_receipt() + receipt_hash = _write_receipt(receipt_path, receipt) + return { + "status": "published", + "revision": revision, + "imageRef": image_ref, + "imageId": config_digest, + "receiptSha256": receipt_hash, + } + + +def _reject_docker_environment(environ: dict[str, str]) -> None: + forbidden = { + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + "BUILDX_BUILDER", + "BUILDKIT_HOST", + "DOCKER_AUTH_CONFIG", + "BUILDX_CONFIG", + } + if any(key in environ for key in forbidden): + raise PublishError("Docker endpoint, builder, or credential-path overrides are not allowed") + if environ.get("HOME") != "/root": + raise PublishError("publisher must use the existing host-root Docker credential context") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("revision", help="exact frozen protected-main revision") + args = parser.parse_args(argv) + try: + _validate_revision(args.revision) + _reject_docker_environment(dict(os.environ)) + if os.name != "posix" or not hasattr(os, "geteuid") or os.geteuid() != 0: + raise PublishError("publisher requires POSIX host root") + runner = _docker_runner + initial_engine, initial_container = _preflight_engine(runner) + _verify_release_volume(runner) + frozen = _attest_official_freeze(args.revision, runner) + source_root = FIXED_RELEASES_ROOT / args.revision + _verify_local_release_root(source_root, args.revision) + _require_publication_slot(FIXED_STATE_ROOT, args.revision) + _ensure_dedicated_builder(runner, frozen) + + def reattest_before_receipt() -> None: + final_engine, final_container = _preflight_engine(runner) + _verify_release_volume(runner) + final_frozen = _attest_official_freeze(args.revision, runner) + if (initial_engine, initial_container) != ( + final_engine, + final_container, + ) or final_frozen != frozen: + raise PublishError( + "controller, Docker engine, or source freeze changed during publication" + ) + + result = _publish_frozen_revision( + args.revision, + frozen=frozen, + docker_runner=runner, + source_root=source_root, + state_root=FIXED_STATE_ROOT, + script_path=Path(__file__), + before_receipt=reattest_before_receipt, + ) + print(json.dumps(result, sort_keys=True)) + return 0 + except PublishError as exc: + print(f"publication denied: {exc}", file=sys.stderr) + return 2 + except (OSError, ValueError): + print("publication denied: fixed publication precondition failed", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/security_scan.py b/scripts/security_scan.py index 04082ae..2276b3c 100644 --- a/scripts/security_scan.py +++ b/scripts/security_scan.py @@ -11,6 +11,8 @@ from __future__ import annotations import argparse import json import re +import shutil +import subprocess from pathlib import Path from typing import Any @@ -29,37 +31,36 @@ _PLACEHOLDER = re.compile( def _tracked_text_files() -> list[Path]: files: list[Path] = [] - for path in ROOT.rglob("*"): - if not path.is_file() or set(path.parts) & { - ".git", - ".venv", - "node_modules", - "dist", - "coverage", - }: + git = shutil.which("git") + if git is None: + raise FileNotFoundError("git is required to enumerate tracked source files") + result = subprocess.run( # noqa: S603 + [git, "ls-files", "-z"], + cwd=ROOT, + capture_output=True, + check=True, + ) + for relative in result.stdout.decode("utf-8").split("\0"): + if not relative: continue - relative_parts = path.relative_to(ROOT).parts - if relative_parts[:2] == (".claude", "worktrees"): - # Git-ignored linked worktrees (.git/info/exclude) are full nested - # checkouts of a branch's tracked content under a different path - # prefix, so a fixture file like tests/fixtures/secret-like.md - # reappears at .claude/worktrees//tests/fixtures/secret-like.md - # and fails the tests/-prefix fixture-placeholder match below even - # though it's the same already-tracked, already-scanned content. - continue - if path.suffix.lower() in { - ".py", - ".ts", - ".tsx", - ".js", - ".jsx", - ".json", - ".yaml", - ".yml", - ".md", - ".toml", - ".env", - }: + path = ROOT / relative + if ( + path.suffix.lower() + in { + ".py", + ".ts", + ".tsx", + ".js", + ".jsx", + ".json", + ".yaml", + ".yml", + ".md", + ".toml", + ".env", + } + and path.is_file() + ): files.append(path) return sorted(files) diff --git a/scripts/verify.py b/scripts/verify.py index 4c25136..c62bebf 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -83,6 +83,9 @@ def main() -> int: "scripts/restore_state.py", "scripts/compose_release_check.py", "scripts/promotion_preflight.py", + "scripts/publish_mcp_hub_ingestion.py", + "scripts/build_mcp_hub_release_images.py", + "scripts/package_mcp_hub_release_artifact.py", "scripts/export_release.py", "scripts/render_release_record.py", "scripts/verify_demo_token_flow.py", @@ -107,6 +110,11 @@ def main() -> int: "tests/test_contextforge_integration.py", "tests/test_bootstrap.py", "tests/test_compose.py", + "tests/test_database_candidate_provenance.py", + "tests/test_mcp_hub_ingestion_publisher.py", + "tests/test_mcp_hub_release_images.py", + "tests/test_mcp_hub_release_artifact.py", + "tests/test_release_images_workflow.py", "tests/test_domain.py", "tests/test_migrations.py", "tests/test_persistence.py", @@ -294,6 +302,9 @@ def main() -> int: "scripts/restore_state.py", "scripts/compose_release_check.py", "scripts/promotion_preflight.py", + "scripts/publish_mcp_hub_ingestion.py", + "scripts/build_mcp_hub_release_images.py", + "scripts/package_mcp_hub_release_artifact.py", "scripts/discover_unraid.py", "scripts/revoke_demo_token.py", "scripts/verify_demo_token_runtime.py", diff --git a/tests/test_compose.py b/tests/test_compose.py index 8734f12..85114a8 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -19,6 +19,35 @@ class ComposeContractTests(unittest.TestCase): def setUpClass(cls) -> None: cls.document = yaml.safe_load(COMPOSE.read_text(encoding="utf-8")) + def assert_database_candidate_pin( + self, + name: str, + image_ref: str, + image_evidence: dict, + publication: dict, + publication_candidate: dict, + ) -> None: + artifact = image_evidence["artifact"] + published = artifact["publishedImage"] + self.assertRegex( + image_ref, + rf"^gitea\.itworx\.tech/jens/itworx-mcp-hub/{name}:[^@]+@sha256:[0-9a-f]{{64}}$", + ) + image_name, digest = image_ref.split("@", maxsplit=1) + self.assertEqual(published["reference"], image_ref) + self.assertEqual(published["taggedReference"], image_name) + self.assertEqual(published["manifestDigest"], digest) + self.assertEqual(published["imageId"], artifact["imageId"]) + self.assertTrue(published["immutablePullConfigMatchedScannedImage"]) + self.assertEqual(published["sourceReceiptSha256"], publication["sourceReceiptSha256"]) + self.assertEqual(publication_candidate["reference"], image_ref) + self.assertEqual(publication_candidate["manifestDigest"], digest) + self.assertEqual(publication_candidate["scannedImageId"], artifact["imageId"]) + self.assertEqual(publication_candidate["pulledImageId"], artifact["imageId"]) + self.assertTrue(publication_candidate["immutablePullConfigMatchedScannedImage"]) + self.assertEqual(image_evidence["humanRiskAcceptance"], "pending") + self.assertFalse(image_evidence["productionDeployment"]) + def test_all_images_are_immutable_or_local_dev(self) -> None: for name, service in self.document["services"].items(): image = service.get("image", "") @@ -68,9 +97,94 @@ class ComposeContractTests(unittest.TestCase): for name in ("postgres", "redis"): with self.subTest(service=name): image = self.document["services"][name]["image"] - self.assertRegex(image, r"^[a-z]+:[0-9.]+@sha256:[0-9a-f]{64}$") + self.assertRegex( + image, + r"^gitea\.itworx\.tech/jens/itworx-mcp-hub/" + rf"{name}:[^@]+@sha256:[0-9a-f]{{64}}$", + ) self.assertEqual(integration["services"][name]["image"], image) + def test_database_candidate_pins_bind_to_published_scan_identities(self) -> None: + evidence_path = ROOT / "evidence/WP-400/DATABASE_REMEDIATION_SCANS.json" + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + publication = evidence["publication"] + publication_proof = json.loads( + (ROOT / "evidence/WP-400/DATABASE_CANDIDATE_PUBLICATION_20261003.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual( + publication, + {key: value for key, value in publication_proof.items() if key != "candidates"}, + ) + self.assertFalse(evidence["productionDeployment"]) + self.assertFalse(publication["productionDeployment"]) + self.assertFalse(publication["cveRiskAccepted"]) + self.assertEqual(publication["status"], "published_digest_verified") + self.assertRegex(publication["sourceReceiptSha256"], r"^[0-9a-f]{64}$") + + integration = yaml.safe_load( + (ROOT / "infrastructure/compose/docker-compose.wp050-integration.yml").read_text( + encoding="utf-8" + ) + ) + for name in ("postgres", "redis"): + with self.subTest(service=name): + image_evidence = evidence["images"][name] + image_ref = image_evidence["artifact"]["publishedImage"]["reference"] + candidate_proof = publication_proof["candidates"][name] + self.assert_database_candidate_pin( + name, + self.document["services"][name]["image"], + image_evidence, + publication, + candidate_proof, + ) + self.assert_database_candidate_pin( + name, + integration["services"][name]["image"], + image_evidence, + publication, + candidate_proof, + ) + self.assertEqual(image_ref, self.document["services"][name]["image"]) + self.assertEqual(image_ref, integration["services"][name]["image"]) + + def test_database_candidate_pin_contract_rejects_tag_only_and_config_drift(self) -> None: + evidence = json.loads( + (ROOT / "evidence/WP-400/DATABASE_REMEDIATION_SCANS.json").read_text(encoding="utf-8") + ) + publication_proof = json.loads( + (ROOT / "evidence/WP-400/DATABASE_CANDIDATE_PUBLICATION_20261003.json").read_text( + encoding="utf-8" + ) + ) + for name in ("postgres", "redis"): + with self.subTest(service=name): + artifact = evidence["images"][name]["artifact"] + published = artifact["publishedImage"] + valid_ref = published["reference"] + tag_only = valid_ref.split("@", maxsplit=1)[0] + with self.assertRaises(AssertionError): + self.assert_database_candidate_pin( + name, + tag_only, + evidence["images"][name], + evidence["publication"], + publication_proof["candidates"][name], + ) + mismatched_image = dict(published, imageId="sha256:" + "0" * 64) + drifted_evidence = dict(evidence["images"][name]) + drifted_evidence["artifact"] = dict(artifact, publishedImage=mismatched_image) + with self.assertRaises(AssertionError): + self.assert_database_candidate_pin( + name, + valid_ref, + drifted_evidence, + evidence["publication"], + publication_proof["candidates"][name], + ) + def test_inspector_is_lan_only_optional_profile(self) -> None: inspector = self.document["services"]["inspector"] self.assertEqual(inspector.get("profiles"), ["inspector"]) diff --git a/tests/test_database_candidate_provenance.py b/tests/test_database_candidate_provenance.py new file mode 100644 index 0000000..f024200 --- /dev/null +++ b/tests/test_database_candidate_provenance.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BUNDLE = ROOT / "evidence/WP-400/database-image-candidate-20261003" +SCAN_EVIDENCE = ROOT / "evidence/WP-400/DATABASE_REMEDIATION_SCANS.json" +PUBLICATION_EVIDENCE = ROOT / "evidence/WP-400/DATABASE_CANDIDATE_PUBLICATION_20261003.json" + + +def read_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def bundle_files(bundle: Path) -> set[str]: + return { + path.relative_to(bundle).as_posix() + for path in bundle.rglob("*") + if path.is_file() and path != bundle / "FILE_HASHES.json" + } + + +def run_git(cwd: Path, *arguments: str) -> bytes: + result = subprocess.run( # noqa: S603 + ["git", *arguments], # noqa: S607 + cwd=cwd, + check=True, + capture_output=True, + ) + return result.stdout + + +class DatabaseCandidateProvenanceTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.manifest = read_json(BUNDLE / "candidate-provenance.json") + cls.scan_evidence = read_json(SCAN_EVIDENCE) + cls.publication = read_json(PUBLICATION_EVIDENCE) + + def test_retained_recipe_sources_match_the_exact_build_recipe_hashes(self) -> None: + for service in ("postgres", "redis"): + with self.subTest(service=service): + item = self.manifest["images"][service] + build = read_json(BUNDLE / item["buildReceiptFile"]) + self.assertEqual(set(item["sourceFiles"]), set(build["recipeHashes"])) + plan = ( + read_json(BUNDLE / "recipes" / service / "plan.json") + if service == "postgres" + else None + ) + for name, expected in item["sourceFiles"].items(): + source = BUNDLE / "recipes" / service / name + if name.endswith(".tar.gz"): + self.assertFalse( + source.exists(), "the binary upstream archive must stay excluded" + ) + self.assertFalse( + self.manifest["trackedBundle"]["binaryGosuSourceArchiveIncluded"] + ) + self.assertEqual(expected, plan["source"]["archive_sha256"]) + else: + self.assertTrue(source.is_file(), name) + self.assertEqual(sha256(source), expected, name) + self.assertEqual(build["recipeHashes"][name], expected, name) + + def test_build_export_scan_and_publication_bind_the_same_candidates(self) -> None: + self.assertFalse(self.manifest["releaseEligible"]) + self.assertFalse(self.manifest["reviewBoundary"]["releaseEligible"]) + self.assertEqual(self.manifest["reviewBoundary"]["humanCveRiskAcceptance"], "pending") + self.assertFalse(self.manifest["reviewBoundary"]["agentMayApproveCveExceptions"]) + self.assertFalse(self.manifest["reviewBoundary"]["bitForBitRebuildClaim"]) + self.assertFalse(self.scan_evidence["productionDeployment"]) + self.assertFalse(self.publication["productionDeployment"]) + self.assertFalse(self.publication["cveRiskAccepted"]) + + for service in ("postgres", "redis"): + with self.subTest(service=service): + item = self.manifest["images"][service] + build = read_json(BUNDLE / item["buildReceiptFile"]) + download = read_json(BUNDLE / item["downloadReceiptFile"]) + scan = read_json(BUNDLE / item["scanReceiptFile"]) + summary = read_json(BUNDLE / item["vulnerabilityScan"]["summaryFile"]) + secret = read_json(BUNDLE / item["secretScan"]["sanitizedReceiptFile"]) + image = build["image"] + published_ref = self.manifest["publishedImageRefs"][service] + published = self.publication["candidates"][service] + scanned = self.scan_evidence["images"][service] + + self.assertEqual(item["publishedImageRef"], published_ref) + self.assertIsNone(item["productionImageRef"]) + self.assertEqual(item["candidateOnly"], True) + self.assertIsNone(item["productionImageRef"]) + self.assertEqual(published["reference"], published_ref) + self.assertEqual(scanned["artifact"]["publishedImage"]["reference"], published_ref) + self.assertEqual(published["scannedImageId"], image["id"]) + self.assertEqual(published["pulledImageId"], image["id"]) + self.assertTrue(published["immutablePullConfigMatchedScannedImage"]) + self.assertEqual(item["vulnerabilityScan"]["artifactRef"]["imageId"], image["id"]) + self.assertEqual( + item["vulnerabilityScan"]["rawReportSha256"], + scan["rawVulnerabilityReportSha256"], + ) + + self.assertEqual(download["artifactRef"]["imageId"], image["id"]) + self.assertEqual(download["artifactRef"]["tag"], image["tag"]) + self.assertEqual(download["localSha256"], build["archive"]["sha256"]) + self.assertEqual(download["remoteSha256"], build["archive"]["sha256"]) + self.assertEqual(scan["artifactRef"]["imageId"], image["id"]) + self.assertEqual(scan["artifactRef"]["archiveSha256"], build["archive"]["sha256"]) + self.assertEqual(scan["reportIdentity"]["ImageID"], image["id"]) + self.assertEqual(scan["ignoreUnfixed"], False) + self.assertEqual(scan["findingFilters"], []) + self.assertEqual(scan["tlsVerification"], True) + self.assertEqual(scan["packageCount"], summary["packageCount"]) + self.assertGreater(summary["packageCount"], 0) + inventory = summary["strictVersionInventory"] + self.assertTrue(inventory["allNamesAndVersionsPresent"]) + self.assertEqual(inventory["packageEntriesChecked"], summary["packageCount"]) + self.assertEqual(secret["ImageID"], image["id"]) + self.assertEqual(secret["FindingCount"], 0) + self.assertFalse(secret["SensitiveFindingDetailsPersisted"]) + self.assertEqual(scanned["humanRiskAcceptance"], "pending") + self.assertFalse(scanned["productionDeployment"]) + + def test_bundle_hash_manifest_covers_retained_recipe_and_receipt_files(self) -> None: + evidence = self.scan_evidence["recipeProvenance"] + self.assertEqual( + evidence["manifestPath"], + "evidence/WP-400/database-image-candidate-20261003/candidate-provenance.json", + ) + self.assertEqual(evidence["manifestSha256"], sha256(BUNDLE / "candidate-provenance.json")) + self.assertEqual( + evidence["fileHashesPath"], + "evidence/WP-400/database-image-candidate-20261003/FILE_HASHES.json", + ) + hashes_path = BUNDLE / "FILE_HASHES.json" + self.assertEqual(evidence["fileHashesSha256"], sha256(hashes_path)) + hashes = read_json(hashes_path) + actual_files = bundle_files(BUNDLE) + self.assertEqual(set(hashes["files"]), actual_files) + for relative, expected in hashes["files"].items(): + with self.subTest(path=relative): + self.assertEqual(sha256(BUNDLE / relative), expected) + + def test_ci_compilation_preserves_strict_bundle_inventory(self) -> None: + with tempfile.TemporaryDirectory(prefix="wp400-compiled-provenance-") as temporary: + bundle = Path(temporary) / "evidence" + recipe = bundle / "recipes" / "test_recipe_contract.py" + recipe.parent.mkdir(parents=True) + recipe.write_text("assert True\n", encoding="utf-8") + source_hash = sha256(recipe) + cache_root = Path(temporary) / "managed-bytecode" + subprocess.run( # noqa: S603 + [sys.executable, "-m", "py_compile", str(recipe)], + env=dict(os.environ, PYTHONPYCACHEPREFIX=str(cache_root)), + check=True, + capture_output=True, + ) + self.assertTrue(any(cache_root.rglob("*.pyc"))) + self.assertEqual(sha256(recipe), source_hash) + self.assertEqual(bundle_files(bundle), {"recipes/test_recipe_contract.py"}) + + unexpected = recipe.parent / "__pycache__" / "unexpected.json" + unexpected.parent.mkdir() + unexpected.write_text("{}\n", encoding="utf-8") + matching_cache = unexpected.with_name( + f"test_recipe_contract.{sys.implementation.cache_tag}.pyc" + ) + matching_cache.write_bytes(b"unexpected cache bytes") + self.assertEqual( + bundle_files(bundle), + { + "recipes/test_recipe_contract.py", + "recipes/__pycache__/unexpected.json", + f"recipes/__pycache__/{matching_cache.name}", + }, + ) + workflow = (ROOT / ".gitea/workflows/managed-validation.yml").read_text(encoding="utf-8") + self.assertIn( + 'PYTHONPYCACHEPREFIX="${RUNNER_TEMP}/managed-bytecode" ' + "xargs -0 -r python -m py_compile", + workflow, + ) + + def test_git_objects_archives_and_autocrlf_checkouts_preserve_bundle_hashes(self) -> None: + file_hashes = read_json(BUNDLE / "FILE_HASHES.json")["files"] + expected_hashes = dict(file_hashes) + expected_hashes["FILE_HASHES.json"] = self.scan_evidence["recipeProvenance"][ + "fileHashesSha256" + ] + bundle_path = Path("evidence/WP-400/database-image-candidate-20261003") + + with tempfile.TemporaryDirectory(prefix="wp400-git-byte-provenance-") as temporary: + temporary_root = Path(temporary) + source = temporary_root / "source" + source.mkdir() + shutil.copyfile(ROOT / ".gitattributes", source / ".gitattributes") + for relative in expected_hashes: + destination = source / bundle_path / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(BUNDLE / relative, destination) + + run_git(source, "init", "--quiet") + run_git(source, "config", "user.name", "WP-400 provenance test") + run_git(source, "config", "user.email", "wp400-provenance@example.invalid") + run_git(source, "config", "core.autocrlf", "true") + run_git(source, "config", "core.safecrlf", "false") + hooks = source / ".test-hooks" + hooks.mkdir() + run_git(source, "config", "core.hooksPath", ".test-hooks") + run_git(source, "add", "--all") + run_git(source, "commit", "--quiet", "-m", "byte-preserving provenance fixture") + + archive = run_git(source, "archive", "--format=tar", "HEAD", str(bundle_path)) + archive_prefix = bundle_path.as_posix() + "/" + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as archived: + archived_files = { + member.name.removeprefix(archive_prefix): member + for member in archived.getmembers() + if member.isfile() and member.name.startswith(archive_prefix) + } + self.assertEqual(set(archived_files), set(expected_hashes)) + for relative, expected in expected_hashes.items(): + member_file = archived.extractfile(archived_files[relative]) + self.assertIsNotNone(member_file) + self.assertEqual( + hashlib.sha256(member_file.read()).hexdigest(), expected, relative + ) + + for autocrlf in ("false", "true"): + with self.subTest(core_autocrlf=autocrlf): + checkout = temporary_root / f"checkout-{autocrlf}" + run_git( + temporary_root, + "clone", + "--quiet", + "--no-checkout", + str(source), + str(checkout), + ) + run_git(checkout, "config", "core.autocrlf", autocrlf) + run_git(checkout, "config", "core.safecrlf", "false") + run_git(checkout, "checkout", "--force", "HEAD") + for relative, expected in expected_hashes.items(): + self.assertEqual(sha256(checkout / bundle_path / relative), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mcp_hub_ingestion_publisher.py b/tests/test_mcp_hub_ingestion_publisher.py new file mode 100644 index 0000000..0dd0e5e --- /dev/null +++ b/tests/test_mcp_hub_ingestion_publisher.py @@ -0,0 +1,812 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any, cast + +ROOT = Path(__file__).resolve().parents[1] +PUBLISHER_PATH = ROOT / "scripts" / "publish_mcp_hub_ingestion.py" +FIXED_REPOSITORY = "jens/itworx_mcp_hub" +IMAGE_REPOSITORY = "gitea.example.com/jens/itworx-mcp-hub/connector-ingestion" +SOURCE_URI = "https://gitea.example.com/jens/itworx-mcp-hub" +REVISION = "a" * 40 +TREE = "b" * 40 +MANIFEST_DIGEST = "sha256:" + "c" * 64 +CONFIG_DIGEST = "sha256:" + "d" * 64 +BUILT_AT = "2026-10-03T01:02:03.004Z" +EPOCH = "1790980806" +ENGINE_ID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +CONTAINER_ID = "f" * 64 +_OMITTED = object() + + +def _load_publisher() -> Any: + spec = importlib.util.spec_from_file_location("test_publish_mcp_hub_ingestion", PUBLISHER_PATH) + if spec is None or spec.loader is None: + raise AssertionError("fixed ingestion publisher script could not be loaded") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class PublisherHarness: + def __init__(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.volume_root = self.root / "volume" + self.releases_root = self.volume_root / "releases" + self.source = self.releases_root / REVISION + self.publisher = _load_publisher() + self.publisher.FIXED_VOLUME_ROOT = self.volume_root + self.publisher.FIXED_RELEASES_ROOT = self.releases_root + self.state_root = self.root / "state" + self.publisher.FIXED_STATE_ROOT = self.state_root + dockerfile = self.source / "infrastructure" / "images" / "python-service" / "Dockerfile" + dockerfile.parent.mkdir(parents=True) + dockerfile.write_text("FROM python:3.13-slim\n", encoding="utf-8") + self.script = self.source / "scripts" / "publish_mcp_hub_ingestion.py" + self.script.parent.mkdir(parents=True) + self.script.write_text("# frozen publisher\n", encoding="utf-8") + self.state_root.mkdir(mode=0o700) + self.freeze_build = { + "id": REVISION[:12], + "repository": FIXED_REPOSITORY, + "revision": REVISION, + "tree": TREE, + "treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(), + "builtAt": BUILT_AT, + } + self.frozen = self.publisher.FrozenSource( + repository=FIXED_REPOSITORY, + revision=REVISION, + build=dict(self.freeze_build), + source_date_epoch=EPOCH, + freeze_sha256="e" * 64, + controller_source_path=f"/controller-hub-releases/releases/{REVISION}", + ) + self.command_calls: list[list[str]] = [] + self.builder_present = False + self.builder_bootstrapped = False + self.builder_container_present = False + self.bad_builder_image = False + self.bad_builder_container_image = False + self.bad_builder_actual_image = False + self.builder_runtime_overrides: dict[str, Any] = {} + self.bad_labels = False + self.bad_manifest = False + self.bad_config = False + self.redaction_sentinel = "sentinel-value-must-not-escape" + if os.name != "nt": + for path in sorted( + self.source.rglob("*"), key=lambda item: len(item.parts), reverse=True + ): + if path.is_file(): + path.chmod(0o444) + elif path.is_dir(): + path.chmod(0o555) + self.source.chmod(0o555) + self.volume_root.chmod(0o755) + self.releases_root.chmod(0o755) + + def close(self) -> None: + if os.name != "nt": + for path in sorted( + self.root.rglob("*"), key=lambda item: len(item.parts), reverse=True + ): + try: + path.chmod(0o700 if path.is_dir() else 0o600) + except OSError: + pass + self.temporary.cleanup() + + def docker_runner( + self, command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + self.command_calls.append(list(command)) + operation = command[3:] + if operation[:2] == ["buildx", "ls"]: + rows = ["default|docker", "default|default"] + if self.builder_present: + rows += [ + "itworx-mcp-hub-ingestion|docker-container", + "itworx-mcp-hub-ingestion0|unix:///var/run/docker.sock", + ] + return subprocess.CompletedProcess(command, 0, "\n".join(rows), "") + if operation[:2] == ["buildx", "create"]: + self.assert_no_global_builder_switch(command) + self.builder_present = True + return subprocess.CompletedProcess(command, 0, "itworx-mcp-hub-ingestion", "") + if operation[:2] == ["container", "ls"]: + names = ( + ["buildx_buildkit_itworx-mcp-hub-ingestion0"] + if self.builder_container_present + else [] + ) + return subprocess.CompletedProcess(command, 0, "\n".join(names), "") + if operation[:2] == ["buildx", "inspect"]: + if "--bootstrap" in operation and timeout < 120: + raise AssertionError("pinned builder bootstrap needs a bounded pull timeout") + text = ( + "Name: itworx-mcp-hub-ingestion\n" + "Driver: docker-container\n\n" + "Nodes:\n" + "Name: itworx-mcp-hub-ingestion0\n" + "Endpoint: unix:///var/run/docker.sock\n" + f'Driver Options: image="{self._builder_image_option()}"\n' + ) + if "--bootstrap" in operation: + self.builder_bootstrapped = True + self.builder_container_present = True + if self.builder_bootstrapped: + text += "Status: running\nBuildKit version: v0.33.0\n" + else: + text += "Status: inactive\n" + return subprocess.CompletedProcess(command, 0, text, "") + if operation[:2] == ["inspect", "--format"]: + target = command[-1] + if target == "buildx_buildkit_itworx-mcp-hub-ingestion0": + return self._builder_container_inspection(command) + if ( + operation[:3] == ["image", "inspect", "--format"] + and command[-1] == self.publisher.FIXED_BUILDKIT_IMAGE + ): + image_record = { + "Id": CONFIG_DIGEST, + "Config.Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "BUILDKIT_SETUP_CGROUPV2_ROOT=1", + ], + "Config.Entrypoint": self.publisher.FIXED_BUILDKIT_ENTRYPOINT, + "Config.User": "", + "Config.WorkingDir": "", + "Config.Labels": {"org.opencontainers.image.title": "BuildKit"}, + } + raw = "\n".join( + json.dumps(image_record[field], separators=(",", ":")) + for field in self.publisher.PINNED_BUILDKIT_IMAGE_FIELDS + ) + return subprocess.CompletedProcess(command, 0, raw, "") + if operation[:2] == ["buildx", "build"]: + self.builder_bootstrapped = True + metadata_path = Path(command[command.index("--metadata-file") + 1]) + metadata = { + "containerimage.digest": MANIFEST_DIGEST, + "containerimage.config.digest": "sha256:" + "9" * 64 + if self.bad_config + else CONFIG_DIGEST, + } + metadata_path.write_text(json.dumps(metadata), encoding="utf-8") + return subprocess.CompletedProcess(command, 0, "", "") + if operation[:4] == ["buildx", "imagetools", "inspect", "--format"]: + descriptor = { + "digest": "sha256:" + "e" * 64 if self.bad_manifest else MANIFEST_DIGEST, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "size": 2458, + } + return subprocess.CompletedProcess(command, 0, json.dumps(descriptor), "") + if operation[:2] == ["pull", "--quiet"]: + return subprocess.CompletedProcess(command, 0, "", "") + if operation[:2] == ["image", "inspect"]: + values = [ + CONFIG_DIGEST, + "10001:10001", + SOURCE_URI, + "f" * 40 if self.bad_labels else REVISION, + TREE, + REVISION[:12], + BUILT_AT, + EPOCH, + ] + return subprocess.CompletedProcess(command, 0, "|".join(values), "") + if self.redaction_sentinel in operation: + raise AssertionError("redaction sentinel must never be an argument") + raise AssertionError(f"unexpected fake Docker command: {command!r}") + + def _builder_container_inspection(self, command: list[str]) -> subprocess.CompletedProcess[str]: + record: dict[str, Any] = { + "Image": "sha256:" + "0" * 64 if self.bad_builder_actual_image else CONFIG_DIGEST, + "Config.Image": "moby/buildkit:unexpected" + if self.bad_builder_container_image + else self.publisher.FIXED_BUILDKIT_IMAGE, + "Config.Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "BUILDKIT_SETUP_CGROUPV2_ROOT=1", + ], + "Config.Entrypoint": self.publisher.FIXED_BUILDKIT_ENTRYPOINT, + "Config.Cmd": self.publisher.FIXED_BUILDKIT_COMMAND, + "Config.User": "", + "Config.WorkingDir": "", + "Config.Labels": {"org.opencontainers.image.title": "BuildKit"}, + "State.Running": self.builder_bootstrapped, + "HostConfig.Privileged": True, + "HostConfig.Init": True, + "HostConfig.RestartPolicy.Name": "unless-stopped", + "HostConfig.RestartPolicy.MaximumRetryCount": 0, + "HostConfig.NetworkMode": "default", + "HostConfig.UsernsMode": "", + "HostConfig.CgroupParent": "/docker/buildx", + "HostConfig.CgroupnsMode": "private", + "HostConfig.Mounts": self._host_mount(), + "Mounts": [ + { + "Type": "volume", + "Name": self.publisher.FIXED_BUILDKIT_STATE_VOLUME, + "Source": ( + f"{self.publisher.FIXED_DOCKER_ROOT}/volumes/" + f"{self.publisher.FIXED_BUILDKIT_STATE_VOLUME}/_data" + ), + "Destination": self.publisher.FIXED_BUILDKIT_STATE_PATH, + "Driver": "local", + "Mode": "", + "RW": True, + "Propagation": "", + } + ], + "HostConfig.Binds": None, + "HostConfig.Devices": None, + "HostConfig.DeviceRequests": None, + "HostConfig.CapAdd": None, + "HostConfig.CapDrop": None, + "HostConfig.SecurityOpt": None, + "HostConfig.PortBindings": None, + "HostConfig.PublishAllPorts": False, + "HostConfig.ReadonlyRootfs": False, + "HostConfig.PidMode": "", + "HostConfig.IpcMode": "private", + "HostConfig.UTSMode": "", + "HostConfig.AutoRemove": False, + "HostConfig.Runtime": "runc", + "HostConfig.Memory": 0, + "HostConfig.MemorySwap": 0, + "HostConfig.CpuShares": 0, + "HostConfig.CpuQuota": 0, + "HostConfig.CpuPeriod": 0, + "HostConfig.CpusetCpus": "", + "HostConfig.CpusetMems": "", + "HostConfig.OomKillDisable": None, + } + record.update(self.builder_runtime_overrides) + raw = "\n".join( + json.dumps(record[field], separators=(",", ":")) + for field in self.publisher.BUILDKIT_CONTAINER_FIELDS + ) + return subprocess.CompletedProcess(command, 0, raw, "") + + def _host_mount(self, read_only: Any = _OMITTED) -> list[dict[str, Any]]: + mount: dict[str, Any] = { + "Type": "volume", + "Source": self.publisher.FIXED_BUILDKIT_STATE_VOLUME, + "Target": self.publisher.FIXED_BUILDKIT_STATE_PATH, + } + if read_only is not _OMITTED: + mount["ReadOnly"] = read_only + return [mount] + + def assert_no_global_builder_switch(self, command: list[str]) -> None: + if "--use" in command: + raise AssertionError("publisher must not switch the global Buildx builder") + if "--name" not in command or "itworx-mcp-hub-ingestion" not in command: + raise AssertionError("publisher must use only its fixed dedicated builder") + if f"image={self.publisher.FIXED_BUILDKIT_IMAGE}" not in command: + raise AssertionError("publisher must pin the dedicated BuildKit image") + + def _builder_image_option(self) -> str: + return "wrong" if self.bad_builder_image else self.publisher.FIXED_BUILDKIT_IMAGE + + def publisher_module(self) -> Any: + return self.publisher + + +class FixedIngestionPublisherTests(unittest.TestCase): + def setUp(self) -> None: + self.harness = PublisherHarness() + self.addCleanup(self.harness.close) + self.publisher = self.harness.publisher_module() + + def prepare_builder(self) -> None: + self.publisher._ensure_dedicated_builder(self.harness.docker_runner, self.harness.frozen) + + def publish(self) -> dict[str, str]: + self.publisher._verify_local_release_root(self.harness.source, REVISION) + self.publisher._ensure_dedicated_builder(self.harness.docker_runner, self.harness.frozen) + return cast( + dict[str, str], + self.publisher._publish_frozen_revision( + REVISION, + frozen=self.harness.frozen, + docker_runner=self.harness.docker_runner, + source_root=self.harness.source, + state_root=self.harness.state_root, + script_path=self.harness.script, + ), + ) + + def receipt_path(self) -> Path: + return ( + self.harness.state_root + / "mcp-hub-ingestion-publications" + / REVISION + / "publication.json" + ) + + def test_builds_fixed_service_with_frozen_labels_pushes_by_digest_and_attests_readback( + self, + ) -> None: + original = { + path.relative_to(self.harness.source).as_posix(): path.read_bytes() + for path in self.harness.source.rglob("*") + if path.is_file() + } + result = self.publish() + + build = next( + command for command in self.harness.command_calls if command[3:5] == ["buildx", "build"] + ) + self.assertEqual(build[3:5], ["buildx", "build"]) + self.assertIn("--builder", build) + self.assertIn("itworx-mcp-hub-ingestion", build) + self.assertEqual(build[build.index("--builder") + 1], self.publisher.FIXED_BUILDER_NAME) + self.assertIn("--platform", build) + self.assertEqual(build[build.index("--platform") + 1], "linux/amd64") + self.assertIn("--provenance=false", build) + output = build[build.index("--output") + 1] + self.assertIn("push-by-digest=true", output) + self.assertNotIn("--tag", build) + self.assertEqual( + {build[index + 1] for index, value in enumerate(build[:-1]) if value == "--build-arg"}, + { + "SERVICE_NAME=connector-ingestion", + f"GIT_REVISION={REVISION}", + f"GIT_SOURCE_TREE={TREE}", + f"IMAGE_VERSION={REVISION[:12]}", + f"SOURCE_DATE_EPOCH={EPOCH}", + f"BUILD_DATE={BUILT_AT}", + }, + ) + self.assertEqual(result["imageRef"], f"{IMAGE_REPOSITORY}@{MANIFEST_DIGEST}") + self.assertEqual(result["imageId"], CONFIG_DIGEST) + receipt = json.loads(self.receipt_path().read_text(encoding="utf-8")) + payload = receipt["receipt"] + self.assertEqual(payload["imageRef"], result["imageRef"]) + self.assertEqual(payload["sourceFreezeSha256"], self.harness.frozen.freeze_sha256) + self.assertEqual( + receipt["receiptSha256"], + hashlib.sha256( + (json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode() + ).hexdigest(), + ) + self.assertNotIn( + self.harness.redaction_sentinel, + self.receipt_path().read_text(encoding="utf-8"), + ) + self.assertFalse( + any(command[3:5] == ["login", ""] for command in self.harness.command_calls) + ) + after = { + path.relative_to(self.harness.source).as_posix(): path.read_bytes() + for path in self.harness.source.rglob("*") + if path.is_file() + } + self.assertEqual(original, after) + if os.name != "nt": + info = self.receipt_path().stat() + self.assertEqual(info.st_uid, 0) + self.assertEqual(info.st_mode & 0o777, 0o600) + + def test_official_attestation_accepts_only_fixed_repository_and_revision(self) -> None: + document = { + "schema": self.publisher.ATTESTATION_SCHEMA, + "repository": FIXED_REPOSITORY, + "revision": REVISION, + "build": dict(self.harness.freeze_build), + "sourceDateEpoch": EPOCH, + "freezeSha256": "e" * 64, + "sourcePath": f"/controller-hub-releases/releases/{REVISION}", + } + frozen = self.publisher._validate_frozen_attestation(document, REVISION) + self.assertEqual(frozen.build, self.harness.freeze_build) + document["repository"] = "attacker/project" + with self.assertRaises(self.publisher.PublishError): + self.publisher._validate_frozen_attestation(document, REVISION) + + def test_official_attestation_command_is_fixed_read_only_controller_probe(self) -> None: + expected = { + "schema": self.publisher.ATTESTATION_SCHEMA, + "repository": FIXED_REPOSITORY, + "revision": REVISION, + "build": dict(self.harness.freeze_build), + "sourceDateEpoch": EPOCH, + "freezeSha256": "e" * 64, + "sourcePath": f"/controller-hub-releases/releases/{REVISION}", + } + + def runner( + command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + self.assertEqual(command[0:4], ["/usr/bin/docker", "--context", "default", "exec"]) + self.assertIn(self.publisher.FIXED_CONTAINER, command) + self.assertIn("python3", command) + self.assertIn("-c", command) + self.assertNotIn("bash", command) + return subprocess.CompletedProcess(command, 0, json.dumps(expected), "") + + result = self.publisher._attest_official_freeze(REVISION, runner) + self.assertEqual(result.revision, REVISION) + + def test_engine_preflight_requires_same_fixed_host_and_controller_socket_engine(self) -> None: + def runner( + command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + if command[1:3] == ["context", "show"]: + return subprocess.CompletedProcess(command, 0, "default", "") + if command[1:3] == ["context", "inspect"]: + return subprocess.CompletedProcess(command, 0, "unix:///var/run/docker.sock", "") + operation = command[3:] + if operation[:1] == ["info"]: + return subprocess.CompletedProcess( + command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", "" + ) + if operation[:2] == ["exec", "--user"]: + return subprocess.CompletedProcess( + command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", "" + ) + if operation[:2] == ["inspect", "--format"]: + return subprocess.CompletedProcess(command, 0, f"{CONTAINER_ID}|true", "") + if operation[:2] == ["buildx", "version"]: + return subprocess.CompletedProcess( + command, 0, "github.com/docker/buildx v0.20.1 abc123", "" + ) + raise AssertionError(f"unexpected preflight command: {command!r}") + + engine, container = self.publisher._preflight_engine(runner) + self.assertEqual(engine, ENGINE_ID) + self.assertEqual(container, CONTAINER_ID) + + def test_engine_preflight_rejects_unreviewed_buildx_before_publication(self) -> None: + def runner( + command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + if command[1:3] == ["context", "show"]: + return subprocess.CompletedProcess(command, 0, "default", "") + if command[1:3] == ["context", "inspect"]: + return subprocess.CompletedProcess(command, 0, "unix:///var/run/docker.sock", "") + operation = command[3:] + if operation[:1] == ["info"]: + return subprocess.CompletedProcess( + command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", "" + ) + if operation[:2] == ["exec", "--user"]: + return subprocess.CompletedProcess( + command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", "" + ) + if operation[:2] == ["inspect", "--format"]: + return subprocess.CompletedProcess(command, 0, f"{CONTAINER_ID}|true", "") + if operation[:2] == ["buildx", "version"]: + return subprocess.CompletedProcess( + command, 0, "github.com/docker/buildx v0.21.3", "" + ) + raise AssertionError(f"unexpected preflight command: {command!r}") + + with self.assertRaises(self.publisher.PublishError): + self.publisher._preflight_engine(runner) + + def test_engine_preflight_rejects_controller_socket_mismatch_before_build(self) -> None: + def runner( + command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + if command[1:3] == ["context", "show"]: + return subprocess.CompletedProcess(command, 0, "default", "") + if command[1:3] == ["context", "inspect"]: + return subprocess.CompletedProcess(command, 0, "unix:///var/run/docker.sock", "") + operation = command[3:] + if operation[:1] == ["info"]: + return subprocess.CompletedProcess( + command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", "" + ) + if operation[:2] == ["exec", "--user"]: + return subprocess.CompletedProcess( + command, 0, f"{'0' * 64}|/var/lib/docker|27.5.1|Tower", "" + ) + raise AssertionError("preflight proceeded after an engine mismatch") + + with self.assertRaises(self.publisher.PublishError): + self.publisher._preflight_engine(runner) + + def test_engine_identity_requires_fixed_tower_hostname(self) -> None: + with self.assertRaises(self.publisher.PublishError): + self.publisher._parse_engine_identity( + f"{ENGINE_ID}|/var/lib/docker|27.5.1|another-host", source="host" + ) + + def test_environment_overrides_cannot_redirect_engine_builder_or_auth_config(self) -> None: + for name in ( + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "BUILDX_BUILDER", + "BUILDX_CONFIG", + "DOCKER_AUTH_CONFIG", + ): + with self.subTest(name=name), self.assertRaises(self.publisher.PublishError): + self.publisher._reject_docker_environment({"HOME": "/root", name: "sentinel"}) + + def test_dedicated_builder_is_created_only_after_freeze_and_does_not_switch_default( + self, + ) -> None: + self.publisher._ensure_dedicated_builder(self.harness.docker_runner, self.harness.frozen) + create = next( + command + for command in self.harness.command_calls + if command[3:5] == ["buildx", "create"] + ) + self.harness.assert_no_global_builder_switch(create) + self.assertIn("docker-container", create) + self.assertIn("default", create) + self.assertTrue(self.harness.builder_present) + + def test_mismatched_preexisting_dedicated_builder_is_denied(self) -> None: + self.harness.builder_present = True + + def wrong_builder( + command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + self.harness.command_calls.append(list(command)) + if command[3:5] == ["buildx", "ls"]: + return subprocess.CompletedProcess( + command, + 0, + "default|docker\ndefault|default\nitworx-mcp-hub-ingestion|docker\nitworx-mcp-hub-ingestion0|default", + "", + ) + if command[3:5] == ["container", "ls"]: + return subprocess.CompletedProcess(command, 0, "", "") + raise AssertionError("mismatched builder should be rejected from inventory") + + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder(wrong_builder, self.harness.frozen) + self.assertFalse( + any(command[3:5] == ["buildx", "build"] for command in self.harness.command_calls) + ) + + def test_preexisting_builder_container_image_is_checked_before_bootstrap(self) -> None: + self.harness.builder_present = True + self.harness.builder_container_present = True + self.harness.bad_builder_container_image = True + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertFalse( + any( + command[3:6] == ["buildx", "inspect", "--bootstrap"] + or command[3:5] == ["buildx", "build"] + for command in self.harness.command_calls + ) + ) + + def test_preexisting_builder_container_actual_image_id_is_checked_before_bootstrap( + self, + ) -> None: + self.harness.builder_present = True + self.harness.builder_container_present = True + self.harness.bad_builder_actual_image = True + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertFalse( + any( + command[3:6] == ["buildx", "inspect", "--bootstrap"] + or command[3:5] == ["buildx", "build"] + for command in self.harness.command_calls + ) + ) + + def test_preexisting_builder_container_unsafe_runtime_is_denied_before_bootstrap(self) -> None: + unsafe_configurations: tuple[dict[str, Any], ...] = ( + {"HostConfig.Privileged": False}, + {"HostConfig.SecurityOpt": ["seccomp=unconfined"]}, + {"HostConfig.NetworkMode": "host"}, + {"HostConfig.IpcMode": "host"}, + {"HostConfig.Runtime": "foreign"}, + {"HostConfig.CgroupnsMode": "host"}, + {"HostConfig.Mounts": []}, + {"Config.Env": ["TOKEN=sentinel-value-must-not-escape"]}, + ) + for override in unsafe_configurations: + with self.subTest(override=tuple(override)): + self.harness.command_calls.clear() + self.harness.builder_present = True + self.harness.builder_container_present = True + self.harness.builder_bootstrapped = False + self.harness.builder_runtime_overrides = override + with self.assertRaises(self.publisher.PublishError) as raised: + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertNotIn("sentinel-value-must-not-escape", str(raised.exception)) + self.assertFalse( + any( + command[3:6] == ["buildx", "inspect", "--bootstrap"] + or command[3:5] == ["buildx", "build"] + for command in self.harness.command_calls + ) + ) + + def test_writable_builder_state_mount_accepts_only_omitted_or_boolean_false(self) -> None: + self.harness.builder_present = True + self.harness.builder_container_present = True + self.harness.builder_bootstrapped = True + self.publisher._verify_dedicated_builder_container( + self.harness.docker_runner, require_running=True + ) + + self.harness.builder_runtime_overrides = { + "HostConfig.Mounts": self.harness._host_mount(False) + } + self.publisher._verify_dedicated_builder_container( + self.harness.docker_runner, require_running=True + ) + + for unsafe_value in (True, None, 0, "false"): + with self.subTest(read_only=unsafe_value): + self.harness.command_calls.clear() + self.harness.builder_runtime_overrides = { + "HostConfig.Mounts": self.harness._host_mount(unsafe_value) + } + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertFalse( + any( + command[3:6] == ["buildx", "inspect", "--bootstrap"] + or command[3:5] == ["buildx", "build"] + for command in self.harness.command_calls + ) + ) + + def test_default_buildkit_config_file_blocks_builder_before_docker_commands(self) -> None: + config = self.harness.root / "buildkitd.default.toml" + config.write_text("[worker.oci]\n", encoding="utf-8") + self.publisher.FIXED_BUILDKIT_CONFIG_PATH = config + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertEqual(self.harness.command_calls, []) + + @unittest.skipIf(os.name == "nt", "symlink behavior is POSIX-specific") + def test_default_buildkit_config_symlink_blocks_builder_before_docker_commands(self) -> None: + target = self.harness.root / "outside.toml" + target.write_text("[worker.oci]\n", encoding="utf-8") + link = self.harness.root / "buildkitd.default.toml" + link.symlink_to(target) + self.publisher.FIXED_BUILDKIT_CONFIG_PATH = link + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertEqual(self.harness.command_calls, []) + + def test_orphan_reserved_container_blocks_create_bootstrap_and_build(self) -> None: + self.harness.builder_container_present = True + self.harness.bad_builder_container_image = True + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder( + self.harness.docker_runner, self.harness.frozen + ) + self.assertFalse( + any( + command[3:5] == ["buildx", "create"] + or command[3:6] == ["buildx", "inspect", "--bootstrap"] + or command[3:5] == ["buildx", "build"] + for command in self.harness.command_calls + ) + ) + + def test_builder_creation_requires_a_validated_frozen_source(self) -> None: + invalid = self.harness.frozen.__class__( + repository="attacker/project", + revision=REVISION, + build=dict(self.harness.freeze_build), + source_date_epoch=EPOCH, + freeze_sha256="e" * 64, + controller_source_path=f"/controller-hub-releases/releases/{REVISION}", + ) + with self.assertRaises(self.publisher.PublishError): + self.publisher._ensure_dedicated_builder(self.harness.docker_runner, invalid) + self.assertEqual(self.harness.command_calls, []) + + def test_rejects_stale_or_writable_staged_source_before_build(self) -> None: + dockerfile = ( + self.harness.source / "infrastructure" / "images" / "python-service" / "Dockerfile" + ) + if os.name != "nt": + dockerfile.chmod(0o644) + with self.assertRaises(self.publisher.PublishError): + self.publish() + else: + with self.assertRaises(self.publisher.PublishError): + self.publisher._verify_local_release_root(self.harness.source.parent, REVISION) + self.assertFalse( + any(command[3:5] == ["buildx", "build"] for command in self.harness.command_calls) + ) + self.assertFalse(self.receipt_path().exists()) + + def test_rejects_bad_build_metadata_and_registry_manifest_digest(self) -> None: + self.harness.bad_manifest = True + with self.assertRaises(self.publisher.PublishError): + self.publish() + self.assertFalse(self.receipt_path().exists()) + + def test_rejects_wrong_config_id_or_provenance_labels_without_receipt(self) -> None: + self.harness.bad_config = True + with self.assertRaises(self.publisher.PublishError): + self.publish() + self.assertFalse(self.receipt_path().exists()) + self.harness.bad_config = False + self.harness.bad_labels = True + with self.assertRaises(self.publisher.PublishError): + self.publish() + self.assertFalse(self.receipt_path().exists()) + + def test_child_error_output_is_scrubbed_and_not_saved(self) -> None: + self.prepare_builder() + + def failed_runner( + command: list[str], *, cwd: Path | None, timeout: int + ) -> subprocess.CompletedProcess[str]: + if command[3:5] == ["buildx", "build"]: + return subprocess.CompletedProcess(command, 1, "", self.harness.redaction_sentinel) + return self.harness.docker_runner(command, cwd=cwd, timeout=timeout) + + with self.assertRaises(self.publisher.PublishError) as raised: + self.publisher._publish_frozen_revision( + REVISION, + frozen=self.harness.frozen, + docker_runner=failed_runner, + source_root=self.harness.source, + state_root=self.harness.state_root, + script_path=self.harness.script, + ) + self.assertNotIn(self.harness.redaction_sentinel, str(raised.exception)) + self.assertFalse(self.receipt_path().exists()) + + def test_preexisting_receipt_is_never_overwritten_or_rebuilt(self) -> None: + self.prepare_builder() + receipt = self.receipt_path() + receipt.parent.mkdir(parents=True, mode=0o700) + receipt.write_text("prior immutable receipt", encoding="utf-8") + before = len(self.harness.command_calls) + with self.assertRaises(self.publisher.PublishError): + self.publisher._publish_frozen_revision( + REVISION, + frozen=self.harness.frozen, + docker_runner=self.harness.docker_runner, + source_root=self.harness.source, + state_root=self.harness.state_root, + script_path=self.harness.script, + ) + self.assertEqual(receipt.read_text(encoding="utf-8"), "prior immutable receipt") + self.assertEqual(len(self.harness.command_calls), before) + + def test_production_slot_check_denies_existing_receipt_without_docker_commands(self) -> None: + receipt = self.receipt_path() + receipt.parent.mkdir(parents=True, mode=0o700) + receipt.write_text("prior immutable receipt", encoding="utf-8") + with self.assertRaises(self.publisher.PublishError): + self.publisher._require_publication_slot(self.harness.state_root, REVISION) + self.assertEqual(self.harness.command_calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mcp_hub_release_artifact.py b/tests/test_mcp_hub_release_artifact.py new file mode 100644 index 0000000..9384d23 --- /dev/null +++ b/tests/test_mcp_hub_release_artifact.py @@ -0,0 +1,364 @@ +"""Behavioral tests for the exact protected-run release artifact payload.""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from scripts import package_mcp_hub_release_artifact as packager + +SERVICES = ( + "connector-appops", + "connector-gitea", + "connector-ingestion", + "connector-knowledge", + "connector-mobilityops", + "connector-projectbrain", + "connector-stitch", + "connector-unraid", + "contextforge", + "control-api", + "dataplane-router", + "docker-read-broker", + "ingestion-clamd", + "postgres", + "redis", + "web", + "worker", +) +REPORT_NAMES = { + "trivy-db-metadata.json", + *( + name + for service in SERVICES + for name in (f"vulnerability-{service}.json", f"secret-{service}.json") + ), +} +REVISION = "a" * 40 +TREE = "b" * 40 +RUN_CREATED_AT = "2026-10-03T10:11:12.123Z" + + +def _json_bytes(value: object) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def _fixture(tmp_path: Path) -> tuple[dict[str, object], dict[str, dict[str, str]], Path, Path]: + image_tar = tmp_path / "inputs" / "images.tar" + report_dir = tmp_path / "inputs" / "reports" + report_dir.mkdir(parents=True) + image_tar.parent.mkdir(parents=True, exist_ok=True) + image_tar.write_bytes(b"fixture-docker-save-archive\n") + images: dict[str, dict[str, str]] = {} + report_time = "2026-10-03T10:12:00Z" + for index, service in enumerate(SERVICES, start=1): + image_id = f"sha256:{index:064x}" + image_ref = f"registry.example/itworx/{service}:protected-main@sha256:{index:064x}" + images[service] = { + "imageRef": image_ref, + "imageId": image_id, + "archiveRef": f"registry.example/itworx/{service}:archive-{REVISION[:12]}", + } + vulnerability = { + "SchemaVersion": 2, + "CreatedAt": report_time, + "Trivy": {"Version": "0.73.0"}, + "Metadata": { + "ImageID": image_id, + "ImageConfig": {"os": "linux", "architecture": "amd64"}, + }, + "Results": [ + { + "Packages": [{"Name": "fixture-package", "Version": "1.0"}], + "Vulnerabilities": [], + } + ], + } + secret = { + "ImageID": image_id, + "CreatedAt": report_time, + "TrivyVersion": "0.73.0", + "FindingCount": 0, + "SensitiveFindingDetailsPersisted": False, + "Results": [{"FindingCount": 0}], + } + (report_dir / f"vulnerability-{service}.json").write_bytes(_json_bytes(vulnerability)) + (report_dir / f"secret-{service}.json").write_bytes(_json_bytes(secret)) + database = { + "UpdatedAt": report_time, + "DownloadedAt": report_time, + "Version": 2, + "NextUpdate": "2026-10-04T10:12:00Z", + } + (report_dir / "trivy-db-metadata.json").write_bytes(_json_bytes(database)) + build = { + "id": REVISION[:12], + "repository": "jens/itworx_mcp_hub", + "revision": REVISION, + "tree": TREE, + "treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(), + "builtAt": RUN_CREATED_AT, + } + context: dict[str, object] = { + "repository": "jens/itworx_mcp_hub", + "revision": REVISION, + "tree": TREE, + "workflowPath": ".gitea/workflows/managed-validation.yml", + "workflowRunId": 2468, + "workflowRunCreatedAt": RUN_CREATED_AT, + "artifactName": f"hub-release-candidate-{REVISION}", + "build": build, + "sourceDateEpoch": "1791022272", + } + return context, images, image_tar, report_dir + + +def _package( + output_dir: Path, + context: dict[str, object], + images: dict[str, dict[str, str]], + image_tar: Path, + report_dir: Path, +) -> dict[str, object]: + return packager.package_candidate_artifact( + output_dir=output_dir, + manifest_context=context, + images=images, + image_tar_path=image_tar, + report_dir=report_dir, + ) + + +@pytest.fixture(autouse=True) +def _bind_fixture_to_test_git(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + packager, + "_git_source_identity", + lambda: (REVISION, TREE, "1791022272"), + ) + + +def test_package_contains_exact_importer_payload_and_binds_original_bytes(tmp_path: Path) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + output = tmp_path / "artifact" + + returned_manifest = _package(output, context, images, image_tar, report_dir) + + expected_names = {"candidate-manifest.json", "images.tar", *REPORT_NAMES} + assert {path.name for path in output.iterdir()} == expected_names + assert all(path.is_file() and not path.is_symlink() for path in output.iterdir()) + manifest_bytes = (output / "candidate-manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + tar_bytes = image_tar.read_bytes() + assert manifest == returned_manifest + assert set(manifest) == { + "schema", + "repository", + "sourceRevision", + "sourceTree", + "workflowPath", + "workflowRunId", + "artifactName", + "build", + "sourceDateEpoch", + "dockerTarBytes", + "dockerTarSha256", + "images", + "reports", + } + assert manifest["schema"] == "itworx.mcp-hub-image-artifact/v1" + assert manifest["repository"] == context["repository"] + assert manifest["sourceRevision"] == context["revision"] + assert manifest["sourceTree"] == context["tree"] + assert manifest["workflowPath"] == context["workflowPath"] + assert manifest["workflowRunId"] == context["workflowRunId"] + assert manifest["artifactName"] == context["artifactName"] + assert manifest["build"] == context["build"] + assert manifest["sourceDateEpoch"] == context["sourceDateEpoch"] + assert manifest["dockerTarBytes"] == len(tar_bytes) + assert manifest["dockerTarSha256"] == hashlib.sha256(tar_bytes).hexdigest() + assert manifest["images"] == images + assert manifest["reports"] == { + name: { + "sha256": hashlib.sha256((report_dir / name).read_bytes()).hexdigest(), + "bytes": (report_dir / name).stat().st_size, + } + for name in sorted(REPORT_NAMES) + } + assert (output / "images.tar").read_bytes() == tar_bytes + for name in REPORT_NAMES: + assert (output / name).read_bytes() == (report_dir / name).read_bytes() + + +def test_canonicalizes_the_exact_workflow_run_timestamp_to_importer_format( + tmp_path: Path, +) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + context["workflowRunCreatedAt"] = "2026-10-03T10:11:12Z" + build = dict(context["build"]) + build["builtAt"] = "2026-10-03T10:11:12.000Z" + context["build"] = build + + manifest = _package(tmp_path / "artifact", context, images, image_tar, report_dir) + + assert manifest["build"]["builtAt"] == "2026-10-03T10:11:12.000Z" + + +@pytest.mark.parametrize( + ("mutation", "value"), + [ + ("revision", "c" * 40), + ("tree", "d" * 40), + ("run_id", 0), + ("run_time", "2026-10-03T10:11:12Z"), + ("artifact_name", "hub-release-candidate-wrong"), + ("epoch", "01791022272"), + ], +) +def test_rejects_unbound_frozen_context_without_publishing( + tmp_path: Path, mutation: str, value: object +) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + context = copy.deepcopy(context) + if mutation == "revision": + context["revision"] = value + elif mutation == "tree": + context["tree"] = value + elif mutation == "run_id": + context["workflowRunId"] = value + elif mutation == "run_time": + context["workflowRunCreatedAt"] = value + elif mutation == "artifact_name": + context["artifactName"] = value + else: + context["sourceDateEpoch"] = value + output = tmp_path / "artifact" + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert not output.exists() + assert not list(tmp_path.glob(".hub-release-artifact-*")) + + +@pytest.mark.parametrize("report_change", ["missing", "extra", "oversized", "wrong_image"]) +def test_rejects_invalid_fixed_report_inventory_without_partial_output( + tmp_path: Path, report_change: str +) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + if report_change == "missing": + (report_dir / "secret-worker.json").unlink() + elif report_change == "extra": + (report_dir / "unlisted.json").write_text("{}", encoding="utf-8") + elif report_change == "oversized": + (report_dir / "secret-worker.json").write_bytes(b" " * (2 * 1024**2 + 1)) + else: + data = json.loads((report_dir / "vulnerability-worker.json").read_text(encoding="utf-8")) + data["Metadata"]["ImageID"] = "sha256:" + "f" * 64 + (report_dir / "vulnerability-worker.json").write_bytes(_json_bytes(data)) + output = tmp_path / "artifact" + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert not output.exists() + + +def test_rejects_secret_findings_instead_of_redacting_or_rewriting_them(tmp_path: Path) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + secret_path = report_dir / "secret-worker.json" + secret = json.loads(secret_path.read_text(encoding="utf-8")) + secret["FindingCount"] = 1 + secret["SensitiveFindingDetailsPersisted"] = True + secret_path.write_bytes(_json_bytes(secret)) + original = secret_path.read_bytes() + output = tmp_path / "artifact" + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert not output.exists() + assert secret_path.read_bytes() == original + + +def test_rejects_output_collision_without_overwriting_existing_files(tmp_path: Path) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + output = tmp_path / "artifact" + output.mkdir() + marker = output / "keep.txt" + marker.write_text("preserve", encoding="utf-8") + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert marker.read_text(encoding="utf-8") == "preserve" + assert {path.name for path in output.iterdir()} == {"keep.txt"} + + +def test_rejects_output_directory_created_at_atomic_publish_boundary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + output = tmp_path / "artifact" + original_publish = packager._rename_noreplace + + def create_collision_then_publish(source: Path, destination: Path) -> None: + destination.mkdir() + original_publish(source, destination) + + monkeypatch.setattr(packager, "_rename_noreplace", create_collision_then_publish) + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert output.is_dir() + assert list(output.iterdir()) == [] + assert not list(tmp_path.glob(".hub-release-artifact-*")) + + +def test_rejects_wrong_image_inventory_without_partial_output(tmp_path: Path) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + images.pop("worker") + output = tmp_path / "artifact" + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert not output.exists() + + +def test_rejects_source_identity_that_does_not_match_checkout( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + monkeypatch.setattr( + packager, + "_git_source_identity", + lambda: ("f" * 40, TREE, "1791022272"), + ) + output = tmp_path / "artifact" + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert not output.exists() + + +def test_rejects_hardlinked_report_without_publishing(tmp_path: Path) -> None: + context, images, image_tar, report_dir = _fixture(tmp_path) + secret_path = report_dir / "secret-worker.json" + secret_path.unlink() + try: + secret_path.hardlink_to(report_dir / "secret-web.json") + except OSError as exc: + pytest.skip(f"filesystem does not support the hard-link fixture: {exc}") + output = tmp_path / "artifact" + + with pytest.raises(packager.ArtifactRejected): + _package(output, context, images, image_tar, report_dir) + + assert not output.exists() diff --git a/tests/test_mcp_hub_release_images.py b/tests/test_mcp_hub_release_images.py new file mode 100644 index 0000000..e9d7f14 --- /dev/null +++ b/tests/test_mcp_hub_release_images.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import hashlib +import io +import json +import os +import subprocess +import sys +import tarfile +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from scripts import build_mcp_hub_release_images as producer + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "build_mcp_hub_release_images.py" + + +def _write_archive_fixture(path: Path, members: dict[str, bytes | None]) -> None: + with tarfile.open(path, "w:", format=tarfile.USTAR_FORMAT) as archive: + for name, payload in members.items(): + info = tarfile.TarInfo( + name + ("/" if payload is None and not name.endswith("/") else "") + ) + if payload is None: + info.type = tarfile.DIRTYPE + info.mode = 0o755 + archive.addfile(info) + else: + info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + + +def _docker29_oci_fixture( + tmp_path: Path, +) -> tuple[Path, dict[str, bytes | None], dict[str, dict[str, str]], dict[str, str]]: + tmp_path.mkdir(parents=True, exist_ok=True) + revision = "a" * 40 + metadata = { + "revision": revision, + "tree": "b" * 40, + "buildId": revision[:12], + "builtAt": "2026-10-03T12:34:56.000Z", + "sourceDateEpoch": "1791027296", + } + layer_file = io.BytesIO() + with tarfile.open(fileobj=layer_file, mode="w:", format=tarfile.USTAR_FORMAT) as layer_archive: + content = b"synthetic Docker 29 image layer" + item = tarfile.TarInfo("fixture.txt") + item.size = len(content) + layer_archive.addfile(item, io.BytesIO(content)) + layer = layer_file.getvalue() + layer_digest = "sha256:" + hashlib.sha256(layer).hexdigest() + layer_name = "blobs/sha256/" + layer_digest.split(":", 1)[1] + members: dict[str, bytes | None] = {"blobs": None, "blobs/sha256": None, layer_name: layer} + images: dict[str, dict[str, str]] = {} + docker_manifest: list[dict[str, object]] = [] + index_descriptors: list[dict[str, object]] = [] + + for service in producer.ALL_SERVICES: + if service in producer.LOCAL_BUILD_SERVICES: + reference = f"{producer.IMAGE_REPOSITORIES[service]}:{revision}" + elif service == "contextforge": + reference = f"itworx-mcp-hub/contextforge-schema:{revision}" + elif service == "connector-ingestion": + reference = f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-123-attempt-1" + else: + reference = producer.EXTERNAL_IMAGES[service][1] + + labels: dict[str, str] = {} + if service not in producer.EXTERNAL_IMAGES: + for key, label in producer.OCI_LABELS.items(): + labels[key] = ( + "https://gitea.example.com/jens/itworx-mcp-hub" + if key == "org.opencontainers.image.source" + else metadata[label] + ) + config = { + "architecture": "amd64", + "os": "linux", + "config": {"User": "10001:10001", "Labels": labels, "Cmd": [service]}, + "rootfs": {"type": "layers", "diff_ids": [layer_digest]}, + } + config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() + image_id = "sha256:" + hashlib.sha256(config_bytes).hexdigest() + config_name = f"blobs/sha256/{image_id.split(':', 1)[1]}" + members[config_name] = config_bytes + images[service] = {"imageId": image_id, "archiveRef": reference, "imageRef": reference} + docker_manifest.append( + { + "Config": config_name, + "RepoTags": [reference], + "Layers": [layer_name], + "LayerSources": { + layer_digest: { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "size": len(layer), + "digest": layer_digest, + } + }, + } + ) + image_manifest = { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": image_id, + "size": len(config_bytes), + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "digest": layer_digest, + "size": len(layer), + } + ], + } + image_manifest_bytes = json.dumps( + image_manifest, sort_keys=True, separators=(",", ":") + ).encode() + manifest_digest = hashlib.sha256(image_manifest_bytes).hexdigest() + members[f"blobs/sha256/{manifest_digest}"] = image_manifest_bytes + index_descriptors.append( + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:" + manifest_digest, + "size": len(image_manifest_bytes), + "annotations": {"org.opencontainers.image.ref.name": reference}, + "platform": {"os": "linux", "architecture": "amd64"}, + } + ) + + members["oci-layout"] = b'{"imageLayoutVersion":"1.0.0"}' + members["index.json"] = json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": index_descriptors, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + members["manifest.json"] = json.dumps(docker_manifest, separators=(",", ":")).encode() + raw = tmp_path / "docker29-oci-save.tar" + _write_archive_fixture(raw, members) + return raw, members, images, metadata + + +def test_missing_registry_credential_is_rejected_before_building(tmp_path: Path) -> None: + env = { + key: value + for key, value in os.environ.items() + if key not in {"GITEA_TOKEN", "HUB_RELEASE_REGISTRY_TOKEN"} + } + env.update( + { + "GITEA_TOKEN": "metadata-token-test-value", + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "Jens/ITWorx_MCP_Hub", + } + ) + + result = subprocess.run( # noqa: S603 -- executes only this repository's fixed producer script. + [ + sys.executable, + str(SCRIPT), + "--revision", + "a" * 40, + "--run-id", + "123", + "--output", + str(tmp_path / "candidate"), + ], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode != 0 + assert "HUB_RELEASE_REGISTRY_TOKEN" in result.stderr + assert "metadata-token-test-value" not in result.stdout + result.stderr + assert not (tmp_path / "candidate").exists() + + +def test_run_binding_requires_the_one_successful_full_gate() -> None: + revision = "a" * 40 + run_id = 123 + run = { + "id": run_id, + "path": producer.WORKFLOW_PATH, + "event": "push", + "head_branch": "main", + "head_sha": revision, + "repository": {"full_name": producer.DISPLAY_REPOSITORY}, + "head_repository": {"full_name": producer.DISPLAY_REPOSITORY}, + "pull_requests": [], + "status": "in_progress", + "run_attempt": 1, + "created_at": "2026-10-03T12:34:56Z", + } + jobs = { + "jobs": [ + { + "name": "full", + "run_id": run_id, + "run_attempt": 1, + "head_sha": revision, + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ] + } + branch = {"commit": {"id": revision}} + + result = producer._validate_run(run, jobs, branch, revision=revision, run_id=run_id) + + assert result["workflowRunCreatedAt"] == "2026-10-03T12:34:56.000Z" + assert result["runAttempt"] == 1 + with pytest.raises(producer.BuildRejected): + producer._validate_run( + run, {"jobs": jobs["jobs"] * 2}, branch, revision=revision, run_id=run_id + ) + bad_job = {"jobs": [dict(jobs["jobs"][0], conclusion="failure")]} + with pytest.raises(producer.BuildRejected): + producer._validate_run(run, bad_job, branch, revision=revision, run_id=run_id) + + +def test_cli_environment_rejects_docker_endpoint_overrides() -> None: + revision = "b" * 40 + env = { + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": producer.DISPLAY_REPOSITORY, + "GITHUB_SHA": revision, + "GITHUB_RUN_ID": "5", + "DOCKER_HOST": "tcp://attacker.invalid:2375", + } + with pytest.raises(producer.BuildRejected): + producer._validate_cli_environment(revision, 5, env) + + +@pytest.mark.parametrize( + ("buildx", "compose", "accepted"), + [ + ("github.com/docker/buildx v0.37.2", "v5.5.1", True), + ("github.com/docker/buildx 0.37.2", "5.5.1", True), + ("github.com/docker/buildx v0.37.20", "v5.5.1", False), + ("github.com/docker/buildx v0.37.2", "v5.5.10", False), + ], +) +def test_job_daemon_requires_exact_buildx_and_compose_versions( + buildx: str, + compose: str, + accepted: bool, +) -> None: + def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + if command[1:3] == ["info", "--format"]: + output = 'daemon-identity-012345|29.8.2|overlay2|["name=rootless"]' + elif command[1:] == ["buildx", "version"]: + output = buildx + elif command[1:] == ["compose", "version", "--short"]: + output = compose + else: + pytest.fail("version admission invoked an unexpected command") + return subprocess.CompletedProcess(command, 0, output, "") + + if accepted: + assert producer._check_job_daemon({}, Path.cwd(), runner=runner) == "daemon-identity-012345" + else: + with pytest.raises(producer.BuildRejected): + producer._check_job_daemon({}, Path.cwd(), runner=runner) + + +def test_fixed_image_contract_is_17_services_with_12_compose_builds() -> None: + revision = "c" * 40 + services = { + service: {"image": f"{repository}:{revision}"} + for service, repository in producer.IMAGE_REPOSITORIES.items() + } + services["contextforge"] = {"image": f"itworx-mcp-hub/contextforge-schema:{revision}"} + services["connector-ingestion"] = { + "image": f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-pending@sha256:{'0' * 64}" + } + for service, (image, _archive) in producer.EXTERNAL_IMAGES.items(): + services[service] = {"image": image} + model = { + "services": { + service: { + **definition, + **({"build": {"context": "."}} if service in producer.LOCAL_BUILD_SERVICES else {}), + } + for service, definition in services.items() + } + } + + assert producer._validate_local_refs(model, revision) == { + service: services[service]["image"] for service in producer.LOCAL_BUILD_SERVICES + } + model["services"]["connector-ingestion"]["build"] = {"context": "."} + with pytest.raises(producer.BuildRejected): + producer._validate_compose_model(model) + wrong_ref = {"services": {name: dict(row) for name, row in model["services"].items()}} + wrong_ref["services"]["control-api"]["image"] = "attacker.invalid/control:mutable" + with pytest.raises(producer.BuildRejected): + producer._validate_local_refs(wrong_ref, revision) + + +def test_manifest_descriptor_is_strict_and_digest_bound() -> None: + digest = "sha256:" + "d" * 64 + descriptor = { + "digest": digest, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 120, + } + assert producer._parse_manifest_descriptor(json.dumps(descriptor)) == digest + with pytest.raises(producer.BuildRejected): + producer._parse_manifest_descriptor(json.dumps({**descriptor, "extra": "x"})) + with pytest.raises(producer.BuildRejected): + producer._parse_manifest_descriptor(json.dumps({**descriptor, "digest": "sha256:bad"})) + + +def test_docker29_oci_archive_normalizes_index_blobs_to_exact_ustar_importer_shape( + tmp_path: Path, +) -> None: + raw, _members, images, metadata = _docker29_oci_fixture(tmp_path) + normalized = tmp_path / "normalized.tar" + + producer._normalize_docker_archive(raw, normalized, images, metadata=metadata) + + with tarfile.open(normalized, "r:") as archive: + members = archive.getmembers() + names = {member.name.rstrip("/") for member in members} + image_ids = {row["imageId"].split(":", 1)[1] for row in images.values()} + manifest_stream = archive.extractfile("manifest.json") + assert manifest_stream is not None + manifest_rows = json.loads(manifest_stream.read()) + expected_names = {"manifest.json", *(f"{image_id}.json" for image_id in image_ids)} + for row in manifest_rows: + expected_names.update(layer[:-10] for layer in row["Layers"]) + expected_names.update(row["Layers"]) + assert names == expected_names + assert len(manifest_rows) == len(producer.ALL_SERVICES) + assert {row["RepoTags"][0] for row in manifest_rows} == { + row["archiveRef"] for row in images.values() + } + assert all(member.type in {tarfile.REGTYPE, tarfile.DIRTYPE} for member in members) + assert all(member.pax_headers == {} for member in members) + with normalized.open("rb") as source: + assert source.read(512)[257:263] == b"ustar\x00" + if os.name != "nt": + assert normalized.stat().st_mode & 0o777 == 0o600 + + +def test_docker_oci_index_can_canonicalize_external_reference_aliases(tmp_path: Path) -> None: + raw, members, images, metadata = _docker29_oci_fixture(tmp_path) + index = json.loads(members["index.json"]) + postgres_ref = images["postgres"]["archiveRef"] + descriptor = next( + item + for item in index["manifests"] + if item["annotations"]["org.opencontainers.image.ref.name"] == postgres_ref + ) + descriptor["annotations"]["org.opencontainers.image.ref.name"] = ( + "docker.io/library/" + postgres_ref + ) + members["index.json"] = json.dumps(index, sort_keys=True, separators=(",", ":")).encode() + _write_archive_fixture(raw, members) + + normalized = tmp_path / "canonical-alias-normalized.tar" + producer._normalize_docker_archive(raw, normalized, images, metadata=metadata) + + assert normalized.is_file() + + +def test_docker29_oci_normalizer_rejects_unreferenced_blobs_and_config_mismatch( + tmp_path: Path, +) -> None: + raw, members, images, metadata = _docker29_oci_fixture(tmp_path) + orphan_name = "blobs/sha256/" + "f" * 64 + members[orphan_name] = b"unreferenced" + orphan_archive = tmp_path / "orphan.tar" + _write_archive_fixture(orphan_archive, members) + with pytest.raises(producer.BuildRejected): + producer._normalize_docker_archive( + orphan_archive, + tmp_path / "orphan-normalized.tar", + images, + metadata=metadata, + ) + + raw, _members, images, metadata = _docker29_oci_fixture(tmp_path / "mismatch") + images["web"]["imageId"] = "sha256:" + "0" * 64 + with pytest.raises(producer.BuildRejected): + producer._normalize_docker_archive( + raw, + tmp_path / "mismatch-normalized.tar", + images, + metadata=metadata, + ) + + +def test_secret_scan_summary_keeps_finding_details_out_of_persisted_json() -> None: + image_id = "sha256:" + "e" * 64 + raw_secret = "super-secret-value-for-test" # noqa: S105 -- synthetic redaction fixture. + raw = json.dumps({"Metadata": {"ImageID": image_id}, "Results": [{"Secrets": []}]}).encode() + + summary = producer._secret_summary(raw, image_id) + + encoded = json.dumps(summary) + assert summary["FindingCount"] == 0 + assert summary["SensitiveFindingDetailsPersisted"] is False + assert raw_secret not in encoded + finding = json.dumps( + { + "Metadata": {"ImageID": image_id}, + "Results": [{"Secrets": [{"Match": raw_secret, "RuleID": "test"}]}], + } + ).encode() + with pytest.raises(producer.BuildRejected): + producer._secret_summary(finding, image_id) + + +@pytest.mark.skipif( + os.name != "posix", reason="bounded scanner stream requires POSIX process groups" +) +def test_scanner_stdout_bound_does_not_cap_or_capture_cache_files(tmp_path: Path) -> None: + def fake_scanner(command: list[str], **kwargs: object) -> subprocess.Popen[bytes]: + assert "preexec_fn" not in kwargs + assert kwargs.get("stdout") == subprocess.PIPE + code = ( + "from pathlib import Path; import os; " + "Path('cache.bin').write_bytes(b'x' * (3 * 1024 * 1024)); " + "os.write(1, b'{\"complete\":true}')" + ) + return subprocess.Popen( # noqa: S603 -- executes the fixed synthetic scanner fixture. + [sys.executable, "-c", code], + **kwargs, # type: ignore[arg-type] + ) + + output = producer._run_scanner_capture( + ["fake-trivy"], + env=os.environ, + cwd=tmp_path, + timeout=10, + popen=fake_scanner, + maximum=64, + ) + + assert output == b'{"complete":true}' + assert (tmp_path / "cache.bin").stat().st_size == 3 * 1024 * 1024 + + +@pytest.mark.skipif( + os.name != "posix", reason="bounded scanner stream requires POSIX process groups" +) +def test_scanner_stdout_over_limit_is_rejected_without_persisting_bytes(tmp_path: Path) -> None: + def fake_scanner(command: list[str], **kwargs: object) -> subprocess.Popen[bytes]: + code = "import os; os.write(1, b'x' * 65)" + return subprocess.Popen( # noqa: S603 -- executes the fixed synthetic scanner fixture. + [sys.executable, "-c", code], + **kwargs, # type: ignore[arg-type] + ) + + with pytest.raises(producer.BuildRejected): + producer._run_scanner_capture( + ["fake-trivy"], + env=os.environ, + cwd=tmp_path, + timeout=10, + popen=fake_scanner, + maximum=64, + ) + assert list(tmp_path.iterdir()) == [] + + +def test_ingestion_publication_tag_is_bound_to_authenticated_attempt() -> None: + revision = "d" * 40 + first = producer._ingestion_tag(revision, 456, 1) + retry = producer._ingestion_tag(revision, 456, 2) + assert first == f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-456-attempt-1" + assert retry == f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-456-attempt-2" + assert first != retry + with pytest.raises(producer.BuildRejected): + producer._ingestion_tag("../" + revision, 456, 1) + + +def test_registry_absence_probe_accepts_only_the_exact_missing_reference() -> None: + reference = f"{producer.IMAGE_PREFIX}/connector-ingestion:{'e' * 40}-run-7-attempt-1" + + def missing(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + assert reference in command + return subprocess.CompletedProcess(command, 1, "", f"ERROR: {reference}: not found") + + producer._registry_tag_absent(reference, {}, Path.cwd(), runner=missing) + + def denied(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(command, 1, "", "unauthorized: authentication required") + + with pytest.raises(producer.BuildRejected): + producer._registry_tag_absent(reference, {}, Path.cwd(), runner=denied) + + def exists(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(command, 0, "{}", "") + + with pytest.raises(producer.BuildRejected): + producer._registry_tag_absent(reference, {}, Path.cwd(), runner=exists) + + +def test_vulnerability_report_requires_all_packages_and_rejects_fixable_critical() -> None: + image_id = "sha256:" + "f" * 64 + report = { + "SchemaVersion": 2, + "CreatedAt": datetime.now(UTC).isoformat(), + "Metadata": {"ImageID": image_id, "ImageConfig": {"os": "linux", "architecture": "amd64"}}, + "Trivy": {"Version": producer.TRIVY_VERSION}, + "Results": [{"Packages": [{"Name": "pkg", "Version": "1.0"}], "Vulnerabilities": []}], + } + producer._validate_vulnerability(report, image_id) + report["Results"][0]["Vulnerabilities"] = [{"Severity": "CRITICAL", "FixedVersion": "2.0"}] + with pytest.raises(producer.BuildRejected): + producer._validate_vulnerability(report, image_id) + report["Results"][0]["Vulnerabilities"] = [] + report["Results"][0]["Packages"] = [] + with pytest.raises(producer.BuildRejected): + producer._validate_vulnerability(report, image_id) diff --git a/tests/test_release_images_workflow.py b/tests/test_release_images_workflow.py new file mode 100644 index 0000000..5df31bb --- /dev/null +++ b/tests/test_release_images_workflow.py @@ -0,0 +1,63 @@ +"""Keep publication credentials and builds behind the protected release trigger.""" + +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +CHECKOUT = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" +UPLOAD = "ChristopherHX/gitea-upload-artifact@62ac910c5d3dfa85c7cb2df15afe2e342b2407c2" + + +def release_job() -> dict: + workflow = yaml.safe_load( + (ROOT / ".gitea/workflows/managed-validation.yml").read_text(encoding="utf-8") + ) + return workflow["jobs"]["release-images"] + + +def test_release_build_requires_successful_validation_and_exact_main_push() -> None: + job = release_job() + assert job["name"] == "release-images" + assert job["needs"] == ["full"] + assert job["if"] == ( + "${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' " + "&& gitea.repository == 'Jens/ITWorx_MCP_Hub' }}" + ) + assert job["runs-on"] == "itworx-ci-build" + assert 0 < job["timeout-minutes"] <= 150 + assert job["permissions"] == {"contents": "read", "actions": "read"} + assert "env" not in job and "container" not in job and "services" not in job + + +def test_release_credentials_only_reach_the_bounded_producer() -> None: + steps = release_job()["steps"] + assert len(steps) == 3 + assert steps[0]["uses"] == CHECKOUT + assert steps[0]["with"]["persist-credentials"] is False + assert steps[0]["with"]["fetch-depth"] == 1 + assert "env" not in steps[0] and "env" not in steps[2] + assert steps[1]["env"] == { + "GITEA_TOKEN": "${{ secrets.GITEA_TOKEN }}", + "HUB_RELEASE_REGISTRY_TOKEN": "${{ secrets.HUB_RELEASE_REGISTRY_TOKEN }}", + } + command = steps[1]["run"] + assert "set -euo pipefail" in command + assert "set -x" not in command and "docker login" not in command + assert 'python3 scripts/build_mcp_hub_release_images.py --revision "$GITHUB_SHA"' in command + assert '--run-id "$GITHUB_RUN_ID"' in command + assert '--output "${RUNNER_TEMP}/hub-release-candidate-${GITHUB_SHA}"' in command + + +def test_release_artifact_is_pinned_uncompressed_and_cannot_overwrite() -> None: + upload = release_job()["steps"][2] + assert upload["uses"] == UPLOAD + assert "if" not in upload + assert upload["with"] == { + "name": "hub-release-candidate-${{ github.sha }}", + "path": "${{ runner.temp }}/hub-release-candidate-${{ github.sha }}/*", + "if-no-files-found": "error", + "retention-days": 90, + "compression-level": 0, + "overwrite": False, + } diff --git a/tests/test_wp160_security.py b/tests/test_wp160_security.py index e692dfa..3d52b42 100644 --- a/tests/test_wp160_security.py +++ b/tests/test_wp160_security.py @@ -1,9 +1,12 @@ from __future__ import annotations import json +import shutil +import subprocess import tempfile import unittest from pathlib import Path +from unittest.mock import patch from uuid import uuid4 from packages.connector_kit.gitea import GiteaSettings @@ -16,7 +19,12 @@ from packages.connector_kit.knowledge import ( from packages.connector_kit.redaction import safe_content from packages.hub_contracts.dataplane import AliasRecord from scripts.generate_sbom import generate -from scripts.security_scan import _BEARER, scan_repository +from scripts.security_scan import ( + _BEARER, + _tracked_text_files, + scan_repository, + scan_secret_patterns, +) from services.control_api.catalog.models import CatalogState from services.control_api.catalog.planner import build_plan from services.control_api.contextforge.models import GatewayDesiredState, GatewaySnapshot @@ -34,6 +42,44 @@ class SecurityPlanTests(unittest.TestCase): self.assertEqual(report["secrets"]["findings"], []) self.assertTrue(report["invariants"]["denyByDefault"]) + def test_secret_scan_ignores_untracked_scratch_checkout_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + git = shutil.which("git") + self.assertIsNotNone(git) + assert git is not None + subprocess.run([git, "init", "--quiet"], cwd=repository, check=True) # noqa: S603 + tracked = repository / "tracked.md" + tracked.write_text("ordinary tracked documentation\n", encoding="utf-8") + subprocess.run([git, "add", "tracked.md"], cwd=repository, check=True) # noqa: S603 + tracked_secret = repository / "tracked-secret.md" + tracked_secret.write_text( + 'password = "tracked-sensitive-value-9283746501"\n', encoding="utf-8" + ) + subprocess.run([git, "add", "tracked-secret.md"], cwd=repository, check=True) # noqa: S603 + scratch = repository / ".tmp" / "source-checkout" + scratch.mkdir(parents=True) + (scratch / "ignored.md").write_text( + 'password = "untracked-sensitive-value-9283746501"\n', encoding="utf-8" + ) + + with patch("scripts.security_scan.ROOT", repository): + self.assertCountEqual(_tracked_text_files(), [tracked, tracked_secret]) + findings = scan_secret_patterns()["findings"] + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0]["path"], "tracked-secret.md") + + def test_git_enumeration_failure_fails_closed(self) -> None: + error = subprocess.CalledProcessError(2, ["git", "ls-files", "-z"]) + with patch("scripts.security_scan.subprocess.run", side_effect=error): + with self.assertRaises(subprocess.CalledProcessError): + _tracked_text_files() + + def test_missing_git_fails_closed(self) -> None: + with patch("scripts.security_scan.shutil.which", return_value=None): + with self.assertRaises(FileNotFoundError): + _tracked_text_files() + def test_sbom_is_deterministic_and_contains_locked_dependencies_and_images(self) -> None: report = generate() self.assertEqual(report["bomFormat"], "CycloneDX")