Publish curated ITWorx_MCP_Hub source from 5d65966d0393e868ed5df1d95a21412a4c587d78

This commit is contained in:
Public Source Publisher committed 2026-10-03 20:13:28 +00:00
1 parent 378ae83d30
commit 570089b749
14 files changed
+3491 -78

No files matched your search

+115 -1
View File
@@ -19,6 +19,35 @@ class ComposeContractTests(unittest.TestCase):
def setUpClass(cls) -> None:
cls.document = yaml.safe_load(COMPOSE.read_text(encoding="utf-8"))
def assert_database_candidate_pin(
self,
name: str,
image_ref: str,
image_evidence: dict,
publication: dict,
publication_candidate: dict,
) -> None:
artifact = image_evidence["artifact"]
published = artifact["publishedImage"]
self.assertRegex(
image_ref,
rf"^gitea\.itworx\.tech/jens/itworx-mcp-hub/{name}:[^@]+@sha256:[0-9a-f]{{64}}$",
)
image_name, digest = image_ref.split("@", maxsplit=1)
self.assertEqual(published["reference"], image_ref)
self.assertEqual(published["taggedReference"], image_name)
self.assertEqual(published["manifestDigest"], digest)
self.assertEqual(published["imageId"], artifact["imageId"])
self.assertTrue(published["immutablePullConfigMatchedScannedImage"])
self.assertEqual(published["sourceReceiptSha256"], publication["sourceReceiptSha256"])
self.assertEqual(publication_candidate["reference"], image_ref)
self.assertEqual(publication_candidate["manifestDigest"], digest)
self.assertEqual(publication_candidate["scannedImageId"], artifact["imageId"])
self.assertEqual(publication_candidate["pulledImageId"], artifact["imageId"])
self.assertTrue(publication_candidate["immutablePullConfigMatchedScannedImage"])
self.assertEqual(image_evidence["humanRiskAcceptance"], "pending")
self.assertFalse(image_evidence["productionDeployment"])
def test_all_images_are_immutable_or_local_dev(self) -> None:
for name, service in self.document["services"].items():
image = service.get("image", "")
@@ -68,9 +97,94 @@ class ComposeContractTests(unittest.TestCase):
for name in ("postgres", "redis"):
with self.subTest(service=name):
image = self.document["services"][name]["image"]
self.assertRegex(image, r"^[a-z]+:[0-9.]+@sha256:[0-9a-f]{64}$")
self.assertRegex(
image,
r"^gitea\.itworx\.tech/jens/itworx-mcp-hub/"
rf"{name}:[^@]+@sha256:[0-9a-f]{{64}}$",
)
self.assertEqual(integration["services"][name]["image"], image)
def test_database_candidate_pins_bind_to_published_scan_identities(self) -> None:
evidence_path = ROOT / "evidence/WP-400/DATABASE_REMEDIATION_SCANS.json"
evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
publication = evidence["publication"]
publication_proof = json.loads(
(ROOT / "evidence/WP-400/DATABASE_CANDIDATE_PUBLICATION_20261003.json").read_text(
encoding="utf-8"
)
)
self.assertEqual(
publication,
{key: value for key, value in publication_proof.items() if key != "candidates"},
)
self.assertFalse(evidence["productionDeployment"])
self.assertFalse(publication["productionDeployment"])
self.assertFalse(publication["cveRiskAccepted"])
self.assertEqual(publication["status"], "published_digest_verified")
self.assertRegex(publication["sourceReceiptSha256"], r"^[0-9a-f]{64}$")
integration = yaml.safe_load(
(ROOT / "infrastructure/compose/docker-compose.wp050-integration.yml").read_text(
encoding="utf-8"
)
)
for name in ("postgres", "redis"):
with self.subTest(service=name):
image_evidence = evidence["images"][name]
image_ref = image_evidence["artifact"]["publishedImage"]["reference"]
candidate_proof = publication_proof["candidates"][name]
self.assert_database_candidate_pin(
name,
self.document["services"][name]["image"],
image_evidence,
publication,
candidate_proof,
)
self.assert_database_candidate_pin(
name,
integration["services"][name]["image"],
image_evidence,
publication,
candidate_proof,
)
self.assertEqual(image_ref, self.document["services"][name]["image"])
self.assertEqual(image_ref, integration["services"][name]["image"])
def test_database_candidate_pin_contract_rejects_tag_only_and_config_drift(self) -> None:
evidence = json.loads(
(ROOT / "evidence/WP-400/DATABASE_REMEDIATION_SCANS.json").read_text(encoding="utf-8")
)
publication_proof = json.loads(
(ROOT / "evidence/WP-400/DATABASE_CANDIDATE_PUBLICATION_20261003.json").read_text(
encoding="utf-8"
)
)
for name in ("postgres", "redis"):
with self.subTest(service=name):
artifact = evidence["images"][name]["artifact"]
published = artifact["publishedImage"]
valid_ref = published["reference"]
tag_only = valid_ref.split("@", maxsplit=1)[0]
with self.assertRaises(AssertionError):
self.assert_database_candidate_pin(
name,
tag_only,
evidence["images"][name],
evidence["publication"],
publication_proof["candidates"][name],
)
mismatched_image = dict(published, imageId="sha256:" + "0" * 64)
drifted_evidence = dict(evidence["images"][name])
drifted_evidence["artifact"] = dict(artifact, publishedImage=mismatched_image)
with self.assertRaises(AssertionError):
self.assert_database_candidate_pin(
name,
valid_ref,
drifted_evidence,
evidence["publication"],
publication_proof["candidates"][name],
)
def test_inspector_is_lan_only_optional_profile(self) -> None:
inspector = self.document["services"]["inspector"]
self.assertEqual(inspector.get("profiles"), ["inspector"])
+263
View File
@@ -0,0 +1,263 @@
from __future__ import annotations
import hashlib
import io
import json
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BUNDLE = ROOT / "evidence/WP-400/database-image-candidate-20261003"
SCAN_EVIDENCE = ROOT / "evidence/WP-400/DATABASE_REMEDIATION_SCANS.json"
PUBLICATION_EVIDENCE = ROOT / "evidence/WP-400/DATABASE_CANDIDATE_PUBLICATION_20261003.json"
def read_json(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def bundle_files(bundle: Path) -> set[str]:
return {
path.relative_to(bundle).as_posix()
for path in bundle.rglob("*")
if path.is_file() and path != bundle / "FILE_HASHES.json"
}
def run_git(cwd: Path, *arguments: str) -> bytes:
result = subprocess.run( # noqa: S603
["git", *arguments], # noqa: S607
cwd=cwd,
check=True,
capture_output=True,
)
return result.stdout
class DatabaseCandidateProvenanceTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.manifest = read_json(BUNDLE / "candidate-provenance.json")
cls.scan_evidence = read_json(SCAN_EVIDENCE)
cls.publication = read_json(PUBLICATION_EVIDENCE)
def test_retained_recipe_sources_match_the_exact_build_recipe_hashes(self) -> None:
for service in ("postgres", "redis"):
with self.subTest(service=service):
item = self.manifest["images"][service]
build = read_json(BUNDLE / item["buildReceiptFile"])
self.assertEqual(set(item["sourceFiles"]), set(build["recipeHashes"]))
plan = (
read_json(BUNDLE / "recipes" / service / "plan.json")
if service == "postgres"
else None
)
for name, expected in item["sourceFiles"].items():
source = BUNDLE / "recipes" / service / name
if name.endswith(".tar.gz"):
self.assertFalse(
source.exists(), "the binary upstream archive must stay excluded"
)
self.assertFalse(
self.manifest["trackedBundle"]["binaryGosuSourceArchiveIncluded"]
)
self.assertEqual(expected, plan["source"]["archive_sha256"])
else:
self.assertTrue(source.is_file(), name)
self.assertEqual(sha256(source), expected, name)
self.assertEqual(build["recipeHashes"][name], expected, name)
def test_build_export_scan_and_publication_bind_the_same_candidates(self) -> None:
self.assertFalse(self.manifest["releaseEligible"])
self.assertFalse(self.manifest["reviewBoundary"]["releaseEligible"])
self.assertEqual(self.manifest["reviewBoundary"]["humanCveRiskAcceptance"], "pending")
self.assertFalse(self.manifest["reviewBoundary"]["agentMayApproveCveExceptions"])
self.assertFalse(self.manifest["reviewBoundary"]["bitForBitRebuildClaim"])
self.assertFalse(self.scan_evidence["productionDeployment"])
self.assertFalse(self.publication["productionDeployment"])
self.assertFalse(self.publication["cveRiskAccepted"])
for service in ("postgres", "redis"):
with self.subTest(service=service):
item = self.manifest["images"][service]
build = read_json(BUNDLE / item["buildReceiptFile"])
download = read_json(BUNDLE / item["downloadReceiptFile"])
scan = read_json(BUNDLE / item["scanReceiptFile"])
summary = read_json(BUNDLE / item["vulnerabilityScan"]["summaryFile"])
secret = read_json(BUNDLE / item["secretScan"]["sanitizedReceiptFile"])
image = build["image"]
published_ref = self.manifest["publishedImageRefs"][service]
published = self.publication["candidates"][service]
scanned = self.scan_evidence["images"][service]
self.assertEqual(item["publishedImageRef"], published_ref)
self.assertIsNone(item["productionImageRef"])
self.assertEqual(item["candidateOnly"], True)
self.assertIsNone(item["productionImageRef"])
self.assertEqual(published["reference"], published_ref)
self.assertEqual(scanned["artifact"]["publishedImage"]["reference"], published_ref)
self.assertEqual(published["scannedImageId"], image["id"])
self.assertEqual(published["pulledImageId"], image["id"])
self.assertTrue(published["immutablePullConfigMatchedScannedImage"])
self.assertEqual(item["vulnerabilityScan"]["artifactRef"]["imageId"], image["id"])
self.assertEqual(
item["vulnerabilityScan"]["rawReportSha256"],
scan["rawVulnerabilityReportSha256"],
)
self.assertEqual(download["artifactRef"]["imageId"], image["id"])
self.assertEqual(download["artifactRef"]["tag"], image["tag"])
self.assertEqual(download["localSha256"], build["archive"]["sha256"])
self.assertEqual(download["remoteSha256"], build["archive"]["sha256"])
self.assertEqual(scan["artifactRef"]["imageId"], image["id"])
self.assertEqual(scan["artifactRef"]["archiveSha256"], build["archive"]["sha256"])
self.assertEqual(scan["reportIdentity"]["ImageID"], image["id"])
self.assertEqual(scan["ignoreUnfixed"], False)
self.assertEqual(scan["findingFilters"], [])
self.assertEqual(scan["tlsVerification"], True)
self.assertEqual(scan["packageCount"], summary["packageCount"])
self.assertGreater(summary["packageCount"], 0)
inventory = summary["strictVersionInventory"]
self.assertTrue(inventory["allNamesAndVersionsPresent"])
self.assertEqual(inventory["packageEntriesChecked"], summary["packageCount"])
self.assertEqual(secret["ImageID"], image["id"])
self.assertEqual(secret["FindingCount"], 0)
self.assertFalse(secret["SensitiveFindingDetailsPersisted"])
self.assertEqual(scanned["humanRiskAcceptance"], "pending")
self.assertFalse(scanned["productionDeployment"])
def test_bundle_hash_manifest_covers_retained_recipe_and_receipt_files(self) -> None:
evidence = self.scan_evidence["recipeProvenance"]
self.assertEqual(
evidence["manifestPath"],
"evidence/WP-400/database-image-candidate-20261003/candidate-provenance.json",
)
self.assertEqual(evidence["manifestSha256"], sha256(BUNDLE / "candidate-provenance.json"))
self.assertEqual(
evidence["fileHashesPath"],
"evidence/WP-400/database-image-candidate-20261003/FILE_HASHES.json",
)
hashes_path = BUNDLE / "FILE_HASHES.json"
self.assertEqual(evidence["fileHashesSha256"], sha256(hashes_path))
hashes = read_json(hashes_path)
actual_files = bundle_files(BUNDLE)
self.assertEqual(set(hashes["files"]), actual_files)
for relative, expected in hashes["files"].items():
with self.subTest(path=relative):
self.assertEqual(sha256(BUNDLE / relative), expected)
def test_ci_compilation_preserves_strict_bundle_inventory(self) -> None:
with tempfile.TemporaryDirectory(prefix="wp400-compiled-provenance-") as temporary:
bundle = Path(temporary) / "evidence"
recipe = bundle / "recipes" / "test_recipe_contract.py"
recipe.parent.mkdir(parents=True)
recipe.write_text("assert True\n", encoding="utf-8")
source_hash = sha256(recipe)
cache_root = Path(temporary) / "managed-bytecode"
subprocess.run( # noqa: S603
[sys.executable, "-m", "py_compile", str(recipe)],
env=dict(os.environ, PYTHONPYCACHEPREFIX=str(cache_root)),
check=True,
capture_output=True,
)
self.assertTrue(any(cache_root.rglob("*.pyc")))
self.assertEqual(sha256(recipe), source_hash)
self.assertEqual(bundle_files(bundle), {"recipes/test_recipe_contract.py"})
unexpected = recipe.parent / "__pycache__" / "unexpected.json"
unexpected.parent.mkdir()
unexpected.write_text("{}\n", encoding="utf-8")
matching_cache = unexpected.with_name(
f"test_recipe_contract.{sys.implementation.cache_tag}.pyc"
)
matching_cache.write_bytes(b"unexpected cache bytes")
self.assertEqual(
bundle_files(bundle),
{
"recipes/test_recipe_contract.py",
"recipes/__pycache__/unexpected.json",
f"recipes/__pycache__/{matching_cache.name}",
},
)
workflow = (ROOT / ".gitea/workflows/managed-validation.yml").read_text(encoding="utf-8")
self.assertIn(
'PYTHONPYCACHEPREFIX="${RUNNER_TEMP}/managed-bytecode" '
"xargs -0 -r python -m py_compile",
workflow,
)
def test_git_objects_archives_and_autocrlf_checkouts_preserve_bundle_hashes(self) -> None:
file_hashes = read_json(BUNDLE / "FILE_HASHES.json")["files"]
expected_hashes = dict(file_hashes)
expected_hashes["FILE_HASHES.json"] = self.scan_evidence["recipeProvenance"][
"fileHashesSha256"
]
bundle_path = Path("evidence/WP-400/database-image-candidate-20261003")
with tempfile.TemporaryDirectory(prefix="wp400-git-byte-provenance-") as temporary:
temporary_root = Path(temporary)
source = temporary_root / "source"
source.mkdir()
shutil.copyfile(ROOT / ".gitattributes", source / ".gitattributes")
for relative in expected_hashes:
destination = source / bundle_path / relative
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(BUNDLE / relative, destination)
run_git(source, "init", "--quiet")
run_git(source, "config", "user.name", "WP-400 provenance test")
run_git(source, "config", "user.email", "wp400-provenance@example.invalid")
run_git(source, "config", "core.autocrlf", "true")
run_git(source, "config", "core.safecrlf", "false")
hooks = source / ".test-hooks"
hooks.mkdir()
run_git(source, "config", "core.hooksPath", ".test-hooks")
run_git(source, "add", "--all")
run_git(source, "commit", "--quiet", "-m", "byte-preserving provenance fixture")
archive = run_git(source, "archive", "--format=tar", "HEAD", str(bundle_path))
archive_prefix = bundle_path.as_posix() + "/"
with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as archived:
archived_files = {
member.name.removeprefix(archive_prefix): member
for member in archived.getmembers()
if member.isfile() and member.name.startswith(archive_prefix)
}
self.assertEqual(set(archived_files), set(expected_hashes))
for relative, expected in expected_hashes.items():
member_file = archived.extractfile(archived_files[relative])
self.assertIsNotNone(member_file)
self.assertEqual(
hashlib.sha256(member_file.read()).hexdigest(), expected, relative
)
for autocrlf in ("false", "true"):
with self.subTest(core_autocrlf=autocrlf):
checkout = temporary_root / f"checkout-{autocrlf}"
run_git(
temporary_root,
"clone",
"--quiet",
"--no-checkout",
str(source),
str(checkout),
)
run_git(checkout, "config", "core.autocrlf", autocrlf)
run_git(checkout, "config", "core.safecrlf", "false")
run_git(checkout, "checkout", "--force", "HEAD")
for relative, expected in expected_hashes.items():
self.assertEqual(sha256(checkout / bundle_path / relative), expected)
if __name__ == "__main__":
unittest.main()
+812
View File
@@ -0,0 +1,812 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from typing import Any, cast
ROOT = Path(__file__).resolve().parents[1]
PUBLISHER_PATH = ROOT / "scripts" / "publish_mcp_hub_ingestion.py"
FIXED_REPOSITORY = "jens/itworx_mcp_hub"
IMAGE_REPOSITORY = "gitea.example.com/jens/itworx-mcp-hub/connector-ingestion"
SOURCE_URI = "https://gitea.example.com/jens/itworx-mcp-hub"
REVISION = "a" * 40
TREE = "b" * 40
MANIFEST_DIGEST = "sha256:" + "c" * 64
CONFIG_DIGEST = "sha256:" + "d" * 64
BUILT_AT = "2026-10-03T01:02:03.004Z"
EPOCH = "1790980806"
ENGINE_ID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
CONTAINER_ID = "f" * 64
_OMITTED = object()
def _load_publisher() -> Any:
spec = importlib.util.spec_from_file_location("test_publish_mcp_hub_ingestion", PUBLISHER_PATH)
if spec is None or spec.loader is None:
raise AssertionError("fixed ingestion publisher script could not be loaded")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
class PublisherHarness:
def __init__(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name)
self.volume_root = self.root / "volume"
self.releases_root = self.volume_root / "releases"
self.source = self.releases_root / REVISION
self.publisher = _load_publisher()
self.publisher.FIXED_VOLUME_ROOT = self.volume_root
self.publisher.FIXED_RELEASES_ROOT = self.releases_root
self.state_root = self.root / "state"
self.publisher.FIXED_STATE_ROOT = self.state_root
dockerfile = self.source / "infrastructure" / "images" / "python-service" / "Dockerfile"
dockerfile.parent.mkdir(parents=True)
dockerfile.write_text("FROM python:3.13-slim\n", encoding="utf-8")
self.script = self.source / "scripts" / "publish_mcp_hub_ingestion.py"
self.script.parent.mkdir(parents=True)
self.script.write_text("# frozen publisher\n", encoding="utf-8")
self.state_root.mkdir(mode=0o700)
self.freeze_build = {
"id": REVISION[:12],
"repository": FIXED_REPOSITORY,
"revision": REVISION,
"tree": TREE,
"treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(),
"builtAt": BUILT_AT,
}
self.frozen = self.publisher.FrozenSource(
repository=FIXED_REPOSITORY,
revision=REVISION,
build=dict(self.freeze_build),
source_date_epoch=EPOCH,
freeze_sha256="e" * 64,
controller_source_path=f"/controller-hub-releases/releases/{REVISION}",
)
self.command_calls: list[list[str]] = []
self.builder_present = False
self.builder_bootstrapped = False
self.builder_container_present = False
self.bad_builder_image = False
self.bad_builder_container_image = False
self.bad_builder_actual_image = False
self.builder_runtime_overrides: dict[str, Any] = {}
self.bad_labels = False
self.bad_manifest = False
self.bad_config = False
self.redaction_sentinel = "sentinel-value-must-not-escape"
if os.name != "nt":
for path in sorted(
self.source.rglob("*"), key=lambda item: len(item.parts), reverse=True
):
if path.is_file():
path.chmod(0o444)
elif path.is_dir():
path.chmod(0o555)
self.source.chmod(0o555)
self.volume_root.chmod(0o755)
self.releases_root.chmod(0o755)
def close(self) -> None:
if os.name != "nt":
for path in sorted(
self.root.rglob("*"), key=lambda item: len(item.parts), reverse=True
):
try:
path.chmod(0o700 if path.is_dir() else 0o600)
except OSError:
pass
self.temporary.cleanup()
def docker_runner(
self, command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
self.command_calls.append(list(command))
operation = command[3:]
if operation[:2] == ["buildx", "ls"]:
rows = ["default|docker", "default|default"]
if self.builder_present:
rows += [
"itworx-mcp-hub-ingestion|docker-container",
"itworx-mcp-hub-ingestion0|unix:///var/run/docker.sock",
]
return subprocess.CompletedProcess(command, 0, "\n".join(rows), "")
if operation[:2] == ["buildx", "create"]:
self.assert_no_global_builder_switch(command)
self.builder_present = True
return subprocess.CompletedProcess(command, 0, "itworx-mcp-hub-ingestion", "")
if operation[:2] == ["container", "ls"]:
names = (
["buildx_buildkit_itworx-mcp-hub-ingestion0"]
if self.builder_container_present
else []
)
return subprocess.CompletedProcess(command, 0, "\n".join(names), "")
if operation[:2] == ["buildx", "inspect"]:
if "--bootstrap" in operation and timeout < 120:
raise AssertionError("pinned builder bootstrap needs a bounded pull timeout")
text = (
"Name: itworx-mcp-hub-ingestion\n"
"Driver: docker-container\n\n"
"Nodes:\n"
"Name: itworx-mcp-hub-ingestion0\n"
"Endpoint: unix:///var/run/docker.sock\n"
f'Driver Options: image="{self._builder_image_option()}"\n'
)
if "--bootstrap" in operation:
self.builder_bootstrapped = True
self.builder_container_present = True
if self.builder_bootstrapped:
text += "Status: running\nBuildKit version: v0.33.0\n"
else:
text += "Status: inactive\n"
return subprocess.CompletedProcess(command, 0, text, "")
if operation[:2] == ["inspect", "--format"]:
target = command[-1]
if target == "buildx_buildkit_itworx-mcp-hub-ingestion0":
return self._builder_container_inspection(command)
if (
operation[:3] == ["image", "inspect", "--format"]
and command[-1] == self.publisher.FIXED_BUILDKIT_IMAGE
):
image_record = {
"Id": CONFIG_DIGEST,
"Config.Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"BUILDKIT_SETUP_CGROUPV2_ROOT=1",
],
"Config.Entrypoint": self.publisher.FIXED_BUILDKIT_ENTRYPOINT,
"Config.User": "",
"Config.WorkingDir": "",
"Config.Labels": {"org.opencontainers.image.title": "BuildKit"},
}
raw = "\n".join(
json.dumps(image_record[field], separators=(",", ":"))
for field in self.publisher.PINNED_BUILDKIT_IMAGE_FIELDS
)
return subprocess.CompletedProcess(command, 0, raw, "")
if operation[:2] == ["buildx", "build"]:
self.builder_bootstrapped = True
metadata_path = Path(command[command.index("--metadata-file") + 1])
metadata = {
"containerimage.digest": MANIFEST_DIGEST,
"containerimage.config.digest": "sha256:" + "9" * 64
if self.bad_config
else CONFIG_DIGEST,
}
metadata_path.write_text(json.dumps(metadata), encoding="utf-8")
return subprocess.CompletedProcess(command, 0, "", "")
if operation[:4] == ["buildx", "imagetools", "inspect", "--format"]:
descriptor = {
"digest": "sha256:" + "e" * 64 if self.bad_manifest else MANIFEST_DIGEST,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"size": 2458,
}
return subprocess.CompletedProcess(command, 0, json.dumps(descriptor), "")
if operation[:2] == ["pull", "--quiet"]:
return subprocess.CompletedProcess(command, 0, "", "")
if operation[:2] == ["image", "inspect"]:
values = [
CONFIG_DIGEST,
"10001:10001",
SOURCE_URI,
"f" * 40 if self.bad_labels else REVISION,
TREE,
REVISION[:12],
BUILT_AT,
EPOCH,
]
return subprocess.CompletedProcess(command, 0, "|".join(values), "")
if self.redaction_sentinel in operation:
raise AssertionError("redaction sentinel must never be an argument")
raise AssertionError(f"unexpected fake Docker command: {command!r}")
def _builder_container_inspection(self, command: list[str]) -> subprocess.CompletedProcess[str]:
record: dict[str, Any] = {
"Image": "sha256:" + "0" * 64 if self.bad_builder_actual_image else CONFIG_DIGEST,
"Config.Image": "moby/buildkit:unexpected"
if self.bad_builder_container_image
else self.publisher.FIXED_BUILDKIT_IMAGE,
"Config.Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"BUILDKIT_SETUP_CGROUPV2_ROOT=1",
],
"Config.Entrypoint": self.publisher.FIXED_BUILDKIT_ENTRYPOINT,
"Config.Cmd": self.publisher.FIXED_BUILDKIT_COMMAND,
"Config.User": "",
"Config.WorkingDir": "",
"Config.Labels": {"org.opencontainers.image.title": "BuildKit"},
"State.Running": self.builder_bootstrapped,
"HostConfig.Privileged": True,
"HostConfig.Init": True,
"HostConfig.RestartPolicy.Name": "unless-stopped",
"HostConfig.RestartPolicy.MaximumRetryCount": 0,
"HostConfig.NetworkMode": "default",
"HostConfig.UsernsMode": "",
"HostConfig.CgroupParent": "/docker/buildx",
"HostConfig.CgroupnsMode": "private",
"HostConfig.Mounts": self._host_mount(),
"Mounts": [
{
"Type": "volume",
"Name": self.publisher.FIXED_BUILDKIT_STATE_VOLUME,
"Source": (
f"{self.publisher.FIXED_DOCKER_ROOT}/volumes/"
f"{self.publisher.FIXED_BUILDKIT_STATE_VOLUME}/_data"
),
"Destination": self.publisher.FIXED_BUILDKIT_STATE_PATH,
"Driver": "local",
"Mode": "",
"RW": True,
"Propagation": "",
}
],
"HostConfig.Binds": None,
"HostConfig.Devices": None,
"HostConfig.DeviceRequests": None,
"HostConfig.CapAdd": None,
"HostConfig.CapDrop": None,
"HostConfig.SecurityOpt": None,
"HostConfig.PortBindings": None,
"HostConfig.PublishAllPorts": False,
"HostConfig.ReadonlyRootfs": False,
"HostConfig.PidMode": "",
"HostConfig.IpcMode": "private",
"HostConfig.UTSMode": "",
"HostConfig.AutoRemove": False,
"HostConfig.Runtime": "runc",
"HostConfig.Memory": 0,
"HostConfig.MemorySwap": 0,
"HostConfig.CpuShares": 0,
"HostConfig.CpuQuota": 0,
"HostConfig.CpuPeriod": 0,
"HostConfig.CpusetCpus": "",
"HostConfig.CpusetMems": "",
"HostConfig.OomKillDisable": None,
}
record.update(self.builder_runtime_overrides)
raw = "\n".join(
json.dumps(record[field], separators=(",", ":"))
for field in self.publisher.BUILDKIT_CONTAINER_FIELDS
)
return subprocess.CompletedProcess(command, 0, raw, "")
def _host_mount(self, read_only: Any = _OMITTED) -> list[dict[str, Any]]:
mount: dict[str, Any] = {
"Type": "volume",
"Source": self.publisher.FIXED_BUILDKIT_STATE_VOLUME,
"Target": self.publisher.FIXED_BUILDKIT_STATE_PATH,
}
if read_only is not _OMITTED:
mount["ReadOnly"] = read_only
return [mount]
def assert_no_global_builder_switch(self, command: list[str]) -> None:
if "--use" in command:
raise AssertionError("publisher must not switch the global Buildx builder")
if "--name" not in command or "itworx-mcp-hub-ingestion" not in command:
raise AssertionError("publisher must use only its fixed dedicated builder")
if f"image={self.publisher.FIXED_BUILDKIT_IMAGE}" not in command:
raise AssertionError("publisher must pin the dedicated BuildKit image")
def _builder_image_option(self) -> str:
return "wrong" if self.bad_builder_image else self.publisher.FIXED_BUILDKIT_IMAGE
def publisher_module(self) -> Any:
return self.publisher
class FixedIngestionPublisherTests(unittest.TestCase):
def setUp(self) -> None:
self.harness = PublisherHarness()
self.addCleanup(self.harness.close)
self.publisher = self.harness.publisher_module()
def prepare_builder(self) -> None:
self.publisher._ensure_dedicated_builder(self.harness.docker_runner, self.harness.frozen)
def publish(self) -> dict[str, str]:
self.publisher._verify_local_release_root(self.harness.source, REVISION)
self.publisher._ensure_dedicated_builder(self.harness.docker_runner, self.harness.frozen)
return cast(
dict[str, str],
self.publisher._publish_frozen_revision(
REVISION,
frozen=self.harness.frozen,
docker_runner=self.harness.docker_runner,
source_root=self.harness.source,
state_root=self.harness.state_root,
script_path=self.harness.script,
),
)
def receipt_path(self) -> Path:
return (
self.harness.state_root
/ "mcp-hub-ingestion-publications"
/ REVISION
/ "publication.json"
)
def test_builds_fixed_service_with_frozen_labels_pushes_by_digest_and_attests_readback(
self,
) -> None:
original = {
path.relative_to(self.harness.source).as_posix(): path.read_bytes()
for path in self.harness.source.rglob("*")
if path.is_file()
}
result = self.publish()
build = next(
command for command in self.harness.command_calls if command[3:5] == ["buildx", "build"]
)
self.assertEqual(build[3:5], ["buildx", "build"])
self.assertIn("--builder", build)
self.assertIn("itworx-mcp-hub-ingestion", build)
self.assertEqual(build[build.index("--builder") + 1], self.publisher.FIXED_BUILDER_NAME)
self.assertIn("--platform", build)
self.assertEqual(build[build.index("--platform") + 1], "linux/amd64")
self.assertIn("--provenance=false", build)
output = build[build.index("--output") + 1]
self.assertIn("push-by-digest=true", output)
self.assertNotIn("--tag", build)
self.assertEqual(
{build[index + 1] for index, value in enumerate(build[:-1]) if value == "--build-arg"},
{
"SERVICE_NAME=connector-ingestion",
f"GIT_REVISION={REVISION}",
f"GIT_SOURCE_TREE={TREE}",
f"IMAGE_VERSION={REVISION[:12]}",
f"SOURCE_DATE_EPOCH={EPOCH}",
f"BUILD_DATE={BUILT_AT}",
},
)
self.assertEqual(result["imageRef"], f"{IMAGE_REPOSITORY}@{MANIFEST_DIGEST}")
self.assertEqual(result["imageId"], CONFIG_DIGEST)
receipt = json.loads(self.receipt_path().read_text(encoding="utf-8"))
payload = receipt["receipt"]
self.assertEqual(payload["imageRef"], result["imageRef"])
self.assertEqual(payload["sourceFreezeSha256"], self.harness.frozen.freeze_sha256)
self.assertEqual(
receipt["receiptSha256"],
hashlib.sha256(
(json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n").encode()
).hexdigest(),
)
self.assertNotIn(
self.harness.redaction_sentinel,
self.receipt_path().read_text(encoding="utf-8"),
)
self.assertFalse(
any(command[3:5] == ["login", ""] for command in self.harness.command_calls)
)
after = {
path.relative_to(self.harness.source).as_posix(): path.read_bytes()
for path in self.harness.source.rglob("*")
if path.is_file()
}
self.assertEqual(original, after)
if os.name != "nt":
info = self.receipt_path().stat()
self.assertEqual(info.st_uid, 0)
self.assertEqual(info.st_mode & 0o777, 0o600)
def test_official_attestation_accepts_only_fixed_repository_and_revision(self) -> None:
document = {
"schema": self.publisher.ATTESTATION_SCHEMA,
"repository": FIXED_REPOSITORY,
"revision": REVISION,
"build": dict(self.harness.freeze_build),
"sourceDateEpoch": EPOCH,
"freezeSha256": "e" * 64,
"sourcePath": f"/controller-hub-releases/releases/{REVISION}",
}
frozen = self.publisher._validate_frozen_attestation(document, REVISION)
self.assertEqual(frozen.build, self.harness.freeze_build)
document["repository"] = "attacker/project"
with self.assertRaises(self.publisher.PublishError):
self.publisher._validate_frozen_attestation(document, REVISION)
def test_official_attestation_command_is_fixed_read_only_controller_probe(self) -> None:
expected = {
"schema": self.publisher.ATTESTATION_SCHEMA,
"repository": FIXED_REPOSITORY,
"revision": REVISION,
"build": dict(self.harness.freeze_build),
"sourceDateEpoch": EPOCH,
"freezeSha256": "e" * 64,
"sourcePath": f"/controller-hub-releases/releases/{REVISION}",
}
def runner(
command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
self.assertEqual(command[0:4], ["/usr/bin/docker", "--context", "default", "exec"])
self.assertIn(self.publisher.FIXED_CONTAINER, command)
self.assertIn("python3", command)
self.assertIn("-c", command)
self.assertNotIn("bash", command)
return subprocess.CompletedProcess(command, 0, json.dumps(expected), "")
result = self.publisher._attest_official_freeze(REVISION, runner)
self.assertEqual(result.revision, REVISION)
def test_engine_preflight_requires_same_fixed_host_and_controller_socket_engine(self) -> None:
def runner(
command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
if command[1:3] == ["context", "show"]:
return subprocess.CompletedProcess(command, 0, "default", "")
if command[1:3] == ["context", "inspect"]:
return subprocess.CompletedProcess(command, 0, "unix:///var/run/docker.sock", "")
operation = command[3:]
if operation[:1] == ["info"]:
return subprocess.CompletedProcess(
command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", ""
)
if operation[:2] == ["exec", "--user"]:
return subprocess.CompletedProcess(
command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", ""
)
if operation[:2] == ["inspect", "--format"]:
return subprocess.CompletedProcess(command, 0, f"{CONTAINER_ID}|true", "")
if operation[:2] == ["buildx", "version"]:
return subprocess.CompletedProcess(
command, 0, "github.com/docker/buildx v0.20.1 abc123", ""
)
raise AssertionError(f"unexpected preflight command: {command!r}")
engine, container = self.publisher._preflight_engine(runner)
self.assertEqual(engine, ENGINE_ID)
self.assertEqual(container, CONTAINER_ID)
def test_engine_preflight_rejects_unreviewed_buildx_before_publication(self) -> None:
def runner(
command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
if command[1:3] == ["context", "show"]:
return subprocess.CompletedProcess(command, 0, "default", "")
if command[1:3] == ["context", "inspect"]:
return subprocess.CompletedProcess(command, 0, "unix:///var/run/docker.sock", "")
operation = command[3:]
if operation[:1] == ["info"]:
return subprocess.CompletedProcess(
command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", ""
)
if operation[:2] == ["exec", "--user"]:
return subprocess.CompletedProcess(
command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", ""
)
if operation[:2] == ["inspect", "--format"]:
return subprocess.CompletedProcess(command, 0, f"{CONTAINER_ID}|true", "")
if operation[:2] == ["buildx", "version"]:
return subprocess.CompletedProcess(
command, 0, "github.com/docker/buildx v0.21.3", ""
)
raise AssertionError(f"unexpected preflight command: {command!r}")
with self.assertRaises(self.publisher.PublishError):
self.publisher._preflight_engine(runner)
def test_engine_preflight_rejects_controller_socket_mismatch_before_build(self) -> None:
def runner(
command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
if command[1:3] == ["context", "show"]:
return subprocess.CompletedProcess(command, 0, "default", "")
if command[1:3] == ["context", "inspect"]:
return subprocess.CompletedProcess(command, 0, "unix:///var/run/docker.sock", "")
operation = command[3:]
if operation[:1] == ["info"]:
return subprocess.CompletedProcess(
command, 0, f"{ENGINE_ID}|/var/lib/docker|27.5.1|Tower", ""
)
if operation[:2] == ["exec", "--user"]:
return subprocess.CompletedProcess(
command, 0, f"{'0' * 64}|/var/lib/docker|27.5.1|Tower", ""
)
raise AssertionError("preflight proceeded after an engine mismatch")
with self.assertRaises(self.publisher.PublishError):
self.publisher._preflight_engine(runner)
def test_engine_identity_requires_fixed_tower_hostname(self) -> None:
with self.assertRaises(self.publisher.PublishError):
self.publisher._parse_engine_identity(
f"{ENGINE_ID}|/var/lib/docker|27.5.1|another-host", source="host"
)
def test_environment_overrides_cannot_redirect_engine_builder_or_auth_config(self) -> None:
for name in (
"DOCKER_HOST",
"DOCKER_CONTEXT",
"DOCKER_CONFIG",
"BUILDX_BUILDER",
"BUILDX_CONFIG",
"DOCKER_AUTH_CONFIG",
):
with self.subTest(name=name), self.assertRaises(self.publisher.PublishError):
self.publisher._reject_docker_environment({"HOME": "/root", name: "sentinel"})
def test_dedicated_builder_is_created_only_after_freeze_and_does_not_switch_default(
self,
) -> None:
self.publisher._ensure_dedicated_builder(self.harness.docker_runner, self.harness.frozen)
create = next(
command
for command in self.harness.command_calls
if command[3:5] == ["buildx", "create"]
)
self.harness.assert_no_global_builder_switch(create)
self.assertIn("docker-container", create)
self.assertIn("default", create)
self.assertTrue(self.harness.builder_present)
def test_mismatched_preexisting_dedicated_builder_is_denied(self) -> None:
self.harness.builder_present = True
def wrong_builder(
command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
self.harness.command_calls.append(list(command))
if command[3:5] == ["buildx", "ls"]:
return subprocess.CompletedProcess(
command,
0,
"default|docker\ndefault|default\nitworx-mcp-hub-ingestion|docker\nitworx-mcp-hub-ingestion0|default",
"",
)
if command[3:5] == ["container", "ls"]:
return subprocess.CompletedProcess(command, 0, "", "")
raise AssertionError("mismatched builder should be rejected from inventory")
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(wrong_builder, self.harness.frozen)
self.assertFalse(
any(command[3:5] == ["buildx", "build"] for command in self.harness.command_calls)
)
def test_preexisting_builder_container_image_is_checked_before_bootstrap(self) -> None:
self.harness.builder_present = True
self.harness.builder_container_present = True
self.harness.bad_builder_container_image = True
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertFalse(
any(
command[3:6] == ["buildx", "inspect", "--bootstrap"]
or command[3:5] == ["buildx", "build"]
for command in self.harness.command_calls
)
)
def test_preexisting_builder_container_actual_image_id_is_checked_before_bootstrap(
self,
) -> None:
self.harness.builder_present = True
self.harness.builder_container_present = True
self.harness.bad_builder_actual_image = True
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertFalse(
any(
command[3:6] == ["buildx", "inspect", "--bootstrap"]
or command[3:5] == ["buildx", "build"]
for command in self.harness.command_calls
)
)
def test_preexisting_builder_container_unsafe_runtime_is_denied_before_bootstrap(self) -> None:
unsafe_configurations: tuple[dict[str, Any], ...] = (
{"HostConfig.Privileged": False},
{"HostConfig.SecurityOpt": ["seccomp=unconfined"]},
{"HostConfig.NetworkMode": "host"},
{"HostConfig.IpcMode": "host"},
{"HostConfig.Runtime": "foreign"},
{"HostConfig.CgroupnsMode": "host"},
{"HostConfig.Mounts": []},
{"Config.Env": ["TOKEN=sentinel-value-must-not-escape"]},
)
for override in unsafe_configurations:
with self.subTest(override=tuple(override)):
self.harness.command_calls.clear()
self.harness.builder_present = True
self.harness.builder_container_present = True
self.harness.builder_bootstrapped = False
self.harness.builder_runtime_overrides = override
with self.assertRaises(self.publisher.PublishError) as raised:
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertNotIn("sentinel-value-must-not-escape", str(raised.exception))
self.assertFalse(
any(
command[3:6] == ["buildx", "inspect", "--bootstrap"]
or command[3:5] == ["buildx", "build"]
for command in self.harness.command_calls
)
)
def test_writable_builder_state_mount_accepts_only_omitted_or_boolean_false(self) -> None:
self.harness.builder_present = True
self.harness.builder_container_present = True
self.harness.builder_bootstrapped = True
self.publisher._verify_dedicated_builder_container(
self.harness.docker_runner, require_running=True
)
self.harness.builder_runtime_overrides = {
"HostConfig.Mounts": self.harness._host_mount(False)
}
self.publisher._verify_dedicated_builder_container(
self.harness.docker_runner, require_running=True
)
for unsafe_value in (True, None, 0, "false"):
with self.subTest(read_only=unsafe_value):
self.harness.command_calls.clear()
self.harness.builder_runtime_overrides = {
"HostConfig.Mounts": self.harness._host_mount(unsafe_value)
}
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertFalse(
any(
command[3:6] == ["buildx", "inspect", "--bootstrap"]
or command[3:5] == ["buildx", "build"]
for command in self.harness.command_calls
)
)
def test_default_buildkit_config_file_blocks_builder_before_docker_commands(self) -> None:
config = self.harness.root / "buildkitd.default.toml"
config.write_text("[worker.oci]\n", encoding="utf-8")
self.publisher.FIXED_BUILDKIT_CONFIG_PATH = config
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertEqual(self.harness.command_calls, [])
@unittest.skipIf(os.name == "nt", "symlink behavior is POSIX-specific")
def test_default_buildkit_config_symlink_blocks_builder_before_docker_commands(self) -> None:
target = self.harness.root / "outside.toml"
target.write_text("[worker.oci]\n", encoding="utf-8")
link = self.harness.root / "buildkitd.default.toml"
link.symlink_to(target)
self.publisher.FIXED_BUILDKIT_CONFIG_PATH = link
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertEqual(self.harness.command_calls, [])
def test_orphan_reserved_container_blocks_create_bootstrap_and_build(self) -> None:
self.harness.builder_container_present = True
self.harness.bad_builder_container_image = True
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(
self.harness.docker_runner, self.harness.frozen
)
self.assertFalse(
any(
command[3:5] == ["buildx", "create"]
or command[3:6] == ["buildx", "inspect", "--bootstrap"]
or command[3:5] == ["buildx", "build"]
for command in self.harness.command_calls
)
)
def test_builder_creation_requires_a_validated_frozen_source(self) -> None:
invalid = self.harness.frozen.__class__(
repository="attacker/project",
revision=REVISION,
build=dict(self.harness.freeze_build),
source_date_epoch=EPOCH,
freeze_sha256="e" * 64,
controller_source_path=f"/controller-hub-releases/releases/{REVISION}",
)
with self.assertRaises(self.publisher.PublishError):
self.publisher._ensure_dedicated_builder(self.harness.docker_runner, invalid)
self.assertEqual(self.harness.command_calls, [])
def test_rejects_stale_or_writable_staged_source_before_build(self) -> None:
dockerfile = (
self.harness.source / "infrastructure" / "images" / "python-service" / "Dockerfile"
)
if os.name != "nt":
dockerfile.chmod(0o644)
with self.assertRaises(self.publisher.PublishError):
self.publish()
else:
with self.assertRaises(self.publisher.PublishError):
self.publisher._verify_local_release_root(self.harness.source.parent, REVISION)
self.assertFalse(
any(command[3:5] == ["buildx", "build"] for command in self.harness.command_calls)
)
self.assertFalse(self.receipt_path().exists())
def test_rejects_bad_build_metadata_and_registry_manifest_digest(self) -> None:
self.harness.bad_manifest = True
with self.assertRaises(self.publisher.PublishError):
self.publish()
self.assertFalse(self.receipt_path().exists())
def test_rejects_wrong_config_id_or_provenance_labels_without_receipt(self) -> None:
self.harness.bad_config = True
with self.assertRaises(self.publisher.PublishError):
self.publish()
self.assertFalse(self.receipt_path().exists())
self.harness.bad_config = False
self.harness.bad_labels = True
with self.assertRaises(self.publisher.PublishError):
self.publish()
self.assertFalse(self.receipt_path().exists())
def test_child_error_output_is_scrubbed_and_not_saved(self) -> None:
self.prepare_builder()
def failed_runner(
command: list[str], *, cwd: Path | None, timeout: int
) -> subprocess.CompletedProcess[str]:
if command[3:5] == ["buildx", "build"]:
return subprocess.CompletedProcess(command, 1, "", self.harness.redaction_sentinel)
return self.harness.docker_runner(command, cwd=cwd, timeout=timeout)
with self.assertRaises(self.publisher.PublishError) as raised:
self.publisher._publish_frozen_revision(
REVISION,
frozen=self.harness.frozen,
docker_runner=failed_runner,
source_root=self.harness.source,
state_root=self.harness.state_root,
script_path=self.harness.script,
)
self.assertNotIn(self.harness.redaction_sentinel, str(raised.exception))
self.assertFalse(self.receipt_path().exists())
def test_preexisting_receipt_is_never_overwritten_or_rebuilt(self) -> None:
self.prepare_builder()
receipt = self.receipt_path()
receipt.parent.mkdir(parents=True, mode=0o700)
receipt.write_text("prior immutable receipt", encoding="utf-8")
before = len(self.harness.command_calls)
with self.assertRaises(self.publisher.PublishError):
self.publisher._publish_frozen_revision(
REVISION,
frozen=self.harness.frozen,
docker_runner=self.harness.docker_runner,
source_root=self.harness.source,
state_root=self.harness.state_root,
script_path=self.harness.script,
)
self.assertEqual(receipt.read_text(encoding="utf-8"), "prior immutable receipt")
self.assertEqual(len(self.harness.command_calls), before)
def test_production_slot_check_denies_existing_receipt_without_docker_commands(self) -> None:
receipt = self.receipt_path()
receipt.parent.mkdir(parents=True, mode=0o700)
receipt.write_text("prior immutable receipt", encoding="utf-8")
with self.assertRaises(self.publisher.PublishError):
self.publisher._require_publication_slot(self.harness.state_root, REVISION)
self.assertEqual(self.harness.command_calls, [])
if __name__ == "__main__":
unittest.main()
+364
View File
@@ -0,0 +1,364 @@
"""Behavioral tests for the exact protected-run release artifact payload."""
from __future__ import annotations
import copy
import hashlib
import json
from pathlib import Path
import pytest
from scripts import package_mcp_hub_release_artifact as packager
SERVICES = (
"connector-appops",
"connector-gitea",
"connector-ingestion",
"connector-knowledge",
"connector-mobilityops",
"connector-projectbrain",
"connector-stitch",
"connector-unraid",
"contextforge",
"control-api",
"dataplane-router",
"docker-read-broker",
"ingestion-clamd",
"postgres",
"redis",
"web",
"worker",
)
REPORT_NAMES = {
"trivy-db-metadata.json",
*(
name
for service in SERVICES
for name in (f"vulnerability-{service}.json", f"secret-{service}.json")
),
}
REVISION = "a" * 40
TREE = "b" * 40
RUN_CREATED_AT = "2026-10-03T10:11:12.123Z"
def _json_bytes(value: object) -> bytes:
return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()
def _fixture(tmp_path: Path) -> tuple[dict[str, object], dict[str, dict[str, str]], Path, Path]:
image_tar = tmp_path / "inputs" / "images.tar"
report_dir = tmp_path / "inputs" / "reports"
report_dir.mkdir(parents=True)
image_tar.parent.mkdir(parents=True, exist_ok=True)
image_tar.write_bytes(b"fixture-docker-save-archive\n")
images: dict[str, dict[str, str]] = {}
report_time = "2026-10-03T10:12:00Z"
for index, service in enumerate(SERVICES, start=1):
image_id = f"sha256:{index:064x}"
image_ref = f"registry.example/itworx/{service}:protected-main@sha256:{index:064x}"
images[service] = {
"imageRef": image_ref,
"imageId": image_id,
"archiveRef": f"registry.example/itworx/{service}:archive-{REVISION[:12]}",
}
vulnerability = {
"SchemaVersion": 2,
"CreatedAt": report_time,
"Trivy": {"Version": "0.73.0"},
"Metadata": {
"ImageID": image_id,
"ImageConfig": {"os": "linux", "architecture": "amd64"},
},
"Results": [
{
"Packages": [{"Name": "fixture-package", "Version": "1.0"}],
"Vulnerabilities": [],
}
],
}
secret = {
"ImageID": image_id,
"CreatedAt": report_time,
"TrivyVersion": "0.73.0",
"FindingCount": 0,
"SensitiveFindingDetailsPersisted": False,
"Results": [{"FindingCount": 0}],
}
(report_dir / f"vulnerability-{service}.json").write_bytes(_json_bytes(vulnerability))
(report_dir / f"secret-{service}.json").write_bytes(_json_bytes(secret))
database = {
"UpdatedAt": report_time,
"DownloadedAt": report_time,
"Version": 2,
"NextUpdate": "2026-10-04T10:12:00Z",
}
(report_dir / "trivy-db-metadata.json").write_bytes(_json_bytes(database))
build = {
"id": REVISION[:12],
"repository": "jens/itworx_mcp_hub",
"revision": REVISION,
"tree": TREE,
"treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(),
"builtAt": RUN_CREATED_AT,
}
context: dict[str, object] = {
"repository": "jens/itworx_mcp_hub",
"revision": REVISION,
"tree": TREE,
"workflowPath": ".gitea/workflows/managed-validation.yml",
"workflowRunId": 2468,
"workflowRunCreatedAt": RUN_CREATED_AT,
"artifactName": f"hub-release-candidate-{REVISION}",
"build": build,
"sourceDateEpoch": "1791022272",
}
return context, images, image_tar, report_dir
def _package(
output_dir: Path,
context: dict[str, object],
images: dict[str, dict[str, str]],
image_tar: Path,
report_dir: Path,
) -> dict[str, object]:
return packager.package_candidate_artifact(
output_dir=output_dir,
manifest_context=context,
images=images,
image_tar_path=image_tar,
report_dir=report_dir,
)
@pytest.fixture(autouse=True)
def _bind_fixture_to_test_git(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
packager,
"_git_source_identity",
lambda: (REVISION, TREE, "1791022272"),
)
def test_package_contains_exact_importer_payload_and_binds_original_bytes(tmp_path: Path) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
output = tmp_path / "artifact"
returned_manifest = _package(output, context, images, image_tar, report_dir)
expected_names = {"candidate-manifest.json", "images.tar", *REPORT_NAMES}
assert {path.name for path in output.iterdir()} == expected_names
assert all(path.is_file() and not path.is_symlink() for path in output.iterdir())
manifest_bytes = (output / "candidate-manifest.json").read_bytes()
manifest = json.loads(manifest_bytes)
tar_bytes = image_tar.read_bytes()
assert manifest == returned_manifest
assert set(manifest) == {
"schema",
"repository",
"sourceRevision",
"sourceTree",
"workflowPath",
"workflowRunId",
"artifactName",
"build",
"sourceDateEpoch",
"dockerTarBytes",
"dockerTarSha256",
"images",
"reports",
}
assert manifest["schema"] == "itworx.mcp-hub-image-artifact/v1"
assert manifest["repository"] == context["repository"]
assert manifest["sourceRevision"] == context["revision"]
assert manifest["sourceTree"] == context["tree"]
assert manifest["workflowPath"] == context["workflowPath"]
assert manifest["workflowRunId"] == context["workflowRunId"]
assert manifest["artifactName"] == context["artifactName"]
assert manifest["build"] == context["build"]
assert manifest["sourceDateEpoch"] == context["sourceDateEpoch"]
assert manifest["dockerTarBytes"] == len(tar_bytes)
assert manifest["dockerTarSha256"] == hashlib.sha256(tar_bytes).hexdigest()
assert manifest["images"] == images
assert manifest["reports"] == {
name: {
"sha256": hashlib.sha256((report_dir / name).read_bytes()).hexdigest(),
"bytes": (report_dir / name).stat().st_size,
}
for name in sorted(REPORT_NAMES)
}
assert (output / "images.tar").read_bytes() == tar_bytes
for name in REPORT_NAMES:
assert (output / name).read_bytes() == (report_dir / name).read_bytes()
def test_canonicalizes_the_exact_workflow_run_timestamp_to_importer_format(
tmp_path: Path,
) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
context["workflowRunCreatedAt"] = "2026-10-03T10:11:12Z"
build = dict(context["build"])
build["builtAt"] = "2026-10-03T10:11:12.000Z"
context["build"] = build
manifest = _package(tmp_path / "artifact", context, images, image_tar, report_dir)
assert manifest["build"]["builtAt"] == "2026-10-03T10:11:12.000Z"
@pytest.mark.parametrize(
("mutation", "value"),
[
("revision", "c" * 40),
("tree", "d" * 40),
("run_id", 0),
("run_time", "2026-10-03T10:11:12Z"),
("artifact_name", "hub-release-candidate-wrong"),
("epoch", "01791022272"),
],
)
def test_rejects_unbound_frozen_context_without_publishing(
tmp_path: Path, mutation: str, value: object
) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
context = copy.deepcopy(context)
if mutation == "revision":
context["revision"] = value
elif mutation == "tree":
context["tree"] = value
elif mutation == "run_id":
context["workflowRunId"] = value
elif mutation == "run_time":
context["workflowRunCreatedAt"] = value
elif mutation == "artifact_name":
context["artifactName"] = value
else:
context["sourceDateEpoch"] = value
output = tmp_path / "artifact"
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert not output.exists()
assert not list(tmp_path.glob(".hub-release-artifact-*"))
@pytest.mark.parametrize("report_change", ["missing", "extra", "oversized", "wrong_image"])
def test_rejects_invalid_fixed_report_inventory_without_partial_output(
tmp_path: Path, report_change: str
) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
if report_change == "missing":
(report_dir / "secret-worker.json").unlink()
elif report_change == "extra":
(report_dir / "unlisted.json").write_text("{}", encoding="utf-8")
elif report_change == "oversized":
(report_dir / "secret-worker.json").write_bytes(b" " * (2 * 1024**2 + 1))
else:
data = json.loads((report_dir / "vulnerability-worker.json").read_text(encoding="utf-8"))
data["Metadata"]["ImageID"] = "sha256:" + "f" * 64
(report_dir / "vulnerability-worker.json").write_bytes(_json_bytes(data))
output = tmp_path / "artifact"
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert not output.exists()
def test_rejects_secret_findings_instead_of_redacting_or_rewriting_them(tmp_path: Path) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
secret_path = report_dir / "secret-worker.json"
secret = json.loads(secret_path.read_text(encoding="utf-8"))
secret["FindingCount"] = 1
secret["SensitiveFindingDetailsPersisted"] = True
secret_path.write_bytes(_json_bytes(secret))
original = secret_path.read_bytes()
output = tmp_path / "artifact"
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert not output.exists()
assert secret_path.read_bytes() == original
def test_rejects_output_collision_without_overwriting_existing_files(tmp_path: Path) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
output = tmp_path / "artifact"
output.mkdir()
marker = output / "keep.txt"
marker.write_text("preserve", encoding="utf-8")
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert marker.read_text(encoding="utf-8") == "preserve"
assert {path.name for path in output.iterdir()} == {"keep.txt"}
def test_rejects_output_directory_created_at_atomic_publish_boundary(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
output = tmp_path / "artifact"
original_publish = packager._rename_noreplace
def create_collision_then_publish(source: Path, destination: Path) -> None:
destination.mkdir()
original_publish(source, destination)
monkeypatch.setattr(packager, "_rename_noreplace", create_collision_then_publish)
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert output.is_dir()
assert list(output.iterdir()) == []
assert not list(tmp_path.glob(".hub-release-artifact-*"))
def test_rejects_wrong_image_inventory_without_partial_output(tmp_path: Path) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
images.pop("worker")
output = tmp_path / "artifact"
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert not output.exists()
def test_rejects_source_identity_that_does_not_match_checkout(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
monkeypatch.setattr(
packager,
"_git_source_identity",
lambda: ("f" * 40, TREE, "1791022272"),
)
output = tmp_path / "artifact"
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert not output.exists()
def test_rejects_hardlinked_report_without_publishing(tmp_path: Path) -> None:
context, images, image_tar, report_dir = _fixture(tmp_path)
secret_path = report_dir / "secret-worker.json"
secret_path.unlink()
try:
secret_path.hardlink_to(report_dir / "secret-web.json")
except OSError as exc:
pytest.skip(f"filesystem does not support the hard-link fixture: {exc}")
output = tmp_path / "artifact"
with pytest.raises(packager.ArtifactRejected):
_package(output, context, images, image_tar, report_dir)
assert not output.exists()
+531
View File
@@ -0,0 +1,531 @@
from __future__ import annotations
import hashlib
import io
import json
import os
import subprocess
import sys
import tarfile
from datetime import UTC, datetime
from pathlib import Path
import pytest
from scripts import build_mcp_hub_release_images as producer
ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "scripts" / "build_mcp_hub_release_images.py"
def _write_archive_fixture(path: Path, members: dict[str, bytes | None]) -> None:
with tarfile.open(path, "w:", format=tarfile.USTAR_FORMAT) as archive:
for name, payload in members.items():
info = tarfile.TarInfo(
name + ("/" if payload is None and not name.endswith("/") else "")
)
if payload is None:
info.type = tarfile.DIRTYPE
info.mode = 0o755
archive.addfile(info)
else:
info.size = len(payload)
archive.addfile(info, io.BytesIO(payload))
def _docker29_oci_fixture(
tmp_path: Path,
) -> tuple[Path, dict[str, bytes | None], dict[str, dict[str, str]], dict[str, str]]:
tmp_path.mkdir(parents=True, exist_ok=True)
revision = "a" * 40
metadata = {
"revision": revision,
"tree": "b" * 40,
"buildId": revision[:12],
"builtAt": "2026-10-03T12:34:56.000Z",
"sourceDateEpoch": "1791027296",
}
layer_file = io.BytesIO()
with tarfile.open(fileobj=layer_file, mode="w:", format=tarfile.USTAR_FORMAT) as layer_archive:
content = b"synthetic Docker 29 image layer"
item = tarfile.TarInfo("fixture.txt")
item.size = len(content)
layer_archive.addfile(item, io.BytesIO(content))
layer = layer_file.getvalue()
layer_digest = "sha256:" + hashlib.sha256(layer).hexdigest()
layer_name = "blobs/sha256/" + layer_digest.split(":", 1)[1]
members: dict[str, bytes | None] = {"blobs": None, "blobs/sha256": None, layer_name: layer}
images: dict[str, dict[str, str]] = {}
docker_manifest: list[dict[str, object]] = []
index_descriptors: list[dict[str, object]] = []
for service in producer.ALL_SERVICES:
if service in producer.LOCAL_BUILD_SERVICES:
reference = f"{producer.IMAGE_REPOSITORIES[service]}:{revision}"
elif service == "contextforge":
reference = f"itworx-mcp-hub/contextforge-schema:{revision}"
elif service == "connector-ingestion":
reference = f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-123-attempt-1"
else:
reference = producer.EXTERNAL_IMAGES[service][1]
labels: dict[str, str] = {}
if service not in producer.EXTERNAL_IMAGES:
for key, label in producer.OCI_LABELS.items():
labels[key] = (
"https://gitea.example.com/jens/itworx-mcp-hub"
if key == "org.opencontainers.image.source"
else metadata[label]
)
config = {
"architecture": "amd64",
"os": "linux",
"config": {"User": "10001:10001", "Labels": labels, "Cmd": [service]},
"rootfs": {"type": "layers", "diff_ids": [layer_digest]},
}
config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode()
image_id = "sha256:" + hashlib.sha256(config_bytes).hexdigest()
config_name = f"blobs/sha256/{image_id.split(':', 1)[1]}"
members[config_name] = config_bytes
images[service] = {"imageId": image_id, "archiveRef": reference, "imageRef": reference}
docker_manifest.append(
{
"Config": config_name,
"RepoTags": [reference],
"Layers": [layer_name],
"LayerSources": {
layer_digest: {
"mediaType": "application/vnd.oci.image.layer.v1.tar",
"size": len(layer),
"digest": layer_digest,
}
},
}
)
image_manifest = {
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"config": {
"mediaType": "application/vnd.oci.image.config.v1+json",
"digest": image_id,
"size": len(config_bytes),
},
"layers": [
{
"mediaType": "application/vnd.oci.image.layer.v1.tar",
"digest": layer_digest,
"size": len(layer),
}
],
}
image_manifest_bytes = json.dumps(
image_manifest, sort_keys=True, separators=(",", ":")
).encode()
manifest_digest = hashlib.sha256(image_manifest_bytes).hexdigest()
members[f"blobs/sha256/{manifest_digest}"] = image_manifest_bytes
index_descriptors.append(
{
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"digest": "sha256:" + manifest_digest,
"size": len(image_manifest_bytes),
"annotations": {"org.opencontainers.image.ref.name": reference},
"platform": {"os": "linux", "architecture": "amd64"},
}
)
members["oci-layout"] = b'{"imageLayoutVersion":"1.0.0"}'
members["index.json"] = json.dumps(
{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.index.v1+json",
"manifests": index_descriptors,
},
sort_keys=True,
separators=(",", ":"),
).encode()
members["manifest.json"] = json.dumps(docker_manifest, separators=(",", ":")).encode()
raw = tmp_path / "docker29-oci-save.tar"
_write_archive_fixture(raw, members)
return raw, members, images, metadata
def test_missing_registry_credential_is_rejected_before_building(tmp_path: Path) -> None:
env = {
key: value
for key, value in os.environ.items()
if key not in {"GITEA_TOKEN", "HUB_RELEASE_REGISTRY_TOKEN"}
}
env.update(
{
"GITEA_TOKEN": "metadata-token-test-value",
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
"GITHUB_REPOSITORY": "Jens/ITWorx_MCP_Hub",
}
)
result = subprocess.run( # noqa: S603 -- executes only this repository's fixed producer script.
[
sys.executable,
str(SCRIPT),
"--revision",
"a" * 40,
"--run-id",
"123",
"--output",
str(tmp_path / "candidate"),
],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode != 0
assert "HUB_RELEASE_REGISTRY_TOKEN" in result.stderr
assert "metadata-token-test-value" not in result.stdout + result.stderr
assert not (tmp_path / "candidate").exists()
def test_run_binding_requires_the_one_successful_full_gate() -> None:
revision = "a" * 40
run_id = 123
run = {
"id": run_id,
"path": producer.WORKFLOW_PATH,
"event": "push",
"head_branch": "main",
"head_sha": revision,
"repository": {"full_name": producer.DISPLAY_REPOSITORY},
"head_repository": {"full_name": producer.DISPLAY_REPOSITORY},
"pull_requests": [],
"status": "in_progress",
"run_attempt": 1,
"created_at": "2026-10-03T12:34:56Z",
}
jobs = {
"jobs": [
{
"name": "full",
"run_id": run_id,
"run_attempt": 1,
"head_sha": revision,
"head_branch": "main",
"status": "completed",
"conclusion": "success",
}
]
}
branch = {"commit": {"id": revision}}
result = producer._validate_run(run, jobs, branch, revision=revision, run_id=run_id)
assert result["workflowRunCreatedAt"] == "2026-10-03T12:34:56.000Z"
assert result["runAttempt"] == 1
with pytest.raises(producer.BuildRejected):
producer._validate_run(
run, {"jobs": jobs["jobs"] * 2}, branch, revision=revision, run_id=run_id
)
bad_job = {"jobs": [dict(jobs["jobs"][0], conclusion="failure")]}
with pytest.raises(producer.BuildRejected):
producer._validate_run(run, bad_job, branch, revision=revision, run_id=run_id)
def test_cli_environment_rejects_docker_endpoint_overrides() -> None:
revision = "b" * 40
env = {
"GITHUB_EVENT_NAME": "push",
"GITHUB_REF": "refs/heads/main",
"GITHUB_REPOSITORY": producer.DISPLAY_REPOSITORY,
"GITHUB_SHA": revision,
"GITHUB_RUN_ID": "5",
"DOCKER_HOST": "tcp://attacker.invalid:2375",
}
with pytest.raises(producer.BuildRejected):
producer._validate_cli_environment(revision, 5, env)
@pytest.mark.parametrize(
("buildx", "compose", "accepted"),
[
("github.com/docker/buildx v0.37.2", "v5.5.1", True),
("github.com/docker/buildx 0.37.2", "5.5.1", True),
("github.com/docker/buildx v0.37.20", "v5.5.1", False),
("github.com/docker/buildx v0.37.2", "v5.5.10", False),
],
)
def test_job_daemon_requires_exact_buildx_and_compose_versions(
buildx: str,
compose: str,
accepted: bool,
) -> None:
def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
if command[1:3] == ["info", "--format"]:
output = 'daemon-identity-012345|29.8.2|overlay2|["name=rootless"]'
elif command[1:] == ["buildx", "version"]:
output = buildx
elif command[1:] == ["compose", "version", "--short"]:
output = compose
else:
pytest.fail("version admission invoked an unexpected command")
return subprocess.CompletedProcess(command, 0, output, "")
if accepted:
assert producer._check_job_daemon({}, Path.cwd(), runner=runner) == "daemon-identity-012345"
else:
with pytest.raises(producer.BuildRejected):
producer._check_job_daemon({}, Path.cwd(), runner=runner)
def test_fixed_image_contract_is_17_services_with_12_compose_builds() -> None:
revision = "c" * 40
services = {
service: {"image": f"{repository}:{revision}"}
for service, repository in producer.IMAGE_REPOSITORIES.items()
}
services["contextforge"] = {"image": f"itworx-mcp-hub/contextforge-schema:{revision}"}
services["connector-ingestion"] = {
"image": f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-pending@sha256:{'0' * 64}"
}
for service, (image, _archive) in producer.EXTERNAL_IMAGES.items():
services[service] = {"image": image}
model = {
"services": {
service: {
**definition,
**({"build": {"context": "."}} if service in producer.LOCAL_BUILD_SERVICES else {}),
}
for service, definition in services.items()
}
}
assert producer._validate_local_refs(model, revision) == {
service: services[service]["image"] for service in producer.LOCAL_BUILD_SERVICES
}
model["services"]["connector-ingestion"]["build"] = {"context": "."}
with pytest.raises(producer.BuildRejected):
producer._validate_compose_model(model)
wrong_ref = {"services": {name: dict(row) for name, row in model["services"].items()}}
wrong_ref["services"]["control-api"]["image"] = "attacker.invalid/control:mutable"
with pytest.raises(producer.BuildRejected):
producer._validate_local_refs(wrong_ref, revision)
def test_manifest_descriptor_is_strict_and_digest_bound() -> None:
digest = "sha256:" + "d" * 64
descriptor = {
"digest": digest,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"size": 120,
}
assert producer._parse_manifest_descriptor(json.dumps(descriptor)) == digest
with pytest.raises(producer.BuildRejected):
producer._parse_manifest_descriptor(json.dumps({**descriptor, "extra": "x"}))
with pytest.raises(producer.BuildRejected):
producer._parse_manifest_descriptor(json.dumps({**descriptor, "digest": "sha256:bad"}))
def test_docker29_oci_archive_normalizes_index_blobs_to_exact_ustar_importer_shape(
tmp_path: Path,
) -> None:
raw, _members, images, metadata = _docker29_oci_fixture(tmp_path)
normalized = tmp_path / "normalized.tar"
producer._normalize_docker_archive(raw, normalized, images, metadata=metadata)
with tarfile.open(normalized, "r:") as archive:
members = archive.getmembers()
names = {member.name.rstrip("/") for member in members}
image_ids = {row["imageId"].split(":", 1)[1] for row in images.values()}
manifest_stream = archive.extractfile("manifest.json")
assert manifest_stream is not None
manifest_rows = json.loads(manifest_stream.read())
expected_names = {"manifest.json", *(f"{image_id}.json" for image_id in image_ids)}
for row in manifest_rows:
expected_names.update(layer[:-10] for layer in row["Layers"])
expected_names.update(row["Layers"])
assert names == expected_names
assert len(manifest_rows) == len(producer.ALL_SERVICES)
assert {row["RepoTags"][0] for row in manifest_rows} == {
row["archiveRef"] for row in images.values()
}
assert all(member.type in {tarfile.REGTYPE, tarfile.DIRTYPE} for member in members)
assert all(member.pax_headers == {} for member in members)
with normalized.open("rb") as source:
assert source.read(512)[257:263] == b"ustar\x00"
if os.name != "nt":
assert normalized.stat().st_mode & 0o777 == 0o600
def test_docker_oci_index_can_canonicalize_external_reference_aliases(tmp_path: Path) -> None:
raw, members, images, metadata = _docker29_oci_fixture(tmp_path)
index = json.loads(members["index.json"])
postgres_ref = images["postgres"]["archiveRef"]
descriptor = next(
item
for item in index["manifests"]
if item["annotations"]["org.opencontainers.image.ref.name"] == postgres_ref
)
descriptor["annotations"]["org.opencontainers.image.ref.name"] = (
"docker.io/library/" + postgres_ref
)
members["index.json"] = json.dumps(index, sort_keys=True, separators=(",", ":")).encode()
_write_archive_fixture(raw, members)
normalized = tmp_path / "canonical-alias-normalized.tar"
producer._normalize_docker_archive(raw, normalized, images, metadata=metadata)
assert normalized.is_file()
def test_docker29_oci_normalizer_rejects_unreferenced_blobs_and_config_mismatch(
tmp_path: Path,
) -> None:
raw, members, images, metadata = _docker29_oci_fixture(tmp_path)
orphan_name = "blobs/sha256/" + "f" * 64
members[orphan_name] = b"unreferenced"
orphan_archive = tmp_path / "orphan.tar"
_write_archive_fixture(orphan_archive, members)
with pytest.raises(producer.BuildRejected):
producer._normalize_docker_archive(
orphan_archive,
tmp_path / "orphan-normalized.tar",
images,
metadata=metadata,
)
raw, _members, images, metadata = _docker29_oci_fixture(tmp_path / "mismatch")
images["web"]["imageId"] = "sha256:" + "0" * 64
with pytest.raises(producer.BuildRejected):
producer._normalize_docker_archive(
raw,
tmp_path / "mismatch-normalized.tar",
images,
metadata=metadata,
)
def test_secret_scan_summary_keeps_finding_details_out_of_persisted_json() -> None:
image_id = "sha256:" + "e" * 64
raw_secret = "super-secret-value-for-test" # noqa: S105 -- synthetic redaction fixture.
raw = json.dumps({"Metadata": {"ImageID": image_id}, "Results": [{"Secrets": []}]}).encode()
summary = producer._secret_summary(raw, image_id)
encoded = json.dumps(summary)
assert summary["FindingCount"] == 0
assert summary["SensitiveFindingDetailsPersisted"] is False
assert raw_secret not in encoded
finding = json.dumps(
{
"Metadata": {"ImageID": image_id},
"Results": [{"Secrets": [{"Match": raw_secret, "RuleID": "test"}]}],
}
).encode()
with pytest.raises(producer.BuildRejected):
producer._secret_summary(finding, image_id)
@pytest.mark.skipif(
os.name != "posix", reason="bounded scanner stream requires POSIX process groups"
)
def test_scanner_stdout_bound_does_not_cap_or_capture_cache_files(tmp_path: Path) -> None:
def fake_scanner(command: list[str], **kwargs: object) -> subprocess.Popen[bytes]:
assert "preexec_fn" not in kwargs
assert kwargs.get("stdout") == subprocess.PIPE
code = (
"from pathlib import Path; import os; "
"Path('cache.bin').write_bytes(b'x' * (3 * 1024 * 1024)); "
"os.write(1, b'{\"complete\":true}')"
)
return subprocess.Popen( # noqa: S603 -- executes the fixed synthetic scanner fixture.
[sys.executable, "-c", code],
**kwargs, # type: ignore[arg-type]
)
output = producer._run_scanner_capture(
["fake-trivy"],
env=os.environ,
cwd=tmp_path,
timeout=10,
popen=fake_scanner,
maximum=64,
)
assert output == b'{"complete":true}'
assert (tmp_path / "cache.bin").stat().st_size == 3 * 1024 * 1024
@pytest.mark.skipif(
os.name != "posix", reason="bounded scanner stream requires POSIX process groups"
)
def test_scanner_stdout_over_limit_is_rejected_without_persisting_bytes(tmp_path: Path) -> None:
def fake_scanner(command: list[str], **kwargs: object) -> subprocess.Popen[bytes]:
code = "import os; os.write(1, b'x' * 65)"
return subprocess.Popen( # noqa: S603 -- executes the fixed synthetic scanner fixture.
[sys.executable, "-c", code],
**kwargs, # type: ignore[arg-type]
)
with pytest.raises(producer.BuildRejected):
producer._run_scanner_capture(
["fake-trivy"],
env=os.environ,
cwd=tmp_path,
timeout=10,
popen=fake_scanner,
maximum=64,
)
assert list(tmp_path.iterdir()) == []
def test_ingestion_publication_tag_is_bound_to_authenticated_attempt() -> None:
revision = "d" * 40
first = producer._ingestion_tag(revision, 456, 1)
retry = producer._ingestion_tag(revision, 456, 2)
assert first == f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-456-attempt-1"
assert retry == f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-456-attempt-2"
assert first != retry
with pytest.raises(producer.BuildRejected):
producer._ingestion_tag("../" + revision, 456, 1)
def test_registry_absence_probe_accepts_only_the_exact_missing_reference() -> None:
reference = f"{producer.IMAGE_PREFIX}/connector-ingestion:{'e' * 40}-run-7-attempt-1"
def missing(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
assert reference in command
return subprocess.CompletedProcess(command, 1, "", f"ERROR: {reference}: not found")
producer._registry_tag_absent(reference, {}, Path.cwd(), runner=missing)
def denied(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
return subprocess.CompletedProcess(command, 1, "", "unauthorized: authentication required")
with pytest.raises(producer.BuildRejected):
producer._registry_tag_absent(reference, {}, Path.cwd(), runner=denied)
def exists(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
return subprocess.CompletedProcess(command, 0, "{}", "")
with pytest.raises(producer.BuildRejected):
producer._registry_tag_absent(reference, {}, Path.cwd(), runner=exists)
def test_vulnerability_report_requires_all_packages_and_rejects_fixable_critical() -> None:
image_id = "sha256:" + "f" * 64
report = {
"SchemaVersion": 2,
"CreatedAt": datetime.now(UTC).isoformat(),
"Metadata": {"ImageID": image_id, "ImageConfig": {"os": "linux", "architecture": "amd64"}},
"Trivy": {"Version": producer.TRIVY_VERSION},
"Results": [{"Packages": [{"Name": "pkg", "Version": "1.0"}], "Vulnerabilities": []}],
}
producer._validate_vulnerability(report, image_id)
report["Results"][0]["Vulnerabilities"] = [{"Severity": "CRITICAL", "FixedVersion": "2.0"}]
with pytest.raises(producer.BuildRejected):
producer._validate_vulnerability(report, image_id)
report["Results"][0]["Vulnerabilities"] = []
report["Results"][0]["Packages"] = []
with pytest.raises(producer.BuildRejected):
producer._validate_vulnerability(report, image_id)
+63
View File
@@ -0,0 +1,63 @@
"""Keep publication credentials and builds behind the protected release trigger."""
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[1]
CHECKOUT = "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
UPLOAD = "ChristopherHX/gitea-upload-artifact@62ac910c5d3dfa85c7cb2df15afe2e342b2407c2"
def release_job() -> dict:
workflow = yaml.safe_load(
(ROOT / ".gitea/workflows/managed-validation.yml").read_text(encoding="utf-8")
)
return workflow["jobs"]["release-images"]
def test_release_build_requires_successful_validation_and_exact_main_push() -> None:
job = release_job()
assert job["name"] == "release-images"
assert job["needs"] == ["full"]
assert job["if"] == (
"${{ gitea.event_name == 'push' && gitea.ref == 'refs/heads/main' "
"&& gitea.repository == 'Jens/ITWorx_MCP_Hub' }}"
)
assert job["runs-on"] == "itworx-ci-build"
assert 0 < job["timeout-minutes"] <= 150
assert job["permissions"] == {"contents": "read", "actions": "read"}
assert "env" not in job and "container" not in job and "services" not in job
def test_release_credentials_only_reach_the_bounded_producer() -> None:
steps = release_job()["steps"]
assert len(steps) == 3
assert steps[0]["uses"] == CHECKOUT
assert steps[0]["with"]["persist-credentials"] is False
assert steps[0]["with"]["fetch-depth"] == 1
assert "env" not in steps[0] and "env" not in steps[2]
assert steps[1]["env"] == {
"GITEA_TOKEN": "${{ secrets.GITEA_TOKEN }}",
"HUB_RELEASE_REGISTRY_TOKEN": "${{ secrets.HUB_RELEASE_REGISTRY_TOKEN }}",
}
command = steps[1]["run"]
assert "set -euo pipefail" in command
assert "set -x" not in command and "docker login" not in command
assert 'python3 scripts/build_mcp_hub_release_images.py --revision "$GITHUB_SHA"' in command
assert '--run-id "$GITHUB_RUN_ID"' in command
assert '--output "${RUNNER_TEMP}/hub-release-candidate-${GITHUB_SHA}"' in command
def test_release_artifact_is_pinned_uncompressed_and_cannot_overwrite() -> None:
upload = release_job()["steps"][2]
assert upload["uses"] == UPLOAD
assert "if" not in upload
assert upload["with"] == {
"name": "hub-release-candidate-${{ github.sha }}",
"path": "${{ runner.temp }}/hub-release-candidate-${{ github.sha }}/*",
"if-no-files-found": "error",
"retention-days": 90,
"compression-level": 0,
"overwrite": False,
}
+47 -1
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
import json
import shutil
import subprocess
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from uuid import uuid4
from packages.connector_kit.gitea import GiteaSettings
@@ -16,7 +19,12 @@ from packages.connector_kit.knowledge import (
from packages.connector_kit.redaction import safe_content
from packages.hub_contracts.dataplane import AliasRecord
from scripts.generate_sbom import generate
from scripts.security_scan import _BEARER, scan_repository
from scripts.security_scan import (
_BEARER,
_tracked_text_files,
scan_repository,
scan_secret_patterns,
)
from services.control_api.catalog.models import CatalogState
from services.control_api.catalog.planner import build_plan
from services.control_api.contextforge.models import GatewayDesiredState, GatewaySnapshot
@@ -34,6 +42,44 @@ class SecurityPlanTests(unittest.TestCase):
self.assertEqual(report["secrets"]["findings"], [])
self.assertTrue(report["invariants"]["denyByDefault"])
def test_secret_scan_ignores_untracked_scratch_checkout_files(self) -> None:
with tempfile.TemporaryDirectory() as directory:
repository = Path(directory)
git = shutil.which("git")
self.assertIsNotNone(git)
assert git is not None
subprocess.run([git, "init", "--quiet"], cwd=repository, check=True) # noqa: S603
tracked = repository / "tracked.md"
tracked.write_text("ordinary tracked documentation\n", encoding="utf-8")
subprocess.run([git, "add", "tracked.md"], cwd=repository, check=True) # noqa: S603
tracked_secret = repository / "tracked-secret.md"
tracked_secret.write_text(
'password = "tracked-sensitive-value-9283746501"\n', encoding="utf-8"
)
subprocess.run([git, "add", "tracked-secret.md"], cwd=repository, check=True) # noqa: S603
scratch = repository / ".tmp" / "source-checkout"
scratch.mkdir(parents=True)
(scratch / "ignored.md").write_text(
'password = "untracked-sensitive-value-9283746501"\n', encoding="utf-8"
)
with patch("scripts.security_scan.ROOT", repository):
self.assertCountEqual(_tracked_text_files(), [tracked, tracked_secret])
findings = scan_secret_patterns()["findings"]
self.assertEqual(len(findings), 1)
self.assertEqual(findings[0]["path"], "tracked-secret.md")
def test_git_enumeration_failure_fails_closed(self) -> None:
error = subprocess.CalledProcessError(2, ["git", "ls-files", "-z"])
with patch("scripts.security_scan.subprocess.run", side_effect=error):
with self.assertRaises(subprocess.CalledProcessError):
_tracked_text_files()
def test_missing_git_fails_closed(self) -> None:
with patch("scripts.security_scan.shutil.which", return_value=None):
with self.assertRaises(FileNotFoundError):
_tracked_text_files()
def test_sbom_is_deterministic_and_contains_locked_dependencies_and_images(self) -> None:
report = generate()
self.assertEqual(report["bomFormat"], "CycloneDX")