808 lines
31 KiB
Python
808 lines
31 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from scripts import build_mcp_hub_release_images as producer
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SCRIPT = ROOT / "scripts" / "build_mcp_hub_release_images.py"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"failure,category,code",
|
|
[("exit", "nonzero_exit", 7), ("timeout", "timeout", None), ("os", "exec_error", None)],
|
|
)
|
|
def test_fixed_stage_failure_reports_only_closed_phase_and_category(
|
|
tmp_path, capsys, failure, category, code
|
|
):
|
|
marker = "fixture-sensitive-output-must-not-escape"
|
|
|
|
def runner(command, **kwargs):
|
|
assert kwargs["stdout"] == subprocess.DEVNULL
|
|
assert kwargs["stderr"] == subprocess.DEVNULL
|
|
if failure == "timeout":
|
|
raise subprocess.TimeoutExpired([marker], 1, output=marker, stderr=marker)
|
|
if failure == "os":
|
|
raise OSError(marker)
|
|
return subprocess.CompletedProcess(command, code, stdout=marker, stderr=marker)
|
|
|
|
env = {
|
|
name: "fixture"
|
|
for name in (
|
|
"GITEA_COMMIT_SHA",
|
|
"APPOPS_SOURCE_TREE",
|
|
"APPOPS_BUILD_ID",
|
|
"APPOPS_BUILD_DATE",
|
|
"SOURCE_DATE_EPOCH",
|
|
)
|
|
}
|
|
with pytest.raises(producer.BuildRejected) as error:
|
|
producer._build_contextforge(tmp_path, env, runner=runner)
|
|
message = str(error.value)
|
|
assert "stage=contextforge_build" in message
|
|
assert f"category={category}" in message
|
|
assert (f"exitCode={code}" in message) if code is not None else "exitCode=" not in message
|
|
assert marker not in message
|
|
assert marker not in "".join(capsys.readouterr())
|
|
|
|
|
|
def test_unknown_stage_is_rejected_without_running_or_echoing_it(tmp_path):
|
|
def runner(*args, **kwargs):
|
|
pytest.fail("unknown stage must not execute")
|
|
|
|
with pytest.raises(producer.BuildRejected, match="^fixed release stage identifier is invalid$"):
|
|
producer._execute(
|
|
["fixture"],
|
|
env={},
|
|
cwd=tmp_path,
|
|
timeout=1,
|
|
stage="fixture-sensitive-stage",
|
|
runner=runner,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("code", [True, "fixture-sensitive-exit", 2**40])
|
|
def test_invalid_exit_code_is_not_echoed(tmp_path, code):
|
|
def runner(command, **kwargs):
|
|
return subprocess.CompletedProcess(command, code)
|
|
|
|
with pytest.raises(producer.BuildRejected, match="category=invalid_exit_code$"):
|
|
producer._execute(
|
|
["fixture"], env={}, cwd=tmp_path, timeout=1, stage="compose_local_build", runner=runner
|
|
)
|
|
|
|
|
|
def test_protected_metadata_uses_fixed_https_without_redirecting_credentials(monkeypatch):
|
|
class Opener:
|
|
def open(self, request, timeout):
|
|
assert request.full_url == (
|
|
"https://gitea.example.com/api/v1/repos/jens/itworx_mcp_hub/actions/runs/123"
|
|
)
|
|
assert request.get_header("Authorization") == "token fixture-token"
|
|
assert timeout == 20
|
|
return io.BytesIO(b'{"id":123}')
|
|
|
|
def build_opener(*handlers):
|
|
assert handlers[0].proxies == {}
|
|
assert (
|
|
handlers[1].redirect_request(None, None, 302, "redirect", {}, "https://other.test")
|
|
is None
|
|
)
|
|
return Opener()
|
|
|
|
monkeypatch.setattr(producer.urllib.request, "build_opener", build_opener)
|
|
assert producer._api_json("fixture-token", "/repos/jens/itworx_mcp_hub/actions/runs/123") == {
|
|
"id": 123
|
|
}
|
|
|
|
|
|
def _write_archive_fixture(path: Path, members: dict[str, bytes | None]) -> None:
|
|
with tarfile.open(path, "w:", format=tarfile.USTAR_FORMAT) as archive:
|
|
for name, payload in members.items():
|
|
info = tarfile.TarInfo(
|
|
name + ("/" if payload is None and not name.endswith("/") else "")
|
|
)
|
|
if payload is None:
|
|
info.type = tarfile.DIRTYPE
|
|
info.mode = 0o755
|
|
archive.addfile(info)
|
|
else:
|
|
info.size = len(payload)
|
|
archive.addfile(info, io.BytesIO(payload))
|
|
|
|
|
|
def _docker29_oci_fixture(
|
|
tmp_path: Path,
|
|
*,
|
|
layer_count: int = 1,
|
|
) -> tuple[Path, dict[str, bytes | None], dict[str, dict[str, str]], dict[str, str]]:
|
|
tmp_path.mkdir(parents=True, exist_ok=True)
|
|
revision = "a" * 40
|
|
metadata = {
|
|
"revision": revision,
|
|
"tree": "b" * 40,
|
|
"buildId": revision[:12],
|
|
"builtAt": "2026-10-03T12:34:56.000Z",
|
|
"sourceDateEpoch": "1791027296",
|
|
}
|
|
layer_file = io.BytesIO()
|
|
with tarfile.open(fileobj=layer_file, mode="w:", format=tarfile.USTAR_FORMAT) as layer_archive:
|
|
content = b"synthetic Docker 29 image layer"
|
|
item = tarfile.TarInfo("fixture.txt")
|
|
item.size = len(content)
|
|
layer_archive.addfile(item, io.BytesIO(content))
|
|
layer = layer_file.getvalue()
|
|
layer_digest = "sha256:" + hashlib.sha256(layer).hexdigest()
|
|
layer_name = "blobs/sha256/" + layer_digest.split(":", 1)[1]
|
|
members: dict[str, bytes | None] = {"blobs": None, "blobs/sha256": None, layer_name: layer}
|
|
layer_rows = [(layer_digest, layer_name, layer)]
|
|
if layer_count == 2:
|
|
stream = io.BytesIO()
|
|
with tarfile.open(fileobj=stream, mode="w:", format=tarfile.USTAR_FORMAT) as archive:
|
|
content = b"second layer verifies prefix chain and legacy parent binding"
|
|
info = tarfile.TarInfo("second.txt")
|
|
info.size = len(content)
|
|
archive.addfile(info, io.BytesIO(content))
|
|
payload = stream.getvalue()
|
|
digest = "sha256:" + hashlib.sha256(payload).hexdigest()
|
|
name = "blobs/sha256/" + digest.split(":", 1)[1]
|
|
members[name] = payload
|
|
layer_rows.append((digest, name, payload))
|
|
images: dict[str, dict[str, str]] = {}
|
|
docker_manifest: list[dict[str, object]] = []
|
|
index_descriptors: list[dict[str, object]] = []
|
|
|
|
for service in producer.ALL_SERVICES:
|
|
if service in producer.LOCAL_BUILD_SERVICES:
|
|
reference = f"{producer.IMAGE_REPOSITORIES[service]}:{revision}"
|
|
elif service == "contextforge":
|
|
reference = f"itworx-mcp-hub/contextforge-schema:{revision}"
|
|
elif service == "connector-ingestion":
|
|
reference = f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-123-attempt-1"
|
|
else:
|
|
reference = producer.EXTERNAL_IMAGES[service][1]
|
|
|
|
labels: dict[str, str] = {}
|
|
if service not in producer.EXTERNAL_IMAGES:
|
|
for key, label in producer.OCI_LABELS.items():
|
|
labels[key] = (
|
|
"https://gitea.example.com/jens/itworx-mcp-hub"
|
|
if key == "org.opencontainers.image.source"
|
|
else metadata[label]
|
|
)
|
|
config = {
|
|
"architecture": "amd64",
|
|
"os": "linux",
|
|
"config": {"User": "10001:10001", "Labels": labels, "Cmd": [service]},
|
|
"rootfs": {"type": "layers", "diff_ids": [row[0] for row in layer_rows]},
|
|
}
|
|
config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode()
|
|
image_id = "sha256:" + hashlib.sha256(config_bytes).hexdigest()
|
|
config_name = f"blobs/sha256/{image_id.split(':', 1)[1]}"
|
|
members[config_name] = config_bytes
|
|
images[service] = {"imageId": image_id, "archiveRef": reference, "imageRef": reference}
|
|
docker_manifest.append(
|
|
{
|
|
"Config": config_name,
|
|
"RepoTags": [reference],
|
|
"Layers": [row[1] for row in layer_rows],
|
|
"LayerSources": {
|
|
digest: {
|
|
"mediaType": "application/vnd.oci.image.layer.v1.tar",
|
|
"size": len(data),
|
|
"digest": digest,
|
|
}
|
|
for digest, _name, data in layer_rows
|
|
},
|
|
}
|
|
)
|
|
image_manifest = {
|
|
"schemaVersion": 2,
|
|
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
|
"config": {
|
|
"mediaType": "application/vnd.oci.image.config.v1+json",
|
|
"digest": image_id,
|
|
"size": len(config_bytes),
|
|
},
|
|
"layers": [
|
|
{
|
|
"mediaType": "application/vnd.oci.image.layer.v1.tar",
|
|
"digest": digest,
|
|
"size": len(data),
|
|
}
|
|
for digest, _name, data in layer_rows
|
|
],
|
|
}
|
|
image_manifest_bytes = json.dumps(
|
|
image_manifest, sort_keys=True, separators=(",", ":")
|
|
).encode()
|
|
manifest_digest = hashlib.sha256(image_manifest_bytes).hexdigest()
|
|
members[f"blobs/sha256/{manifest_digest}"] = image_manifest_bytes
|
|
index_descriptors.append(
|
|
{
|
|
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
|
"digest": "sha256:" + manifest_digest,
|
|
"size": len(image_manifest_bytes),
|
|
"annotations": {"org.opencontainers.image.ref.name": reference},
|
|
"platform": {"os": "linux", "architecture": "amd64"},
|
|
}
|
|
)
|
|
|
|
members["oci-layout"] = b'{"imageLayoutVersion":"1.0.0"}'
|
|
members["index.json"] = json.dumps(
|
|
{
|
|
"schemaVersion": 2,
|
|
"mediaType": "application/vnd.oci.image.index.v1+json",
|
|
"manifests": index_descriptors,
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode()
|
|
members["manifest.json"] = json.dumps(docker_manifest, separators=(",", ":")).encode()
|
|
raw = tmp_path / "docker29-oci-save.tar"
|
|
_write_archive_fixture(raw, members)
|
|
return raw, members, images, metadata
|
|
|
|
|
|
def test_missing_registry_credential_is_rejected_before_building(tmp_path: Path) -> None:
|
|
env = {
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"GITEA_TOKEN", "HUB_RELEASE_REGISTRY_TOKEN"}
|
|
}
|
|
env.update(
|
|
{
|
|
"GITEA_TOKEN": "metadata-token-test-value",
|
|
"GITHUB_EVENT_NAME": "push",
|
|
"GITHUB_REF": "refs/heads/main",
|
|
"GITHUB_REPOSITORY": "Jens/ITWorx_MCP_Hub",
|
|
}
|
|
)
|
|
|
|
result = subprocess.run( # noqa: S603 -- executes only this repository's fixed producer script.
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--revision",
|
|
"a" * 40,
|
|
"--run-id",
|
|
"123",
|
|
"--output",
|
|
str(tmp_path / "candidate"),
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert "HUB_RELEASE_REGISTRY_TOKEN" in result.stderr
|
|
assert "metadata-token-test-value" not in result.stdout + result.stderr
|
|
assert not (tmp_path / "candidate").exists()
|
|
|
|
|
|
def test_run_binding_requires_the_one_successful_full_gate() -> None:
|
|
revision = "a" * 40
|
|
run_id = 123
|
|
run = {
|
|
"id": run_id,
|
|
"path": producer.WORKFLOW_PATH,
|
|
"event": "push",
|
|
"head_branch": "main",
|
|
"head_sha": revision,
|
|
"repository": {"full_name": producer.DISPLAY_REPOSITORY},
|
|
"head_repository": {"full_name": producer.DISPLAY_REPOSITORY},
|
|
"pull_requests": [],
|
|
"status": "in_progress",
|
|
"run_attempt": 1,
|
|
"created_at": "2026-10-03T12:34:56Z",
|
|
}
|
|
jobs = {
|
|
"jobs": [
|
|
{
|
|
"name": "full",
|
|
"run_id": run_id,
|
|
"run_attempt": 1,
|
|
"head_sha": revision,
|
|
"head_branch": "main",
|
|
"status": "completed",
|
|
"conclusion": "success",
|
|
}
|
|
]
|
|
}
|
|
branch = {"commit": {"id": revision}}
|
|
|
|
result = producer._validate_run(run, jobs, branch, revision=revision, run_id=run_id)
|
|
|
|
assert result["workflowRunCreatedAt"] == "2026-10-03T12:34:56.000Z"
|
|
assert result["runAttempt"] == 1
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_run(
|
|
run, {"jobs": jobs["jobs"] * 2}, branch, revision=revision, run_id=run_id
|
|
)
|
|
bad_job = {"jobs": [dict(jobs["jobs"][0], conclusion="failure")]}
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_run(run, bad_job, branch, revision=revision, run_id=run_id)
|
|
|
|
|
|
def test_cli_environment_rejects_docker_endpoint_overrides() -> None:
|
|
revision = "b" * 40
|
|
env = {
|
|
"GITHUB_EVENT_NAME": "push",
|
|
"GITHUB_REF": "refs/heads/main",
|
|
"GITHUB_REPOSITORY": producer.DISPLAY_REPOSITORY,
|
|
"GITHUB_SHA": revision,
|
|
"GITHUB_RUN_ID": "5",
|
|
"DOCKER_HOST": "tcp://attacker.invalid:2375",
|
|
}
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_cli_environment(revision, 5, env)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("buildx", "compose", "accepted"),
|
|
[
|
|
("github.com/docker/buildx v0.37.2", "v5.5.1", True),
|
|
("github.com/docker/buildx 0.37.2", "5.5.1", True),
|
|
("github.com/docker/buildx v0.37.20", "v5.5.1", False),
|
|
("github.com/docker/buildx v0.37.2", "v5.5.10", False),
|
|
],
|
|
)
|
|
def test_job_daemon_requires_exact_buildx_and_compose_versions(
|
|
buildx: str,
|
|
compose: str,
|
|
accepted: bool,
|
|
) -> None:
|
|
def runner(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
|
if command[1:3] == ["info", "--format"]:
|
|
output = 'daemon-identity-012345|29.8.2|overlay2|["name=rootless"]'
|
|
elif command[1:] == ["buildx", "version"]:
|
|
output = buildx
|
|
elif command[1:] == ["compose", "version", "--short"]:
|
|
output = compose
|
|
else:
|
|
pytest.fail("version admission invoked an unexpected command")
|
|
return subprocess.CompletedProcess(command, 0, output, "")
|
|
|
|
if accepted:
|
|
assert producer._check_job_daemon({}, Path.cwd(), runner=runner) == "daemon-identity-012345"
|
|
else:
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._check_job_daemon({}, Path.cwd(), runner=runner)
|
|
|
|
|
|
def test_fixed_image_contract_is_17_services_with_12_compose_builds() -> None:
|
|
revision = "c" * 40
|
|
services = {
|
|
service: {"image": f"{repository}:{revision}"}
|
|
for service, repository in producer.IMAGE_REPOSITORIES.items()
|
|
}
|
|
services["contextforge"] = {"image": f"itworx-mcp-hub/contextforge-schema:{revision}"}
|
|
services["connector-ingestion"] = {
|
|
"image": f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-pending@sha256:{'0' * 64}"
|
|
}
|
|
for service, (image, _archive) in producer.EXTERNAL_IMAGES.items():
|
|
services[service] = {"image": image}
|
|
model = {
|
|
"services": {
|
|
service: {
|
|
**definition,
|
|
**({"build": {"context": "."}} if service in producer.LOCAL_BUILD_SERVICES else {}),
|
|
}
|
|
for service, definition in services.items()
|
|
}
|
|
}
|
|
|
|
assert producer._validate_local_refs(model, revision) == {
|
|
service: services[service]["image"] for service in producer.LOCAL_BUILD_SERVICES
|
|
}
|
|
model["services"]["connector-ingestion"]["build"] = {"context": "."}
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_compose_model(model)
|
|
wrong_ref = {"services": {name: dict(row) for name, row in model["services"].items()}}
|
|
wrong_ref["services"]["control-api"]["image"] = "attacker.invalid/control:mutable"
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_local_refs(wrong_ref, revision)
|
|
|
|
|
|
def test_manifest_descriptor_is_strict_and_digest_bound() -> None:
|
|
digest = "sha256:" + "d" * 64
|
|
descriptor = {
|
|
"digest": digest,
|
|
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
|
"size": 120,
|
|
}
|
|
assert producer._parse_manifest_descriptor(json.dumps(descriptor)) == digest
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._parse_manifest_descriptor(json.dumps({**descriptor, "extra": "x"}))
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._parse_manifest_descriptor(json.dumps({**descriptor, "digest": "sha256:bad"}))
|
|
|
|
|
|
def test_docker29_oci_archive_normalizes_index_blobs_to_exact_ustar_importer_shape(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
raw, _members, images, metadata = _docker29_oci_fixture(tmp_path)
|
|
normalized = tmp_path / "normalized.tar"
|
|
|
|
producer._normalize_docker_archive(raw, normalized, images, metadata=metadata)
|
|
|
|
with tarfile.open(normalized, "r:") as archive:
|
|
members = archive.getmembers()
|
|
names = {member.name.rstrip("/") for member in members}
|
|
image_ids = {row["imageId"].split(":", 1)[1] for row in images.values()}
|
|
manifest_stream = archive.extractfile("manifest.json")
|
|
assert manifest_stream is not None
|
|
manifest_rows = json.loads(manifest_stream.read())
|
|
expected_names = {"manifest.json", *(f"{image_id}.json" for image_id in image_ids)}
|
|
for row in manifest_rows:
|
|
expected_names.update(layer[:-10] for layer in row["Layers"])
|
|
expected_names.update(row["Layers"])
|
|
assert names == expected_names
|
|
assert len(manifest_rows) == len(producer.ALL_SERVICES)
|
|
assert {row["RepoTags"][0] for row in manifest_rows} == {
|
|
row["archiveRef"] for row in images.values()
|
|
}
|
|
assert all(member.type in {tarfile.REGTYPE, tarfile.DIRTYPE} for member in members)
|
|
assert all(member.pax_headers == {} for member in members)
|
|
with normalized.open("rb") as source:
|
|
assert source.read(512)[257:263] == b"ustar\x00"
|
|
if os.name != "nt":
|
|
assert normalized.stat().st_mode & 0o777 == 0o600
|
|
|
|
|
|
def test_docker_oci_index_can_canonicalize_external_reference_aliases(tmp_path: Path) -> None:
|
|
raw, members, images, metadata = _docker29_oci_fixture(tmp_path)
|
|
index = json.loads(members["index.json"])
|
|
postgres_ref = images["postgres"]["archiveRef"]
|
|
descriptor = next(
|
|
item
|
|
for item in index["manifests"]
|
|
if item["annotations"]["org.opencontainers.image.ref.name"] == postgres_ref
|
|
)
|
|
descriptor["annotations"]["org.opencontainers.image.ref.name"] = (
|
|
"docker.io/library/" + postgres_ref
|
|
)
|
|
members["index.json"] = json.dumps(index, sort_keys=True, separators=(",", ":")).encode()
|
|
_write_archive_fixture(raw, members)
|
|
|
|
normalized = tmp_path / "canonical-alias-normalized.tar"
|
|
producer._normalize_docker_archive(raw, normalized, images, metadata=metadata)
|
|
|
|
assert normalized.is_file()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"case",
|
|
json.loads((ROOT / "tests/fixtures/moby_v1_golden.json").read_text(encoding="utf-8"))["cases"],
|
|
)
|
|
def test_independent_go_reference_matches_exact_legacy_wire_bytes(case):
|
|
expected = {
|
|
row["name"]: base64.b64decode(row["payloadBase64"], validate=True) for row in case["blobs"]
|
|
}
|
|
actual = producer._moby_v1_metadata(
|
|
case["config"], legacy_onbuild=case["dialect"] == "onbuild_null"
|
|
)
|
|
assert actual == expected
|
|
|
|
|
|
def test_go_reference_fixture_is_bound_to_its_reproducible_generator():
|
|
fixture = json.loads((ROOT / "tests/fixtures/moby_v1_golden.json").read_text(encoding="utf-8"))
|
|
assert fixture["schema"] == "itworx.moby-v1-wire-golden/v1"
|
|
assert fixture["goVersion"] == "go1.27.2"
|
|
assert (
|
|
fixture["referenceMainSha256"]
|
|
== hashlib.sha256(
|
|
(ROOT / "tests/fixtures/moby_v1_reference/main.go").read_bytes()
|
|
).hexdigest()
|
|
)
|
|
assert fixture["sourceCommits"] == [
|
|
"89c5e8fd66634b6128fc4c0e6f1236e2540e46e0",
|
|
"5834c866f83c0bae2d0be28f38b92f32cd07ca38",
|
|
]
|
|
|
|
|
|
def _moby_v1_fixture(members, *, legacy_onbuild=True):
|
|
# Faithful classic Moby saveConfig/CreateID serialization for the fixed
|
|
# single-layer images above, independent of the production normalizer.
|
|
zero = {
|
|
"Hostname": "",
|
|
"Domainname": "",
|
|
"User": "",
|
|
"AttachStdin": False,
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"OpenStdin": False,
|
|
"StdinOnce": False,
|
|
"Env": None,
|
|
"Cmd": None,
|
|
"Image": "",
|
|
"Volumes": None,
|
|
"WorkingDir": "",
|
|
"Entrypoint": None,
|
|
"OnBuild": None,
|
|
"Labels": None,
|
|
}
|
|
if not legacy_onbuild:
|
|
del zero["OnBuild"]
|
|
result = dict(members)
|
|
added = []
|
|
for row in json.loads(members["manifest.json"]):
|
|
config = json.loads(members[row["Config"]])
|
|
runtime = {**zero, **config["config"]}
|
|
parent = ""
|
|
chain = ""
|
|
diffs = config["rootfs"]["diff_ids"]
|
|
for index, diff in enumerate(diffs):
|
|
chain = (
|
|
diff
|
|
if not chain
|
|
else "sha256:" + hashlib.sha256((chain + " " + diff).encode()).hexdigest()
|
|
)
|
|
if index == len(diffs) - 1:
|
|
body = {
|
|
"created": None,
|
|
"container_config": zero,
|
|
"config": runtime,
|
|
"architecture": "amd64",
|
|
"os": "linux",
|
|
}
|
|
else:
|
|
body = {"created": "1970-01-01T00:00:00Z", "container_config": zero}
|
|
preimage = {**body, "layer_id": chain}
|
|
if parent:
|
|
preimage["parent"] = "sha256:" + parent
|
|
preimage = dict(sorted(preimage.items()))
|
|
legacy_id = hashlib.sha256(
|
|
json.dumps(preimage, separators=(",", ":")).encode()
|
|
).hexdigest()
|
|
saved = {"id": legacy_id}
|
|
if parent:
|
|
saved["parent"] = parent
|
|
saved.update(body)
|
|
saved["os"] = "linux"
|
|
payload = json.dumps(saved, separators=(",", ":")).encode()
|
|
name = "blobs/sha256/" + hashlib.sha256(payload).hexdigest()
|
|
result[name] = payload
|
|
added.append(name)
|
|
parent = legacy_id
|
|
return result, added
|
|
|
|
|
|
@pytest.mark.parametrize("legacy_onbuild", [True, False])
|
|
@pytest.mark.parametrize("layer_count", [1, 2])
|
|
def test_moby_v1_compatibility_metadata_preserves_exact_importer_bytes(
|
|
tmp_path, legacy_onbuild, layer_count
|
|
):
|
|
raw, members, images, metadata = _docker29_oci_fixture(tmp_path, layer_count=layer_count)
|
|
expected = tmp_path / "expected.tar"
|
|
producer._normalize_docker_archive(raw, expected, images, metadata=metadata)
|
|
extended, _added = _moby_v1_fixture(members, legacy_onbuild=legacy_onbuild)
|
|
raw_compat = tmp_path / "with-compat.tar"
|
|
_write_archive_fixture(raw_compat, extended)
|
|
actual = tmp_path / "actual.tar"
|
|
producer._normalize_docker_archive(raw_compat, actual, images, metadata=metadata)
|
|
assert actual.read_bytes() == expected.read_bytes()
|
|
|
|
|
|
def test_content_addressed_foreign_json_is_still_rejected(tmp_path):
|
|
_raw, members, images, metadata = _docker29_oci_fixture(tmp_path)
|
|
extended, _added = _moby_v1_fixture(members)
|
|
payload = b'{"id":"foreign","created":null,"os":"linux"}'
|
|
extended["blobs/sha256/" + hashlib.sha256(payload).hexdigest()] = payload
|
|
raw = tmp_path / "foreign.tar"
|
|
_write_archive_fixture(raw, extended)
|
|
output = tmp_path / "foreign-output.tar"
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._normalize_docker_archive(raw, output, images, metadata=metadata)
|
|
assert not output.exists()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change", ["default", "unknown", "parent", "chain", "config", "order", "id"]
|
|
)
|
|
def test_self_consistent_foreign_v1_metadata_never_extends_archive_inventory(tmp_path, change):
|
|
_raw, members, images, metadata = _docker29_oci_fixture(tmp_path, layer_count=2)
|
|
extended, added = _moby_v1_fixture(members)
|
|
original_name = added[-1]
|
|
body = json.loads(extended.pop(original_name))
|
|
first = json.loads(members["manifest.json"])[0]
|
|
diffs = json.loads(members[first["Config"]])["rootfs"]["diff_ids"]
|
|
chain = "sha256:" + hashlib.sha256((diffs[0] + " " + diffs[1]).encode()).hexdigest()
|
|
if change == "default":
|
|
body["container_config"]["Tty"] = True
|
|
elif change == "unknown":
|
|
body["unexpected"] = "fixture-sensitive-foreign-field"
|
|
elif change == "parent":
|
|
body["parent"] = "e" * 64
|
|
elif change == "chain":
|
|
chain = "sha256:" + "e" * 64
|
|
elif change == "config":
|
|
body["config"]["User"] = "root"
|
|
elif change == "order":
|
|
body["container_config"] = dict(sorted(body["container_config"].items()))
|
|
preimage = {key: value for key, value in body.items() if key not in {"id", "parent"}}
|
|
preimage.update(layer_id=chain, parent="sha256:" + body["parent"])
|
|
body["id"] = hashlib.sha256(
|
|
json.dumps(dict(sorted(preimage.items())), separators=(",", ":")).encode()
|
|
).hexdigest()
|
|
if change == "id":
|
|
body["id"] = "e" * 64
|
|
payload = json.dumps(body, separators=(",", ":")).encode()
|
|
extended["blobs/sha256/" + hashlib.sha256(payload).hexdigest()] = payload
|
|
raw = tmp_path / "spoof.tar"
|
|
_write_archive_fixture(raw, extended)
|
|
output = tmp_path / "spoof-output.tar"
|
|
with pytest.raises(producer.BuildRejected) as error:
|
|
producer._normalize_docker_archive(raw, output, images, metadata=metadata)
|
|
assert "fixture-sensitive" not in str(error.value)
|
|
assert not output.exists()
|
|
|
|
|
|
def test_docker29_oci_normalizer_rejects_unreferenced_blobs_and_config_mismatch(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
raw, members, images, metadata = _docker29_oci_fixture(tmp_path)
|
|
orphan_name = "blobs/sha256/" + "f" * 64
|
|
members[orphan_name] = b"unreferenced"
|
|
orphan_archive = tmp_path / "orphan.tar"
|
|
_write_archive_fixture(orphan_archive, members)
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._normalize_docker_archive(
|
|
orphan_archive,
|
|
tmp_path / "orphan-normalized.tar",
|
|
images,
|
|
metadata=metadata,
|
|
)
|
|
|
|
raw, _members, images, metadata = _docker29_oci_fixture(tmp_path / "mismatch")
|
|
images["web"]["imageId"] = "sha256:" + "0" * 64
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._normalize_docker_archive(
|
|
raw,
|
|
tmp_path / "mismatch-normalized.tar",
|
|
images,
|
|
metadata=metadata,
|
|
)
|
|
|
|
|
|
def test_secret_scan_summary_keeps_finding_details_out_of_persisted_json() -> None:
|
|
image_id = "sha256:" + "e" * 64
|
|
raw_secret = "super-secret-value-for-test" # noqa: S105 -- synthetic redaction fixture.
|
|
raw = json.dumps({"Metadata": {"ImageID": image_id}, "Results": [{"Secrets": []}]}).encode()
|
|
|
|
summary = producer._secret_summary(raw, image_id)
|
|
|
|
encoded = json.dumps(summary)
|
|
assert summary["FindingCount"] == 0
|
|
assert summary["SensitiveFindingDetailsPersisted"] is False
|
|
assert raw_secret not in encoded
|
|
finding = json.dumps(
|
|
{
|
|
"Metadata": {"ImageID": image_id},
|
|
"Results": [{"Secrets": [{"Match": raw_secret, "RuleID": "test"}]}],
|
|
}
|
|
).encode()
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._secret_summary(finding, image_id)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
os.name != "posix", reason="bounded scanner stream requires POSIX process groups"
|
|
)
|
|
def test_scanner_stdout_bound_does_not_cap_or_capture_cache_files(tmp_path: Path) -> None:
|
|
def fake_scanner(command: list[str], **kwargs: object) -> subprocess.Popen[bytes]:
|
|
assert "preexec_fn" not in kwargs
|
|
assert kwargs.get("stdout") == subprocess.PIPE
|
|
assert kwargs.get("umask") == 0o077
|
|
code = (
|
|
"from pathlib import Path; import os; "
|
|
"Path('cache.bin').write_bytes(b'x' * (3 * 1024 * 1024)); "
|
|
"os.write(1, b'{\"complete\":true}')"
|
|
)
|
|
return subprocess.Popen( # noqa: S603 -- executes the fixed synthetic scanner fixture.
|
|
[sys.executable, "-c", code],
|
|
**kwargs, # type: ignore[arg-type]
|
|
)
|
|
|
|
output = producer._run_scanner_capture(
|
|
["fake-trivy"],
|
|
env=os.environ,
|
|
cwd=tmp_path,
|
|
timeout=10,
|
|
popen=fake_scanner,
|
|
maximum=64,
|
|
)
|
|
|
|
assert output == b'{"complete":true}'
|
|
assert (tmp_path / "cache.bin").stat().st_size == 3 * 1024 * 1024
|
|
assert (tmp_path / "cache.bin").stat().st_mode & 0o777 == 0o600
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
os.name != "posix", reason="bounded scanner stream requires POSIX process groups"
|
|
)
|
|
def test_scanner_stdout_over_limit_is_rejected_without_persisting_bytes(tmp_path: Path) -> None:
|
|
def fake_scanner(command: list[str], **kwargs: object) -> subprocess.Popen[bytes]:
|
|
code = "import os; os.write(1, b'x' * 65)"
|
|
return subprocess.Popen( # noqa: S603 -- executes the fixed synthetic scanner fixture.
|
|
[sys.executable, "-c", code],
|
|
**kwargs, # type: ignore[arg-type]
|
|
)
|
|
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._run_scanner_capture(
|
|
["fake-trivy"],
|
|
env=os.environ,
|
|
cwd=tmp_path,
|
|
timeout=10,
|
|
popen=fake_scanner,
|
|
maximum=64,
|
|
)
|
|
assert list(tmp_path.iterdir()) == []
|
|
|
|
|
|
def test_ingestion_publication_tag_is_bound_to_authenticated_attempt() -> None:
|
|
revision = "d" * 40
|
|
first = producer._ingestion_tag(revision, 456, 1)
|
|
retry = producer._ingestion_tag(revision, 456, 2)
|
|
assert first == f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-456-attempt-1"
|
|
assert retry == f"{producer.IMAGE_PREFIX}/connector-ingestion:{revision}-run-456-attempt-2"
|
|
assert first != retry
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._ingestion_tag("../" + revision, 456, 1)
|
|
|
|
|
|
def test_registry_absence_probe_accepts_only_the_exact_missing_reference() -> None:
|
|
reference = f"{producer.IMAGE_PREFIX}/connector-ingestion:{'e' * 40}-run-7-attempt-1"
|
|
|
|
def missing(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
|
assert reference in command
|
|
return subprocess.CompletedProcess(command, 1, "", f"ERROR: {reference}: not found")
|
|
|
|
producer._registry_tag_absent(reference, {}, Path.cwd(), runner=missing)
|
|
|
|
def denied(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.CompletedProcess(command, 1, "", "unauthorized: authentication required")
|
|
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._registry_tag_absent(reference, {}, Path.cwd(), runner=denied)
|
|
|
|
def exists(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.CompletedProcess(command, 0, "{}", "")
|
|
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._registry_tag_absent(reference, {}, Path.cwd(), runner=exists)
|
|
|
|
|
|
def test_vulnerability_report_requires_all_packages_and_rejects_fixable_critical() -> None:
|
|
image_id = "sha256:" + "f" * 64
|
|
report = {
|
|
"SchemaVersion": 2,
|
|
"CreatedAt": datetime.now(UTC).isoformat(),
|
|
"Metadata": {"ImageID": image_id, "ImageConfig": {"os": "linux", "architecture": "amd64"}},
|
|
"Trivy": {"Version": producer.TRIVY_VERSION},
|
|
"Results": [{"Packages": [{"Name": "pkg", "Version": "1.0"}], "Vulnerabilities": []}],
|
|
}
|
|
producer._validate_vulnerability(report, image_id)
|
|
report["Results"][0]["Vulnerabilities"] = [{"Severity": "CRITICAL", "FixedVersion": "2.0"}]
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_vulnerability(report, image_id)
|
|
report["Results"][0]["Vulnerabilities"] = []
|
|
report["Results"][0]["Packages"] = []
|
|
with pytest.raises(producer.BuildRejected):
|
|
producer._validate_vulnerability(report, image_id)
|