Files
Unraid-AppOps-Gateway-Public/backend/tests/test_hub_candidate_import.py
T

1171 lines
41 KiB
Python

from __future__ import annotations
import hashlib
import importlib
import io
import json
import contextlib
import stat
import struct
import sys
import tarfile
import types
import zipfile
from datetime import datetime, timedelta, timezone
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
BROKER_DIR = ROOT / "ops" / "gitea-deploy-broker"
@pytest.fixture
def modules(monkeypatch):
if "fcntl" not in sys.modules:
shim = types.ModuleType("fcntl")
shim.LOCK_EX, shim.LOCK_UN, shim.LOCK_NB = 2, 8, 4
shim.flock = lambda *_args: None
monkeypatch.setitem(sys.modules, "fcntl", shim)
monkeypatch.syspath_prepend(str(BROKER_DIR))
for name in ("deploy", "prepare_mcp_hub_candidate", "hub_candidate_import"):
sys.modules.pop(name, None)
controller = importlib.import_module("deploy")
prepare = importlib.import_module("prepare_mcp_hub_candidate")
importer = importlib.import_module("hub_candidate_import")
monkeypatch.setattr(prepare, "_is_host_root", lambda: True)
yield types.SimpleNamespace(
controller=controller, prepare=prepare, importer=importer
)
for name in ("hub_candidate_import", "prepare_mcp_hub_candidate", "deploy"):
sys.modules.pop(name, None)
def _zip_stream_member(name: str, payload: bytes, offset: int) -> tuple[bytes, bytes]:
encoded_name = name.encode("ascii")
crc = zipfile.crc32(payload) & 0xFFFFFFFF
local = struct.pack(
"<IHHHHHIIIHH",
0x04034B50,
20,
0x8,
0,
0,
0,
0,
0,
0,
len(encoded_name),
0,
) + encoded_name + payload + struct.pack(
"<IIII", 0x08074B50, crc, len(payload), len(payload)
)
central = struct.pack(
"<IHHHHHHIIIHHHHHII",
0x02014B50,
20,
20,
0x8,
0,
0,
0,
crc,
len(payload),
len(payload),
len(encoded_name),
0,
0,
0,
0,
(stat.S_IFREG | 0o600) << 16,
offset,
) + encoded_name
return local, central
def _streamed_zip_bytes(importer, *, extra_member: bool = False) -> bytearray:
names = sorted(importer._expected_zip_members())
if extra_member:
names.append("unexpected.txt")
members = [(name, b"{}" if name != "images.tar" else b"tar-data") for name in names]
output = bytearray()
centrals: list[bytes] = []
for name, payload in members:
local, central = _zip_stream_member(name, payload, len(output))
output.extend(local)
centrals.append(central)
central_offset = len(output)
for central in centrals:
output.extend(central)
central_size = len(output) - central_offset
output.extend(
struct.pack(
"<IHHHHIIH",
0x06054B50,
0,
0,
len(members),
len(members),
central_size,
central_offset,
0,
)
)
return output
def _private_file(path: Path, data: bytes) -> None:
path.write_bytes(data)
path.chmod(0o600)
def _synthetic_scan_reports(
services, images, *, created_at: str, finding_service: str | None = None
) -> dict[str, bytes]:
report_data: dict[str, bytes] = {}
for service in services:
image_id = images[service]["imageId"]
vulnerabilities = (
[
{
"Severity": "HIGH",
"PkgName": "package",
"VulnerabilityID": "CVE-2099-0001",
"InstalledVersion": "1.0",
}
]
if service == finding_service
else []
)
vulnerability = {
"SchemaVersion": 2,
"Metadata": {
"ImageID": image_id,
"ImageConfig": {"os": "linux", "architecture": "amd64"},
},
"CreatedAt": created_at,
"Trivy": {"Version": "0.65.0"},
"Results": [
{
"Packages": [{"Name": "package", "Version": "1.0"}],
"Vulnerabilities": vulnerabilities,
}
],
}
secret = {
"ImageID": image_id,
"CreatedAt": created_at,
"TrivyVersion": "0.65.0",
"FindingCount": 0,
"SensitiveFindingDetailsPersisted": False,
"Results": [{"FindingCount": 0}],
}
report_data[f"vulnerability-{service}.json"] = json.dumps(
vulnerability
).encode()
report_data[f"secret-{service}.json"] = json.dumps(secret).encode()
report_data["trivy-db-metadata.json"] = json.dumps(
{"UpdatedAt": created_at, "DownloadedAt": created_at}
).encode()
return report_data
def test_run_identity_requires_exact_protected_push_and_no_pull_request(modules):
importer = modules.importer
revision = "a" * 40
run = {
"path": importer.RUN_PATH,
"event": "push",
"head_branch": "main",
"head_sha": revision,
"repository": {"full_name": "Jens/ITWorx_MCP_Hub"},
"head_repository": None,
"pull_requests": [],
}
assert importer._run_matches(run, revision)
for change in (
{"event": "workflow_dispatch"},
{"path": "other.yml@refs/heads/main"},
{"head_repository": {"full_name": "attacker/fork"}},
{"pull_requests": [{"number": 12}]},
{"head_sha": "b" * 40},
{"repository": {"full_name": "attacker/repository"}},
):
assert not importer._run_matches({**run, **change}, revision)
def test_connector_ingestion_binding_is_scoped_to_authenticated_run_attempt(modules):
controller = modules.controller
revision = "a" * 40
run_id, attempt = 21531, 2
repository = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES["connector-ingestion"]
archive_ref = f"{repository}:{revision}-run-{run_id}-attempt-{attempt}"
digest = "d" * 64
image_ref = f"{archive_ref}@sha256:{digest}"
image_id = "sha256:" + "e" * 64
binding = controller.mcp_hub_connector_ingestion_binding(
revision, run_id, attempt, image_ref, image_id
)
assert binding == {
"workflowRunId": run_id,
"workflowRunAttempt": attempt,
"archiveRef": archive_ref,
"imageRef": image_ref,
"imageId": image_id,
}
for invalid_ref in (
f"{repository}:{revision}-run-{run_id}@sha256:{digest}",
f"{repository}:{revision}-run-{run_id}-attempt-1@sha256:{digest}",
f"attacker.example/{repository.rsplit('/', 1)[-1]}:{revision}-run-{run_id}-attempt-{attempt}@sha256:{digest}",
f"{archive_ref}@sha256:{'F' * 64}",
f"{archive_ref}@sha256:{digest}:extra",
):
with pytest.raises(controller.DeployError):
controller.mcp_hub_connector_ingestion_binding(
revision, run_id, attempt, invalid_ref, image_id
)
with pytest.raises(controller.DeployError):
controller.mcp_hub_connector_ingestion_binding(
revision, run_id, 0, image_ref, image_id
)
def test_candidate_ingestion_binding_overrides_only_fixed_hub_render_environment(
modules,
):
controller = modules.controller
revision, tree = "a" * 40, "b" * 40
run_id, attempt = 21531, 2
repository = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES["connector-ingestion"]
archive_ref = f"{repository}:{revision}-run-{run_id}-attempt-{attempt}"
image_ref = f"{archive_ref}@sha256:{'d' * 64}"
binding = controller.mcp_hub_connector_ingestion_binding(
revision, run_id, attempt, image_ref, "sha256:" + "e" * 64
)
project = {
"compose_project": controller.MCP_HUB_PROJECT,
"_mcp_hub_contract": controller.MCP_HUB_TOPOLOGY_CONTRACT,
"_mcp_hub_connector_ingestion_binding": binding,
"environment": {"CONNECTOR_INGESTION_IMAGE": "attacker.invalid/image:latest"},
}
metadata = {
"id": revision[:12],
"repository": controller.MCP_HUB_REPOSITORY,
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-10-03T12:00:00.000Z",
}
environment = controller.deployment_environment(
project, revision, controller.MCP_HUB_REPOSITORY, metadata
)
assert environment["CONNECTOR_INGESTION_IMAGE"] == image_ref
project["_mcp_hub_connector_ingestion_binding"]["archiveRef"] += "-tampered"
with pytest.raises(controller.DeployError):
controller.deployment_environment(
project, revision, controller.MCP_HUB_REPOSITORY, metadata
)
def test_registered_hub_cannot_supply_private_ingestion_binding(modules):
controller = modules.controller
raw = {
"branch": "main",
"slug": controller.MCP_HUB_PROJECT,
"compose_project": controller.MCP_HUB_PROJECT,
"containers": list(controller.MCP_HUB_LEGACY_CONTAINERS),
"source_dir": (
controller.MCP_HUB_APPDATA_ROOT / "deployment" / "repository"
).as_posix(),
"compose_dir": "infrastructure/compose",
"compose_files": list(controller.MCP_HUB_LEGACY_COMPOSE_FILES),
"env_source": (controller.MCP_HUB_APPDATA_ROOT / "deploy.env").as_posix(),
"_mcp_hub_connector_ingestion_binding": {"imageRef": "attacker.invalid/x"},
}
effective = controller.effective_mcp_hub_project(
controller.MCP_HUB_REPOSITORY, raw
)
assert "_mcp_hub_connector_ingestion_binding" not in effective
def test_import_manifest_ingestion_binding_rejects_wrong_run_or_archive_tag(modules):
importer = modules.importer
revision, tree = "a" * 40, "b" * 40
run = {"id": 21531, "run_attempt": 2}
repository = importer.controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES[
"connector-ingestion"
]
archive_ref = f"{repository}:{revision}-run-{run['id']}-attempt-{run['run_attempt']}"
row = {
"imageRef": f"{archive_ref}@sha256:{'d' * 64}",
"archiveRef": archive_ref,
"imageId": "sha256:" + "e" * 64,
}
manifest = {
"schema": importer.ARTIFACT_SCHEMA,
"repository": importer.REPOSITORY,
"sourceRevision": revision,
"sourceTree": tree,
"workflowPath": importer.WORKFLOW_PATH,
"workflowRunId": run["id"],
"artifactName": importer.ARTIFACT_PREFIX + revision,
"images": {
service: row if service == "connector-ingestion" else {}
for service in importer.controller.MCP_HUB_SERVICES
},
}
binding = importer._manifest_ingestion_binding(
manifest, revision=revision, tree=tree, run=run
)
assert binding["imageRef"] == row["imageRef"]
assert binding["archiveRef"] == archive_ref
for changed in (
{**manifest, "workflowRunId": run["id"] + 1},
{
**manifest,
"images": {
**manifest["images"],
"connector-ingestion": {
**row,
"archiveRef": archive_ref.replace("-attempt-2", "-attempt-1"),
},
},
},
):
with pytest.raises(importer.CandidateRejected):
importer._manifest_ingestion_binding(
changed, revision=revision, tree=tree, run=run
)
@pytest.mark.parametrize("drift", [None, "image_id", "repo_digest", "provenance"])
def test_import_immutably_pulls_and_reads_back_connector_ingestion(
modules, monkeypatch, drift
):
importer = modules.importer
controller = modules.controller
revision, tree, run_id, attempt = "a" * 40, "b" * 40, 21531, 2
built_at = "2026-10-03T12:00:00.000Z"
build = {
"id": revision[:12],
"repository": controller.MCP_HUB_REPOSITORY,
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": built_at,
}
labels = {
"org.opencontainers.image.source": controller.MCP_HUB_OCI_IMAGE_SOURCE,
"org.opencontainers.image.revision": revision,
"org.opencontainers.image.source-tree": tree,
"org.opencontainers.image.version": revision[:12],
"org.opencontainers.image.created": built_at,
"org.opencontainers.image.source-date-epoch": "1790985600",
}
images = {}
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1):
external = importer.EXTERNAL_IMAGES.get(service)
if external:
image_ref, archive_ref = external[0], external[1]
elif service == "connector-ingestion":
repo = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES[service]
archive_ref = f"{repo}:{revision}-run-{run_id}-attempt-{attempt}"
image_ref = f"{archive_ref}@sha256:{'d' * 64}"
elif service == "contextforge":
image_ref = archive_ref = f"itworx-mcp-hub/contextforge-schema:{revision}"
else:
image_ref = archive_ref = f"fixture/{service}:{revision}"
images[service] = {
"imageRef": image_ref,
"archiveRef": archive_ref,
"imageId": "sha256:" + f"{index:064x}",
}
ingestion = images["connector-ingestion"]
immutable_ref = ingestion["imageRef"]
repo = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES["connector-ingestion"]
immutable_inspect = {
"Id": ingestion["imageId"],
"RepoDigests": [f"{repo}@sha256:{'d' * 64}"],
"Config": {"User": "10001:10001", "Labels": dict(labels)},
}
tagged_inspect = {"Id": ingestion["imageId"], "Config": {"Labels": dict(labels)}}
if drift == "image_id":
immutable_inspect["Id"] = "sha256:" + "f" * 64
elif drift == "repo_digest":
immutable_inspect["RepoDigests"] = []
elif drift == "provenance":
immutable_inspect["Config"]["Labels"][
"org.opencontainers.image.source-tree"
] = "c" * 40
monkeypatch.setattr(
importer,
"_inspect_image",
lambda ref: tagged_inspect if ref == ingestion["archiveRef"] else immutable_inspect,
)
monkeypatch.setattr(
importer,
"_require_loaded_image",
lambda _service, row, **_kwargs: {
"Id": row["imageId"],
"Config": {"User": "10001:10001", "Labels": dict(labels)},
},
)
docker_calls = []
monkeypatch.setattr(
importer,
"_fixed_docker",
lambda *args, **kwargs: docker_calls.append((args, kwargs)),
)
if drift is None:
result = importer._verify_loaded_images(
images,
revision,
build,
"1790985600",
pull_external=True,
ingestion_run_id=run_id,
ingestion_run_attempt=attempt,
)
assert result["connector-ingestion"] == ingestion["imageId"]
assert any(
call[0] == ("pull", immutable_ref)
for call in docker_calls
)
else:
with pytest.raises(importer.CandidateRejected):
importer._verify_loaded_images(
images,
revision,
build,
"1790985600",
pull_external=True,
ingestion_run_id=run_id,
ingestion_run_attempt=attempt,
)
def test_selected_run_and_jobs_accept_the_live_gitea_job_shape(modules):
importer = modules.importer
revision = "a" * 40
now = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace(
"+00:00", "Z"
)
run = {
"id": 21531,
"path": importer.RUN_PATH,
"event": "push",
"head_branch": "main",
"head_sha": revision,
"repository": {"full_name": "Jens/ITWorx_MCP_Hub"},
"head_repository": None,
"pull_requests": [],
"created_at": now,
"run_attempt": 1,
"status": "completed",
"conclusion": "success",
}
job_rows = [
{
"id": 37145 + index,
"name": name,
"run_id": 21531,
"status": "completed",
"conclusion": "success",
"runner_id": 22,
"started_at": now,
"completed_at": now,
}
for index, name in enumerate(("full", "release-images"))
]
calls = []
def api(path, *, query):
calls.append((path, query))
if path.endswith("/actions/runs"):
return {"total_count": 1, "workflow_runs": [run]}
if path.endswith("/actions/runs/21531/jobs"):
return {"total_count": 2, "jobs": job_rows}
raise AssertionError(f"unexpected fixed API path: {path}")
selected, built_at = importer._selected_run(api, revision)
assert selected == run
assert built_at.endswith("Z")
assert importer._jobs_ready(api, revision, selected)
assert calls[0][1] == {
"event": "push",
"branch": "main",
"head_sha": revision,
"page": "1",
"limit": "100",
}
assert calls[1][1] == {"page": "1", "limit": "100"}
def test_required_candidate_jobs_must_both_succeed_and_bind_run(modules):
importer = modules.importer
run = {
"id": 5,
"status": "completed",
"conclusion": "success",
"run_attempt": 1,
}
rows = [
{"name": name, "run_id": 5, "status": "completed", "conclusion": "success"}
for name in ("full", "release-images")
]
def api(_path, *, query):
assert query == {"page": "1", "limit": "100"}
return {"total_count": len(rows), "jobs": rows}
assert importer._jobs_ready(api, "a" * 40, run)
rows[1] = {**rows[1], "conclusion": "failure"}
with pytest.raises(importer.CandidateRejected):
importer._jobs_ready(api, "a" * 40, run)
def test_recheck_rejects_a_new_attempt_for_the_same_workflow_run(modules, monkeypatch):
importer = modules.importer
revision = "a" * 40
original = {"id": 21531, "run_attempt": 1}
restarted = {"id": 21531, "run_attempt": 2}
artifact = {"id": 730}
monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: "b" * 40)
monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None)
monkeypatch.setattr(
importer,
"_selected_run",
lambda *_args: (restarted, "2026-10-03T10:00:00.000Z"),
)
monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True)
with pytest.raises(importer.CandidateRejected, match="run changed"):
importer._recheck_protected_artifact(
lambda *_args, **_kwargs: None,
revision=revision,
tree="b" * 40,
initial_run=original,
initial_built_at="2026-10-03T10:00:00.000Z",
initial_artifact=artifact,
)
def test_unfixable_vulnerability_finding_is_bound_to_fixed_image_reference(modules):
importer = modules.importer
controller = modules.controller
created_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace(
"+00:00", "Z"
)
images = {
service: {
"imageId": f"sha256:{index:064x}",
"imageRef": f"registry.example/{service}:fixed",
}
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1)
}
report_data = _synthetic_scan_reports(
controller.MCP_HUB_SERVICES,
images,
created_at=created_at,
finding_service=controller.MCP_HUB_SERVICES[0],
)
_reports, findings = importer._validate_reports(images, report_data)
assert len(findings) == 1
assert findings[0]["imageRef"] == images[controller.MCP_HUB_SERVICES[0]]["imageRef"]
summary = importer._candidate_findings_summary(findings, images)
assert summary["unresolvedCriticalOrHigh"][0]["affectedImages"] == [
findings[0]["imageRef"]
]
@pytest.mark.parametrize("finding_present", [False, True])
@pytest.mark.parametrize("rerun_during_load", [False, True])
def test_fake_local_import_publishes_candidate_and_requires_review_for_findings(
modules, monkeypatch, tmp_path, finding_present, rerun_during_load
):
importer = modules.importer
prepare = modules.prepare
controller = modules.controller
revision, tree, epoch = "a" * 40, "b" * 40, "1790985600"
built_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace(
"+00:00", "Z"
)
build = {
"id": revision[:12],
"repository": importer.REPOSITORY,
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": built_at,
}
compose_sha = "c" * 64
model_services = {}
images = {}
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1):
external = importer.EXTERNAL_IMAGES.get(service)
if external is not None:
image_ref, archive_ref = external[0], external[1]
elif service == "connector-ingestion":
repository = controller.MCP_HUB_OWNED_IMAGE_REPOSITORIES[service]
archive_ref = f"{repository}:{revision}-run-21531-attempt-2"
image_ref = f"{archive_ref}@sha256:{'f' * 64}"
else:
image_ref = f"registry.example/itworx/{service}:{revision[:12]}"
archive_ref = image_ref
model_services[service] = {"image": image_ref}
images[service] = {
"imageRef": image_ref,
"archiveRef": archive_ref,
"imageId": f"sha256:{index:064x}",
}
model = {"services": model_services}
created_at = built_at
finding_service = controller.MCP_HUB_SERVICES[0] if finding_present else None
report_data = _synthetic_scan_reports(
controller.MCP_HUB_SERVICES,
images,
created_at=created_at,
finding_service=finding_service,
)
_reports, findings = importer._validate_reports(images, report_data)
summary = importer._candidate_findings_summary(findings, images)
digests = {
"images": {
service: {
"imageId": images[service]["imageId"],
"tag": images[service]["imageRef"],
}
for service in controller.MCP_HUB_SERVICES
}
}
exception_rows = [
{
"package": finding["package"],
"cve": finding["cve"],
"installedVersion": finding["installedVersion"],
"severity": finding["severity"],
"affectedImageDigests": {
finding["imageRef"]: images_by_ref["imageId"]
for images_by_ref in [
next(
row
for row in images.values()
if row["imageRef"] == finding["imageRef"]
)
]
},
}
for finding in findings
]
draft = {"schemaVersion": "1.0", "exceptions": exception_rows}
state_root = tmp_path / "state"
state_root.mkdir(mode=0o700)
state_root.chmod(0o700)
releases_root = tmp_path / "releases"
source = releases_root / revision
source.mkdir(parents=True)
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases_root)
monkeypatch.setattr(controller, "GLOBAL_DEPLOY_LOCK", state_root / "deploy.lock")
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda *_args: None)
monkeypatch.setattr(controller, "_root_owned", lambda *_args: True)
def create_private_directory(path, *, create=False, root=None):
del root
if create:
path.mkdir(mode=0o700, parents=True, exist_ok=True)
path.chmod(0o700)
if path.is_symlink() or not path.is_dir():
raise controller.DeployError("test private directory missing")
def candidate_directory(candidate_revision):
directory = state_root / "mcp-hub-release-candidates" / candidate_revision
for path in (directory.parent, directory):
if path.is_symlink() or not path.is_dir():
raise controller.DeployError("test candidate directory missing")
return directory
monkeypatch.setattr(controller, "_mcp_hub_private_directory", create_private_directory)
monkeypatch.setattr(controller, "_mcp_hub_candidate_directory", candidate_directory)
project, repo = {"branch": "main"}, tmp_path / "repo"
monkeypatch.setattr(prepare, "_fixed_target", lambda *_args: (project, repo))
monkeypatch.setattr(controller, "deployment_metadata", lambda *_args, **_kwargs: build)
monkeypatch.setattr(controller, "mcp_hub_source_date_epoch", lambda *_args: epoch)
monkeypatch.setattr(controller, "mcp_hub_stage_release_source", lambda *_args: source)
monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True)
monkeypatch.setattr(controller, "_mcp_hub_verify_release_tree_metadata", lambda *_args: None)
monkeypatch.setattr(
prepare,
"_render",
lambda *_args, **_kwargs: (model, compose_sha),
)
monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: tree)
monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None)
run = {"id": 21531, "run_attempt": 2}
run_state = {"attempt": 2}
monkeypatch.setattr(
importer,
"_selected_run",
lambda *_args: ({**run, "run_attempt": run_state["attempt"]}, built_at),
)
monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True)
artifact_bytes = b"bounded fake Gitea artifact"
artifact = {
"id": 730,
"size_in_bytes": len(artifact_bytes),
"name": importer.ARTIFACT_PREFIX + revision,
"workflow_run": {"id": run["id"]},
}
monkeypatch.setattr(importer, "_candidate_artifact", lambda *_args: artifact)
monkeypatch.setattr(importer, "_docker_root", lambda: tmp_path)
monkeypatch.setattr(importer, "_check_disk_headroom", lambda **_kwargs: None)
artifact_manifest = {
"schema": importer.ARTIFACT_SCHEMA,
"repository": importer.REPOSITORY,
"sourceRevision": revision,
"sourceTree": tree,
"workflowPath": importer.WORKFLOW_PATH,
"workflowRunId": run["id"],
"artifactName": importer.ARTIFACT_PREFIX + revision,
"images": {
service: {
"imageRef": row["imageRef"],
"imageId": row["imageId"],
"archiveRef": row["archiveRef"],
}
for service, row in images.items()
},
}
def fake_parse_artifact(*_args, **_kwargs):
return (
artifact_manifest,
report_data,
hashlib.sha256(artifact_bytes).hexdigest(),
"d" * 64,
"e" * 64,
123,
)
monkeypatch.setattr(importer, "_parse_artifact_zip", fake_parse_artifact)
monkeypatch.setattr(
importer,
"_validate_manifest",
lambda *_args, **_kwargs: (images, {}),
)
monkeypatch.setattr(importer, "_validate_docker_save", lambda *_args, **_kwargs: None)
monkeypatch.setattr(importer, "_bounded_global_deploy_lock", contextlib.nullcontext)
monkeypatch.setattr(importer, "_image_preflight", lambda *_args: None)
def fake_fixed_docker(*args, **_kwargs):
if rerun_during_load and args and args[0] == "load":
run_state["attempt"] = 3
return object()
monkeypatch.setattr(importer, "_fixed_docker", fake_fixed_docker)
verified_runs = []
def verify_loaded(_images, _revision, _build, _epoch, **kwargs):
verified_runs.append(kwargs)
return {}
monkeypatch.setattr(importer, "_verify_loaded_images", verify_loaded)
monkeypatch.setattr(
controller,
"_mcp_hub_check_candidate_exceptions",
lambda *_args: None,
)
monkeypatch.setattr(
prepare,
"_draft_pending_exceptions",
lambda *_args: draft,
)
planned_bindings = []
def fake_plan(
_repository,
_revision,
*,
built_at,
staging_directory,
connector_ingestion_binding,
**_kwargs,
):
planned_bindings.append(connector_ingestion_binding)
Path(staging_directory).mkdir(mode=0o700)
frozen = {
"schema": prepare.BUILD_SCHEMA,
"repository": importer.REPOSITORY,
"revision": revision,
"build": build,
"sourceDateEpoch": epoch,
"composeSha256": compose_sha,
}
controller._mcp_hub_write_private_new(
Path(staging_directory) / "build.json", prepare._json_bytes(frozen)
)
return {"status": "planned", "revision": revision, "composeSha256": compose_sha}
monkeypatch.setattr(prepare, "plan_candidate", fake_plan)
def fake_capture(
_repository,
_revision,
*,
staging_directory,
connector_ingestion_binding,
**_kwargs,
):
assert connector_ingestion_binding == planned_bindings[0]
image_rows = {
service: {
"imageRef": images[service]["imageRef"],
"imageId": images[service]["imageId"],
"vulnerabilitySha256": hashlib.sha256(
report_data[f"vulnerability-{service}.json"]
).hexdigest(),
"secretSha256": hashlib.sha256(
report_data[f"secret-{service}.json"]
).hexdigest(),
}
for service in controller.MCP_HUB_SERVICES
}
candidate = {
"schema": prepare.CANDIDATE_SCHEMA,
"repository": importer.REPOSITORY,
"revision": revision,
"build": build,
"sourceDateEpoch": epoch,
"composeSha256": compose_sha,
"images": image_rows,
"databaseMetadataSha256": hashlib.sha256(
report_data["trivy-db-metadata.json"]
).hexdigest(),
"summarySha256": hashlib.sha256(prepare._json_bytes(summary)).hexdigest(),
"digestsSha256": hashlib.sha256(prepare._json_bytes(digests)).hexdigest(),
}
outputs = {
"candidate.json": candidate,
"scan-summary.json": summary,
"image-digests.json": digests,
"exceptions.json": draft,
}
for filename, value in outputs.items():
controller._mcp_hub_write_private_new(
Path(staging_directory) / filename, prepare._json_bytes(value)
)
candidate_sha = hashlib.sha256(prepare._json_bytes(candidate)).hexdigest()
exceptions_sha = hashlib.sha256(prepare._json_bytes(draft)).hexdigest()
return {
"candidateSha256": candidate_sha,
"exceptionsSha256": exceptions_sha,
}
monkeypatch.setattr(prepare, "capture_candidate", fake_capture)
def download_artifact(_artifact_id, destination, _maximum):
_private_file(destination, artifact_bytes)
return None
def fake_no_overwrite_publish(stage, final_directory):
if final_directory.exists() or final_directory.is_symlink():
importer._reject(
"MCP Hub candidate destination already exists; import never overwrites"
)
final_directory.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
stage.rename(final_directory)
monkeypatch.setattr(importer, "_publish_candidate", fake_no_overwrite_publish)
candidate_directory = state_root / "mcp-hub-release-candidates" / revision
if rerun_during_load:
with pytest.raises(importer.CandidateRejected, match="run changed"):
importer.prepare_candidate(
importer.REPOSITORY,
revision,
api_json=lambda *_args, **_kwargs: None,
download_artifact=download_artifact,
)
assert not candidate_directory.exists()
return
first = importer.prepare_candidate(
importer.REPOSITORY,
revision,
api_json=lambda *_args, **_kwargs: None,
download_artifact=download_artifact,
)
assert candidate_directory.is_dir()
assert planned_bindings[0]["workflowRunId"] == run["id"]
assert planned_bindings[0]["workflowRunAttempt"] == run["run_attempt"]
assert verified_runs == [
{
"pull_external": True,
"ingestion_run_id": run["id"],
"ingestion_run_attempt": run["run_attempt"],
},
{
"pull_external": False,
"ingestion_run_id": run["id"],
"ingestion_run_attempt": run["run_attempt"],
},
]
if importer.os.name != "nt":
assert stat.S_IMODE(candidate_directory.stat().st_mode) == 0o700
candidate_before_review = (candidate_directory / "candidate.json").read_bytes()
receipt_before_review = (candidate_directory / "import-receipt.json").read_bytes()
collision_stage = tmp_path / "second-candidate"
collision_stage.mkdir(mode=0o700)
with pytest.raises(importer.CandidateRejected, match="never overwrites"):
importer._publish_candidate(collision_stage, candidate_directory)
assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review
if finding_present:
assert first["status"] == "pending_review"
reviewed = {
**draft,
"reviewer": "release-operator",
"exceptions": [
{**row, "justification": "Reviewed unfixable finding"}
for row in draft["exceptions"]
],
}
exceptions_path = candidate_directory / "exceptions.json"
exceptions_path.unlink()
controller._mcp_hub_write_private_new(
exceptions_path, prepare._json_bytes(reviewed)
)
approval = {
"schema": importer.APPROVAL_SCHEMA,
"candidateSha256": first["candidateSha256"],
"exceptionsSha256": hashlib.sha256(
prepare._json_bytes(reviewed)
).hexdigest(),
}
controller._mcp_hub_write_private_new(
candidate_directory / "approval.json", prepare._json_bytes(approval)
)
final = importer.prepare_candidate(
importer.REPOSITORY,
revision,
api_json=lambda *_args, **_kwargs: None,
download_artifact=download_artifact,
)
assert final["status"] == "ready"
else:
assert first["status"] == "ready"
final = first
assert final["candidateSha256"] == hashlib.sha256(candidate_before_review).hexdigest()
assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review
assert (candidate_directory / "import-receipt.json").read_bytes() == receipt_before_review
def test_candidate_artifact_is_exact_bounded_and_unexpired(modules):
importer = modules.importer
revision = "c" * 40
run_id = 11
artifact = {
"id": 22,
"name": importer.ARTIFACT_PREFIX + revision,
"size_in_bytes": 4096,
"expired": False,
"expires_at": (datetime.now(timezone.utc) + timedelta(hours=1))
.isoformat()
.replace("+00:00", "Z"),
"workflow_run": {"id": run_id},
}
result = importer._candidate_artifact(
lambda _path, *, query: {"artifacts": [artifact]}, run_id, revision
)
assert result == artifact
for changed in (
{"expired": True},
{"name": "hub-release-candidate-" + "d" * 40},
{"size_in_bytes": importer.MAX_ARTIFACT_ZIP_BYTES + 1},
{"workflow_run": {"id": run_id + 1}},
):
with pytest.raises(importer.CandidateRejected):
importer._candidate_artifact(
lambda _path, *, query: {"artifacts": [{**artifact, **changed}]},
run_id,
revision,
)
def test_streamed_uploader_zip_descriptors_are_verified(modules, tmp_path):
importer = modules.importer
payload = _streamed_zip_bytes(importer)
zip_path = tmp_path / "candidate.zip"
tar_path = tmp_path / "images.tar"
_private_file(zip_path, bytes(payload))
(
manifest,
reports,
zip_sha,
manifest_sha,
tar_sha,
tar_bytes,
) = importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(payload))
assert manifest == {}
assert len(reports) == 35
assert zip_sha == hashlib.sha256(payload).hexdigest()
assert manifest_sha == hashlib.sha256(b"{}").hexdigest()
assert tar_sha == hashlib.sha256(b"tar-data").hexdigest()
assert tar_bytes == len(b"tar-data")
assert tar_path.read_bytes() == b"tar-data"
tampered = bytearray(payload)
name_size = len("candidate-manifest.json")
descriptor_offset = 30 + name_size + 2
tampered[descriptor_offset + 4] ^= 0x01
_private_file(zip_path, bytes(tampered))
with pytest.raises(importer.CandidateRejected, match="descriptor"):
importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(tampered))
def test_zip_central_directory_is_bounded_before_zipfile_parse(modules, tmp_path, monkeypatch):
importer = modules.importer
payload = _streamed_zip_bytes(importer, extra_member=True)
path = tmp_path / "too-many.zip"
_private_file(path, bytes(payload))
monkeypatch.setattr(
importer.zipfile,
"ZipFile",
lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized member inventory"),
)
with pytest.raises(importer.CandidateRejected, match="central directory"):
importer._zip_central_directory_preflight(path, len(payload))
def test_zip_oversized_central_directory_is_rejected_without_allocating_it(
modules, tmp_path, monkeypatch
):
importer = modules.importer
payload = struct.pack(
"<IHHHHIIH",
0x06054B50,
0,
0,
37,
37,
65 * 1024,
0,
0,
)
path = tmp_path / "oversized-directory.zip"
_private_file(path, payload)
monkeypatch.setattr(
importer.zipfile,
"ZipFile",
lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized central directory"),
)
with pytest.raises(importer.CandidateRejected, match="central directory"):
importer._zip_central_directory_preflight(path, len(payload))
def test_tar_member_count_is_stopped_during_streamed_header_walk(
modules, tmp_path, monkeypatch
):
importer = modules.importer
path = tmp_path / "many-members.tar"
with tarfile.open(path, "w", format=tarfile.USTAR_FORMAT) as archive:
for name in ("one/", "two/"):
info = tarfile.TarInfo(name)
info.type = tarfile.DIRTYPE
archive.addfile(info)
monkeypatch.setattr(importer, "MAX_TAR_MEMBERS", 1)
with pytest.raises(importer.CandidateRejected, match="member-count"):
importer._preflight_uncompressed_tar(path)
def test_tar_preflight_rejects_extension_headers_before_tarfile(modules, tmp_path):
importer = modules.importer
path = tmp_path / "pax.tar"
with tarfile.open(path, "w", format=tarfile.PAX_FORMAT) as archive:
info = tarfile.TarInfo("x" * 120)
archive.addfile(info, io.BytesIO(b"x"))
with pytest.raises(importer.CandidateRejected, match="unsupported member type"):
importer._preflight_uncompressed_tar(path)
def test_remote_head_without_completed_candidate_stays_awaiting(modules, tmp_path, monkeypatch):
controller = modules.controller
prepare = modules.prepare
importer = modules.importer
revision = "a" * 40
tree = "b" * 40
monkeypatch.setattr(
prepare, "_fixed_target", lambda _repository, _revision: ({}, tmp_path)
)
responses = {
"/repos/jens/itworx_mcp_hub/branches/main": {"commit": {"id": revision}},
f"/repos/jens/itworx_mcp_hub/git/commits/{revision}": {
"sha": revision,
"commit": {"tree": {"sha": tree}},
},
importer.WORKFLOW_LIST_PATH: {
"total_count": 1,
"workflows": [
{
"id": importer.WORKFLOW_ID,
"path": importer.WORKFLOW_PATH,
"state": "active",
}
],
},
"/repos/jens/itworx_mcp_hub/actions/runs": {
"total_count": 0,
"workflow_runs": [],
},
}
calls = []
def api(path, *, query):
calls.append((path, query))
return responses[path]
result = importer.prepare_candidate(
controller.MCP_HUB_REPOSITORY,
revision,
api_json=api,
download_artifact=lambda *_args: pytest.fail("an absent run cannot download artifacts"),
)
assert result == {"status": "awaiting_candidate", "revision": revision}
assert len(calls) == 4
def test_fixed_compose_environment_discards_ambient_docker_and_compose_context(
modules, monkeypatch
):
controller = modules.controller
importer = modules.importer
monkeypatch.setattr(
controller,
"deployment_environment",
lambda *_args: {
"PATH": "/usr/bin",
"DOCKER_HOST": "tcp://peer:2375",
"DOCKER_CONTEXT": "attacker",
"DOCKER_TLS_VERIFY": "1",
"COMPOSE_FILE": "/tmp/untrusted.yml",
"COMPOSE_PROFILES": "untrusted",
},
)
env = importer._fixed_compose_environment({"compose_project": "fixed"}, "a" * 40, "jens/itworx_mcp_hub")
assert env == {
"PATH": "/usr/bin",
"DOCKER_HOST": importer.DOCKER_HOST,
"COMPOSE_PROJECT_NAME": "fixed",
"COMPOSE_PROFILES": ",".join(controller.MCP_HUB_COMPOSE_PROFILES),
}