Publish curated Unraid-AppOps-Gateway source from a506550ef83ef2d81651e37c98a89f7ee330c730
This commit is contained in:
Public Source Publisher committed
1 parent
2facb8de34
commit
767e66921e
68 files changed
+15924
-313
No files matched your search
@@ -0,0 +1,837 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import io
|
||||
import json
|
||||
import contextlib
|
||||
import stat
|
||||
import struct
|
||||
import sys
|
||||
import tarfile
|
||||
import types
|
||||
import zipfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
BROKER_DIR = ROOT / "ops" / "gitea-deploy-broker"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def modules(monkeypatch):
|
||||
if "fcntl" not in sys.modules:
|
||||
shim = types.ModuleType("fcntl")
|
||||
shim.LOCK_EX, shim.LOCK_UN, shim.LOCK_NB = 2, 8, 4
|
||||
shim.flock = lambda *_args: None
|
||||
monkeypatch.setitem(sys.modules, "fcntl", shim)
|
||||
monkeypatch.syspath_prepend(str(BROKER_DIR))
|
||||
for name in ("deploy", "prepare_mcp_hub_candidate", "hub_candidate_import"):
|
||||
sys.modules.pop(name, None)
|
||||
controller = importlib.import_module("deploy")
|
||||
prepare = importlib.import_module("prepare_mcp_hub_candidate")
|
||||
importer = importlib.import_module("hub_candidate_import")
|
||||
monkeypatch.setattr(prepare, "_is_host_root", lambda: True)
|
||||
yield types.SimpleNamespace(
|
||||
controller=controller, prepare=prepare, importer=importer
|
||||
)
|
||||
for name in ("hub_candidate_import", "prepare_mcp_hub_candidate", "deploy"):
|
||||
sys.modules.pop(name, None)
|
||||
|
||||
|
||||
def _zip_stream_member(name: str, payload: bytes, offset: int) -> tuple[bytes, bytes]:
|
||||
encoded_name = name.encode("ascii")
|
||||
crc = zipfile.crc32(payload) & 0xFFFFFFFF
|
||||
local = struct.pack(
|
||||
"<IHHHHHIIIHH",
|
||||
0x04034B50,
|
||||
20,
|
||||
0x8,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
len(encoded_name),
|
||||
0,
|
||||
) + encoded_name + payload + struct.pack(
|
||||
"<IIII", 0x08074B50, crc, len(payload), len(payload)
|
||||
)
|
||||
central = struct.pack(
|
||||
"<IHHHHHHIIIHHHHHII",
|
||||
0x02014B50,
|
||||
20,
|
||||
20,
|
||||
0x8,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
crc,
|
||||
len(payload),
|
||||
len(payload),
|
||||
len(encoded_name),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
(stat.S_IFREG | 0o600) << 16,
|
||||
offset,
|
||||
) + encoded_name
|
||||
return local, central
|
||||
|
||||
|
||||
def _streamed_zip_bytes(importer, *, extra_member: bool = False) -> bytearray:
|
||||
names = sorted(importer._expected_zip_members())
|
||||
if extra_member:
|
||||
names.append("unexpected.txt")
|
||||
members = [(name, b"{}" if name != "images.tar" else b"tar-data") for name in names]
|
||||
output = bytearray()
|
||||
centrals: list[bytes] = []
|
||||
for name, payload in members:
|
||||
local, central = _zip_stream_member(name, payload, len(output))
|
||||
output.extend(local)
|
||||
centrals.append(central)
|
||||
central_offset = len(output)
|
||||
for central in centrals:
|
||||
output.extend(central)
|
||||
central_size = len(output) - central_offset
|
||||
output.extend(
|
||||
struct.pack(
|
||||
"<IHHHHIIH",
|
||||
0x06054B50,
|
||||
0,
|
||||
0,
|
||||
len(members),
|
||||
len(members),
|
||||
central_size,
|
||||
central_offset,
|
||||
0,
|
||||
)
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _private_file(path: Path, data: bytes) -> None:
|
||||
path.write_bytes(data)
|
||||
path.chmod(0o600)
|
||||
|
||||
|
||||
def _synthetic_scan_reports(
|
||||
services, images, *, created_at: str, finding_service: str | None = None
|
||||
) -> dict[str, bytes]:
|
||||
report_data: dict[str, bytes] = {}
|
||||
for service in services:
|
||||
image_id = images[service]["imageId"]
|
||||
vulnerabilities = (
|
||||
[
|
||||
{
|
||||
"Severity": "HIGH",
|
||||
"PkgName": "package",
|
||||
"VulnerabilityID": "CVE-2099-0001",
|
||||
"InstalledVersion": "1.0",
|
||||
}
|
||||
]
|
||||
if service == finding_service
|
||||
else []
|
||||
)
|
||||
vulnerability = {
|
||||
"SchemaVersion": 2,
|
||||
"Metadata": {
|
||||
"ImageID": image_id,
|
||||
"ImageConfig": {"os": "linux", "architecture": "amd64"},
|
||||
},
|
||||
"CreatedAt": created_at,
|
||||
"Trivy": {"Version": "0.65.0"},
|
||||
"Results": [
|
||||
{
|
||||
"Packages": [{"Name": "package", "Version": "1.0"}],
|
||||
"Vulnerabilities": vulnerabilities,
|
||||
}
|
||||
],
|
||||
}
|
||||
secret = {
|
||||
"ImageID": image_id,
|
||||
"CreatedAt": created_at,
|
||||
"TrivyVersion": "0.65.0",
|
||||
"FindingCount": 0,
|
||||
"SensitiveFindingDetailsPersisted": False,
|
||||
"Results": [{"FindingCount": 0}],
|
||||
}
|
||||
report_data[f"vulnerability-{service}.json"] = json.dumps(
|
||||
vulnerability
|
||||
).encode()
|
||||
report_data[f"secret-{service}.json"] = json.dumps(secret).encode()
|
||||
report_data["trivy-db-metadata.json"] = json.dumps(
|
||||
{"UpdatedAt": created_at, "DownloadedAt": created_at}
|
||||
).encode()
|
||||
return report_data
|
||||
|
||||
|
||||
def test_run_identity_requires_exact_protected_push_and_no_pull_request(modules):
|
||||
importer = modules.importer
|
||||
revision = "a" * 40
|
||||
run = {
|
||||
"path": importer.RUN_PATH,
|
||||
"event": "push",
|
||||
"head_branch": "main",
|
||||
"head_sha": revision,
|
||||
"repository": {"full_name": "Jens/ITWorx_MCP_Hub"},
|
||||
"head_repository": None,
|
||||
"pull_requests": [],
|
||||
}
|
||||
assert importer._run_matches(run, revision)
|
||||
for change in (
|
||||
{"event": "workflow_dispatch"},
|
||||
{"path": "other.yml@refs/heads/main"},
|
||||
{"head_repository": {"full_name": "attacker/fork"}},
|
||||
{"pull_requests": [{"number": 12}]},
|
||||
{"head_sha": "b" * 40},
|
||||
{"repository": {"full_name": "attacker/repository"}},
|
||||
):
|
||||
assert not importer._run_matches({**run, **change}, revision)
|
||||
|
||||
|
||||
def test_selected_run_and_jobs_accept_the_live_gitea_job_shape(modules):
|
||||
importer = modules.importer
|
||||
revision = "a" * 40
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace(
|
||||
"+00:00", "Z"
|
||||
)
|
||||
run = {
|
||||
"id": 21531,
|
||||
"path": importer.RUN_PATH,
|
||||
"event": "push",
|
||||
"head_branch": "main",
|
||||
"head_sha": revision,
|
||||
"repository": {"full_name": "Jens/ITWorx_MCP_Hub"},
|
||||
"head_repository": None,
|
||||
"pull_requests": [],
|
||||
"created_at": now,
|
||||
"run_attempt": 1,
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
}
|
||||
job_rows = [
|
||||
{
|
||||
"id": 37145 + index,
|
||||
"name": name,
|
||||
"run_id": 21531,
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
"runner_id": 22,
|
||||
"started_at": now,
|
||||
"completed_at": now,
|
||||
}
|
||||
for index, name in enumerate(("full", "release-images"))
|
||||
]
|
||||
calls = []
|
||||
|
||||
def api(path, *, query):
|
||||
calls.append((path, query))
|
||||
if path.endswith("/actions/runs"):
|
||||
return {"total_count": 1, "workflow_runs": [run]}
|
||||
if path.endswith("/actions/runs/21531/jobs"):
|
||||
return {"total_count": 2, "jobs": job_rows}
|
||||
raise AssertionError(f"unexpected fixed API path: {path}")
|
||||
|
||||
selected, built_at = importer._selected_run(api, revision)
|
||||
assert selected == run
|
||||
assert built_at.endswith("Z")
|
||||
assert importer._jobs_ready(api, revision, selected)
|
||||
assert calls[0][1] == {
|
||||
"event": "push",
|
||||
"branch": "main",
|
||||
"head_sha": revision,
|
||||
"page": "1",
|
||||
"limit": "100",
|
||||
}
|
||||
assert calls[1][1] == {"page": "1", "limit": "100"}
|
||||
|
||||
|
||||
def test_required_candidate_jobs_must_both_succeed_and_bind_run(modules):
|
||||
importer = modules.importer
|
||||
run = {
|
||||
"id": 5,
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
"run_attempt": 1,
|
||||
}
|
||||
rows = [
|
||||
{"name": name, "run_id": 5, "status": "completed", "conclusion": "success"}
|
||||
for name in ("full", "release-images")
|
||||
]
|
||||
|
||||
def api(_path, *, query):
|
||||
assert query == {"page": "1", "limit": "100"}
|
||||
return {"total_count": len(rows), "jobs": rows}
|
||||
|
||||
assert importer._jobs_ready(api, "a" * 40, run)
|
||||
rows[1] = {**rows[1], "conclusion": "failure"}
|
||||
with pytest.raises(importer.CandidateRejected):
|
||||
importer._jobs_ready(api, "a" * 40, run)
|
||||
|
||||
|
||||
def test_recheck_rejects_a_new_attempt_for_the_same_workflow_run(modules, monkeypatch):
|
||||
importer = modules.importer
|
||||
revision = "a" * 40
|
||||
original = {"id": 21531, "run_attempt": 1}
|
||||
restarted = {"id": 21531, "run_attempt": 2}
|
||||
artifact = {"id": 730}
|
||||
monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: "b" * 40)
|
||||
monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
importer,
|
||||
"_selected_run",
|
||||
lambda *_args: (restarted, "2026-10-03T10:00:00.000Z"),
|
||||
)
|
||||
monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True)
|
||||
with pytest.raises(importer.CandidateRejected, match="run changed"):
|
||||
importer._recheck_protected_artifact(
|
||||
lambda *_args, **_kwargs: None,
|
||||
revision=revision,
|
||||
tree="b" * 40,
|
||||
initial_run=original,
|
||||
initial_built_at="2026-10-03T10:00:00.000Z",
|
||||
initial_artifact=artifact,
|
||||
)
|
||||
|
||||
|
||||
def test_unfixable_vulnerability_finding_is_bound_to_fixed_image_reference(modules):
|
||||
importer = modules.importer
|
||||
controller = modules.controller
|
||||
created_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace(
|
||||
"+00:00", "Z"
|
||||
)
|
||||
images = {
|
||||
service: {
|
||||
"imageId": f"sha256:{index:064x}",
|
||||
"imageRef": f"registry.example/{service}:fixed",
|
||||
}
|
||||
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1)
|
||||
}
|
||||
report_data = _synthetic_scan_reports(
|
||||
controller.MCP_HUB_SERVICES,
|
||||
images,
|
||||
created_at=created_at,
|
||||
finding_service=controller.MCP_HUB_SERVICES[0],
|
||||
)
|
||||
|
||||
_reports, findings = importer._validate_reports(images, report_data)
|
||||
assert len(findings) == 1
|
||||
assert findings[0]["imageRef"] == images[controller.MCP_HUB_SERVICES[0]]["imageRef"]
|
||||
summary = importer._candidate_findings_summary(findings, images)
|
||||
assert summary["unresolvedCriticalOrHigh"][0]["affectedImages"] == [
|
||||
findings[0]["imageRef"]
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finding_present", [False, True])
|
||||
@pytest.mark.parametrize("rerun_during_load", [False, True])
|
||||
def test_fake_local_import_publishes_candidate_and_requires_review_for_findings(
|
||||
modules, monkeypatch, tmp_path, finding_present, rerun_during_load
|
||||
):
|
||||
importer = modules.importer
|
||||
prepare = modules.prepare
|
||||
controller = modules.controller
|
||||
revision, tree, epoch = "a" * 40, "b" * 40, "1790985600"
|
||||
built_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace(
|
||||
"+00:00", "Z"
|
||||
)
|
||||
build = {
|
||||
"id": revision[:12],
|
||||
"repository": importer.REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
|
||||
"builtAt": built_at,
|
||||
}
|
||||
compose_sha = "c" * 64
|
||||
model_services = {}
|
||||
images = {}
|
||||
for index, service in enumerate(controller.MCP_HUB_SERVICES, start=1):
|
||||
external = importer.EXTERNAL_IMAGES.get(service)
|
||||
image_ref = (
|
||||
external[0]
|
||||
if external is not None
|
||||
else f"registry.example/itworx/{service}:{revision[:12]}"
|
||||
)
|
||||
archive_ref = external[1] if external is not None else image_ref
|
||||
model_services[service] = {"image": image_ref}
|
||||
images[service] = {
|
||||
"imageRef": image_ref,
|
||||
"archiveRef": archive_ref,
|
||||
"imageId": f"sha256:{index:064x}",
|
||||
}
|
||||
model = {"services": model_services}
|
||||
created_at = built_at
|
||||
finding_service = controller.MCP_HUB_SERVICES[0] if finding_present else None
|
||||
report_data = _synthetic_scan_reports(
|
||||
controller.MCP_HUB_SERVICES,
|
||||
images,
|
||||
created_at=created_at,
|
||||
finding_service=finding_service,
|
||||
)
|
||||
_reports, findings = importer._validate_reports(images, report_data)
|
||||
summary = importer._candidate_findings_summary(findings, images)
|
||||
digests = {
|
||||
"images": {
|
||||
service: {
|
||||
"imageId": images[service]["imageId"],
|
||||
"tag": images[service]["imageRef"],
|
||||
}
|
||||
for service in controller.MCP_HUB_SERVICES
|
||||
}
|
||||
}
|
||||
exception_rows = [
|
||||
{
|
||||
"package": finding["package"],
|
||||
"cve": finding["cve"],
|
||||
"installedVersion": finding["installedVersion"],
|
||||
"severity": finding["severity"],
|
||||
"affectedImageDigests": {
|
||||
finding["imageRef"]: images_by_ref["imageId"]
|
||||
for images_by_ref in [
|
||||
next(
|
||||
row
|
||||
for row in images.values()
|
||||
if row["imageRef"] == finding["imageRef"]
|
||||
)
|
||||
]
|
||||
},
|
||||
}
|
||||
for finding in findings
|
||||
]
|
||||
draft = {"schemaVersion": "1.0", "exceptions": exception_rows}
|
||||
state_root = tmp_path / "state"
|
||||
state_root.mkdir(mode=0o700)
|
||||
state_root.chmod(0o700)
|
||||
releases_root = tmp_path / "releases"
|
||||
source = releases_root / revision
|
||||
source.mkdir(parents=True)
|
||||
monkeypatch.setattr(controller, "STATE_ROOT", state_root)
|
||||
monkeypatch.setattr(controller, "MCP_HUB_RELEASES_ROOT", releases_root)
|
||||
monkeypatch.setattr(controller, "GLOBAL_DEPLOY_LOCK", state_root / "deploy.lock")
|
||||
monkeypatch.setattr(controller, "_mcp_hub_require_trusted_state_root", lambda: None)
|
||||
monkeypatch.setattr(controller, "_mcp_hub_require_no_posix_acl", lambda *_args: None)
|
||||
monkeypatch.setattr(controller, "_root_owned", lambda *_args: True)
|
||||
|
||||
def create_private_directory(path, *, create=False, root=None):
|
||||
del root
|
||||
if create:
|
||||
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
path.chmod(0o700)
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise controller.DeployError("test private directory missing")
|
||||
|
||||
def candidate_directory(candidate_revision):
|
||||
directory = state_root / "mcp-hub-release-candidates" / candidate_revision
|
||||
for path in (directory.parent, directory):
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise controller.DeployError("test candidate directory missing")
|
||||
return directory
|
||||
|
||||
monkeypatch.setattr(controller, "_mcp_hub_private_directory", create_private_directory)
|
||||
monkeypatch.setattr(controller, "_mcp_hub_candidate_directory", candidate_directory)
|
||||
project, repo = {"branch": "main"}, tmp_path / "repo"
|
||||
monkeypatch.setattr(prepare, "_fixed_target", lambda *_args: (project, repo))
|
||||
monkeypatch.setattr(controller, "deployment_metadata", lambda *_args, **_kwargs: build)
|
||||
monkeypatch.setattr(controller, "mcp_hub_source_date_epoch", lambda *_args: epoch)
|
||||
monkeypatch.setattr(controller, "mcp_hub_stage_release_source", lambda *_args: source)
|
||||
monkeypatch.setattr(controller, "exact_source_matches_revision", lambda *_args: True)
|
||||
monkeypatch.setattr(controller, "_mcp_hub_verify_release_tree_metadata", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
prepare,
|
||||
"_render",
|
||||
lambda *_args, **_kwargs: (model, compose_sha),
|
||||
)
|
||||
monkeypatch.setattr(importer, "_remote_main_tree", lambda *_args: tree)
|
||||
monkeypatch.setattr(importer, "_fixed_workflow", lambda *_args: None)
|
||||
run = {"id": 21531, "run_attempt": 1}
|
||||
run_state = {"attempt": 1}
|
||||
monkeypatch.setattr(
|
||||
importer,
|
||||
"_selected_run",
|
||||
lambda *_args: ({**run, "run_attempt": run_state["attempt"]}, built_at),
|
||||
)
|
||||
monkeypatch.setattr(importer, "_jobs_ready", lambda *_args: True)
|
||||
artifact_bytes = b"bounded fake Gitea artifact"
|
||||
artifact = {
|
||||
"id": 730,
|
||||
"size_in_bytes": len(artifact_bytes),
|
||||
"name": importer.ARTIFACT_PREFIX + revision,
|
||||
"workflow_run": {"id": run["id"]},
|
||||
}
|
||||
monkeypatch.setattr(importer, "_candidate_artifact", lambda *_args: artifact)
|
||||
monkeypatch.setattr(importer, "_docker_root", lambda: tmp_path)
|
||||
monkeypatch.setattr(importer, "_check_disk_headroom", lambda **_kwargs: None)
|
||||
monkeypatch.setattr(importer, "_parse_artifact_zip", lambda *_args, **_kwargs: (
|
||||
{},
|
||||
report_data,
|
||||
hashlib.sha256(artifact_bytes).hexdigest(),
|
||||
"d" * 64,
|
||||
"e" * 64,
|
||||
123,
|
||||
))
|
||||
monkeypatch.setattr(
|
||||
importer,
|
||||
"_validate_manifest",
|
||||
lambda *_args, **_kwargs: (images, {}),
|
||||
)
|
||||
monkeypatch.setattr(importer, "_validate_docker_save", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(importer, "_bounded_global_deploy_lock", contextlib.nullcontext)
|
||||
monkeypatch.setattr(importer, "_image_preflight", lambda *_args: None)
|
||||
|
||||
def fake_fixed_docker(*args, **_kwargs):
|
||||
if rerun_during_load and args and args[0] == "load":
|
||||
run_state["attempt"] = 2
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(importer, "_fixed_docker", fake_fixed_docker)
|
||||
monkeypatch.setattr(importer, "_verify_loaded_images", lambda *_args, **_kwargs: {})
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_mcp_hub_check_candidate_exceptions",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
prepare,
|
||||
"_draft_pending_exceptions",
|
||||
lambda *_args: draft,
|
||||
)
|
||||
|
||||
def fake_plan(_repository, _revision, *, built_at, staging_directory, **_kwargs):
|
||||
Path(staging_directory).mkdir(mode=0o700)
|
||||
frozen = {
|
||||
"schema": prepare.BUILD_SCHEMA,
|
||||
"repository": importer.REPOSITORY,
|
||||
"revision": revision,
|
||||
"build": build,
|
||||
"sourceDateEpoch": epoch,
|
||||
"composeSha256": compose_sha,
|
||||
}
|
||||
controller._mcp_hub_write_private_new(
|
||||
Path(staging_directory) / "build.json", prepare._json_bytes(frozen)
|
||||
)
|
||||
return {"status": "planned", "revision": revision, "composeSha256": compose_sha}
|
||||
|
||||
monkeypatch.setattr(prepare, "plan_candidate", fake_plan)
|
||||
|
||||
def fake_capture(_repository, _revision, *, staging_directory, **_kwargs):
|
||||
image_rows = {
|
||||
service: {
|
||||
"imageRef": images[service]["imageRef"],
|
||||
"imageId": images[service]["imageId"],
|
||||
"vulnerabilitySha256": hashlib.sha256(
|
||||
report_data[f"vulnerability-{service}.json"]
|
||||
).hexdigest(),
|
||||
"secretSha256": hashlib.sha256(
|
||||
report_data[f"secret-{service}.json"]
|
||||
).hexdigest(),
|
||||
}
|
||||
for service in controller.MCP_HUB_SERVICES
|
||||
}
|
||||
candidate = {
|
||||
"schema": prepare.CANDIDATE_SCHEMA,
|
||||
"repository": importer.REPOSITORY,
|
||||
"revision": revision,
|
||||
"build": build,
|
||||
"sourceDateEpoch": epoch,
|
||||
"composeSha256": compose_sha,
|
||||
"images": image_rows,
|
||||
"databaseMetadataSha256": hashlib.sha256(
|
||||
report_data["trivy-db-metadata.json"]
|
||||
).hexdigest(),
|
||||
"summarySha256": hashlib.sha256(prepare._json_bytes(summary)).hexdigest(),
|
||||
"digestsSha256": hashlib.sha256(prepare._json_bytes(digests)).hexdigest(),
|
||||
}
|
||||
outputs = {
|
||||
"candidate.json": candidate,
|
||||
"scan-summary.json": summary,
|
||||
"image-digests.json": digests,
|
||||
"exceptions.json": draft,
|
||||
}
|
||||
for filename, value in outputs.items():
|
||||
controller._mcp_hub_write_private_new(
|
||||
Path(staging_directory) / filename, prepare._json_bytes(value)
|
||||
)
|
||||
candidate_sha = hashlib.sha256(prepare._json_bytes(candidate)).hexdigest()
|
||||
exceptions_sha = hashlib.sha256(prepare._json_bytes(draft)).hexdigest()
|
||||
return {
|
||||
"candidateSha256": candidate_sha,
|
||||
"exceptionsSha256": exceptions_sha,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(prepare, "capture_candidate", fake_capture)
|
||||
|
||||
def download_artifact(_artifact_id, destination, _maximum):
|
||||
_private_file(destination, artifact_bytes)
|
||||
return None
|
||||
|
||||
candidate_directory = state_root / "mcp-hub-release-candidates" / revision
|
||||
if rerun_during_load:
|
||||
with pytest.raises(importer.CandidateRejected, match="run changed"):
|
||||
importer.prepare_candidate(
|
||||
importer.REPOSITORY,
|
||||
revision,
|
||||
api_json=lambda *_args, **_kwargs: None,
|
||||
download_artifact=download_artifact,
|
||||
)
|
||||
assert not candidate_directory.exists()
|
||||
return
|
||||
|
||||
first = importer.prepare_candidate(
|
||||
importer.REPOSITORY,
|
||||
revision,
|
||||
api_json=lambda *_args, **_kwargs: None,
|
||||
download_artifact=download_artifact,
|
||||
)
|
||||
assert candidate_directory.is_dir()
|
||||
assert stat.S_IMODE(candidate_directory.stat().st_mode) == 0o700
|
||||
candidate_before_review = (candidate_directory / "candidate.json").read_bytes()
|
||||
receipt_before_review = (candidate_directory / "import-receipt.json").read_bytes()
|
||||
collision_stage = tmp_path / "second-candidate"
|
||||
collision_stage.mkdir(mode=0o700)
|
||||
with pytest.raises(importer.CandidateRejected, match="never overwrites"):
|
||||
importer._publish_candidate(collision_stage, candidate_directory)
|
||||
assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review
|
||||
if finding_present:
|
||||
assert first["status"] == "pending_review"
|
||||
reviewed = {
|
||||
**draft,
|
||||
"reviewer": "release-operator",
|
||||
"exceptions": [
|
||||
{**row, "justification": "Reviewed unfixable finding"}
|
||||
for row in draft["exceptions"]
|
||||
],
|
||||
}
|
||||
exceptions_path = candidate_directory / "exceptions.json"
|
||||
exceptions_path.unlink()
|
||||
controller._mcp_hub_write_private_new(
|
||||
exceptions_path, prepare._json_bytes(reviewed)
|
||||
)
|
||||
approval = {
|
||||
"schema": importer.APPROVAL_SCHEMA,
|
||||
"candidateSha256": first["candidateSha256"],
|
||||
"exceptionsSha256": hashlib.sha256(
|
||||
prepare._json_bytes(reviewed)
|
||||
).hexdigest(),
|
||||
}
|
||||
controller._mcp_hub_write_private_new(
|
||||
candidate_directory / "approval.json", prepare._json_bytes(approval)
|
||||
)
|
||||
final = importer.prepare_candidate(
|
||||
importer.REPOSITORY,
|
||||
revision,
|
||||
api_json=lambda *_args, **_kwargs: None,
|
||||
download_artifact=download_artifact,
|
||||
)
|
||||
assert final["status"] == "ready"
|
||||
else:
|
||||
assert first["status"] == "ready"
|
||||
final = first
|
||||
assert final["candidateSha256"] == hashlib.sha256(candidate_before_review).hexdigest()
|
||||
assert (candidate_directory / "candidate.json").read_bytes() == candidate_before_review
|
||||
assert (candidate_directory / "import-receipt.json").read_bytes() == receipt_before_review
|
||||
|
||||
|
||||
def test_candidate_artifact_is_exact_bounded_and_unexpired(modules):
|
||||
importer = modules.importer
|
||||
revision = "c" * 40
|
||||
run_id = 11
|
||||
artifact = {
|
||||
"id": 22,
|
||||
"name": importer.ARTIFACT_PREFIX + revision,
|
||||
"size_in_bytes": 4096,
|
||||
"expired": False,
|
||||
"expires_at": (datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
"workflow_run": {"id": run_id},
|
||||
}
|
||||
result = importer._candidate_artifact(
|
||||
lambda _path, *, query: {"artifacts": [artifact]}, run_id, revision
|
||||
)
|
||||
assert result == artifact
|
||||
for changed in (
|
||||
{"expired": True},
|
||||
{"name": "hub-release-candidate-" + "d" * 40},
|
||||
{"size_in_bytes": importer.MAX_ARTIFACT_ZIP_BYTES + 1},
|
||||
{"workflow_run": {"id": run_id + 1}},
|
||||
):
|
||||
with pytest.raises(importer.CandidateRejected):
|
||||
importer._candidate_artifact(
|
||||
lambda _path, *, query: {"artifacts": [{**artifact, **changed}]},
|
||||
run_id,
|
||||
revision,
|
||||
)
|
||||
|
||||
|
||||
def test_streamed_uploader_zip_descriptors_are_verified(modules, tmp_path):
|
||||
importer = modules.importer
|
||||
payload = _streamed_zip_bytes(importer)
|
||||
zip_path = tmp_path / "candidate.zip"
|
||||
tar_path = tmp_path / "images.tar"
|
||||
_private_file(zip_path, bytes(payload))
|
||||
(
|
||||
manifest,
|
||||
reports,
|
||||
zip_sha,
|
||||
manifest_sha,
|
||||
tar_sha,
|
||||
tar_bytes,
|
||||
) = importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(payload))
|
||||
assert manifest == {}
|
||||
assert len(reports) == 35
|
||||
assert zip_sha == hashlib.sha256(payload).hexdigest()
|
||||
assert manifest_sha == hashlib.sha256(b"{}").hexdigest()
|
||||
assert tar_sha == hashlib.sha256(b"tar-data").hexdigest()
|
||||
assert tar_bytes == len(b"tar-data")
|
||||
assert tar_path.read_bytes() == b"tar-data"
|
||||
|
||||
tampered = bytearray(payload)
|
||||
name_size = len("candidate-manifest.json")
|
||||
descriptor_offset = 30 + name_size + 2
|
||||
tampered[descriptor_offset + 4] ^= 0x01
|
||||
_private_file(zip_path, bytes(tampered))
|
||||
with pytest.raises(importer.CandidateRejected, match="descriptor"):
|
||||
importer._parse_artifact_zip(zip_path, tar_path, expected_size=len(tampered))
|
||||
|
||||
|
||||
def test_zip_central_directory_is_bounded_before_zipfile_parse(modules, tmp_path, monkeypatch):
|
||||
importer = modules.importer
|
||||
payload = _streamed_zip_bytes(importer, extra_member=True)
|
||||
path = tmp_path / "too-many.zip"
|
||||
_private_file(path, bytes(payload))
|
||||
monkeypatch.setattr(
|
||||
importer.zipfile,
|
||||
"ZipFile",
|
||||
lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized member inventory"),
|
||||
)
|
||||
with pytest.raises(importer.CandidateRejected, match="central directory"):
|
||||
importer._zip_central_directory_preflight(path, len(payload))
|
||||
|
||||
|
||||
def test_zip_oversized_central_directory_is_rejected_without_allocating_it(
|
||||
modules, tmp_path, monkeypatch
|
||||
):
|
||||
importer = modules.importer
|
||||
payload = struct.pack(
|
||||
"<IHHHHIIH",
|
||||
0x06054B50,
|
||||
0,
|
||||
0,
|
||||
37,
|
||||
37,
|
||||
65 * 1024,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
path = tmp_path / "oversized-directory.zip"
|
||||
_private_file(path, payload)
|
||||
monkeypatch.setattr(
|
||||
importer.zipfile,
|
||||
"ZipFile",
|
||||
lambda *_args, **_kwargs: pytest.fail("ZipFile parsed an oversized central directory"),
|
||||
)
|
||||
with pytest.raises(importer.CandidateRejected, match="central directory"):
|
||||
importer._zip_central_directory_preflight(path, len(payload))
|
||||
|
||||
|
||||
def test_tar_member_count_is_stopped_during_streamed_header_walk(
|
||||
modules, tmp_path, monkeypatch
|
||||
):
|
||||
importer = modules.importer
|
||||
path = tmp_path / "many-members.tar"
|
||||
with tarfile.open(path, "w", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for name in ("one/", "two/"):
|
||||
info = tarfile.TarInfo(name)
|
||||
info.type = tarfile.DIRTYPE
|
||||
archive.addfile(info)
|
||||
monkeypatch.setattr(importer, "MAX_TAR_MEMBERS", 1)
|
||||
with pytest.raises(importer.CandidateRejected, match="member-count"):
|
||||
importer._preflight_uncompressed_tar(path)
|
||||
|
||||
|
||||
def test_tar_preflight_rejects_extension_headers_before_tarfile(modules, tmp_path):
|
||||
importer = modules.importer
|
||||
path = tmp_path / "pax.tar"
|
||||
with tarfile.open(path, "w", format=tarfile.PAX_FORMAT) as archive:
|
||||
info = tarfile.TarInfo("x" * 120)
|
||||
archive.addfile(info, io.BytesIO(b"x"))
|
||||
with pytest.raises(importer.CandidateRejected, match="unsupported member type"):
|
||||
importer._preflight_uncompressed_tar(path)
|
||||
|
||||
|
||||
def test_remote_head_without_completed_candidate_stays_awaiting(modules, tmp_path, monkeypatch):
|
||||
controller = modules.controller
|
||||
prepare = modules.prepare
|
||||
importer = modules.importer
|
||||
revision = "a" * 40
|
||||
tree = "b" * 40
|
||||
monkeypatch.setattr(
|
||||
prepare, "_fixed_target", lambda _repository, _revision: ({}, tmp_path)
|
||||
)
|
||||
responses = {
|
||||
"/repos/jens/itworx_mcp_hub/branches/main": {"commit": {"id": revision}},
|
||||
f"/repos/jens/itworx_mcp_hub/git/commits/{revision}": {
|
||||
"sha": revision,
|
||||
"commit": {"tree": {"sha": tree}},
|
||||
},
|
||||
importer.WORKFLOW_LIST_PATH: {
|
||||
"total_count": 1,
|
||||
"workflows": [
|
||||
{
|
||||
"id": importer.WORKFLOW_ID,
|
||||
"path": importer.WORKFLOW_PATH,
|
||||
"state": "active",
|
||||
}
|
||||
],
|
||||
},
|
||||
"/repos/jens/itworx_mcp_hub/actions/runs": {
|
||||
"total_count": 0,
|
||||
"workflow_runs": [],
|
||||
},
|
||||
}
|
||||
calls = []
|
||||
|
||||
def api(path, *, query):
|
||||
calls.append((path, query))
|
||||
return responses[path]
|
||||
|
||||
result = importer.prepare_candidate(
|
||||
controller.MCP_HUB_REPOSITORY,
|
||||
revision,
|
||||
api_json=api,
|
||||
download_artifact=lambda *_args: pytest.fail("an absent run cannot download artifacts"),
|
||||
)
|
||||
assert result == {"status": "awaiting_candidate", "revision": revision}
|
||||
assert len(calls) == 4
|
||||
|
||||
|
||||
def test_fixed_compose_environment_discards_ambient_docker_and_compose_context(
|
||||
modules, monkeypatch
|
||||
):
|
||||
controller = modules.controller
|
||||
importer = modules.importer
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"deployment_environment",
|
||||
lambda *_args: {
|
||||
"PATH": "/usr/bin",
|
||||
"DOCKER_HOST": "tcp://peer:2375",
|
||||
"DOCKER_CONTEXT": "attacker",
|
||||
"DOCKER_TLS_VERIFY": "1",
|
||||
"COMPOSE_FILE": "/tmp/untrusted.yml",
|
||||
"COMPOSE_PROFILES": "untrusted",
|
||||
},
|
||||
)
|
||||
env = importer._fixed_compose_environment({"compose_project": "fixed"}, "a" * 40, "jens/itworx_mcp_hub")
|
||||
assert env == {
|
||||
"PATH": "/usr/bin",
|
||||
"DOCKER_HOST": importer.DOCKER_HOST,
|
||||
"COMPOSE_PROJECT_NAME": "fixed",
|
||||
"COMPOSE_PROFILES": ",".join(controller.MCP_HUB_COMPOSE_PROFILES),
|
||||
}
|
||||
Reference in new issue
Block a user