Publish curated ITWorx_MCP_Hub source from 2aa63ba527a98b38b4d28f2a1bb85b1b1dbe7f6d
This commit is contained in:
1 parent
49627c7a7c
commit
378ae83d30
22 files changed
+4055
-81
No files matched your search
@@ -1,13 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
# Candidate-only mode emits a local SHA-scoped image and does not publish or deploy it.
|
||||
MODE="legacy"
|
||||
if [[ "$#" -gt 0 ]]; then
|
||||
[[ "$#" -eq 1 && "$1" == "--release-candidate" ]] || {
|
||||
echo "FATAL: supported mode is --release-candidate (or no arguments for legacy mode)" >&2
|
||||
exit 64
|
||||
}
|
||||
MODE="release-candidate"
|
||||
fi
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
MANIFEST="$ROOT/infrastructure/images/contextforge/compat-image.env"
|
||||
CONTRACT="$ROOT/infrastructure/images/contextforge/runtime-contract.json"
|
||||
ENTRYPOINT="$ROOT/infrastructure/images/contextforge/entrypoint.sh"
|
||||
RUNTIME_VERIFIER="$ROOT/scripts/contextforge_runtime_contract.py"
|
||||
SECURITY_REQUIREMENTS="$ROOT/infrastructure/images/contextforge/security-requirements.txt"
|
||||
CANDIDATE_RECIPE="$ROOT/scripts/contextforge_candidate_recipe.py"
|
||||
|
||||
for required in "$MANIFEST" "$CONTRACT" "$ENTRYPOINT" "$RUNTIME_VERIFIER"; do
|
||||
for required in "$MANIFEST" "$CONTRACT" "$ENTRYPOINT" "$RUNTIME_VERIFIER" "$SECURITY_REQUIREMENTS"; do
|
||||
[[ -f "$required" ]] || {
|
||||
echo "FATAL: required ContextForge compatibility input is missing: $required" >&2
|
||||
exit 66
|
||||
@@ -22,15 +34,87 @@ source "$MANIFEST"
|
||||
: "${CONTEXTFORGE_MCP_SDK:?missing MCP SDK version}"
|
||||
: "${CONTEXTFORGE_LOCAL_IMAGE:?missing local output image}"
|
||||
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
[[ -f "$CANDIDATE_RECIPE" ]] || {
|
||||
echo "FATAL: the source-locked candidate recipe helper is missing" >&2
|
||||
exit 66
|
||||
}
|
||||
# Validate every interpolated value before invoking Git, Docker, or Python.
|
||||
# These fixed-format fields become Docker tags, Git comparisons, and labels.
|
||||
: "${APPOPS_SOURCE_REPOSITORY:?missing APPOPS_SOURCE_REPOSITORY}"
|
||||
: "${APPOPS_SOURCE_REVISION:?missing APPOPS_SOURCE_REVISION}"
|
||||
: "${APPOPS_SOURCE_TREE:?missing APPOPS_SOURCE_TREE}"
|
||||
: "${APPOPS_BUILD_ID:?missing APPOPS_BUILD_ID}"
|
||||
: "${APPOPS_BUILD_DATE:?missing APPOPS_BUILD_DATE}"
|
||||
: "${SOURCE_DATE_EPOCH:?missing SOURCE_DATE_EPOCH}"
|
||||
: "${GITEA_REPOSITORY:?missing GITEA_REPOSITORY}"
|
||||
: "${GITEA_COMMIT_SHA:?missing GITEA_COMMIT_SHA}"
|
||||
|
||||
[[ "$APPOPS_SOURCE_REPOSITORY" == "jens/itworx_mcp_hub" ]] || {
|
||||
echo "FATAL: release-candidate source repository is not the protected Hub repository" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$GITEA_REPOSITORY" == "$APPOPS_SOURCE_REPOSITORY" ]] || {
|
||||
echo "FATAL: Gitea and AppOps repositories disagree" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$APPOPS_SOURCE_REVISION" =~ ^[0-9a-f]{40}$ ]] || {
|
||||
echo "FATAL: APPOPS_SOURCE_REVISION must be a full lowercase Git SHA" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$GITEA_COMMIT_SHA" == "$APPOPS_SOURCE_REVISION" ]] || {
|
||||
echo "FATAL: Gitea and AppOps revisions disagree" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$APPOPS_SOURCE_TREE" =~ ^[0-9a-f]{40}$ ]] || {
|
||||
echo "FATAL: APPOPS_SOURCE_TREE must be a full lowercase Git tree SHA" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$APPOPS_BUILD_ID" == "${APPOPS_SOURCE_REVISION:0:12}" ]] || {
|
||||
echo "FATAL: APPOPS_BUILD_ID must be the first 12 revision characters" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$APPOPS_BUILD_DATE" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z$ ]] || {
|
||||
echo "FATAL: APPOPS_BUILD_DATE must be canonical UTC with millisecond precision" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$SOURCE_DATE_EPOCH" =~ ^[1-9][0-9]{0,11}$ ]] || {
|
||||
echo "FATAL: SOURCE_DATE_EPOCH must be a canonical positive integer of at most 12 digits" >&2
|
||||
exit 64
|
||||
}
|
||||
[[ "$CONTEXTFORGE_LOCAL_IMAGE" == "itworx-mcp-hub/contextforge-schema:12d4a0c7789c" ]] || {
|
||||
echo "FATAL: compatibility manifest image name is not the reviewed Hub image" >&2
|
||||
exit 64
|
||||
}
|
||||
fi
|
||||
|
||||
command -v git >/dev/null 2>&1 || { echo "FATAL: git is required" >&2; exit 69; }
|
||||
command -v docker >/dev/null 2>&1 || { echo "FATAL: docker is required" >&2; exit 69; }
|
||||
command -v python3 >/dev/null 2>&1 || { echo "FATAL: python3 is required" >&2; exit 69; }
|
||||
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
python3 - "$APPOPS_BUILD_DATE" <<'PY'
|
||||
from datetime import datetime
|
||||
import sys
|
||||
|
||||
try:
|
||||
datetime.strptime(sys.argv[1], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
except ValueError as exc:
|
||||
raise SystemExit("FATAL: APPOPS_BUILD_DATE is not a valid UTC calendar timestamp") from exc
|
||||
PY
|
||||
fi
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
SRC="$TMP/contextforge"
|
||||
BASE_IMAGE="${CONTEXTFORGE_LOCAL_IMAGE}-upstream"
|
||||
STAGED_IMAGE="${CONTEXTFORGE_LOCAL_IMAGE}-staged"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
CANDIDATE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}"
|
||||
BASE_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}-upstream"
|
||||
STAGED_IMAGE="itworx-mcp-hub/contextforge-schema:${APPOPS_SOURCE_REVISION}-staged"
|
||||
else
|
||||
BASE_IMAGE="${CONTEXTFORGE_LOCAL_IMAGE}-upstream"
|
||||
STAGED_IMAGE="${CONTEXTFORGE_LOCAL_IMAGE}-staged"
|
||||
fi
|
||||
REVISION_PREFIX="${CONTEXTFORGE_ALEMBIC_HEAD}_"
|
||||
MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"
|
||||
|
||||
@@ -70,12 +154,41 @@ CONTRACT_REQUIRED_SYMBOL="$(read_contract requiredSymbol)"
|
||||
CONTRACT_SHA256="$(sha256_file "$CONTRACT")"
|
||||
ENTRYPOINT_SHA256="$(sha256_file "$ENTRYPOINT")"
|
||||
VERIFIER_SHA256="$(sha256_file "$RUNTIME_VERIFIER")"
|
||||
SECURITY_REQUIREMENTS_SHA256="$(sha256_file "$SECURITY_REQUIREMENTS")"
|
||||
|
||||
[[ "$CONTRACT_SOURCE_COMMIT" == "$CONTEXTFORGE_UPSTREAM_COMMIT" ]] || fail "contract source commit disagrees with compatibility manifest"
|
||||
[[ "$CONTRACT_ALEMBIC_HEAD" == "$CONTEXTFORGE_ALEMBIC_HEAD" ]] || fail "contract Alembic head disagrees with compatibility manifest"
|
||||
[[ "$CONTRACT_MCP_SDK" == "$CONTEXTFORGE_MCP_SDK" ]] || fail "contract MCP SDK disagrees with compatibility manifest"
|
||||
[[ "$CONTRACT_REQUIRED_SYMBOL" == "McpError" ]] || fail "unsupported required MCP symbol: $CONTRACT_REQUIRED_SYMBOL"
|
||||
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
[[ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all)" ]] || fail "Hub source checkout has changes"
|
||||
[[ "$(git -C "$ROOT" rev-parse HEAD)" == "$APPOPS_SOURCE_REVISION" ]] || fail "checked-out Hub revision differs from frozen release metadata"
|
||||
[[ "$(git -C "$ROOT" rev-parse 'HEAD^{tree}')" == "$APPOPS_SOURCE_TREE" ]] || fail "checked-out Hub source tree differs from frozen release metadata"
|
||||
[[ "$(git -C "$ROOT" show -s --format=%ct HEAD)" == "$SOURCE_DATE_EPOCH" ]] || fail "source date epoch differs from frozen release metadata"
|
||||
|
||||
ensure_candidate_tag_absent() {
|
||||
local image="$1"
|
||||
local diagnostic="$TMP/docker-image-inspect.err"
|
||||
if docker image inspect "$image" >/dev/null 2>"$diagnostic"; then
|
||||
fail "release-candidate image tag already exists: $image"
|
||||
fi
|
||||
local error_line
|
||||
IFS= read -r error_line < "$diagnostic" || true
|
||||
case "$error_line" in
|
||||
*"No such image"*|*"no such image"*|*"No such object"*|*"no such object"*|*"reference does not exist"*) ;;
|
||||
*) fail "could not prove release-candidate image tag is absent: $image" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Never build over or retag an existing revision-scoped image. Docker errors
|
||||
# other than its explicit not-found response fail closed.
|
||||
ensure_candidate_tag_absent "$BASE_IMAGE"
|
||||
ensure_candidate_tag_absent "$STAGED_IMAGE"
|
||||
ensure_candidate_tag_absent "$CANDIDATE_IMAGE"
|
||||
|
||||
fi
|
||||
|
||||
echo "=== FETCH EXACT CONTEXTFORGE SOURCE ==="
|
||||
git init -q "$SRC"
|
||||
git -C "$SRC" remote add origin https://github.com/IBM/mcp-context-forge.git
|
||||
@@ -97,13 +210,28 @@ echo "contract sha : $CONTRACT_SHA256"
|
||||
|
||||
echo
|
||||
echo "=== BUILD UPSTREAM SOURCE IMAGE ==="
|
||||
docker build --pull=false -f "$SRC/Containerfile" -t "$BASE_IMAGE" "$SRC"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
python3 "$CANDIDATE_RECIPE" prepare --source-dir "$SRC"
|
||||
BUILD_ARG_FILE="$TMP/contextforge-build-args.txt"
|
||||
python3 "$CANDIDATE_RECIPE" build-args > "$BUILD_ARG_FILE"
|
||||
mapfile -t CANDIDATE_BUILD_ARGS < "$BUILD_ARG_FILE"
|
||||
[[ "${#CANDIDATE_BUILD_ARGS[@]}" -eq 9 ]] || fail "candidate base/build argument contract is incomplete"
|
||||
CANDIDATE_DOCKER_ARGS=()
|
||||
for build_arg in "${CANDIDATE_BUILD_ARGS[@]}"; do
|
||||
[[ "$build_arg" =~ ^[A-Z_]+=([A-Za-z0-9./:@_-]+)$ ]] || fail "candidate build argument is malformed"
|
||||
CANDIDATE_DOCKER_ARGS+=(--build-arg "$build_arg")
|
||||
done
|
||||
docker build --pull=false --platform linux/amd64 "${CANDIDATE_DOCKER_ARGS[@]}" -f "$SRC/Containerfile.itworx-candidate" -t "$BASE_IMAGE" "$SRC"
|
||||
else
|
||||
docker build --pull=false -f "$SRC/Containerfile" -t "$BASE_IMAGE" "$SRC"
|
||||
fi
|
||||
|
||||
BASE_USER="$(docker image inspect -f '{{.Config.User}}' "$BASE_IMAGE")"
|
||||
LAYER="$TMP/Containerfile.compat"
|
||||
cp "$CONTRACT" "$TMP/contextforge-runtime-contract.json"
|
||||
cp "$ENTRYPOINT" "$TMP/contextforge-entrypoint.sh"
|
||||
cp "$RUNTIME_VERIFIER" "$TMP/contextforge-runtime-contract.py"
|
||||
cp "$SECURITY_REQUIREMENTS" "$TMP/contextforge-security-requirements.txt"
|
||||
|
||||
{
|
||||
echo "FROM $BASE_IMAGE"
|
||||
@@ -111,6 +239,7 @@ cp "$RUNTIME_VERIFIER" "$TMP/contextforge-runtime-contract.py"
|
||||
echo 'COPY contextforge-runtime-contract.json /opt/itworx/contextforge-runtime-contract.json'
|
||||
echo 'COPY contextforge-entrypoint.sh /opt/itworx/contextforge-entrypoint.sh'
|
||||
echo 'COPY contextforge-runtime-contract.py /opt/itworx/contextforge-runtime-contract.py'
|
||||
echo 'COPY contextforge-security-requirements.txt /opt/itworx/contextforge-security-requirements.txt'
|
||||
echo 'RUN chmod 0555 /opt/itworx/contextforge-entrypoint.sh /opt/itworx/contextforge-runtime-contract.py'
|
||||
printf 'LABEL io.itworx.contextforge.contract-sha256="%s" \\\n' "$CONTRACT_SHA256"
|
||||
printf ' io.itworx.contextforge.entrypoint-sha256="%s" \\\n' "$ENTRYPOINT_SHA256"
|
||||
@@ -118,8 +247,24 @@ cp "$RUNTIME_VERIFIER" "$TMP/contextforge-runtime-contract.py"
|
||||
printf ' io.itworx.contextforge.source-commit="%s" \\\n' "$CONTEXTFORGE_UPSTREAM_COMMIT"
|
||||
printf ' io.itworx.contextforge.alembic-head="%s" \\\n' "$CONTEXTFORGE_ALEMBIC_HEAD"
|
||||
printf ' io.itworx.contextforge.mcp-sdk="%s"\n' "$CONTEXTFORGE_MCP_SDK"
|
||||
printf 'RUN /app/.venv/bin/python -m pip install --no-cache-dir --upgrade %q \\\n' "$MCP_REQUIREMENT"
|
||||
printf '%s\n' ' && /app/.venv/bin/python -c "import importlib.metadata as md; from mcp import McpError; assert md.version(\"mcp\") == \"'"${CONTEXTFORGE_MCP_SDK}"'\"; print(\"MCP SDK\", md.version(\"mcp\"), \"McpError OK\")"'
|
||||
printf 'LABEL io.itworx.contextforge.security-requirements-sha256="%s"\n' "$SECURITY_REQUIREMENTS_SHA256"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
printf 'LABEL org.opencontainers.image.source="https://gitea.example.com/jens/itworx-mcp-hub" \\\n'
|
||||
printf ' org.opencontainers.image.revision="%s" \\\n' "$APPOPS_SOURCE_REVISION"
|
||||
printf ' org.opencontainers.image.source-tree="%s" \\\n' "$APPOPS_SOURCE_TREE"
|
||||
printf ' org.opencontainers.image.version="%s" \\\n' "$APPOPS_BUILD_ID"
|
||||
printf ' org.opencontainers.image.source-date-epoch="%s" \\\n' "$SOURCE_DATE_EPOCH"
|
||||
printf ' org.opencontainers.image.created="%s"\n' "$APPOPS_BUILD_DATE"
|
||||
fi
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
echo 'RUN /app/.venv/bin/python -c "import importlib.metadata as md; import importlib.util; from mcp import McpError; assert md.version(\"mcp\") == \"1.30.0\"; assert md.version(\"PyJWT\") == \"2.15.0\"; assert md.version(\"urllib3\") == \"2.8.0\"; assert all(importlib.util.find_spec(name) is None for name in (\"pip\", \"setuptools\", \"wheel\", \"uv\"))"'
|
||||
else
|
||||
printf 'RUN /app/.venv/bin/python -m pip install --no-cache-dir --upgrade %q \\\n' "$MCP_REQUIREMENT"
|
||||
printf '%s\n' ' && /app/.venv/bin/python -c "import importlib.metadata as md; from mcp import McpError; assert md.version(\"mcp\") == \"'"${CONTEXTFORGE_MCP_SDK}"'\"; print(\"MCP SDK\", md.version(\"mcp\"), \"McpError OK\")"'
|
||||
echo 'RUN /app/.venv/bin/python -m pip install --no-cache-dir --no-deps --require-hashes --only-binary=:all: -r /opt/itworx/contextforge-security-requirements.txt \'
|
||||
echo ' && /app/.venv/bin/python -m pip check \'
|
||||
echo ' && /app/.venv/bin/python -c "import importlib.metadata as md; assert md.version(\"PyJWT\") == \"2.15.0\"; assert md.version(\"urllib3\") == \"2.8.0\""'
|
||||
fi
|
||||
if [[ -n "$BASE_USER" ]]; then
|
||||
printf 'USER %s\n' "$BASE_USER"
|
||||
fi
|
||||
@@ -142,6 +287,7 @@ embedded_sha() {
|
||||
[[ "$(embedded_sha /opt/itworx/contextforge-runtime-contract.json)" == "$CONTRACT_SHA256" ]] || fail "embedded runtime contract digest mismatch"
|
||||
[[ "$(embedded_sha /opt/itworx/contextforge-entrypoint.sh)" == "$ENTRYPOINT_SHA256" ]] || fail "embedded entrypoint digest mismatch"
|
||||
[[ "$(embedded_sha /opt/itworx/contextforge-runtime-contract.py)" == "$VERIFIER_SHA256" ]] || fail "embedded verifier digest mismatch"
|
||||
[[ "$(embedded_sha /opt/itworx/contextforge-security-requirements.txt)" == "$SECURITY_REQUIREMENTS_SHA256" ]] || fail "embedded security requirements digest mismatch"
|
||||
|
||||
label_value() {
|
||||
docker image inspect -f "{{ index .Config.Labels \"$1\" }}" "$STAGED_IMAGE"
|
||||
@@ -152,19 +298,43 @@ label_value() {
|
||||
[[ "$(label_value io.itworx.contextforge.source-commit)" == "$CONTEXTFORGE_UPSTREAM_COMMIT" ]] || fail "source label mismatch"
|
||||
[[ "$(label_value io.itworx.contextforge.alembic-head)" == "$CONTEXTFORGE_ALEMBIC_HEAD" ]] || fail "Alembic label mismatch"
|
||||
[[ "$(label_value io.itworx.contextforge.mcp-sdk)" == "$CONTEXTFORGE_MCP_SDK" ]] || fail "MCP SDK label mismatch"
|
||||
[[ "$(label_value io.itworx.contextforge.security-requirements-sha256)" == "$SECURITY_REQUIREMENTS_SHA256" ]] || fail "security requirements label mismatch"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
[[ "$(label_value org.opencontainers.image.source)" == "https://gitea.example.com/jens/itworx-mcp-hub" ]] || fail "OCI source label mismatch"
|
||||
[[ "$(label_value org.opencontainers.image.revision)" == "$APPOPS_SOURCE_REVISION" ]] || fail "OCI revision label mismatch"
|
||||
[[ "$(label_value org.opencontainers.image.source-tree)" == "$APPOPS_SOURCE_TREE" ]] || fail "OCI source tree label mismatch"
|
||||
[[ "$(label_value org.opencontainers.image.version)" == "$APPOPS_BUILD_ID" ]] || fail "OCI version label mismatch"
|
||||
[[ "$(label_value org.opencontainers.image.source-date-epoch)" == "$SOURCE_DATE_EPOCH" ]] || fail "OCI source date epoch label mismatch"
|
||||
[[ "$(label_value org.opencontainers.image.created)" == "$APPOPS_BUILD_DATE" ]] || fail "OCI created label mismatch"
|
||||
fi
|
||||
|
||||
echo "MCP SDK : $CONTEXTFORGE_MCP_SDK"
|
||||
echo "Alembic head : $IMAGE_HEAD"
|
||||
echo "Contract SHA : $CONTRACT_SHA256"
|
||||
|
||||
docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
# Recheck immediately before publishing. There is intentionally no cleanup or
|
||||
# retagging of any static compatibility image in candidate mode.
|
||||
ensure_candidate_tag_absent "$CANDIDATE_IMAGE"
|
||||
docker tag "$STAGED_IMAGE" "$CANDIDATE_IMAGE"
|
||||
else
|
||||
docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "======================================================"
|
||||
echo " SUCCESS: CONTEXTFORGE COMPATIBILITY IMAGE VERIFIED"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
echo " CANDIDATE IMAGE BUILT AND VERIFIED (NOT APPROVED OR DEPLOYED)"
|
||||
else
|
||||
echo " SUCCESS: CONTEXTFORGE COMPATIBILITY IMAGE VERIFIED"
|
||||
fi
|
||||
echo " source commit : $CONTEXTFORGE_UPSTREAM_COMMIT"
|
||||
echo " Alembic head : $CONTEXTFORGE_ALEMBIC_HEAD"
|
||||
echo " MCP SDK : $CONTEXTFORGE_MCP_SDK"
|
||||
echo " contract SHA : $CONTRACT_SHA256"
|
||||
echo " local image : $CONTEXTFORGE_LOCAL_IMAGE"
|
||||
if [[ "$MODE" == "release-candidate" ]]; then
|
||||
echo " candidate tag : $CANDIDATE_IMAGE"
|
||||
else
|
||||
echo " local image : $CONTEXTFORGE_LOCAL_IMAGE"
|
||||
fi
|
||||
echo "======================================================"
|
||||
+190
-22
@@ -12,9 +12,8 @@ Specifically fails on:
|
||||
grant);
|
||||
- a matching exception that has expired (expiryDate in the past) or is past
|
||||
its mandatory rescan date;
|
||||
- a matching exception whose recorded affectedImageDigests disagree with
|
||||
--image-digests for any service both documents name (the digest changed --
|
||||
the exception was reviewed against a build that no longer exists);
|
||||
- a matching exception without a valid image binding for each affected
|
||||
service, or whose binding differs from the current image inventory;
|
||||
- an exception missing any required field.
|
||||
|
||||
See docs/09-release/CVE_EXCEPTIONS.md for the full policy and
|
||||
@@ -25,6 +24,7 @@ context. This script has no "--force"/override flag by design.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
@@ -36,6 +36,8 @@ if str(_ROOT_DIR) not in sys.path:
|
||||
|
||||
from scripts.packlib import load_json # noqa: E402
|
||||
|
||||
_SHA256_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
REQUIRED_FIELDS = (
|
||||
"id",
|
||||
"cve",
|
||||
@@ -57,24 +59,134 @@ REQUIRED_FIELDS = (
|
||||
)
|
||||
|
||||
|
||||
def _findings(scan: dict[str, Any]) -> list[tuple[str, str, str]]:
|
||||
"""Return (package, cve, severity) for every CRITICAL/HIGH finding."""
|
||||
def _findings(scan: dict[str, Any]) -> list[tuple[dict[str, Any], str]]:
|
||||
"""Return (scan row, CVE) pairs for every CRITICAL/HIGH finding."""
|
||||
findings = []
|
||||
for row in scan.get("unresolvedCriticalOrHigh", []):
|
||||
for cve in row["cves"]:
|
||||
findings.append((row["package"], cve, row.get("severity", "UNKNOWN")))
|
||||
findings.append((row, cve))
|
||||
return findings
|
||||
|
||||
|
||||
def _current_image_inventory(
|
||||
image_digests: Any,
|
||||
) -> tuple[dict[str, dict[str, str]], list[str]]:
|
||||
"""Validate immutable current image bindings while retaining optional tags."""
|
||||
errors: list[str] = []
|
||||
if not isinstance(image_digests, dict) or not image_digests:
|
||||
return {}, ["IMAGE INVENTORY INVALID: --image-digests must contain a non-empty images map"]
|
||||
|
||||
inventory: dict[str, dict[str, str]] = {}
|
||||
for service, entry in image_digests.items():
|
||||
if not isinstance(service, str) or not service.strip():
|
||||
errors.append("IMAGE INVENTORY INVALID: every image service needs a non-empty name")
|
||||
continue
|
||||
if not isinstance(entry, dict):
|
||||
errors.append(f"IMAGE INVENTORY INVALID: {service} must be an image evidence object")
|
||||
continue
|
||||
|
||||
image_id = entry.get("imageId")
|
||||
if not isinstance(image_id, str) or not _SHA256_IMAGE_ID.fullmatch(image_id):
|
||||
errors.append(f"CURRENT IMAGE ID INVALID: {service} needs a sha256 imageId")
|
||||
continue
|
||||
|
||||
if "digest" in entry:
|
||||
digest = entry.get("digest")
|
||||
if not isinstance(digest, str) or not _SHA256_IMAGE_ID.fullmatch(digest):
|
||||
errors.append(f"CURRENT IMAGE DIGEST INVALID: {service} has a malformed digest")
|
||||
continue
|
||||
binding = digest
|
||||
else:
|
||||
# Older v1 inventory evidence records the immutable local image ID
|
||||
# instead of a registry manifest digest.
|
||||
binding = image_id
|
||||
|
||||
tag = entry.get("tag")
|
||||
if tag is not None and (not isinstance(tag, str) or not tag.strip()):
|
||||
errors.append(f"IMAGE INVENTORY INVALID: {service} has a malformed tag")
|
||||
continue
|
||||
inventory[service] = {"binding": binding, "imageId": image_id}
|
||||
if isinstance(tag, str):
|
||||
inventory[service]["tag"] = tag
|
||||
|
||||
if not inventory:
|
||||
errors.append("IMAGE INVENTORY INVALID: no valid current image bindings were supplied")
|
||||
return inventory, errors
|
||||
|
||||
|
||||
def _affected_services(
|
||||
row: dict[str, Any], inventory: dict[str, dict[str, str]]
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Resolve scan image tags to services, or use the legacy full inventory."""
|
||||
if "affectedImages" not in row:
|
||||
# Legacy v1 scan rows had no affectedImages field. Requiring the
|
||||
# exception to cover the whole supplied inventory is the safe fallback.
|
||||
return list(inventory), []
|
||||
|
||||
affected_images = row.get("affectedImages")
|
||||
if (
|
||||
not isinstance(affected_images, list)
|
||||
or not affected_images
|
||||
or any(not isinstance(image, str) or not image.strip() for image in affected_images)
|
||||
or len(set(affected_images)) != len(affected_images)
|
||||
):
|
||||
return [], ["SCAN AFFECTED IMAGES INVALID: expected a non-empty list of unique image tags"]
|
||||
|
||||
services_by_tag: dict[str, list[str]] = {}
|
||||
for service, evidence in inventory.items():
|
||||
tag = evidence.get("tag")
|
||||
if tag:
|
||||
services_by_tag.setdefault(tag, []).append(service)
|
||||
|
||||
services: list[str] = []
|
||||
errors: list[str] = []
|
||||
for image in affected_images:
|
||||
matches = services_by_tag.get(image, [])
|
||||
if not matches:
|
||||
errors.append(
|
||||
f"AFFECTED IMAGE UNBOUND: scan image {image} has no current service binding"
|
||||
)
|
||||
elif len(matches) > 1:
|
||||
errors.append(f"AFFECTED IMAGE AMBIGUOUS: scan image {image} maps to multiple services")
|
||||
else:
|
||||
services.append(matches[0])
|
||||
return services, errors
|
||||
|
||||
|
||||
def check(
|
||||
scan: dict[str, Any],
|
||||
exceptions_doc: dict[str, Any],
|
||||
image_digests: dict[str, str] | None,
|
||||
image_digests: dict[str, Any] | None,
|
||||
today: date,
|
||||
) -> list[str]:
|
||||
errors: list[str] = []
|
||||
by_key: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
image_inventory: dict[str, dict[str, str]] = {}
|
||||
if image_digests is not None:
|
||||
image_inventory, inventory_errors = _current_image_inventory(image_digests)
|
||||
errors.extend(inventory_errors)
|
||||
|
||||
by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
raw_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
conflicting_keys: set[tuple[str, str, str]] = set()
|
||||
for exception in exceptions_doc.get("exceptions", []):
|
||||
if not isinstance(exception, dict):
|
||||
errors.append("exception entry must be an object")
|
||||
continue
|
||||
package = exception.get("package")
|
||||
cve = exception.get("cve")
|
||||
version = exception.get("installedVersion")
|
||||
key: tuple[str, str, str] | None = None
|
||||
if isinstance(package, str) and isinstance(cve, str) and isinstance(version, str):
|
||||
if package and cve and version:
|
||||
key = (package, cve, version)
|
||||
if key is not None and key in raw_by_key and raw_by_key[key] != exception:
|
||||
conflicting_keys.add(key)
|
||||
errors.append(
|
||||
f"CONFLICTING DUPLICATE: multiple exception entries disagree for {package}/{cve}"
|
||||
)
|
||||
elif key is not None:
|
||||
raw_by_key.setdefault(key, exception)
|
||||
|
||||
missing = [field for field in REQUIRED_FIELDS if not exception.get(field)]
|
||||
# riskOwner/approver/compensatingControls may be legitimately empty
|
||||
# only while approvalStatus is "pending" -- once "approved" they are
|
||||
@@ -87,12 +199,44 @@ def check(
|
||||
f"exception {exception.get('id', '<no id>')} is missing required fields: {missing}"
|
||||
)
|
||||
continue
|
||||
key = (exception["package"], exception["cve"])
|
||||
by_key[key] = exception
|
||||
if key is not None:
|
||||
by_key.setdefault(key, exception)
|
||||
|
||||
for package, cve, severity in _findings(scan):
|
||||
key = (package, cve)
|
||||
exception = by_key.get(key)
|
||||
for key in conflicting_keys:
|
||||
by_key.pop(key, None)
|
||||
|
||||
by_pair: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
for (package, cve, _version), exception in by_key.items():
|
||||
by_pair.setdefault((package, cve), []).append(exception)
|
||||
|
||||
for row, cve in _findings(scan):
|
||||
package = row["package"]
|
||||
severity = row.get("severity", "UNKNOWN")
|
||||
reviewed = by_pair.get((package, cve), [])
|
||||
if "installedVersion" in row:
|
||||
scanned_version = row.get("installedVersion")
|
||||
if not isinstance(scanned_version, str) or not scanned_version:
|
||||
errors.append(
|
||||
f"SCAN VERSION MISSING: {package}/{cve} has no valid installedVersion"
|
||||
)
|
||||
continue
|
||||
exception = by_key.get((package, cve, scanned_version))
|
||||
if exception is None and reviewed:
|
||||
errors.append(
|
||||
f"VERSION MISMATCH: {package}/{cve} scan found {scanned_version}, "
|
||||
"but no exception reviewed that exact version"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
# Existing v1 evidence may omit the installed version. It remains
|
||||
# usable only when one unambiguous reviewed version exists.
|
||||
if len(reviewed) > 1:
|
||||
errors.append(
|
||||
f"AMBIGUOUS VERSION: {package}/{cve} legacy scan omits installedVersion "
|
||||
"and multiple versions were reviewed"
|
||||
)
|
||||
continue
|
||||
exception = reviewed[0] if reviewed else None
|
||||
if exception is None:
|
||||
errors.append(
|
||||
f"UNAPPROVED: {severity} finding {package}/{cve} has no exception recorded at all"
|
||||
@@ -128,11 +272,38 @@ def check(
|
||||
f"{rescan_due.isoformat()} has passed"
|
||||
)
|
||||
|
||||
if image_digests:
|
||||
if image_digests is not None:
|
||||
legacy_full_inventory = "affectedImages" not in row
|
||||
services, scope_errors = _affected_services(row, image_inventory)
|
||||
errors.extend(scope_errors)
|
||||
recorded = exception.get("affectedImageDigests", {})
|
||||
for service, current_digest in image_digests.items():
|
||||
if not isinstance(recorded, dict):
|
||||
errors.append(
|
||||
f"IMAGE BINDING INVALID: {package}/{cve} affectedImageDigests must be an object"
|
||||
)
|
||||
continue
|
||||
if legacy_full_inventory and set(recorded) != set(image_inventory):
|
||||
errors.append(
|
||||
f"IMAGE INVENTORY SCOPE CHANGED: {package}/{cve} uses a legacy scan "
|
||||
"without affectedImages and its reviewed service set does not exactly "
|
||||
"match the current full inventory"
|
||||
)
|
||||
for service in services:
|
||||
current_digest = image_inventory[service]["binding"]
|
||||
recorded_digest = recorded.get(service)
|
||||
if recorded_digest and recorded_digest != current_digest:
|
||||
if recorded_digest is None or recorded_digest == "":
|
||||
errors.append(
|
||||
f"IMAGE BINDING MISSING: {package}/{cve} has no valid reviewed digest "
|
||||
f"for {service}"
|
||||
)
|
||||
elif not isinstance(recorded_digest, str) or not _SHA256_IMAGE_ID.fullmatch(
|
||||
recorded_digest
|
||||
):
|
||||
errors.append(
|
||||
f"IMAGE BINDING INVALID: {package}/{cve} has a malformed reviewed digest "
|
||||
f"for {service}"
|
||||
)
|
||||
elif recorded_digest != current_digest:
|
||||
errors.append(
|
||||
f"DIGEST CHANGED: {package}/{cve} exception recorded "
|
||||
f"{service}={recorded_digest}, but the current build is "
|
||||
@@ -151,19 +322,16 @@ def main() -> int:
|
||||
"--image-digests",
|
||||
type=Path,
|
||||
help="evidence/<packet>/image-digests.json; when given, exceptions whose "
|
||||
"recorded digest for a named service disagrees with the current build fail.",
|
||||
"current image IDs or reviewed bindings are missing, malformed, or changed fail.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
scan = load_json(args.scan_summary)
|
||||
exceptions_doc = load_json(args.exceptions)
|
||||
image_digests: dict[str, str] | None = None
|
||||
image_digests: dict[str, Any] | None = None
|
||||
if args.image_digests:
|
||||
digest_doc = load_json(args.image_digests)
|
||||
image_digests = {
|
||||
service: (entry.get("digest") or entry.get("imageId"))
|
||||
for service, entry in digest_doc.get("images", {}).items()
|
||||
}
|
||||
image_digests = digest_doc.get("images", {})
|
||||
|
||||
errors = check(scan, exceptions_doc, image_digests, date.today())
|
||||
if errors:
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a hash-locked, source-checked ContextForge candidate build recipe."""
|
||||
|
||||
# Exact image references, source anchors, and Dockerfile instructions exceed 100 columns.
|
||||
# ruff: noqa: E501
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOCK_DIR = ROOT / "infrastructure" / "images" / "contextforge" / "candidate"
|
||||
LOCK_MANIFEST = LOCK_DIR / "locks.json"
|
||||
PRODUCTION_LOCK = LOCK_DIR / "production-requirements.lock"
|
||||
BUILD_TOOLS_LOCK = LOCK_DIR / "build-tools.lock"
|
||||
|
||||
UPSTREAM_COMMIT = "1e58a0434a78bbdbe1e975abeb0bed1043ff14c1"
|
||||
UV_VERSION = "0.12.10"
|
||||
LOCK_TIMESTAMP = "2026-10-02T00:00:00Z"
|
||||
GLOBAL_LOCK_CUTOFF = "2026-09-22T00:00:00Z"
|
||||
EXTRAS = ("redis", "postgres", "observability", "granian", "plugins", "llmchat")
|
||||
CONSTRAINTS = ("mcp==1.30.0", "PyJWT==2.15.0", "urllib3==2.8.0")
|
||||
BUILD_TOOLS = ("pip==26.2.1", "setuptools==82.0.1", "wheel==0.46.2")
|
||||
|
||||
SOURCE_HASHES = {
|
||||
"Containerfile": "2ab0187f3a63eced093a1dec3141ec3e30ef879eb7de0dfd9e4bc5fce33f64f2",
|
||||
"pyproject.toml": "cd365566f922546e569c65475d78cd53e5d49f10df2c82534efa607ffcf55e4b",
|
||||
"uv.lock": "c5297a4701e4a0ecb9903bd3f4aa1c8cf40cf4e5568880fcb8f4078703589df1",
|
||||
}
|
||||
|
||||
BASE_IMAGES = (
|
||||
{
|
||||
"arg": "UBI_BASE",
|
||||
"sourceTag": "gitea.example.com/jens/itworx-mcp-hub/contextforge-os-builder:sha256-c6bb70a7a340ed1047e7cb934aa55b086ee68a5eb9ba6b2ad2e6825c7184a3e8",
|
||||
"reference": "gitea.example.com/jens/itworx-mcp-hub/contextforge-os-builder@sha256:e23a733abd28376eb3328676a3ab1e6d153122228110de0c881bb5f3c38a4483",
|
||||
"parentReference": "registry.access.redhat.com/ubi10@sha256:1e32d829a4de713a835d702d3bc05f0c448a08590796281b7dec181912e9aeb8",
|
||||
"imageId": "sha256:c6bb70a7a340ed1047e7cb934aa55b086ee68a5eb9ba6b2ad2e6825c7184a3e8",
|
||||
"rpmInventorySha256": "6a8c0d4f66fc5a38008541ad15614711e7f536cd18c70c02d2eb0691552b88b0",
|
||||
},
|
||||
{
|
||||
"arg": "NODEJS_IMAGE",
|
||||
"sourceTag": "registry.access.redhat.com/ubi10/nodejs-24:1784092609",
|
||||
"reference": "registry.access.redhat.com/ubi10/nodejs-24@sha256:0f110e68682f4a49188ba4db786eb4f207b119420913fc76bb5ba9512ffbef06",
|
||||
},
|
||||
{
|
||||
"arg": "UBI_MINIMAL",
|
||||
"sourceTag": "gitea.example.com/jens/itworx-mcp-hub/contextforge-os-runtime:sha256-fd94e2643dfc0401790d0528d3740bb2f0c04709e743f6c774a4e8a38606e40b",
|
||||
"reference": "gitea.example.com/jens/itworx-mcp-hub/contextforge-os-runtime@sha256:9b23d8152f253b10e47ce077bb9c5f8c3c370f3e0d5533f8a4a0e986ff176236",
|
||||
"parentReference": "registry.access.redhat.com/ubi10/ubi-minimal@sha256:afbfb5c18c628796fed3ad3122dcf096e0b2dec9dad3fbabe8620de06bc6f078",
|
||||
"imageId": "sha256:fd94e2643dfc0401790d0528d3740bb2f0c04709e743f6c774a4e8a38606e40b",
|
||||
"rpmInventorySha256": "a74d60ecb43d1aa561874305d1a9c62ac35c99a14c1dc275ba1656f14df7d034",
|
||||
},
|
||||
)
|
||||
|
||||
LOCK_OUTPUT_NAMES = {
|
||||
"production-requirements.lock": "contextforge-candidate-production-requirements.txt",
|
||||
"build-tools.lock": "contextforge-candidate-build-tools.txt",
|
||||
}
|
||||
|
||||
INSTALL_START = (
|
||||
"RUN set -euo pipefail \\\n"
|
||||
" && . /etc/profile.d/use-openssl.sh \\\n"
|
||||
" && python3 -m venv /app/.venv"
|
||||
)
|
||||
INSTALL_END = "\n# ----------------------------------------------------------------------------\n# Copy Rust runtime binaries and frontend static output from their builder\n"
|
||||
LOCK_COPY_ANCHOR = "COPY pyproject.toml /app/\n"
|
||||
|
||||
|
||||
class CandidateRecipeError(ValueError):
|
||||
"""Raised when a candidate recipe input is absent or differs from contract."""
|
||||
|
||||
|
||||
def sha256_hex(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _require_file(path: Path) -> bytes:
|
||||
if not path.is_file() or path.is_symlink():
|
||||
raise CandidateRecipeError(f"required regular file is missing: {path.name}")
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def verify_upstream_source(
|
||||
source_dir: Path, expected_hashes: dict[str, str] | None = None
|
||||
) -> dict[str, bytes]:
|
||||
files: dict[str, bytes] = {}
|
||||
for name, expected in (expected_hashes or SOURCE_HASHES).items():
|
||||
content = _require_file(source_dir / name)
|
||||
actual = sha256_hex(content)
|
||||
if actual != expected:
|
||||
raise CandidateRecipeError(f"pinned upstream {name} hash mismatch")
|
||||
files[name] = content
|
||||
return files
|
||||
|
||||
|
||||
def _table_range(text: str, table: str) -> tuple[int, int, str]:
|
||||
header = f"[{table}]"
|
||||
matches = list(re.finditer(rf"(?m)^{re.escape(header)}\s*$", text))
|
||||
if len(matches) != 1:
|
||||
raise CandidateRecipeError(f"expected exactly one TOML table {header}")
|
||||
start = matches[0].start()
|
||||
next_header = re.search(r"(?m)^\[[^\r\n]+\]\s*$", text[matches[0].end() :])
|
||||
end = matches[0].end() + next_header.start() if next_header else len(text)
|
||||
return start, end, text[start:end]
|
||||
|
||||
|
||||
def candidate_pyproject(source_text: str) -> str:
|
||||
"""Apply the narrowly scoped candidate-only dependency resolution policy."""
|
||||
try:
|
||||
import tomllib
|
||||
|
||||
source = tomllib.loads(source_text)
|
||||
except (ImportError, ValueError) as exc:
|
||||
raise CandidateRecipeError("upstream pyproject is not valid TOML") from exc
|
||||
|
||||
uv = source.get("tool", {}).get("uv", {})
|
||||
if uv.get("exclude-newer") != "10 days":
|
||||
raise CandidateRecipeError("upstream uv freshness policy changed")
|
||||
groups = source.get("dependency-groups", {})
|
||||
if set(groups) != {"dev"}:
|
||||
raise CandidateRecipeError("upstream dependency-group shape changed")
|
||||
sources = uv.get("sources", {})
|
||||
if sources != {
|
||||
"compliance-reference-server": {
|
||||
"path": "mcp-servers/python/compliance_reference_server",
|
||||
"editable": True,
|
||||
}
|
||||
}:
|
||||
raise CandidateRecipeError("upstream uv source override shape changed")
|
||||
|
||||
text = source_text.replace("\r\n", "\n")
|
||||
source_start, source_end, source_table = _table_range(text, "tool.uv.sources")
|
||||
source_lines = source_table.strip().splitlines()
|
||||
expected_source_lines = [
|
||||
"[tool.uv.sources]",
|
||||
'compliance-reference-server = { path = "mcp-servers/python/compliance_reference_server", editable = true }',
|
||||
]
|
||||
if source_lines[:2] != expected_source_lines or any(
|
||||
line.strip() and not line.lstrip().startswith("#") for line in source_lines[2:]
|
||||
):
|
||||
raise CandidateRecipeError("upstream local development source table changed")
|
||||
text = text[:source_start] + text[source_end:]
|
||||
|
||||
group_start, group_end, _group_table = _table_range(text, "dependency-groups")
|
||||
text = text[:group_start] + text[group_end:]
|
||||
|
||||
uv_anchor = 'exclude-newer = "10 days"\n'
|
||||
if text.count(uv_anchor) != 1:
|
||||
raise CandidateRecipeError("upstream uv table anchor changed")
|
||||
text = text.replace(
|
||||
uv_anchor,
|
||||
f'exclude-newer = "{GLOBAL_LOCK_CUTOFF}"\n'
|
||||
+ 'constraint-dependencies = ["mcp==1.30.0", "PyJWT==2.15.0", "urllib3==2.8.0"]\n',
|
||||
1,
|
||||
)
|
||||
|
||||
package_table = "[tool.uv.exclude-newer-package]\n"
|
||||
if text.count(package_table) != 1:
|
||||
raise CandidateRecipeError("upstream package freshness table is missing")
|
||||
package_overrides = "".join(
|
||||
f'{name.lower()} = "{LOCK_TIMESTAMP}"\n'
|
||||
for name in ("mcp", "pyjwt", "urllib3", "pip", "setuptools", "wheel")
|
||||
)
|
||||
text = text.replace(package_table, package_table + package_overrides, 1)
|
||||
|
||||
text = text.rstrip() + "\n\n[dependency-groups]\nbuild-tools = [\n"
|
||||
text += "".join(f' "{requirement}",\n' for requirement in BUILD_TOOLS)
|
||||
text += "]\n"
|
||||
try:
|
||||
tomllib.loads(text)
|
||||
except ValueError as exc:
|
||||
raise CandidateRecipeError("generated candidate pyproject is invalid TOML") from exc
|
||||
return text
|
||||
|
||||
|
||||
def _run_uv(args: list[str], cwd: Path) -> None:
|
||||
completed = subprocess.run( # noqa: S603 - fixed executable and structured arguments
|
||||
[_uv_binary(), *args],
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if completed.returncode:
|
||||
diagnostic = completed.stderr.strip().splitlines()[-1:] or ["uv command failed"]
|
||||
raise CandidateRecipeError(diagnostic[0])
|
||||
|
||||
|
||||
def _uv_binary() -> str:
|
||||
binary = shutil.which("uv")
|
||||
if binary is None or not Path(binary).is_absolute():
|
||||
raise CandidateRecipeError("uv executable is missing or non-absolute")
|
||||
return binary
|
||||
|
||||
|
||||
def generate_locks(source_dir: Path, output_dir: Path) -> None:
|
||||
"""Regenerate locks from the exact upstream project using the pinned uv tool."""
|
||||
source = verify_upstream_source(source_dir)
|
||||
version = subprocess.run( # noqa: S603 - fixed version check before lock generation
|
||||
[_uv_binary(), "--version"], check=False, capture_output=True, text=True
|
||||
)
|
||||
version_parts = version.stdout.strip().split()
|
||||
if version.returncode or version_parts[:2] != ["uv", UV_VERSION]:
|
||||
raise CandidateRecipeError(f"lock generation requires uv {UV_VERSION}")
|
||||
|
||||
output_dir = output_dir.resolve()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="contextforge-lock-") as temp_name:
|
||||
temp = Path(temp_name)
|
||||
(temp / "pyproject.toml").write_text(
|
||||
candidate_pyproject(source["pyproject.toml"].decode("utf-8")), encoding="utf-8"
|
||||
)
|
||||
(temp / "uv.lock").write_bytes(source["uv.lock"])
|
||||
_run_uv(["lock", "--project", str(temp)], temp)
|
||||
|
||||
production_path = output_dir / "production-requirements.lock"
|
||||
build_path = output_dir / "build-tools.lock"
|
||||
_run_uv(
|
||||
[
|
||||
"export",
|
||||
"--project",
|
||||
str(temp),
|
||||
"--format",
|
||||
"requirements-txt",
|
||||
"--locked",
|
||||
"--no-dev",
|
||||
"--no-default-groups",
|
||||
"--no-emit-project",
|
||||
"--no-header",
|
||||
"--no-annotate",
|
||||
*[value for extra in EXTRAS for value in ("--extra", extra)],
|
||||
"--output-file",
|
||||
str(production_path),
|
||||
],
|
||||
temp,
|
||||
)
|
||||
_run_uv(
|
||||
[
|
||||
"export",
|
||||
"--project",
|
||||
str(temp),
|
||||
"--format",
|
||||
"requirements-txt",
|
||||
"--locked",
|
||||
"--only-group",
|
||||
"build-tools",
|
||||
"--no-default-groups",
|
||||
"--no-emit-project",
|
||||
"--no-header",
|
||||
"--no-annotate",
|
||||
"--output-file",
|
||||
str(build_path),
|
||||
],
|
||||
temp,
|
||||
)
|
||||
|
||||
manifest = _make_manifest(
|
||||
source_hashes=SOURCE_HASHES,
|
||||
production_hash=sha256_hex(_require_file(output_dir / "production-requirements.lock")),
|
||||
build_tools_hash=sha256_hex(_require_file(output_dir / "build-tools.lock")),
|
||||
)
|
||||
_write_atomic(output_dir / "locks.json", _canonical_json(manifest))
|
||||
|
||||
|
||||
def _make_manifest(
|
||||
*, source_hashes: dict[str, str], production_hash: str, build_tools_hash: str
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": "itworx.contextforge-candidate-locks/v1",
|
||||
"upstreamCommit": UPSTREAM_COMMIT,
|
||||
"upstreamFiles": source_hashes,
|
||||
"uvVersion": UV_VERSION,
|
||||
"python": "3.12",
|
||||
"platform": "linux/amd64",
|
||||
"extras": list(EXTRAS),
|
||||
"constraints": list(CONSTRAINTS),
|
||||
"buildTools": list(BUILD_TOOLS),
|
||||
"globalExcludeNewerUtc": GLOBAL_LOCK_CUTOFF,
|
||||
"freshnessOverrideUtc": LOCK_TIMESTAMP,
|
||||
"requirements": {
|
||||
"production-requirements.lock": production_hash,
|
||||
"build-tools.lock": build_tools_hash,
|
||||
},
|
||||
"baseImages": [
|
||||
{
|
||||
**base,
|
||||
"platform": "linux/amd64",
|
||||
"childManifestDigest": base["reference"].split("@", 1)[1],
|
||||
}
|
||||
for base in BASE_IMAGES
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: Any) -> bytes:
|
||||
return (json.dumps(value, sort_keys=True, indent=2) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def _write_atomic(path: Path, content: bytes) -> None:
|
||||
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
try:
|
||||
temporary.write_bytes(content)
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _read_lock_manifest() -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(_require_file(LOCK_MANIFEST).decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise CandidateRecipeError("candidate lock manifest is invalid JSON") from exc
|
||||
expected = _make_manifest(
|
||||
source_hashes=SOURCE_HASHES,
|
||||
production_hash=str(
|
||||
payload.get("requirements", {}).get("production-requirements.lock", "")
|
||||
),
|
||||
build_tools_hash=str(payload.get("requirements", {}).get("build-tools.lock", "")),
|
||||
)
|
||||
if payload != expected:
|
||||
raise CandidateRecipeError("candidate lock manifest differs from the reviewed contract")
|
||||
for name, expected_hash in payload["requirements"].items():
|
||||
if sha256_hex(_require_file(LOCK_DIR / name)) != expected_hash:
|
||||
raise CandidateRecipeError(f"candidate lock hash mismatch: {name}")
|
||||
return payload
|
||||
|
||||
|
||||
def verify_lock_contents(manifest: dict[str, Any]) -> None:
|
||||
production = _require_file(LOCK_DIR / "production-requirements.lock").decode("utf-8")
|
||||
build_tools = _require_file(LOCK_DIR / "build-tools.lock").decode("utf-8")
|
||||
for requirement in CONSTRAINTS:
|
||||
name, version = requirement.split("==", 1)
|
||||
match = re.search(rf"(?im)^{re.escape(name)}==([^ ;\\]+)", production)
|
||||
if match is None or match.group(1) != version:
|
||||
raise CandidateRecipeError(f"production lock does not pin {name}=={version}")
|
||||
for requirement in BUILD_TOOLS:
|
||||
name, version = requirement.split("==", 1)
|
||||
match = re.search(rf"(?im)^{re.escape(name)}==([^ ;\\]+)", build_tools)
|
||||
if match is None or match.group(1) != version:
|
||||
raise CandidateRecipeError(f"build-tool lock does not pin {name}=={version}")
|
||||
for name, requirements in (
|
||||
("production", production),
|
||||
("build-tool", build_tools),
|
||||
):
|
||||
blocks = re.split(r"(?m)(?=^[A-Za-z0-9_.-]+==)", requirements)
|
||||
blocks = [block for block in blocks if re.match(r"(?m)^[A-Za-z0-9_.-]+==", block)]
|
||||
if not blocks or any(
|
||||
not re.search(r"(?m)^\s*--hash=sha256:[0-9a-f]{64}\s*$", block) for block in blocks
|
||||
):
|
||||
raise CandidateRecipeError(f"candidate {name} requirements must pin every wheel hash")
|
||||
if "pip==26.2.1" not in build_tools:
|
||||
raise CandidateRecipeError("candidate build tools must include the removable pip bootstrap")
|
||||
if manifest["platform"] != "linux/amd64":
|
||||
raise CandidateRecipeError("candidate lock platform differs from the fixed build platform")
|
||||
|
||||
|
||||
def freeze_os_layers(text: str) -> str:
|
||||
"""Replace the four upstream RPM transactions with frozen-base assertions."""
|
||||
guards = (
|
||||
(
|
||||
'RUN if [ "$ENABLE_RUST" = "true" ]; then',
|
||||
'RUN test "$ENABLE_RUST" = "false" && test "$ENABLE_RUST_MCP_RMCP" = "false"',
|
||||
),
|
||||
(
|
||||
"RUN set -euo pipefail",
|
||||
'RUN test "$PYTHON_VERSION" = "3.12" && test -s /usr/share/itworx/rpm-inventory.txt',
|
||||
),
|
||||
(
|
||||
"RUN microdnf upgrade",
|
||||
'RUN test "$PYTHON_VERSION" = "3.12" && test "$ENABLE_PROFILING" = "false" '
|
||||
'&& test "$(id -u app):$(id -g app)" = "10001:10001" '
|
||||
"&& test -s /usr/share/itworx/rpm-inventory.txt",
|
||||
),
|
||||
('RUN if [ "$ENABLE_FIPS" = "true" ]; then', 'RUN test "$ENABLE_FIPS" = "false"'),
|
||||
)
|
||||
transactions = [
|
||||
match
|
||||
for match in re.finditer(r"(?m)^RUN [^\n]*(?:\n(?<=\\\n)[^\n]*)*", text)
|
||||
if re.search(r"\b(?:microdnf|dnf) (?:upgrade|install)\b", match.group())
|
||||
]
|
||||
if len(transactions) != len(guards):
|
||||
raise CandidateRecipeError("upstream RPM transaction count changed")
|
||||
for match, (prefix, guard) in reversed(list(zip(transactions, guards, strict=True))):
|
||||
if not match.group().startswith(prefix):
|
||||
raise CandidateRecipeError("upstream RPM transaction order changed")
|
||||
text = text[: match.start()] + guard + text[match.end() :]
|
||||
return text
|
||||
|
||||
|
||||
def transform_containerfile(containerfile: bytes, expected_hash: str | None = None) -> bytes:
|
||||
if sha256_hex(containerfile) != (expected_hash or SOURCE_HASHES["Containerfile"]):
|
||||
raise CandidateRecipeError("pinned upstream Containerfile hash mismatch")
|
||||
text = freeze_os_layers(containerfile.decode("utf-8"))
|
||||
if text.count(LOCK_COPY_ANCHOR) != 1:
|
||||
raise CandidateRecipeError("upstream dependency-copy anchor count changed")
|
||||
text = text.replace(
|
||||
LOCK_COPY_ANCHOR,
|
||||
LOCK_COPY_ANCHOR
|
||||
+ "COPY contextforge-candidate-production-requirements.txt /tmp/contextforge-production-requirements.txt\n"
|
||||
+ "COPY contextforge-candidate-build-tools.txt /tmp/contextforge-build-tools.txt\n",
|
||||
1,
|
||||
)
|
||||
|
||||
start = text.find(INSTALL_START)
|
||||
end = text.find(INSTALL_END, start)
|
||||
if start < 0 or end < 0 or text.find(INSTALL_START, start + 1) >= 0:
|
||||
raise CandidateRecipeError("upstream dependency-install block anchors changed")
|
||||
locked_install = """RUN set -euo pipefail \\
|
||||
&& . /etc/profile.d/use-openssl.sh \\
|
||||
&& python3 -m venv /app/.venv \\
|
||||
&& /app/.venv/bin/python -m ensurepip --upgrade \\
|
||||
&& /app/.venv/bin/python -m pip install --disable-pip-version-check --no-cache-dir --require-hashes --only-binary=:all: -r /tmp/contextforge-build-tools.txt \\
|
||||
&& /app/.venv/bin/python -m pip install --disable-pip-version-check --no-cache-dir --require-hashes --only-binary=:all: -r /tmp/contextforge-production-requirements.txt \\
|
||||
&& /app/.venv/bin/python -m pip install --disable-pip-version-check --no-cache-dir --no-deps --no-build-isolation ".[redis,postgres,observability,granian,plugins,llmchat]" \\
|
||||
&& /app/.venv/bin/python -m pip check \\
|
||||
&& /app/.venv/bin/python -c "import importlib.metadata as md; assert md.version('mcp') == '1.30.0'; assert md.version('PyJWT') == '2.15.0'; assert md.version('urllib3') == '2.8.0'" \\
|
||||
&& /app/.venv/bin/python -m pip uninstall --yes pip setuptools wheel \\
|
||||
&& (find /app/.venv -name "*.dist-info" -type d \\
|
||||
\\( -name "pip-*" -o -name "setuptools-*" -o -name "wheel-*" \\) \\
|
||||
-exec rm -rf {} + 2>/dev/null || true) \\
|
||||
&& rm -rf /app/.venv/share/python-wheels \\
|
||||
&& rm -rf "/app/"*.egg-info /app/build /app/dist /app/.eggs \\
|
||||
&& rm -rf /root/.cache /var/cache/dnf
|
||||
"""
|
||||
text = text[:start] + locked_install + text[end:]
|
||||
return text.encode("utf-8")
|
||||
|
||||
|
||||
def prepare_source(source_dir: Path) -> None:
|
||||
source = verify_upstream_source(source_dir)
|
||||
manifest = _read_lock_manifest()
|
||||
verify_lock_contents(manifest)
|
||||
transformed = transform_containerfile(source["Containerfile"])
|
||||
|
||||
for source_name, target_name in LOCK_OUTPUT_NAMES.items():
|
||||
_write_atomic(source_dir / target_name, _require_file(LOCK_DIR / source_name))
|
||||
_write_atomic(source_dir / "Containerfile.itworx-candidate", transformed)
|
||||
|
||||
|
||||
def print_build_args() -> None:
|
||||
for key, value in candidate_build_args().items():
|
||||
print(f"{key}={value}")
|
||||
|
||||
|
||||
def candidate_build_args() -> dict[str, str]:
|
||||
manifest = _read_lock_manifest()
|
||||
verify_lock_contents(manifest)
|
||||
overrides = {item["arg"]: item["reference"] for item in BASE_IMAGES}
|
||||
overrides.update(
|
||||
{
|
||||
"WHEELS_REF": overrides["UBI_MINIMAL"],
|
||||
"PYTHON_VERSION": "3.12",
|
||||
"ENABLE_RUST": "false",
|
||||
"ENABLE_RUST_MCP_RMCP": "false",
|
||||
"ENABLE_PROFILING": "false",
|
||||
"ENABLE_FIPS": "false",
|
||||
}
|
||||
)
|
||||
return overrides
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
prepare = commands.add_parser(
|
||||
"prepare", help="verify and transform an exact fetched source tree"
|
||||
)
|
||||
prepare.add_argument("--source-dir", type=Path, required=True)
|
||||
generate = commands.add_parser("generate-locks", help="regenerate lock files with pinned uv")
|
||||
generate.add_argument("--source-dir", type=Path, required=True)
|
||||
generate.add_argument("--output-dir", type=Path, default=LOCK_DIR)
|
||||
commands.add_parser("build-args", help="print fixed platform/base image build arguments")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
if args.command == "prepare":
|
||||
prepare_source(args.source_dir)
|
||||
elif args.command == "generate-locks":
|
||||
generate_locks(args.source_dir, args.output_dir)
|
||||
else:
|
||||
print_build_args()
|
||||
except (CandidateRecipeError, OSError, subprocess.SubprocessError) as exc:
|
||||
parser.exit(1, f"FATAL: {exc}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -16,6 +16,7 @@ satisfying the release gate for that finding.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from datetime import date, timedelta
|
||||
@@ -28,6 +29,52 @@ if str(_ROOT_DIR) not in sys.path:
|
||||
from scripts.packlib import brussels_now, load_json # noqa: E402
|
||||
|
||||
|
||||
def _affected_image_digests(row: dict[str, object], images: dict[str, object]) -> dict[str, str]:
|
||||
"""Select current image bindings by scan tag, with a legacy full-inventory fallback."""
|
||||
if not images:
|
||||
raise ValueError("--image-digests must contain a non-empty images map")
|
||||
|
||||
if "affectedImages" not in row:
|
||||
services = list(images)
|
||||
else:
|
||||
affected_images = row.get("affectedImages")
|
||||
if (
|
||||
not isinstance(affected_images, list)
|
||||
or not affected_images
|
||||
or any(not isinstance(tag, str) or not tag.strip() for tag in affected_images)
|
||||
or len(set(affected_images)) != len(affected_images)
|
||||
):
|
||||
raise ValueError("scan affectedImages must be a non-empty list of unique image tags")
|
||||
|
||||
services_by_tag: dict[str, list[str]] = {}
|
||||
for service, entry in images.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
tag = entry.get("tag")
|
||||
if isinstance(tag, str) and tag:
|
||||
services_by_tag.setdefault(tag, []).append(service)
|
||||
|
||||
services = []
|
||||
for tag in affected_images:
|
||||
matches = services_by_tag.get(tag, [])
|
||||
if len(matches) != 1:
|
||||
raise ValueError(
|
||||
f"scan image {tag} must resolve to exactly one current image service"
|
||||
)
|
||||
services.append(matches[0])
|
||||
|
||||
bindings: dict[str, str] = {}
|
||||
for service in services:
|
||||
entry = images.get(service)
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError(f"image evidence for {service} must be an object")
|
||||
digest = entry.get("digest") or entry.get("imageId")
|
||||
if not isinstance(digest, str) or not digest:
|
||||
raise ValueError(f"image evidence for {service} has no immutable digest or imageId")
|
||||
bindings[service] = digest
|
||||
return bindings
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--scan-summary", required=True, type=Path)
|
||||
@@ -46,25 +93,41 @@ def main() -> int:
|
||||
scan_date = scan.get("scannedAt", brussels_now())[:10]
|
||||
scanner = f"trivy {scan.get('trivyVersion', 'unknown')}"
|
||||
|
||||
affected_digests: dict[str, str] = {}
|
||||
image_inventory: dict[str, object] = {}
|
||||
if args.image_digests:
|
||||
digest_doc = load_json(args.image_digests)
|
||||
for service, entry in digest_doc.get("images", {}).items():
|
||||
digest = entry.get("digest") or entry.get("imageId")
|
||||
if digest:
|
||||
affected_digests[service] = digest
|
||||
raw_images = digest_doc.get("images", {})
|
||||
if isinstance(raw_images, dict):
|
||||
image_inventory = raw_images
|
||||
|
||||
today = date.fromisoformat(scan_date) if len(scan_date) == 10 else date.today()
|
||||
expiry = (today + timedelta(days=args.expiry_days)).isoformat()
|
||||
rescan = (today + timedelta(days=args.rescan_days)).isoformat()
|
||||
|
||||
versions_by_pair: dict[tuple[str, str], set[str]] = {}
|
||||
for row in scan.get("unresolvedCriticalOrHigh", []):
|
||||
for cve in row["cves"]:
|
||||
versions_by_pair.setdefault((row["package"], cve), set()).add(
|
||||
str(row.get("installedVersion"))
|
||||
)
|
||||
exceptions = []
|
||||
for row in scan.get("unresolvedCriticalOrHigh", []):
|
||||
package = row["package"]
|
||||
try:
|
||||
affected_digests = (
|
||||
_affected_image_digests(row, image_inventory) if args.image_digests else {}
|
||||
)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
for cve in row["cves"]:
|
||||
version_suffix = ""
|
||||
if len(versions_by_pair[(package, cve)]) > 1:
|
||||
version_suffix = (
|
||||
"-" + hashlib.sha256(str(row.get("installedVersion")).encode()).hexdigest()[:12]
|
||||
)
|
||||
exceptions.append(
|
||||
{
|
||||
"id": f"{package}-{cve}",
|
||||
"id": f"{package}-{cve}{version_suffix}",
|
||||
"cve": cve,
|
||||
"package": package,
|
||||
"installedVersion": row.get("installedVersion"),
|
||||
|
||||
Reference in new issue
Block a user