803 lines
33 KiB
Python
803 lines
33 KiB
Python
"""Build a ModelForge release: validate, build, inventory, sign the inventory, package.
|
|
|
|
One command, so that what ships is what was tested rather than whatever happened to be in the
|
|
working tree. Everything it emits is derived from the repository and the built images — nothing is
|
|
typed in by hand, because a release manifest an operator cannot verify is decoration.
|
|
|
|
python scripts/release_build.py --output dist
|
|
|
|
The build refuses to run against a dirty working tree unless told otherwise. A release built from
|
|
uncommitted changes cannot be reproduced, and its recorded source commit would be a lie.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "backend" / "src"))
|
|
|
|
from modelforge_api.domain.release import ( # noqa: E402
|
|
CURRENT_AGENT_PROTOCOL_VERSION,
|
|
MINIMUM_POSTGRES_MAJOR,
|
|
MINIMUM_UPGRADE_SOURCE,
|
|
PRODUCT_NAME,
|
|
PRODUCT_VERSION,
|
|
RELEASE_CHANNEL,
|
|
SUPPORTED_AGENT_PROTOCOL_VERSIONS,
|
|
SUPPORTED_SCHEMA_REVISIONS,
|
|
SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS,
|
|
TARGET_SCHEMA_REVISION,
|
|
)
|
|
|
|
#: image name -> (build context, Dockerfile path), both relative to the repository root. Python
|
|
#: images build from the repository root so they can consume the shared hash-locked toolchain.
|
|
#: Keeping the four distributable images in one inventory makes it impossible for the private
|
|
#: runtime boundary to disappear from a release.
|
|
IMAGES = {
|
|
"modelforge-api": (".", "backend/Dockerfile"),
|
|
"modelforge-web": ("frontend", "frontend/Dockerfile"),
|
|
"modelforge-node-agent": (".", "node-agent/Dockerfile"),
|
|
"modelforge-runtime-worker": (".", "runtime-worker/Dockerfile"),
|
|
}
|
|
CANONICAL_SOURCE_REPOSITORY = "https://git.example.com/example/modelforge"
|
|
PUBLIC_SOURCE_REPOSITORY = "https://git.example.com/example/modelforge"
|
|
OCI_SOURCE_LABEL = "org.opencontainers.image.source"
|
|
SOURCE_DOCKERFILES = (
|
|
"backend/Dockerfile",
|
|
"frontend/Dockerfile",
|
|
"node-agent/Dockerfile",
|
|
"runtime-worker/Dockerfile",
|
|
)
|
|
|
|
#: What a release tarball contains. Deliberately no model weights, no .env, no database dump and no
|
|
#: credential of any kind — see the packaging test, which fails if that changes.
|
|
ARTIFACT_PATHS = (
|
|
"docker-compose.yml",
|
|
"docker-compose.production.yml",
|
|
"docker-compose.rc-ingress.yml",
|
|
"docker-compose.validation-tls.yml",
|
|
"docker-compose.node-agent.yml",
|
|
"docker-compose.runtime-worker.yml",
|
|
"docker-compose.gpu.yml",
|
|
"docker-compose.backup.yml",
|
|
"docker-compose.dr.yml",
|
|
".env.example",
|
|
"VERSION",
|
|
"README.md",
|
|
"config",
|
|
"clients",
|
|
"deploy/postgres",
|
|
"deploy/validation",
|
|
"docs",
|
|
"backend/alembic",
|
|
"backend/alembic.ini",
|
|
"backend/pyproject.toml",
|
|
"backend/src",
|
|
"scripts/bootstrap.py",
|
|
"scripts/m16_chaos.py",
|
|
"scripts/m16_invariants.py",
|
|
"scripts/m16_soak.py",
|
|
"scripts/operator_identity.py",
|
|
"scripts/operator_password.py",
|
|
"scripts/modelforge_scheduled_backup.sh",
|
|
"scripts/preflight.py",
|
|
"scripts/rc_isolation_preflight.py",
|
|
"scripts/rc_stack_retention.py",
|
|
"scripts/upgrade.py",
|
|
)
|
|
|
|
|
|
def run(*args: str, check: bool = True, cwd: Path | None = None) -> str:
|
|
completed = subprocess.run(
|
|
list(args),
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
cwd=cwd or ROOT,
|
|
check=False,
|
|
)
|
|
if check and completed.returncode != 0:
|
|
detail = (completed.stderr or completed.stdout or "").strip()
|
|
raise RuntimeError(
|
|
f"{' '.join(args)} failed ({completed.returncode}): {detail[-2000:]}"
|
|
)
|
|
return (completed.stdout or "").strip()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _is_exact_commit(value: str) -> bool:
|
|
return len(value) == 40 and all(character in "0123456789abcdef" for character in value)
|
|
|
|
|
|
def _load_json_object(path: Path) -> dict[str, Any]:
|
|
"""Load one manifest object while refusing duplicate fields that could obscure provenance."""
|
|
|
|
def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError(f"duplicate JSON key {key!r}")
|
|
result[key] = value
|
|
return result
|
|
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicates)
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
raise SystemExit(f"cannot read provenance manifest {path}: {exc}") from exc
|
|
if not isinstance(payload, dict):
|
|
raise SystemExit(f"provenance manifest {path} must contain a JSON object")
|
|
return payload
|
|
|
|
|
|
def _dockerfile_source_repository(root: Path, relative_path: str) -> str:
|
|
path = root / relative_path
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except (OSError, UnicodeDecodeError) as exc:
|
|
raise SystemExit(f"cannot read release Dockerfile {relative_path}: {exc}") from exc
|
|
labels = re.findall(
|
|
rf'(?m)^LABEL\s+{re.escape(OCI_SOURCE_LABEL)}="([^"]+)"\s*$', text
|
|
)
|
|
if len(labels) != 1:
|
|
raise SystemExit(f"{relative_path} must declare exactly one {OCI_SOURCE_LABEL} label")
|
|
return labels[0]
|
|
|
|
|
|
def _public_source_repository(root: Path, manifest: dict[str, Any]) -> str:
|
|
source_repository = manifest.get("sourceRepository")
|
|
if source_repository != PUBLIC_SOURCE_REPOSITORY:
|
|
raise SystemExit(
|
|
"PUBLIC_SOURCE_MANIFEST.json sourceRepository is not the approved sanitized identity"
|
|
)
|
|
files = manifest.get("files")
|
|
if not isinstance(files, list) or not files:
|
|
raise SystemExit("PUBLIC_SOURCE_MANIFEST.json has no reviewed file inventory")
|
|
|
|
entries: dict[str, dict[str, Any]] = {}
|
|
for entry in files:
|
|
if not isinstance(entry, dict) or not isinstance(entry.get("path"), str):
|
|
raise SystemExit("PUBLIC_SOURCE_MANIFEST.json has a malformed file entry")
|
|
relative_path = entry["path"]
|
|
if relative_path in entries:
|
|
raise SystemExit(f"PUBLIC_SOURCE_MANIFEST.json repeats {relative_path!r}")
|
|
entries[relative_path] = entry
|
|
|
|
for relative_path in SOURCE_DOCKERFILES:
|
|
entry = entries.get(relative_path)
|
|
if entry is None:
|
|
raise SystemExit(
|
|
f"PUBLIC_SOURCE_MANIFEST.json does not bind {relative_path}"
|
|
)
|
|
path = root / relative_path
|
|
try:
|
|
payload = path.read_bytes()
|
|
except OSError as exc:
|
|
raise SystemExit(f"cannot read public source file {relative_path}: {exc}") from exc
|
|
if (
|
|
entry.get("bytes") != len(payload)
|
|
or entry.get("sha256") != hashlib.sha256(payload).hexdigest()
|
|
):
|
|
raise SystemExit(
|
|
f"PUBLIC_SOURCE_MANIFEST.json does not bind the current {relative_path}"
|
|
)
|
|
if _dockerfile_source_repository(root, relative_path) != source_repository:
|
|
raise SystemExit(
|
|
f"{relative_path} source label does not match the manifest sourceRepository"
|
|
)
|
|
return source_repository
|
|
|
|
|
|
def _assert_image_source_identity(
|
|
images: dict[str, dict[str, Any]], source_repository: str
|
|
) -> None:
|
|
for image, record in images.items():
|
|
labels = record.get("labels")
|
|
if not isinstance(labels, dict) or labels.get(OCI_SOURCE_LABEL) != source_repository:
|
|
raise SystemExit(
|
|
f"built image {image} source label does not match source identity "
|
|
f"{source_repository!r}"
|
|
)
|
|
|
|
|
|
def source_identity(
|
|
allow_dirty: bool, canonical_source_commit: str | None = None
|
|
) -> dict[str, Any]:
|
|
build_commit = run("git", "rev-parse", "HEAD")
|
|
dirty = bool(run("git", "status", "--porcelain"))
|
|
if dirty and not allow_dirty:
|
|
raise SystemExit(
|
|
"refusing to build a release from a dirty working tree: the recorded source commit "
|
|
"would not describe what was built. Commit first, or pass --allow-dirty for a "
|
|
"rehearsal build."
|
|
)
|
|
describe = run("git", "describe", "--tags", "--always", check=False)
|
|
identity: dict[str, Any] = {
|
|
"commit": build_commit,
|
|
"describe": describe or None,
|
|
"dirty": dirty,
|
|
"repository": run("git", "remote", "get-url", "origin", check=False) or None,
|
|
"branch": run("git", "rev-parse", "--abbrev-ref", "HEAD", check=False) or None,
|
|
}
|
|
public_manifest = ROOT / "PUBLIC_SOURCE_MANIFEST.json"
|
|
if public_manifest.exists():
|
|
if not public_manifest.is_file() or public_manifest.is_symlink():
|
|
raise SystemExit("PUBLIC_SOURCE_MANIFEST.json must be a regular file")
|
|
if canonical_source_commit is None:
|
|
raise SystemExit(
|
|
"a public-source build requires --canonical-source-commit so the export manifest "
|
|
"can bind it to the canonical source"
|
|
)
|
|
if not _is_exact_commit(canonical_source_commit):
|
|
raise SystemExit(
|
|
"--canonical-source-commit must be an exact lowercase 40-character SHA"
|
|
)
|
|
manifest = _load_json_object(public_manifest)
|
|
if (
|
|
manifest.get("schemaVersion") != 1
|
|
or manifest.get("sourceRevision") != canonical_source_commit
|
|
):
|
|
raise SystemExit(
|
|
"PUBLIC_SOURCE_MANIFEST.json does not bind this build tree to the requested "
|
|
"canonical source commit"
|
|
)
|
|
source_repository = _public_source_repository(ROOT, manifest)
|
|
identity.update(
|
|
{
|
|
"commit": canonical_source_commit,
|
|
"build_commit": build_commit,
|
|
"build_describe": describe or None,
|
|
"build_repository": identity["repository"],
|
|
"build_branch": identity["branch"],
|
|
"public_source_manifest_sha256": sha256_file(public_manifest),
|
|
"source_repository": source_repository,
|
|
}
|
|
)
|
|
# ``describe`` and ``branch`` describe the parentless build tree, not the canonical revision.
|
|
# Do not present either as canonical provenance or expose the private build remote.
|
|
identity["describe"] = None
|
|
identity["repository"] = None
|
|
identity["branch"] = None
|
|
return identity
|
|
|
|
if canonical_source_commit is not None:
|
|
if not _is_exact_commit(canonical_source_commit):
|
|
raise SystemExit("--canonical-source-commit must be an exact lowercase 40-character SHA")
|
|
if canonical_source_commit != build_commit:
|
|
raise SystemExit(
|
|
"canonical source differs from the build tree, but PUBLIC_SOURCE_MANIFEST.json is "
|
|
"absent; refusing an unbound source-label override"
|
|
)
|
|
|
|
source_labels = {
|
|
relative_path: _dockerfile_source_repository(ROOT, relative_path)
|
|
for relative_path in SOURCE_DOCKERFILES
|
|
}
|
|
if any(value != CANONICAL_SOURCE_REPOSITORY for value in source_labels.values()):
|
|
raise SystemExit(
|
|
"private release Dockerfiles do not use the canonical source repository label"
|
|
)
|
|
identity["source_repository"] = CANONICAL_SOURCE_REPOSITORY
|
|
return identity
|
|
|
|
|
|
def source_date_epoch(built_at: str) -> int:
|
|
"""Translate the declared build time into BuildKit's reproducible timestamp contract."""
|
|
|
|
parsed = datetime.fromisoformat(built_at.replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
raise ValueError("build timestamp must include a UTC offset")
|
|
return int(parsed.timestamp())
|
|
|
|
|
|
def manifest_digest(repo_digest: str | None) -> str | None:
|
|
"""Return the pullable OCI digest rather than Docker's local config/image identifier."""
|
|
|
|
if not repo_digest or "@" not in repo_digest:
|
|
return None
|
|
digest = repo_digest.rsplit("@", 1)[1]
|
|
return digest if digest.startswith("sha256:") else None
|
|
|
|
|
|
#: Images whose behaviour depends on the origin the console is served against. Vite inlines
|
|
#: VITE_* variables into the bundle at build time, so this is a property of the artifact rather
|
|
#: than of the deployment that runs it.
|
|
ORIGIN_DEPENDENT_IMAGES = frozenset({"modelforge-web"})
|
|
|
|
|
|
def normalise_public_api_origin(value: str) -> str:
|
|
"""Validate the console's compiled-in API origin, or refuse to build a release without one.
|
|
|
|
v1.2.0 shipped a console that could not reach its own API. The release build never passed
|
|
``VITE_API_BASE_URL``, so Vite compiled the Dockerfile's development default into an immutable
|
|
bundle, and the nginx CSP — derived from the same argument — hardcoded the same wrong origin.
|
|
A convenience default is exactly right for ``docker compose up`` on a laptop and exactly wrong
|
|
for a release artifact, so the release path refuses to guess.
|
|
"""
|
|
|
|
origin = value.strip().rstrip("/")
|
|
if not origin:
|
|
raise SystemExit(
|
|
"a release build needs the public API origin the console will be served against. "
|
|
"Pass --public-api-origin, or set MODELFORGE_PUBLIC_API_ORIGIN. It is compiled into "
|
|
"the bundle and into the CSP, so it cannot be corrected after the fact by the "
|
|
"deployment that runs the image."
|
|
)
|
|
parsed = urlparse(origin)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
raise SystemExit(
|
|
f"public API origin must be an absolute http(s) origin, got {value!r}. "
|
|
"Example: https://modelforge.example.com or http://192.0.2.10:18000"
|
|
)
|
|
if parsed.path or parsed.query or parsed.fragment:
|
|
raise SystemExit(
|
|
f"public API origin must be a bare origin with no path, query or fragment, "
|
|
f"got {value!r}. The console appends its own /api/v1 paths."
|
|
)
|
|
return origin
|
|
|
|
|
|
def build_images(
|
|
version: str, commit: str, built_at: str, public_api_origin: str
|
|
) -> dict[str, dict[str, Any]]:
|
|
built: dict[str, dict[str, Any]] = {}
|
|
epoch = source_date_epoch(built_at)
|
|
for image, (context, dockerfile) in IMAGES.items():
|
|
tag = f"{image}:{version}"
|
|
print(f" building {tag}", flush=True)
|
|
origin_args: tuple[str, ...] = ()
|
|
if image in ORIGIN_DEPENDENT_IMAGES:
|
|
origin_args = ("--build-arg", f"VITE_API_BASE_URL={public_api_origin}")
|
|
print(f" public API origin {public_api_origin}", flush=True)
|
|
with tempfile.TemporaryDirectory(prefix="modelforge-build-") as temporary:
|
|
metadata_path = Path(temporary) / "metadata.json"
|
|
run(
|
|
"docker",
|
|
"build",
|
|
# A release must execute current package-manager upgrade layers. Reusing a layer
|
|
# that predates an upstream security fix would make the packaged image differ
|
|
# from the cache-free image validated by the managed security gate.
|
|
"--pull",
|
|
"--no-cache",
|
|
# BuildKit attestations carry an exporter invocation timestamp and make the local
|
|
# manifest-list ID vary. ModelForge emits its own commit-bound CycloneDX and image
|
|
# provenance below, so suppress the duplicate nondeterministic attestations.
|
|
"--provenance=false",
|
|
"--sbom=false",
|
|
"--metadata-file",
|
|
str(metadata_path),
|
|
"-t",
|
|
tag,
|
|
"-f",
|
|
str(ROOT / dockerfile),
|
|
"--build-arg",
|
|
f"MODELFORGE_VERSION={version}",
|
|
"--build-arg",
|
|
f"MODELFORGE_COMMIT={commit}",
|
|
"--build-arg",
|
|
f"MODELFORGE_BUILT_AT={built_at}",
|
|
"--build-arg",
|
|
f"SOURCE_DATE_EPOCH={epoch}",
|
|
*origin_args,
|
|
str(ROOT / context),
|
|
)
|
|
build_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
image_id = run("docker", "inspect", "--format", "{{.Id}}", tag)
|
|
architecture = run("docker", "inspect", "--format", "{{.Architecture}}", tag)
|
|
labels = json.loads(
|
|
run("docker", "inspect", "--format", "{{json .Config.Labels}}", tag) or "{}"
|
|
)
|
|
repo_digest = (
|
|
run("docker", "inspect", "--format", "{{index .RepoDigests 0}}", tag, check=False)
|
|
or None
|
|
)
|
|
rootfs_layers = json.loads(
|
|
run("docker", "inspect", "--format", "{{json .RootFS.Layers}}", tag) or "[]"
|
|
)
|
|
if not (
|
|
isinstance(rootfs_layers, list)
|
|
and rootfs_layers
|
|
and all(isinstance(layer, str) and layer.startswith("sha256:") for layer in rootfs_layers)
|
|
):
|
|
raise RuntimeError(f"{tag} does not expose a content-addressable RootFS layer set")
|
|
built[image] = {
|
|
"tag": tag,
|
|
"image_id": image_id,
|
|
"image_digest": manifest_digest(repo_digest)
|
|
or build_metadata.get("containerimage.digest"),
|
|
"repo_digest": repo_digest,
|
|
"architecture": architecture,
|
|
"labels": {
|
|
key: value
|
|
for key, value in (labels or {}).items()
|
|
if key.startswith("org.opencontainers.image.")
|
|
},
|
|
# Docker config IDs are daemon-local after ``docker load``. Diff IDs instead bind the
|
|
# transport bundle to the actual uncompressed filesystem layer content on every host.
|
|
"rootfs_layers": rootfs_layers,
|
|
}
|
|
return built
|
|
|
|
|
|
def _bundle_member(tar: tarfile.TarFile, name: str) -> tarfile.TarInfo:
|
|
if not name or name.startswith("/") or ".." in Path(name).parts:
|
|
raise ValueError(f"unsafe image-bundle member {name!r}")
|
|
try:
|
|
member = tar.getmember(name)
|
|
except KeyError as exc:
|
|
raise ValueError(f"image bundle is missing {name!r}") from exc
|
|
if not member.isfile():
|
|
raise ValueError(f"image-bundle member {name!r} is not a regular file")
|
|
return member
|
|
|
|
|
|
def _bundle_json(tar: tarfile.TarFile, name: str) -> Any:
|
|
member = _bundle_member(tar, name)
|
|
stream = tar.extractfile(member)
|
|
if stream is None: # pragma: no cover - tarfile guarantees a stream for regular files
|
|
raise ValueError(f"cannot read image-bundle member {name!r}")
|
|
try:
|
|
return json.loads(stream.read().decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise ValueError(f"image-bundle member {name!r} is not valid JSON") from exc
|
|
|
|
|
|
def verify_image_bundle(bundle: Path, images: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
|
"""Verify the saved images retain exact tags, OCI labels and filesystem layer sets.
|
|
|
|
This deliberately does not compare the Docker image/config ID after a later ``docker load``:
|
|
Docker daemons can rewrite it. The bundle itself contains the stable config and the rootfs diff
|
|
IDs, which are content-addressable and can be rechecked after transport.
|
|
"""
|
|
|
|
expected_tags = {str(record["tag"]): name for name, record in images.items()}
|
|
if set(images) != set(IMAGES) or len(expected_tags) != len(IMAGES):
|
|
raise ValueError("an RC bundle must contain exactly the four declared release images")
|
|
|
|
with tarfile.open(bundle, mode="r") as archive:
|
|
manifest = _bundle_json(archive, "manifest.json")
|
|
if not isinstance(manifest, list) or len(manifest) != len(IMAGES):
|
|
raise ValueError("image bundle must have one manifest entry for each release image")
|
|
observed: dict[str, dict[str, Any]] = {}
|
|
for entry in manifest:
|
|
if not isinstance(entry, dict):
|
|
raise ValueError("image bundle manifest entry is not an object")
|
|
tags = entry.get("RepoTags")
|
|
if not isinstance(tags, list) or len(tags) != 1 or tags[0] not in expected_tags:
|
|
raise ValueError(f"image bundle has an unexpected tag set: {tags!r}")
|
|
tag = tags[0]
|
|
name = expected_tags[tag]
|
|
if name in observed:
|
|
raise ValueError(f"image bundle repeats {tag}")
|
|
config_name = entry.get("Config")
|
|
if not isinstance(config_name, str):
|
|
raise ValueError(f"image bundle has no config for {tag}")
|
|
config = _bundle_json(archive, config_name)
|
|
if not isinstance(config, dict):
|
|
raise ValueError(f"image bundle config for {tag} is not an object")
|
|
layers = entry.get("Layers")
|
|
if not isinstance(layers, list) or not layers:
|
|
raise ValueError(f"image bundle has no filesystem layers for {tag}")
|
|
for layer in layers:
|
|
if not isinstance(layer, str):
|
|
raise ValueError(f"image bundle layer entry for {tag} is not a path")
|
|
_bundle_member(archive, layer)
|
|
rootfs = config.get("rootfs")
|
|
config_layers = rootfs.get("diff_ids") if isinstance(rootfs, dict) else None
|
|
expected = images[name]
|
|
if config_layers != expected.get("rootfs_layers"):
|
|
raise ValueError(f"image bundle layer set does not match built {tag}")
|
|
config_section = config.get("config")
|
|
actual_labels = config_section.get("Labels") if isinstance(config_section, dict) else None
|
|
actual_labels = actual_labels or {}
|
|
if not isinstance(actual_labels, dict) or any(
|
|
actual_labels.get(key) != value for key, value in expected["labels"].items()
|
|
):
|
|
raise ValueError(f"image bundle OCI labels do not match built {tag}")
|
|
if config.get("architecture") != expected.get("architecture"):
|
|
raise ValueError(f"image bundle architecture does not match built {tag}")
|
|
observed[name] = {
|
|
"tag": tag,
|
|
"config": config_name,
|
|
"rootfs_layers": config_layers,
|
|
"layer_members": layers,
|
|
}
|
|
|
|
if set(observed) != set(IMAGES):
|
|
raise ValueError("image bundle did not contain every declared release image")
|
|
return observed
|
|
|
|
|
|
def export_image_bundle(
|
|
output: Path, version: str, images: dict[str, dict[str, Any]]
|
|
) -> dict[str, Any]:
|
|
"""Save and verify a transport bundle without rebuilding or re-tagging any image."""
|
|
|
|
bundle = output / f"modelforge-{version}-images.tar"
|
|
if bundle.exists():
|
|
bundle.unlink()
|
|
tags = [str(images[name]["tag"]) for name in IMAGES]
|
|
run("docker", "image", "save", "--output", str(bundle), *tags)
|
|
verified = verify_image_bundle(bundle, images)
|
|
return {
|
|
"path": bundle.name,
|
|
"sha256": sha256_file(bundle),
|
|
"bytes": bundle.stat().st_size,
|
|
"images": verified,
|
|
}
|
|
|
|
|
|
def package(output: Path, version: str, sbom_source: Path | None = None) -> Path:
|
|
"""Assemble the release tarball from tracked paths only."""
|
|
|
|
staging = output / f"modelforge-{version}"
|
|
if staging.exists():
|
|
shutil.rmtree(staging)
|
|
staging.mkdir(parents=True)
|
|
for relative in ARTIFACT_PATHS:
|
|
source = ROOT / relative
|
|
if not source.exists():
|
|
print(f" note: {relative} is absent and was not packaged", flush=True)
|
|
continue
|
|
target = staging / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
if source.is_dir():
|
|
shutil.copytree(source, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
|
|
else:
|
|
shutil.copy2(source, target)
|
|
|
|
if sbom_source is not None:
|
|
sbom_target = staging / "docs" / "security" / "sbom"
|
|
if sbom_target.exists():
|
|
shutil.rmtree(sbom_target)
|
|
shutil.copytree(sbom_source, sbom_target)
|
|
|
|
archive = output / f"modelforge-{version}.tar.gz"
|
|
if archive.exists():
|
|
archive.unlink()
|
|
# A deterministic archive: sorted entries, and identity metadata normalised so two builds of the
|
|
# same tree produce the same bytes rather than differing by uid and mtime.
|
|
def normalise(info: tarfile.TarInfo) -> tarfile.TarInfo:
|
|
info.uid = info.gid = 0
|
|
info.uname = info.gname = "root"
|
|
info.mtime = 0
|
|
return info
|
|
|
|
# gzip writes the current time into its header, so "w:gz" alone produces a different digest on
|
|
# every build even when the contents are identical. Writing through a GzipFile with mtime=0
|
|
# makes two builds of the same tree byte-identical, which is what makes a published checksum
|
|
# worth anything.
|
|
with (
|
|
archive.open("wb") as raw,
|
|
gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=9, mtime=0) as compressed,
|
|
tarfile.open(fileobj=compressed, mode="w") as tar,
|
|
):
|
|
for path in sorted(staging.rglob("*")):
|
|
# recursive=False matters: tar.add recurses into a directory by default, so adding the
|
|
# directory and then each of its children again put every file in the archive several
|
|
# times over.
|
|
tar.add(
|
|
path,
|
|
arcname=str(path.relative_to(output)),
|
|
filter=normalise,
|
|
recursive=False,
|
|
)
|
|
shutil.rmtree(staging)
|
|
return archive
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", default="dist")
|
|
parser.add_argument("--skip-images", action="store_true", help="package without building")
|
|
parser.add_argument(
|
|
"--image-bundle",
|
|
action="store_true",
|
|
help="save and verify the four exact built images as a Docker transport bundle",
|
|
)
|
|
parser.add_argument(
|
|
"--canonical-source-commit",
|
|
default=None,
|
|
help=(
|
|
"canonical source SHA bound by PUBLIC_SOURCE_MANIFEST.json when building a curated "
|
|
"parentless export; ordinary checkouts may only use their own HEAD"
|
|
),
|
|
)
|
|
parser.add_argument("--allow-dirty", action="store_true")
|
|
parser.add_argument(
|
|
"--built-at",
|
|
default=None,
|
|
help="fixed ISO-8601 build timestamp; reuse it when proving archive reproducibility",
|
|
)
|
|
parser.add_argument(
|
|
"--public-api-origin",
|
|
default=os.environ.get("MODELFORGE_PUBLIC_API_ORIGIN", ""),
|
|
help=(
|
|
"absolute origin the console will reach its API on, for example "
|
|
"https://modelforge.example.com. Vite compiles it into the bundle and the CSP is "
|
|
"derived from it, so a release cannot be corrected here afterwards. Required unless "
|
|
"--skip-images. Also read from MODELFORGE_PUBLIC_API_ORIGIN."
|
|
),
|
|
)
|
|
args = parser.parse_args(argv)
|
|
if args.image_bundle and args.skip_images:
|
|
raise SystemExit("--image-bundle requires built images; do not combine it with --skip-images")
|
|
|
|
output = (ROOT / args.output).resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
built_at = args.built_at or datetime.now(UTC).isoformat()
|
|
version = PRODUCT_VERSION
|
|
|
|
# Validated before anything is built: refusing after three images is worse than refusing first.
|
|
public_api_origin = (
|
|
"" if args.skip_images else normalise_public_api_origin(args.public_api_origin)
|
|
)
|
|
|
|
print(f"{PRODUCT_NAME} {version} — release build", flush=True)
|
|
source = source_identity(args.allow_dirty, args.canonical_source_commit)
|
|
print(f" source commit {source['commit']}"
|
|
f"{' (DIRTY)' if source['dirty'] else ''}", flush=True)
|
|
|
|
images = (
|
|
{}
|
|
if args.skip_images
|
|
else build_images(version, source["commit"], built_at, public_api_origin)
|
|
)
|
|
if images:
|
|
_assert_image_source_identity(images, source["source_repository"])
|
|
|
|
sbom_output = output / "sbom"
|
|
run(
|
|
sys.executable,
|
|
str(ROOT / "scripts" / "m16_sbom.py"),
|
|
"--output",
|
|
str(sbom_output),
|
|
"--version",
|
|
version,
|
|
"--source-commit",
|
|
source["commit"],
|
|
"--generated-at",
|
|
built_at,
|
|
)
|
|
sbom_artifact = output / f"modelforge-{version}-cyclonedx.json"
|
|
provenance_artifact = output / f"modelforge-{version}-image-provenance.json"
|
|
shutil.copy2(sbom_output / "modelforge-cyclonedx.json", sbom_artifact)
|
|
shutil.copy2(sbom_output / "image-provenance.json", provenance_artifact)
|
|
|
|
# Read the artifact, not the source that produced it. v1.2.0 was published with correct
|
|
# labels, a correct manifest and a console bundle pointing at localhost; nothing upstream of
|
|
# here could see that, because the defect only exists once the image is built.
|
|
if images:
|
|
print(" verifying the console image against its declared API origin", flush=True)
|
|
acceptance = run(
|
|
sys.executable,
|
|
str(ROOT / "scripts" / "release_image_acceptance.py"),
|
|
"--image",
|
|
images["modelforge-web"]["tag"],
|
|
"--expect-origin",
|
|
public_api_origin,
|
|
"--version",
|
|
version,
|
|
check=False,
|
|
)
|
|
for line in acceptance.splitlines():
|
|
# A release build must not fail because the host console cannot encode a character in
|
|
# a subprocess's output. Windows defaults to cp1252 here.
|
|
safe = line.encode(sys.stdout.encoding or "utf-8", "replace").decode(
|
|
sys.stdout.encoding or "utf-8", "replace"
|
|
)
|
|
print(f" {safe}", flush=True)
|
|
if "All " not in acceptance:
|
|
raise SystemExit(
|
|
"the built console image does not match the API origin it was built for; "
|
|
"refusing to package a release that cannot reach its own API."
|
|
)
|
|
|
|
image_bundle = None
|
|
if args.image_bundle:
|
|
print(" saving and verifying the exact four-image transport bundle", flush=True)
|
|
image_bundle = export_image_bundle(output, version, images)
|
|
|
|
archive = package(output, version, sbom_output)
|
|
print(f" packaged {archive.name} ({archive.stat().st_size} bytes)", flush=True)
|
|
|
|
manifest: dict[str, Any] = {
|
|
"product": PRODUCT_NAME,
|
|
"version": version,
|
|
"channel": RELEASE_CHANNEL,
|
|
"built_at": built_at,
|
|
"source": source,
|
|
"compatibility": {
|
|
"schema_revision": TARGET_SCHEMA_REVISION,
|
|
"supported_schema_revisions": list(SUPPORTED_SCHEMA_REVISIONS),
|
|
"supported_upgrade_source_schema_revisions": list(
|
|
SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS
|
|
),
|
|
"agent_protocol_version": CURRENT_AGENT_PROTOCOL_VERSION,
|
|
"supported_agent_protocol_versions": list(SUPPORTED_AGENT_PROTOCOL_VERSIONS),
|
|
"minimum_upgrade_source": MINIMUM_UPGRADE_SOURCE,
|
|
"minimum_postgres_major": MINIMUM_POSTGRES_MAJOR,
|
|
},
|
|
"external_dependencies": {
|
|
"postgresql": f">={MINIMUM_POSTGRES_MAJOR}",
|
|
"redis": ">=7",
|
|
"docker_engine": ">=24",
|
|
"nvidia_container_runtime": "required on any GPU node",
|
|
},
|
|
"images": images,
|
|
# The origin compiled into the console bundle and into its CSP. Recorded because it is a
|
|
# property of the artifact that an operator otherwise cannot see without unpacking the
|
|
# image, and because a console pointed at the wrong API is indistinguishable from a
|
|
# healthy one until someone opens it.
|
|
"console": {"public_api_origin": public_api_origin or None},
|
|
"artifacts": {},
|
|
}
|
|
if image_bundle is not None:
|
|
manifest["image_bundle"] = image_bundle
|
|
|
|
manifest["sbom"] = {
|
|
"format": "CycloneDX 1.5",
|
|
"cyclonedx": sha256_file(sbom_artifact),
|
|
"provenance": sha256_file(provenance_artifact),
|
|
}
|
|
|
|
for path in sorted(output.glob(f"modelforge-{version}*")):
|
|
if path.suffix == ".json" or path.name.endswith(".sha256"):
|
|
continue
|
|
manifest["artifacts"][path.name] = {
|
|
"sha256": sha256_file(path),
|
|
"bytes": path.stat().st_size,
|
|
}
|
|
|
|
manifest_path = output / f"modelforge-{version}-release-manifest.json"
|
|
# newline="\n" on every release artifact. Python otherwise translates to the host's line ending,
|
|
# so a Windows-built release and a Linux-built release of the same commit would differ byte for
|
|
# byte — and the recorded hashes with them.
|
|
manifest_path.write_text(
|
|
json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8", newline="\n"
|
|
)
|
|
|
|
# The checksum file covers the manifest too, so a tampered manifest is as detectable as a
|
|
# tampered archive.
|
|
checksums = output / f"modelforge-{version}-SHA256SUMS"
|
|
lines = [
|
|
f"{sha256_file(path)} {path.name}"
|
|
for path in sorted(output.iterdir())
|
|
if path.is_file() and not path.name.endswith("SHA256SUMS")
|
|
]
|
|
# Must be LF. `sha256sum -c` treats a trailing CR as part of the filename, so a CRLF checksum
|
|
# file fails to verify every artifact it covers — on the very command an operator is told to run.
|
|
checksums.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n")
|
|
|
|
print(f" manifest {manifest_path.name}", flush=True)
|
|
print(f" checksums {checksums.name} ({len(lines)} files)", flush=True)
|
|
for line in lines:
|
|
print(f" {line}", flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|