Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

scripts/release_build.py

@@ -17,6 +17,7 @@ import gzip
 import hashlib
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -30,7 +31,7 @@ from urllib.parse import urlparse
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "backend" / "src"))
 
-from modelforge_api.domain.release import (
+from modelforge_api.domain.release import (  # noqa: E402
     CURRENT_AGENT_PROTOCOL_VERSION,
     MINIMUM_POSTGRES_MAJOR,
     MINIMUM_UPGRADE_SOURCE,
@@ -43,22 +44,33 @@ from modelforge_api.domain.release import (
     TARGET_SCHEMA_REVISION,
 )
 
-#: image name -> (build context, Dockerfile path), both relative to the repository root. The Node
-#: Agent and Runtime Worker build from the repository root because their Dockerfiles copy a
-#: component directory from that shared context. Keeping the four distributable images in one
-#: inventory makes it impossible for the private runtime boundary to disappear from a release.
+#: image name -> (build context, Dockerfile path), both relative to the repository root. Python
+#: images build from the repository root so they can consume the shared hash-locked toolchain.
+#: Keeping the four distributable images in one inventory makes it impossible for the private
+#: runtime boundary to disappear from a release.
 IMAGES = {
-    "modelforge-api": ("backend", "backend/Dockerfile"),
+    "modelforge-api": (".", "backend/Dockerfile"),
     "modelforge-web": ("frontend", "frontend/Dockerfile"),
     "modelforge-node-agent": (".", "node-agent/Dockerfile"),
     "modelforge-runtime-worker": (".", "runtime-worker/Dockerfile"),
 }
+CANONICAL_SOURCE_REPOSITORY = "https://git.example.com/example/modelforge"
+PUBLIC_SOURCE_REPOSITORY = "https://git.example.com/example/modelforge"
+OCI_SOURCE_LABEL = "org.opencontainers.image.source"
+SOURCE_DOCKERFILES = (
+    "backend/Dockerfile",
+    "frontend/Dockerfile",
+    "node-agent/Dockerfile",
+    "runtime-worker/Dockerfile",
+)
 
 #: What a release tarball contains. Deliberately no model weights, no .env, no database dump and no
 #: credential of any kind — see the packaging test, which fails if that changes.
 ARTIFACT_PATHS = (
     "docker-compose.yml",
     "docker-compose.production.yml",
+    "docker-compose.rc-ingress.yml",
+    "docker-compose.validation-tls.yml",
     "docker-compose.node-agent.yml",
     "docker-compose.runtime-worker.yml",
     "docker-compose.gpu.yml",
@@ -68,9 +80,25 @@ ARTIFACT_PATHS = (
     "VERSION",
     "README.md",
     "config",
+    "clients",
+    "deploy/postgres",
+    "deploy/validation",
     "docs",
+    "backend/alembic",
+    "backend/alembic.ini",
+    "backend/pyproject.toml",
+    "backend/src",
     "scripts/bootstrap.py",
+    "scripts/m16_chaos.py",
+    "scripts/m16_invariants.py",
+    "scripts/m16_soak.py",
+    "scripts/operator_identity.py",
+    "scripts/operator_password.py",
+    "scripts/modelforge_scheduled_backup.sh",
     "scripts/preflight.py",
+    "scripts/rc_isolation_preflight.py",
+    "scripts/rc_stack_retention.py",
+    "scripts/upgrade.py",
 )
 
 
@@ -85,7 +113,10 @@ def run(*args: str, check: bool = True, cwd: Path | None = None) -> str:
         check=False,
     )
     if check and completed.returncode != 0:
-        raise RuntimeError(f"{' '.join(args)} failed: {completed.stderr.strip()[:500]}")
+        detail = (completed.stderr or completed.stdout or "").strip()
+        raise RuntimeError(
+            f"{' '.join(args)} failed ({completed.returncode}): {detail[-2000:]}"
+        )
     return (completed.stdout or "").strip()
 
 
@@ -97,8 +128,104 @@ def sha256_file(path: Path) -> str:
     return digest.hexdigest()
 
 
-def source_identity(allow_dirty: bool) -> dict[str, Any]:
-    commit = run("git", "rev-parse", "HEAD")
+def _is_exact_commit(value: str) -> bool:
+    return len(value) == 40 and all(character in "0123456789abcdef" for character in value)
+
+
+def _load_json_object(path: Path) -> dict[str, Any]:
+    """Load one manifest object while refusing duplicate fields that could obscure provenance."""
+
+    def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError(f"duplicate JSON key {key!r}")
+            result[key] = value
+        return result
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicates)
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        raise SystemExit(f"cannot read provenance manifest {path}: {exc}") from exc
+    if not isinstance(payload, dict):
+        raise SystemExit(f"provenance manifest {path} must contain a JSON object")
+    return payload
+
+
+def _dockerfile_source_repository(root: Path, relative_path: str) -> str:
+    path = root / relative_path
+    try:
+        text = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as exc:
+        raise SystemExit(f"cannot read release Dockerfile {relative_path}: {exc}") from exc
+    labels = re.findall(
+        rf'(?m)^LABEL\s+{re.escape(OCI_SOURCE_LABEL)}="([^"]+)"\s*$', text
+    )
+    if len(labels) != 1:
+        raise SystemExit(f"{relative_path} must declare exactly one {OCI_SOURCE_LABEL} label")
+    return labels[0]
+
+
+def _public_source_repository(root: Path, manifest: dict[str, Any]) -> str:
+    source_repository = manifest.get("sourceRepository")
+    if source_repository != PUBLIC_SOURCE_REPOSITORY:
+        raise SystemExit(
+            "PUBLIC_SOURCE_MANIFEST.json sourceRepository is not the approved sanitized identity"
+        )
+    files = manifest.get("files")
+    if not isinstance(files, list) or not files:
+        raise SystemExit("PUBLIC_SOURCE_MANIFEST.json has no reviewed file inventory")
+
+    entries: dict[str, dict[str, Any]] = {}
+    for entry in files:
+        if not isinstance(entry, dict) or not isinstance(entry.get("path"), str):
+            raise SystemExit("PUBLIC_SOURCE_MANIFEST.json has a malformed file entry")
+        relative_path = entry["path"]
+        if relative_path in entries:
+            raise SystemExit(f"PUBLIC_SOURCE_MANIFEST.json repeats {relative_path!r}")
+        entries[relative_path] = entry
+
+    for relative_path in SOURCE_DOCKERFILES:
+        entry = entries.get(relative_path)
+        if entry is None:
+            raise SystemExit(
+                f"PUBLIC_SOURCE_MANIFEST.json does not bind {relative_path}"
+            )
+        path = root / relative_path
+        try:
+            payload = path.read_bytes()
+        except OSError as exc:
+            raise SystemExit(f"cannot read public source file {relative_path}: {exc}") from exc
+        if (
+            entry.get("bytes") != len(payload)
+            or entry.get("sha256") != hashlib.sha256(payload).hexdigest()
+        ):
+            raise SystemExit(
+                f"PUBLIC_SOURCE_MANIFEST.json does not bind the current {relative_path}"
+            )
+        if _dockerfile_source_repository(root, relative_path) != source_repository:
+            raise SystemExit(
+                f"{relative_path} source label does not match the manifest sourceRepository"
+            )
+    return source_repository
+
+
+def _assert_image_source_identity(
+    images: dict[str, dict[str, Any]], source_repository: str
+) -> None:
+    for image, record in images.items():
+        labels = record.get("labels")
+        if not isinstance(labels, dict) or labels.get(OCI_SOURCE_LABEL) != source_repository:
+            raise SystemExit(
+                f"built image {image} source label does not match source identity "
+                f"{source_repository!r}"
+            )
+
+
+def source_identity(
+    allow_dirty: bool, canonical_source_commit: str | None = None
+) -> dict[str, Any]:
+    build_commit = run("git", "rev-parse", "HEAD")
     dirty = bool(run("git", "status", "--porcelain"))
     if dirty and not allow_dirty:
         raise SystemExit(
@@ -107,13 +234,73 @@ def source_identity(allow_dirty: bool) -> dict[str, Any]:
             "rehearsal build."
         )
     describe = run("git", "describe", "--tags", "--always", check=False)
-    return {
-        "commit": commit,
+    identity: dict[str, Any] = {
+        "commit": build_commit,
         "describe": describe or None,
         "dirty": dirty,
         "repository": run("git", "remote", "get-url", "origin", check=False) or None,
         "branch": run("git", "rev-parse", "--abbrev-ref", "HEAD", check=False) or None,
     }
+    public_manifest = ROOT / "PUBLIC_SOURCE_MANIFEST.json"
+    if public_manifest.exists():
+        if not public_manifest.is_file() or public_manifest.is_symlink():
+            raise SystemExit("PUBLIC_SOURCE_MANIFEST.json must be a regular file")
+        if canonical_source_commit is None:
+            raise SystemExit(
+                "a public-source build requires --canonical-source-commit so the export manifest "
+                "can bind it to the canonical source"
+            )
+        if not _is_exact_commit(canonical_source_commit):
+            raise SystemExit(
+                "--canonical-source-commit must be an exact lowercase 40-character SHA"
+            )
+        manifest = _load_json_object(public_manifest)
+        if (
+            manifest.get("schemaVersion") != 1
+            or manifest.get("sourceRevision") != canonical_source_commit
+        ):
+            raise SystemExit(
+                "PUBLIC_SOURCE_MANIFEST.json does not bind this build tree to the requested "
+                "canonical source commit"
+            )
+        source_repository = _public_source_repository(ROOT, manifest)
+        identity.update(
+            {
+                "commit": canonical_source_commit,
+                "build_commit": build_commit,
+                "build_describe": describe or None,
+                "build_repository": identity["repository"],
+                "build_branch": identity["branch"],
+                "public_source_manifest_sha256": sha256_file(public_manifest),
+                "source_repository": source_repository,
+            }
+        )
+        # ``describe`` and ``branch`` describe the parentless build tree, not the canonical revision.
+        # Do not present either as canonical provenance or expose the private build remote.
+        identity["describe"] = None
+        identity["repository"] = None
+        identity["branch"] = None
+        return identity
+
+    if canonical_source_commit is not None:
+        if not _is_exact_commit(canonical_source_commit):
+            raise SystemExit("--canonical-source-commit must be an exact lowercase 40-character SHA")
+        if canonical_source_commit != build_commit:
+            raise SystemExit(
+                "canonical source differs from the build tree, but PUBLIC_SOURCE_MANIFEST.json is "
+                "absent; refusing an unbound source-label override"
+            )
+
+    source_labels = {
+        relative_path: _dockerfile_source_repository(ROOT, relative_path)
+        for relative_path in SOURCE_DOCKERFILES
+    }
+    if any(value != CANONICAL_SOURCE_REPOSITORY for value in source_labels.values()):
+        raise SystemExit(
+            "private release Dockerfiles do not use the canonical source repository label"
+        )
+    identity["source_repository"] = CANONICAL_SOURCE_REPOSITORY
+    return identity
 
 
 def source_date_epoch(built_at: str) -> int:
@@ -189,6 +376,11 @@ def build_images(
             run(
                 "docker",
                 "build",
+                # A release must execute current package-manager upgrade layers. Reusing a layer
+                # that predates an upstream security fix would make the packaged image differ
+                # from the cache-free image validated by the managed security gate.
+                "--pull",
+                "--no-cache",
                 # BuildKit attestations carry an exporter invocation timestamp and make the local
                 # manifest-list ID vary. ModelForge emits its own commit-bound CycloneDX and image
                 # provenance below, so suppress the duplicate nondeterministic attestations.
@@ -221,6 +413,15 @@ def build_images(
             run("docker", "inspect", "--format", "{{index .RepoDigests 0}}", tag, check=False)
             or None
         )
+        rootfs_layers = json.loads(
+            run("docker", "inspect", "--format", "{{json .RootFS.Layers}}", tag) or "[]"
+        )
+        if not (
+            isinstance(rootfs_layers, list)
+            and rootfs_layers
+            and all(isinstance(layer, str) and layer.startswith("sha256:") for layer in rootfs_layers)
+        ):
+            raise RuntimeError(f"{tag} does not expose a content-addressable RootFS layer set")
         built[image] = {
             "tag": tag,
             "image_id": image_id,
@@ -233,10 +434,121 @@ def build_images(
                 for key, value in (labels or {}).items()
                 if key.startswith("org.opencontainers.image.")
             },
+            # Docker config IDs are daemon-local after ``docker load``. Diff IDs instead bind the
+            # transport bundle to the actual uncompressed filesystem layer content on every host.
+            "rootfs_layers": rootfs_layers,
         }
     return built
 
 
+def _bundle_member(tar: tarfile.TarFile, name: str) -> tarfile.TarInfo:
+    if not name or name.startswith("/") or ".." in Path(name).parts:
+        raise ValueError(f"unsafe image-bundle member {name!r}")
+    try:
+        member = tar.getmember(name)
+    except KeyError as exc:
+        raise ValueError(f"image bundle is missing {name!r}") from exc
+    if not member.isfile():
+        raise ValueError(f"image-bundle member {name!r} is not a regular file")
+    return member
+
+
+def _bundle_json(tar: tarfile.TarFile, name: str) -> Any:
+    member = _bundle_member(tar, name)
+    stream = tar.extractfile(member)
+    if stream is None:  # pragma: no cover - tarfile guarantees a stream for regular files
+        raise ValueError(f"cannot read image-bundle member {name!r}")
+    try:
+        return json.loads(stream.read().decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise ValueError(f"image-bundle member {name!r} is not valid JSON") from exc
+
+
+def verify_image_bundle(bundle: Path, images: dict[str, dict[str, Any]]) -> dict[str, Any]:
+    """Verify the saved images retain exact tags, OCI labels and filesystem layer sets.
+
+    This deliberately does not compare the Docker image/config ID after a later ``docker load``:
+    Docker daemons can rewrite it. The bundle itself contains the stable config and the rootfs diff
+    IDs, which are content-addressable and can be rechecked after transport.
+    """
+
+    expected_tags = {str(record["tag"]): name for name, record in images.items()}
+    if set(images) != set(IMAGES) or len(expected_tags) != len(IMAGES):
+        raise ValueError("an RC bundle must contain exactly the four declared release images")
+
+    with tarfile.open(bundle, mode="r") as archive:
+        manifest = _bundle_json(archive, "manifest.json")
+        if not isinstance(manifest, list) or len(manifest) != len(IMAGES):
+            raise ValueError("image bundle must have one manifest entry for each release image")
+        observed: dict[str, dict[str, Any]] = {}
+        for entry in manifest:
+            if not isinstance(entry, dict):
+                raise ValueError("image bundle manifest entry is not an object")
+            tags = entry.get("RepoTags")
+            if not isinstance(tags, list) or len(tags) != 1 or tags[0] not in expected_tags:
+                raise ValueError(f"image bundle has an unexpected tag set: {tags!r}")
+            tag = tags[0]
+            name = expected_tags[tag]
+            if name in observed:
+                raise ValueError(f"image bundle repeats {tag}")
+            config_name = entry.get("Config")
+            if not isinstance(config_name, str):
+                raise ValueError(f"image bundle has no config for {tag}")
+            config = _bundle_json(archive, config_name)
+            if not isinstance(config, dict):
+                raise ValueError(f"image bundle config for {tag} is not an object")
+            layers = entry.get("Layers")
+            if not isinstance(layers, list) or not layers:
+                raise ValueError(f"image bundle has no filesystem layers for {tag}")
+            for layer in layers:
+                if not isinstance(layer, str):
+                    raise ValueError(f"image bundle layer entry for {tag} is not a path")
+                _bundle_member(archive, layer)
+            rootfs = config.get("rootfs")
+            config_layers = rootfs.get("diff_ids") if isinstance(rootfs, dict) else None
+            expected = images[name]
+            if config_layers != expected.get("rootfs_layers"):
+                raise ValueError(f"image bundle layer set does not match built {tag}")
+            config_section = config.get("config")
+            actual_labels = config_section.get("Labels") if isinstance(config_section, dict) else None
+            actual_labels = actual_labels or {}
+            if not isinstance(actual_labels, dict) or any(
+                actual_labels.get(key) != value for key, value in expected["labels"].items()
+            ):
+                raise ValueError(f"image bundle OCI labels do not match built {tag}")
+            if config.get("architecture") != expected.get("architecture"):
+                raise ValueError(f"image bundle architecture does not match built {tag}")
+            observed[name] = {
+                "tag": tag,
+                "config": config_name,
+                "rootfs_layers": config_layers,
+                "layer_members": layers,
+            }
+
+    if set(observed) != set(IMAGES):
+        raise ValueError("image bundle did not contain every declared release image")
+    return observed
+
+
+def export_image_bundle(
+    output: Path, version: str, images: dict[str, dict[str, Any]]
+) -> dict[str, Any]:
+    """Save and verify a transport bundle without rebuilding or re-tagging any image."""
+
+    bundle = output / f"modelforge-{version}-images.tar"
+    if bundle.exists():
+        bundle.unlink()
+    tags = [str(images[name]["tag"]) for name in IMAGES]
+    run("docker", "image", "save", "--output", str(bundle), *tags)
+    verified = verify_image_bundle(bundle, images)
+    return {
+        "path": bundle.name,
+        "sha256": sha256_file(bundle),
+        "bytes": bundle.stat().st_size,
+        "images": verified,
+    }
+
+
 def package(output: Path, version: str, sbom_source: Path | None = None) -> Path:
     """Assemble the release tarball from tracked paths only."""
 
@@ -300,6 +612,19 @@ def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--output", default="dist")
     parser.add_argument("--skip-images", action="store_true", help="package without building")
+    parser.add_argument(
+        "--image-bundle",
+        action="store_true",
+        help="save and verify the four exact built images as a Docker transport bundle",
+    )
+    parser.add_argument(
+        "--canonical-source-commit",
+        default=None,
+        help=(
+            "canonical source SHA bound by PUBLIC_SOURCE_MANIFEST.json when building a curated "
+            "parentless export; ordinary checkouts may only use their own HEAD"
+        ),
+    )
     parser.add_argument("--allow-dirty", action="store_true")
     parser.add_argument(
         "--built-at",
@@ -317,6 +642,8 @@ def main(argv: list[str] | None = None) -> int:
         ),
     )
     args = parser.parse_args(argv)
+    if args.image_bundle and args.skip_images:
+        raise SystemExit("--image-bundle requires built images; do not combine it with --skip-images")
 
     output = (ROOT / args.output).resolve()
     output.mkdir(parents=True, exist_ok=True)
@@ -329,7 +656,7 @@ def main(argv: list[str] | None = None) -> int:
     )
 
     print(f"{PRODUCT_NAME} {version} — release build", flush=True)
-    source = source_identity(args.allow_dirty)
+    source = source_identity(args.allow_dirty, args.canonical_source_commit)
     print(f"  source commit {source['commit']}"
           f"{' (DIRTY)' if source['dirty'] else ''}", flush=True)
 
@@ -338,6 +665,8 @@ def main(argv: list[str] | None = None) -> int:
         if args.skip_images
         else build_images(version, source["commit"], built_at, public_api_origin)
     )
+    if images:
+        _assert_image_source_identity(images, source["source_repository"])
 
     sbom_output = output / "sbom"
     run(
@@ -386,6 +715,11 @@ def main(argv: list[str] | None = None) -> int:
                 "refusing to package a release that cannot reach its own API."
             )
 
+    image_bundle = None
+    if args.image_bundle:
+        print("  saving and verifying the exact four-image transport bundle", flush=True)
+        image_bundle = export_image_bundle(output, version, images)
+
     archive = package(output, version, sbom_output)
     print(f"  packaged {archive.name} ({archive.stat().st_size} bytes)", flush=True)
 
@@ -420,6 +754,8 @@ def main(argv: list[str] | None = None) -> int:
         "console": {"public_api_origin": public_api_origin or None},
         "artifacts": {},
     }
+    if image_bundle is not None:
+        manifest["image_bundle"] = image_bundle
 
     manifest["sbom"] = {
         "format": "CycloneDX 1.5",