#!/usr/bin/env python3 """Create a deterministic checksum manifest from canonical Git-tracked content. Every entry is hashed from `git archive` output at a chosen ref (default ``HEAD``), never from raw checked-out filesystem bytes and never from raw `git cat-file` blob content either. This is deliberate, for two reasons: 1. Working-tree bytes are not platform-independent — a Windows checkout (core.autocrlf=true) and a Linux checkout of the same commit can differ byte-for-byte (see the WP-220 evidence and ``.gitattributes`` for why that already caused a false-positive manifest failure once, in commit 95329ca). 2. Raw blob content is *also* not what a real archive extraction or checkout produces for every file: Git always stores blobs LF-normalized internally, but a file with an explicit `eol=crlf` `.gitattributes` override (e.g. scripts/bootstrap.ps1, intentionally always-CRLF for native Windows tooling) gets CRLF from `git archive`/`git checkout` regardless. Hashing raw blobs would make the manifest describe bytes nobody ever actually receives. Hashing `git archive` output instead makes manifest generation and "verify a real archive extraction" (`validate_pack.py --verify-tree`) definitionally the same operation. Both properties hold together because ``.gitattributes`` pins an explicit `eol` for every tracked file type in this repository — no file is left to `core.autocrlf` auto-detection ambiguity, so `git archive` output is fully deterministic across platforms even though it is not identical to the raw blob for CRLF-pinned files. Only files tracked by Git at the chosen ref are ever considered — ignored, untracked, local-only, generated, cache, secret, editor, and agent-setting files can never enter the manifest, because they are not in the tree Git enumerates. ``PACK_MANIFEST.json`` itself is always excluded from its own file list (self-referential exclusion), deterministically, regardless of whether it happens to be tracked. Every git-plumbing helper below takes an explicit ``repo_root`` (defaulting to this repository's root) rather than hardcoding it, so tests can point them at an isolated throwaway repository instead of mutating this one. """ from __future__ import annotations import argparse import hashlib import io import json import subprocess import sys import tarfile from pathlib import Path from typing import cast # Ensure the package import works both for direct CLI execution and imports. _ROOT_DIR = Path(__file__).resolve().parents[1] if str(_ROOT_DIR) not in sys.path: sys.path.insert(0, str(_ROOT_DIR)) from scripts.packlib import ROOT, brussels_now # noqa: E402 SELF_EXCLUDED_PATH = "PACK_MANIFEST.json" class GitError(RuntimeError): """Raised when a required Git plumbing command fails.""" def run_git(args: list[str], repo_root: Path = ROOT) -> str: result = subprocess.run( # noqa: S603 ["git", *args], # noqa: S607 cwd=repo_root, capture_output=True, text=False, check=False, ) if result.returncode != 0: stderr = result.stderr.decode("utf-8", errors="replace") raise GitError(f"git {' '.join(args)} failed: {stderr.strip()}") return result.stdout.decode("utf-8") def resolve_commit(ref: str, repo_root: Path = ROOT) -> str: return run_git(["rev-parse", ref], repo_root).strip() def resolve_tree(ref: str, repo_root: Path = ROOT) -> str: return run_git(["rev-parse", f"{ref}^{{tree}}"], repo_root).strip() def list_tracked_paths(ref: str, repo_root: Path = ROOT) -> list[str]: """Return every path tracked in ``ref``'s tree, sorted, POSIX-separated.""" output = run_git(["ls-tree", "-r", "--name-only", "-z", ref], repo_root) paths = [p for p in output.split("\0") if p] return sorted(paths) def list_dirty_tracked_paths(repo_root: Path = ROOT) -> list[str]: """Return tracked paths with staged or unstaged changes versus HEAD. Untracked files are deliberately excluded (``--untracked-files=no``): they cannot enter the manifest at all, so their presence is not a manifest-integrity concern, only tracked-content drift is. """ output = run_git(["status", "--porcelain", "--untracked-files=no"], repo_root) return [line[3:] for line in output.splitlines() if line.strip()] def batch_blob_contents(ref: str, paths: list[str], repo_root: Path = ROOT) -> dict[str, bytes]: """Fetch every path's content at ``ref`` exactly as `git archive` would produce it. Deliberately sourced from `git archive`, not raw `git cat-file` blob content. Those two differ for any file with an explicit `eol=crlf` (or `eol=lf`) `.gitattributes` override — e.g. scripts/bootstrap.ps1, which is intentionally always-CRLF for native Windows tooling. Git stores every blob LF-normalized internally regardless of that attribute; the attribute only governs the conversion `git archive` and `git checkout` apply on the way *out*. Hashing raw blob bytes would make the manifest describe content nobody ever actually receives from a real archive extraction or checkout — hashing `git archive` output instead makes the manifest and "verify a real archive extraction" (scripts/validate_pack.py --verify-tree) definitionally the same operation, by construction. This is still fully platform-independent: .gitattributes pins an explicit `eol` for every tracked file type, so no file is left to `core.autocrlf` auto-detection ambiguity (verified: a Windows checkout with core.autocrlf=true and a Linux checkout produce byte-identical archives). """ if not paths: return {} wanted = set(paths) result = subprocess.run( # noqa: S603 ["git", "archive", "--format=tar", ref], # noqa: S607 cwd=repo_root, capture_output=True, check=False, ) if result.returncode != 0: stderr = result.stderr.decode("utf-8", errors="replace") raise GitError(f"git archive failed: {stderr.strip()}") contents: dict[str, bytes] = {} with tarfile.open(fileobj=io.BytesIO(result.stdout)) as archive: for member in archive.getmembers(): if not member.isfile() or member.name not in wanted: continue extracted = archive.extractfile(member) assert extracted is not None # isfile() guarantees a readable member contents[member.name] = extracted.read() missing = wanted - set(contents) if missing: raise GitError(f"git archive did not produce expected paths: {sorted(missing)}") return contents def compute_content_digest(files: list[dict[str, object]]) -> str: """Digest every (path, sha256) pair, independent of the manifest's own blob. This is deliberately NOT the Git tree hash. The manifest file itself is tracked and committed alongside the content it describes, so the raw ``HEAD^{tree}`` hash always includes PACK_MANIFEST.json's own blob — and that blob's content (via ``generatedAt``) changes on every regeneration. A staleness check keyed on the raw tree hash therefore can never match itself: the tree recorded *before* the manifest is committed is necessarily different from the tree *after*, since committing the manifest changes the tree by definition. Digesting only the (path, checksum) pairs of every *other* tracked file sidesteps that paradox entirely: this digest is stable across manifest-only commits and changes if and only if some other tracked file's content actually changes. """ hasher = hashlib.sha256() for entry in sorted(files, key=lambda item: item["path"]): # type: ignore[arg-type,return-value] hasher.update(str(entry["path"]).encode("utf-8")) hasher.update(b"\0") hasher.update(str(entry["sha256"]).encode("utf-8")) hasher.update(b"\n") return hasher.hexdigest() def build_manifest(ref: str, repo_root: Path = ROOT) -> dict[str, object]: commit = resolve_commit(ref, repo_root) tree = resolve_tree(ref, repo_root) paths = [p for p in list_tracked_paths(commit, repo_root) if p != SELF_EXCLUDED_PATH] contents = batch_blob_contents(commit, paths, repo_root) files = [] for path in paths: data = contents[path] files.append( { "path": path, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(), } ) return { "schemaVersion": "2.0", "product": "ITWorx MCP Hub — Luna Design & Build Pack", "generatedAt": brussels_now(), "source": { "commit": commit, "tree": tree, "contentDigest": compute_content_digest(files), "description": ( "Every entry below is hashed from `git archive` output at " "this commit, not from checked-out filesystem bytes and not " "from raw `git cat-file` blob content (those differ for " "files with an explicit eol= override, e.g. " "scripts/bootstrap.ps1). Regenerating this manifest from the " "same commit on any platform (Windows checkout, Linux " "checkout, fresh clone, or `git archive` extraction) must " "produce identical checksums for every entry. `commit`/`tree` " "are informational provenance only; staleness is gated on " "`contentDigest`, which deliberately excludes this " "manifest's own tracked blob (see compute_content_digest's " "docstring for why)." ), }, "selfExcluded": SELF_EXCLUDED_PATH, "fileCount": len(files), "totalBytes": sum(cast(int, item["bytes"]) for item in files), "files": files, } def main() -> int: parser = argparse.ArgumentParser( description=("Create a deterministic Git-blob-based checksum manifest for the design pack.") ) parser.add_argument("--output", default="PACK_MANIFEST.json") parser.add_argument( "--ref", default="HEAD", help="Git ref to generate the manifest from (default: HEAD).", ) parser.add_argument( "--root", default=str(ROOT), help="Repository root to generate the manifest from (default: this repo).", ) args = parser.parse_args() repo_root = Path(args.root).resolve() manifest = build_manifest(args.ref, repo_root) output_arg = Path(args.output) output = output_arg if output_arg.is_absolute() else repo_root / output_arg # newline="" writes exactly the LF this content already uses, rather # than letting Python's platform-default newline translation turn it # into CRLF on Windows only to have Git's clean filter convert it back # on the next `git add` (harmless, but needless churn and warning noise). output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8", newline="") try: display_path = output.relative_to(repo_root) except ValueError: display_path = output print( f"Wrote {display_path}: {manifest['fileCount']} files, " f"{manifest['totalBytes']} bytes, source commit " f"{manifest['source']['commit']}" # type: ignore[index] ) return 0 if __name__ == "__main__": raise SystemExit(main())