#!/usr/bin/env python3 """Create a deterministic checksum manifest from canonical Git-tracked content. Every entry is hashed from `git archive` output at a chosen ref (default ``HEAD``), never from raw checked-out filesystem bytes and never from raw `git cat-file` blob content either. This is deliberate, for two reasons: 1. Working-tree bytes are not platform-independent — a Windows checkout (core.autocrlf=true) and a Linux checkout of the same commit can differ byte-for-byte (see the WP-220 evidence and ``.gitattributes`` for why that already caused a false-positive manifest failure once, in commit 95329ca). 2. The bytes received from Git depend on the path's .gitattributes rules. Text files with an explicit eol rule are normalized in the object store and converted on archive/checkout; paths marked -text retain their exact source bytes in both places. Hashing raw blobs would miss the conversion for text files, while hashing git archive output makes manifest generation and "verify a real archive extraction" (validate_pack.py --verify-tree) definitionally the same operation. Both properties hold together because .gitattributes pins explicit behavior for every tracked path: ordinary text files have a pinned eol, and the WP-400 retained-provenance bundle is a scoped -text exact-byte exception. This avoids core.autocrlf auto-detection ambiguity and keeps git archive output deterministic across platforms. Only files tracked by Git at the chosen ref are ever considered — ignored, untracked, local-only, generated, cache, secret, editor, and agent-setting files can never enter the manifest, because they are not in the tree Git enumerates. ``PACK_MANIFEST.json`` itself is always excluded from its own file list (self-referential exclusion), deterministically, regardless of whether it happens to be tracked. Every git-plumbing helper below takes an explicit ``repo_root`` (defaulting to this repository's root) rather than hardcoding it, so tests can point them at an isolated throwaway repository instead of mutating this one. """ from __future__ import annotations import argparse import hashlib import io import json import subprocess import sys import tarfile from pathlib import Path from typing import cast # Ensure the package import works both for direct CLI execution and imports. _ROOT_DIR = Path(__file__).resolve().parents[1] if str(_ROOT_DIR) not in sys.path: sys.path.insert(0, str(_ROOT_DIR)) from scripts.packlib import ROOT, brussels_now # noqa: E402 SELF_EXCLUDED_PATH = "PACK_MANIFEST.json" class GitError(RuntimeError): """Raised when a required Git plumbing command fails.""" def run_git(args: list[str], repo_root: Path = ROOT) -> str: result = subprocess.run( # noqa: S603 ["git", *args], # noqa: S607 cwd=repo_root, capture_output=True, text=False, check=False, ) if result.returncode != 0: stderr = result.stderr.decode("utf-8", errors="replace") raise GitError(f"git {' '.join(args)} failed: {stderr.strip()}") return result.stdout.decode("utf-8") def resolve_commit(ref: str, repo_root: Path = ROOT) -> str: return run_git(["rev-parse", ref], repo_root).strip() def resolve_tree(ref: str, repo_root: Path = ROOT) -> str: return run_git(["rev-parse", f"{ref}^{{tree}}"], repo_root).strip() def list_tracked_paths(ref: str, repo_root: Path = ROOT) -> list[str]: """Return every path tracked in ``ref``'s tree, sorted, POSIX-separated.""" output = run_git(["ls-tree", "-r", "--name-only", "-z", ref], repo_root) paths = [p for p in output.split("\0") if p] return sorted(paths) def list_dirty_tracked_paths(repo_root: Path = ROOT) -> list[str]: """Return tracked paths with staged or unstaged changes versus HEAD. Untracked files are deliberately excluded (``--untracked-files=no``): they cannot enter the manifest at all, so their presence is not a manifest-integrity concern, only tracked-content drift is. """ output = run_git(["status", "--porcelain", "--untracked-files=no"], repo_root) return [line[3:] for line in output.splitlines() if line.strip()] def batch_blob_contents(ref: str, paths: list[str], repo_root: Path = ROOT) -> dict[str, bytes]: """Fetch every path's content at ``ref`` exactly as `git archive` would produce it. Deliberately sourced from `git archive`, not raw `git cat-file` blob content. Those two differ for any file with an explicit `eol=crlf` (or `eol=lf`) `.gitattributes` override — e.g. scripts/bootstrap.ps1, which is intentionally always-CRLF for native Windows tooling. Text paths are normalized in the object store and converted by archive/checkout; -text paths such as the retained WP-400 provenance bundle keep their raw bytes. Raw blobs omit archive-time conversion for text files, while hashing git archive output matches the bytes consumers extract and check out. This makes the manifest and "verify a real archive extraction" (scripts/validate_pack.py --verify-tree) definitionally the same operation. """ if not paths: return {} wanted = set(paths) result = subprocess.run( # noqa: S603 ["git", "archive", "--format=tar", ref], # noqa: S607 cwd=repo_root, capture_output=True, check=False, ) if result.returncode != 0: stderr = result.stderr.decode("utf-8", errors="replace") raise GitError(f"git archive failed: {stderr.strip()}") contents: dict[str, bytes] = {} with tarfile.open(fileobj=io.BytesIO(result.stdout)) as archive: for member in archive.getmembers(): if not member.isfile() or member.name not in wanted: continue extracted = archive.extractfile(member) assert extracted is not None # isfile() guarantees a readable member contents[member.name] = extracted.read() missing = wanted - set(contents) if missing: raise GitError(f"git archive did not produce expected paths: {sorted(missing)}") return contents def compute_content_digest(files: list[dict[str, object]]) -> str: """Digest every (path, sha256) pair, independent of the manifest's own blob. This is deliberately NOT the Git tree hash. The manifest file itself is tracked and committed alongside the content it describes, so the raw ``HEAD^{tree}`` hash always includes PACK_MANIFEST.json's own blob — and that blob's content (via ``generatedAt``) changes on every regeneration. A staleness check keyed on the raw tree hash therefore can never match itself: the tree recorded *before* the manifest is committed is necessarily different from the tree *after*, since committing the manifest changes the tree by definition. Digesting only the (path, checksum) pairs of every *other* tracked file sidesteps that paradox entirely: this digest is stable across manifest-only commits and changes if and only if some other tracked file's content actually changes. """ hasher = hashlib.sha256() for entry in sorted(files, key=lambda item: item["path"]): # type: ignore[arg-type,return-value] hasher.update(str(entry["path"]).encode("utf-8")) hasher.update(b"\0") hasher.update(str(entry["sha256"]).encode("utf-8")) hasher.update(b"\n") return hasher.hexdigest() def build_manifest(ref: str, repo_root: Path = ROOT) -> dict[str, object]: commit = resolve_commit(ref, repo_root) tree = resolve_tree(ref, repo_root) paths = [p for p in list_tracked_paths(commit, repo_root) if p != SELF_EXCLUDED_PATH] contents = batch_blob_contents(commit, paths, repo_root) files = [] for path in paths: data = contents[path] files.append( { "path": path, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(), } ) return { "schemaVersion": "2.0", "product": "ITWorx MCP Hub — Luna Design & Build Pack", "generatedAt": brussels_now(), "source": { "commit": commit, "tree": tree, "contentDigest": compute_content_digest(files), "description": ( "Every entry below is hashed from `git archive` output at " "this commit, not from checked-out filesystem bytes or raw " "blob content for text paths with an eol conversion (e.g. " "scripts/bootstrap.ps1). Regenerating this manifest from the " "same commit on any platform (Windows checkout, Linux " "checkout, fresh clone, or `git archive` extraction) must " "produce identical checksums for every entry. `commit`/`tree` " "are informational provenance only; staleness is gated on " "`contentDigest`, which deliberately excludes this " "manifest's own tracked blob (see compute_content_digest's " "docstring for why)." ), }, "selfExcluded": SELF_EXCLUDED_PATH, "fileCount": len(files), "totalBytes": sum(cast(int, item["bytes"]) for item in files), "files": files, } def main() -> int: parser = argparse.ArgumentParser( description="Create a deterministic Git-archive-based checksum manifest.", ) parser.add_argument("--output", default="PACK_MANIFEST.json") parser.add_argument( "--ref", default="HEAD", help="Git ref to generate the manifest from (default: HEAD).", ) parser.add_argument( "--root", default=str(ROOT), help="Repository root to generate the manifest from (default: this repo).", ) args = parser.parse_args() repo_root = Path(args.root).resolve() manifest = build_manifest(args.ref, repo_root) output_arg = Path(args.output) output = output_arg if output_arg.is_absolute() else repo_root / output_arg # newline="" writes exactly the LF this content already uses, rather # than letting Python's platform-default newline translation turn it # into CRLF on Windows only to have Git's clean filter convert it back # on the next `git add` (harmless, but needless churn and warning noise). output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8", newline="") try: display_path = output.relative_to(repo_root) except ValueError: display_path = output print( f"Wrote {display_path}: {manifest['fileCount']} files, " f"{manifest['totalBytes']} bytes, source commit " f"{manifest['source']['commit']}" # type: ignore[index] ) return 0 if __name__ == "__main__": raise SystemExit(main())