257 lines
11 KiB
Python
257 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Create a deterministic checksum manifest from canonical Git-tracked content.
|
|
|
|
Every entry is hashed from `git archive` output at a chosen ref (default
|
|
``HEAD``), never from raw checked-out filesystem bytes and never from raw
|
|
`git cat-file` blob content either. This is deliberate, for two reasons:
|
|
|
|
1. Working-tree bytes are not platform-independent — a Windows checkout
|
|
(core.autocrlf=true) and a Linux checkout of the same commit can differ
|
|
byte-for-byte (see the WP-220 evidence and ``.gitattributes`` for why
|
|
that already caused a false-positive manifest failure once, in commit
|
|
95329ca).
|
|
2. The bytes received from Git depend on the path's .gitattributes rules.
|
|
Text files with an explicit eol rule are normalized in the object store
|
|
and converted on archive/checkout; paths marked -text retain their exact
|
|
source bytes in both places. Hashing raw blobs would miss the conversion
|
|
for text files, while hashing git archive output makes manifest generation
|
|
and "verify a real archive extraction" (validate_pack.py --verify-tree)
|
|
definitionally the same operation.
|
|
|
|
Both properties hold together because .gitattributes pins explicit behavior
|
|
for every tracked path: ordinary text files have a pinned eol, and the WP-400
|
|
retained-provenance bundle is a scoped -text exact-byte exception. This avoids
|
|
core.autocrlf auto-detection ambiguity and keeps git archive output
|
|
deterministic across platforms.
|
|
|
|
Only files tracked by Git at the chosen ref are ever considered — ignored,
|
|
untracked, local-only, generated, cache, secret, editor, and agent-setting
|
|
files can never enter the manifest, because they are not in the tree Git
|
|
enumerates. ``PACK_MANIFEST.json`` itself is always excluded from its own
|
|
file list (self-referential exclusion), deterministically, regardless of
|
|
whether it happens to be tracked.
|
|
|
|
Every git-plumbing helper below takes an explicit ``repo_root`` (defaulting
|
|
to this repository's root) rather than hardcoding it, so tests can point
|
|
them at an isolated throwaway repository instead of mutating this one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
# Ensure the package import works both for direct CLI execution and imports.
|
|
_ROOT_DIR = Path(__file__).resolve().parents[1]
|
|
if str(_ROOT_DIR) not in sys.path:
|
|
sys.path.insert(0, str(_ROOT_DIR))
|
|
|
|
from scripts.packlib import ROOT, brussels_now # noqa: E402
|
|
|
|
SELF_EXCLUDED_PATH = "PACK_MANIFEST.json"
|
|
|
|
|
|
class GitError(RuntimeError):
|
|
"""Raised when a required Git plumbing command fails."""
|
|
|
|
|
|
def run_git(args: list[str], repo_root: Path = ROOT) -> str:
|
|
result = subprocess.run( # noqa: S603
|
|
["git", *args], # noqa: S607
|
|
cwd=repo_root,
|
|
capture_output=True,
|
|
text=False,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
stderr = result.stderr.decode("utf-8", errors="replace")
|
|
raise GitError(f"git {' '.join(args)} failed: {stderr.strip()}")
|
|
return result.stdout.decode("utf-8")
|
|
|
|
|
|
def resolve_commit(ref: str, repo_root: Path = ROOT) -> str:
|
|
return run_git(["rev-parse", ref], repo_root).strip()
|
|
|
|
|
|
def resolve_tree(ref: str, repo_root: Path = ROOT) -> str:
|
|
return run_git(["rev-parse", f"{ref}^{{tree}}"], repo_root).strip()
|
|
|
|
|
|
def list_tracked_paths(ref: str, repo_root: Path = ROOT) -> list[str]:
|
|
"""Return every path tracked in ``ref``'s tree, sorted, POSIX-separated."""
|
|
output = run_git(["ls-tree", "-r", "--name-only", "-z", ref], repo_root)
|
|
paths = [p for p in output.split("\0") if p]
|
|
return sorted(paths)
|
|
|
|
|
|
def list_dirty_tracked_paths(repo_root: Path = ROOT) -> list[str]:
|
|
"""Return tracked paths with staged or unstaged changes versus HEAD.
|
|
|
|
Untracked files are deliberately excluded (``--untracked-files=no``):
|
|
they cannot enter the manifest at all, so their presence is not a
|
|
manifest-integrity concern, only tracked-content drift is.
|
|
"""
|
|
output = run_git(["status", "--porcelain", "--untracked-files=no"], repo_root)
|
|
return [line[3:] for line in output.splitlines() if line.strip()]
|
|
|
|
|
|
def batch_blob_contents(ref: str, paths: list[str], repo_root: Path = ROOT) -> dict[str, bytes]:
|
|
"""Fetch every path's content at ``ref`` exactly as `git archive` would produce it.
|
|
|
|
Deliberately sourced from `git archive`, not raw `git cat-file` blob
|
|
content. Those two differ for any file with an explicit `eol=crlf` (or
|
|
`eol=lf`) `.gitattributes` override — e.g. scripts/bootstrap.ps1, which
|
|
is intentionally always-CRLF for native Windows tooling. Text paths are
|
|
normalized in the object store and converted by archive/checkout; -text
|
|
paths such as the retained WP-400 provenance bundle keep their raw bytes.
|
|
Raw blobs omit archive-time conversion for text files, while hashing
|
|
git archive output matches the bytes consumers extract and check out.
|
|
This makes the manifest and "verify a real archive extraction"
|
|
(scripts/validate_pack.py --verify-tree) definitionally the same operation.
|
|
"""
|
|
if not paths:
|
|
return {}
|
|
wanted = set(paths)
|
|
result = subprocess.run( # noqa: S603
|
|
["git", "archive", "--format=tar", ref], # noqa: S607
|
|
cwd=repo_root,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
stderr = result.stderr.decode("utf-8", errors="replace")
|
|
raise GitError(f"git archive failed: {stderr.strip()}")
|
|
|
|
contents: dict[str, bytes] = {}
|
|
with tarfile.open(fileobj=io.BytesIO(result.stdout)) as archive:
|
|
for member in archive.getmembers():
|
|
if not member.isfile() or member.name not in wanted:
|
|
continue
|
|
extracted = archive.extractfile(member)
|
|
assert extracted is not None # isfile() guarantees a readable member
|
|
contents[member.name] = extracted.read()
|
|
|
|
missing = wanted - set(contents)
|
|
if missing:
|
|
raise GitError(f"git archive did not produce expected paths: {sorted(missing)}")
|
|
return contents
|
|
|
|
|
|
def compute_content_digest(files: list[dict[str, object]]) -> str:
|
|
"""Digest every (path, sha256) pair, independent of the manifest's own blob.
|
|
|
|
This is deliberately NOT the Git tree hash. The manifest file itself is
|
|
tracked and committed alongside the content it describes, so the raw
|
|
``HEAD^{tree}`` hash always includes PACK_MANIFEST.json's own blob — and
|
|
that blob's content (via ``generatedAt``) changes on every regeneration.
|
|
A staleness check keyed on the raw tree hash therefore can never match
|
|
itself: the tree recorded *before* the manifest is committed is
|
|
necessarily different from the tree *after*, since committing the
|
|
manifest changes the tree by definition. Digesting only the (path,
|
|
checksum) pairs of every *other* tracked file sidesteps that paradox
|
|
entirely: this digest is stable across manifest-only commits and changes
|
|
if and only if some other tracked file's content actually changes.
|
|
"""
|
|
hasher = hashlib.sha256()
|
|
for entry in sorted(files, key=lambda item: item["path"]): # type: ignore[arg-type,return-value]
|
|
hasher.update(str(entry["path"]).encode("utf-8"))
|
|
hasher.update(b"\0")
|
|
hasher.update(str(entry["sha256"]).encode("utf-8"))
|
|
hasher.update(b"\n")
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def build_manifest(ref: str, repo_root: Path = ROOT) -> dict[str, object]:
|
|
commit = resolve_commit(ref, repo_root)
|
|
tree = resolve_tree(ref, repo_root)
|
|
paths = [p for p in list_tracked_paths(commit, repo_root) if p != SELF_EXCLUDED_PATH]
|
|
contents = batch_blob_contents(commit, paths, repo_root)
|
|
|
|
files = []
|
|
for path in paths:
|
|
data = contents[path]
|
|
files.append(
|
|
{
|
|
"path": path,
|
|
"bytes": len(data),
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
}
|
|
)
|
|
|
|
return {
|
|
"schemaVersion": "2.0",
|
|
"product": "ITWorx MCP Hub — Luna Design & Build Pack",
|
|
"generatedAt": brussels_now(),
|
|
"source": {
|
|
"commit": commit,
|
|
"tree": tree,
|
|
"contentDigest": compute_content_digest(files),
|
|
"description": (
|
|
"Every entry below is hashed from `git archive` output at "
|
|
"this commit, not from checked-out filesystem bytes or raw "
|
|
"blob content for text paths with an eol conversion (e.g. "
|
|
"scripts/bootstrap.ps1). Regenerating this manifest from the "
|
|
"same commit on any platform (Windows checkout, Linux "
|
|
"checkout, fresh clone, or `git archive` extraction) must "
|
|
"produce identical checksums for every entry. `commit`/`tree` "
|
|
"are informational provenance only; staleness is gated on "
|
|
"`contentDigest`, which deliberately excludes this "
|
|
"manifest's own tracked blob (see compute_content_digest's "
|
|
"docstring for why)."
|
|
),
|
|
},
|
|
"selfExcluded": SELF_EXCLUDED_PATH,
|
|
"fileCount": len(files),
|
|
"totalBytes": sum(cast(int, item["bytes"]) for item in files),
|
|
"files": files,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Create a deterministic Git-archive-based checksum manifest.",
|
|
)
|
|
parser.add_argument("--output", default="PACK_MANIFEST.json")
|
|
parser.add_argument(
|
|
"--ref",
|
|
default="HEAD",
|
|
help="Git ref to generate the manifest from (default: HEAD).",
|
|
)
|
|
parser.add_argument(
|
|
"--root",
|
|
default=str(ROOT),
|
|
help="Repository root to generate the manifest from (default: this repo).",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
repo_root = Path(args.root).resolve()
|
|
manifest = build_manifest(args.ref, repo_root)
|
|
output_arg = Path(args.output)
|
|
output = output_arg if output_arg.is_absolute() else repo_root / output_arg
|
|
# newline="" writes exactly the LF this content already uses, rather
|
|
# than letting Python's platform-default newline translation turn it
|
|
# into CRLF on Windows only to have Git's clean filter convert it back
|
|
# on the next `git add` (harmless, but needless churn and warning noise).
|
|
output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8", newline="")
|
|
try:
|
|
display_path = output.relative_to(repo_root)
|
|
except ValueError:
|
|
display_path = output
|
|
print(
|
|
f"Wrote {display_path}: {manifest['fileCount']} files, "
|
|
f"{manifest['totalBytes']} bytes, source commit "
|
|
f"{manifest['source']['commit']}" # type: ignore[index]
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|