Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020

scripts/create_manifest.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Create a deterministic checksum manifest from canonical Git-tracked content.
+
+Every entry is hashed from `git archive` output at a chosen ref (default
+``HEAD``), never from raw checked-out filesystem bytes and never from raw
+`git cat-file` blob content either. This is deliberate, for two reasons:
+
+1. Working-tree bytes are not platform-independent — a Windows checkout
+   (core.autocrlf=true) and a Linux checkout of the same commit can differ
+   byte-for-byte (see the WP-220 evidence and ``.gitattributes`` for why
+   that already caused a false-positive manifest failure once, in commit
+   95329ca).
+2. Raw blob content is *also* not what a real archive extraction or checkout
+   produces for every file: Git always stores blobs LF-normalized
+   internally, but a file with an explicit `eol=crlf` `.gitattributes`
+   override (e.g. scripts/bootstrap.ps1, intentionally always-CRLF for
+   native Windows tooling) gets CRLF from `git archive`/`git checkout`
+   regardless. Hashing raw blobs would make the manifest describe bytes
+   nobody ever actually receives. Hashing `git archive` output instead makes
+   manifest generation and "verify a real archive extraction"
+   (`validate_pack.py --verify-tree`) definitionally the same operation.
+
+Both properties hold together because ``.gitattributes`` pins an explicit
+`eol` for every tracked file type in this repository — no file is left to
+`core.autocrlf` auto-detection ambiguity, so `git archive` output is fully
+deterministic across platforms even though it is not identical to the raw
+blob for CRLF-pinned files.
+
+Only files tracked by Git at the chosen ref are ever considered — ignored,
+untracked, local-only, generated, cache, secret, editor, and agent-setting
+files can never enter the manifest, because they are not in the tree Git
+enumerates. ``PACK_MANIFEST.json`` itself is always excluded from its own
+file list (self-referential exclusion), deterministically, regardless of
+whether it happens to be tracked.
+
+Every git-plumbing helper below takes an explicit ``repo_root`` (defaulting
+to this repository's root) rather than hardcoding it, so tests can point
+them at an isolated throwaway repository instead of mutating this one.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import io
+import json
+import subprocess
+import sys
+import tarfile
+from pathlib import Path
+from typing import cast
+
+# Ensure the package import works both for direct CLI execution and imports.
+_ROOT_DIR = Path(__file__).resolve().parents[1]
+if str(_ROOT_DIR) not in sys.path:
+    sys.path.insert(0, str(_ROOT_DIR))
+
+from scripts.packlib import ROOT, brussels_now  # noqa: E402
+
+SELF_EXCLUDED_PATH = "PACK_MANIFEST.json"
+
+
+class GitError(RuntimeError):
+    """Raised when a required Git plumbing command fails."""
+
+
+def run_git(args: list[str], repo_root: Path = ROOT) -> str:
+    result = subprocess.run(  # noqa: S603
+        ["git", *args],  # noqa: S607
+        cwd=repo_root,
+        capture_output=True,
+        text=False,
+        check=False,
+    )
+    if result.returncode != 0:
+        stderr = result.stderr.decode("utf-8", errors="replace")
+        raise GitError(f"git {' '.join(args)} failed: {stderr.strip()}")
+    return result.stdout.decode("utf-8")
+
+
+def resolve_commit(ref: str, repo_root: Path = ROOT) -> str:
+    return run_git(["rev-parse", ref], repo_root).strip()
+
+
+def resolve_tree(ref: str, repo_root: Path = ROOT) -> str:
+    return run_git(["rev-parse", f"{ref}^{{tree}}"], repo_root).strip()
+
+
+def list_tracked_paths(ref: str, repo_root: Path = ROOT) -> list[str]:
+    """Return every path tracked in ``ref``'s tree, sorted, POSIX-separated."""
+    output = run_git(["ls-tree", "-r", "--name-only", "-z", ref], repo_root)
+    paths = [p for p in output.split("\0") if p]
+    return sorted(paths)
+
+
+def list_dirty_tracked_paths(repo_root: Path = ROOT) -> list[str]:
+    """Return tracked paths with staged or unstaged changes versus HEAD.
+
+    Untracked files are deliberately excluded (``--untracked-files=no``):
+    they cannot enter the manifest at all, so their presence is not a
+    manifest-integrity concern, only tracked-content drift is.
+    """
+    output = run_git(["status", "--porcelain", "--untracked-files=no"], repo_root)
+    return [line[3:] for line in output.splitlines() if line.strip()]
+
+
+def batch_blob_contents(ref: str, paths: list[str], repo_root: Path = ROOT) -> dict[str, bytes]:
+    """Fetch every path's content at ``ref`` exactly as `git archive` would produce it.
+
+    Deliberately sourced from `git archive`, not raw `git cat-file` blob
+    content. Those two differ for any file with an explicit `eol=crlf` (or
+    `eol=lf`) `.gitattributes` override — e.g. scripts/bootstrap.ps1, which
+    is intentionally always-CRLF for native Windows tooling. Git stores every
+    blob LF-normalized internally regardless of that attribute; the
+    attribute only governs the conversion `git archive` and `git checkout`
+    apply on the way *out*. Hashing raw blob bytes would make the manifest
+    describe content nobody ever actually receives from a real archive
+    extraction or checkout — hashing `git archive` output instead makes the
+    manifest and "verify a real archive extraction" (scripts/validate_pack.py
+    --verify-tree) definitionally the same operation, by construction. This
+    is still fully platform-independent: .gitattributes pins an explicit
+    `eol` for every tracked file type, so no file is left to `core.autocrlf`
+    auto-detection ambiguity (verified: a Windows checkout with
+    core.autocrlf=true and a Linux checkout produce byte-identical archives).
+    """
+    if not paths:
+        return {}
+    wanted = set(paths)
+    result = subprocess.run(  # noqa: S603
+        ["git", "archive", "--format=tar", ref],  # noqa: S607
+        cwd=repo_root,
+        capture_output=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        stderr = result.stderr.decode("utf-8", errors="replace")
+        raise GitError(f"git archive failed: {stderr.strip()}")
+
+    contents: dict[str, bytes] = {}
+    with tarfile.open(fileobj=io.BytesIO(result.stdout)) as archive:
+        for member in archive.getmembers():
+            if not member.isfile() or member.name not in wanted:
+                continue
+            extracted = archive.extractfile(member)
+            assert extracted is not None  # isfile() guarantees a readable member
+            contents[member.name] = extracted.read()
+
+    missing = wanted - set(contents)
+    if missing:
+        raise GitError(f"git archive did not produce expected paths: {sorted(missing)}")
+    return contents
+
+
+def compute_content_digest(files: list[dict[str, object]]) -> str:
+    """Digest every (path, sha256) pair, independent of the manifest's own blob.
+
+    This is deliberately NOT the Git tree hash. The manifest file itself is
+    tracked and committed alongside the content it describes, so the raw
+    ``HEAD^{tree}`` hash always includes PACK_MANIFEST.json's own blob — and
+    that blob's content (via ``generatedAt``) changes on every regeneration.
+    A staleness check keyed on the raw tree hash therefore can never match
+    itself: the tree recorded *before* the manifest is committed is
+    necessarily different from the tree *after*, since committing the
+    manifest changes the tree by definition. Digesting only the (path,
+    checksum) pairs of every *other* tracked file sidesteps that paradox
+    entirely: this digest is stable across manifest-only commits and changes
+    if and only if some other tracked file's content actually changes.
+    """
+    hasher = hashlib.sha256()
+    for entry in sorted(files, key=lambda item: item["path"]):  # type: ignore[arg-type,return-value]
+        hasher.update(str(entry["path"]).encode("utf-8"))
+        hasher.update(b"\0")
+        hasher.update(str(entry["sha256"]).encode("utf-8"))
+        hasher.update(b"\n")
+    return hasher.hexdigest()
+
+
+def build_manifest(ref: str, repo_root: Path = ROOT) -> dict[str, object]:
+    commit = resolve_commit(ref, repo_root)
+    tree = resolve_tree(ref, repo_root)
+    paths = [p for p in list_tracked_paths(commit, repo_root) if p != SELF_EXCLUDED_PATH]
+    contents = batch_blob_contents(commit, paths, repo_root)
+
+    files = []
+    for path in paths:
+        data = contents[path]
+        files.append(
+            {
+                "path": path,
+                "bytes": len(data),
+                "sha256": hashlib.sha256(data).hexdigest(),
+            }
+        )
+
+    return {
+        "schemaVersion": "2.0",
+        "product": "ITWorx MCP Hub — Luna Design & Build Pack",
+        "generatedAt": brussels_now(),
+        "source": {
+            "commit": commit,
+            "tree": tree,
+            "contentDigest": compute_content_digest(files),
+            "description": (
+                "Every entry below is hashed from `git archive` output at "
+                "this commit, not from checked-out filesystem bytes and not "
+                "from raw `git cat-file` blob content (those differ for "
+                "files with an explicit eol= override, e.g. "
+                "scripts/bootstrap.ps1). Regenerating this manifest from the "
+                "same commit on any platform (Windows checkout, Linux "
+                "checkout, fresh clone, or `git archive` extraction) must "
+                "produce identical checksums for every entry. `commit`/`tree` "
+                "are informational provenance only; staleness is gated on "
+                "`contentDigest`, which deliberately excludes this "
+                "manifest's own tracked blob (see compute_content_digest's "
+                "docstring for why)."
+            ),
+        },
+        "selfExcluded": SELF_EXCLUDED_PATH,
+        "fileCount": len(files),
+        "totalBytes": sum(cast(int, item["bytes"]) for item in files),
+        "files": files,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=("Create a deterministic Git-blob-based checksum manifest for the design pack.")
+    )
+    parser.add_argument("--output", default="PACK_MANIFEST.json")
+    parser.add_argument(
+        "--ref",
+        default="HEAD",
+        help="Git ref to generate the manifest from (default: HEAD).",
+    )
+    parser.add_argument(
+        "--root",
+        default=str(ROOT),
+        help="Repository root to generate the manifest from (default: this repo).",
+    )
+    args = parser.parse_args()
+
+    repo_root = Path(args.root).resolve()
+    manifest = build_manifest(args.ref, repo_root)
+    output_arg = Path(args.output)
+    output = output_arg if output_arg.is_absolute() else repo_root / output_arg
+    # newline="" writes exactly the LF this content already uses, rather
+    # than letting Python's platform-default newline translation turn it
+    # into CRLF on Windows only to have Git's clean filter convert it back
+    # on the next `git add` (harmless, but needless churn and warning noise).
+    output.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8", newline="")
+    try:
+        display_path = output.relative_to(repo_root)
+    except ValueError:
+        display_path = output
+    print(
+        f"Wrote {display_path}: {manifest['fileCount']} files, "
+        f"{manifest['totalBytes']} bytes, source commit "
+        f"{manifest['source']['commit']}"  # type: ignore[index]
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())