Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020

tests/test_wp220_manifest.py

@@ -0,0 +1,445 @@
+"""WP-220A regression tests: Git-tracked-content-only release manifest.
+
+These tests build small, throwaway Git repositories under ``tmp_path`` and
+exercise scripts/create_manifest.py and scripts/validate_pack.py against
+them directly (via their ``repo_root``-parameterized functions) — never
+against this actual repository, so nothing here can corrupt
+PACK_MANIFEST.json or require a clean working tree to run.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from scripts import create_manifest, validate_pack
+
+
+def _git(repo: Path, *args: str) -> str:
+    result = subprocess.run(  # noqa: S603
+        ["git", *args],  # noqa: S607
+        cwd=repo,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def _init_repo(repo: Path) -> None:
+    repo.mkdir(parents=True, exist_ok=True)
+    # Pin the initial branch name explicitly: the local/global
+    # init.defaultBranch config is not guaranteed to be "main" (this
+    # workstation defaults to "master"), and the merge-commit regression
+    # tests below assume a branch literally named "main".
+    _git(repo, "init", "-q", "-b", "main")
+    _git(repo, "config", "user.email", "test@example.invalid")
+    _git(repo, "config", "user.name", "WP-220 Test")
+    _git(repo, "config", "core.autocrlf", "false")
+
+
+def _commit_all(repo: Path, message: str) -> str:
+    _git(repo, "add", "-A")
+    _git(repo, "commit", "-q", "-m", message)
+    return _git(repo, "rev-parse", "HEAD").strip()
+
+
+@pytest.fixture
+def repo(tmp_path: Path) -> Path:
+    path = tmp_path / "repo"
+    _init_repo(path)
+    (path / "README.md").write_text("hello\n", encoding="utf-8")
+    (path / "src").mkdir()
+    (path / "src" / "app.py").write_text("print('hi')\n", encoding="utf-8")
+    _commit_all(path, "initial commit")
+    return path
+
+
+def test_clean_git_content_passes(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    errors: list[str] = []
+    warnings: list[str] = []
+    validate_pack.validate_manifest(errors, warnings, repo_root=repo)
+    assert errors == []
+
+
+def test_ignored_and_untracked_files_cannot_enter_manifest(tmp_path: Path) -> None:
+    path = tmp_path / "repo"
+    _init_repo(path)
+    (path / ".gitignore").write_text("*.env\nsecret.local\n", encoding="utf-8")
+    (path / "app.py").write_text("print(1)\n", encoding="utf-8")
+    _commit_all(path, "initial commit")
+
+    # Present on disk, but ignored/untracked — must never appear.
+    (path / "secret.env").write_text("TOKEN=abc123\n", encoding="utf-8")
+    (path / "secret.local").write_text("also secret\n", encoding="utf-8")
+    (path / "scratch.tmp").write_text("untracked, not even ignored\n", encoding="utf-8")
+
+    manifest = create_manifest.build_manifest("HEAD", path)
+    paths = {entry["path"] for entry in manifest["files"]}
+    assert paths == {".gitignore", "app.py"}
+    assert "secret.env" not in paths
+    assert "secret.local" not in paths
+    assert "scratch.tmp" not in paths
+
+
+def test_manifest_excludes_itself_deterministically(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    paths = {entry["path"] for entry in manifest["files"]}
+    assert "PACK_MANIFEST.json" not in paths
+    assert manifest["selfExcluded"] == "PACK_MANIFEST.json"
+
+
+def test_crlf_vs_lf_checkout_does_not_alter_canonical_checksums(tmp_path: Path) -> None:
+    """Same commit, checked out under different core.autocrlf, must hash identically.
+
+    This requires an explicit `.gitattributes` `eol` override — without one,
+    `git archive` output (which build_manifest hashes) inherits whichever
+    machine's local core.autocrlf ran the archive command, which is exactly
+    the ambiguity `.gitattributes` exists to remove. This mirrors the real
+    repository, which pins `eol=lf` for `*.py` explicitly (see
+    test_ignored_and_untracked_files_cannot_enter_manifest's repo fixture,
+    and the real .gitattributes, for the same pattern).
+
+    The fixture's `.gitattributes` must also govern *itself*, not just
+    `*.py` — without a rule covering `.gitattributes`, Git's default
+    `text=auto` heuristic classifies it as text but leaves its own eol
+    conversion to `core.autocrlf`, so `git archive` output for the
+    `.gitattributes` blob itself differs between the two checkouts (17
+    bytes LF vs 18 bytes CRLF) even though `script.py` matches correctly.
+    Reproduced on Linux (fails there; passed on Windows only because the
+    fixture's own bytes happened to be written LF-only both times). The
+    real repository's `.gitattributes` already has this covered via its
+    top-level `* text=auto eol=lf` catch-all (see `.gitattributes:17`) plus
+    an explicit `*.gitattributes text eol=lf` rule (`.gitattributes:36`) —
+    this was a test-fixture gap, not a real defect in the manifest
+    algorithm or the real repository's attributes.
+    """
+    origin = tmp_path / "origin"
+    _init_repo(origin)
+    (origin / ".gitattributes").write_text(
+        "* text=auto eol=lf\n.gitattributes text eol=lf\n*.py text eol=lf\n", encoding="utf-8"
+    )
+    (origin / "script.py").write_bytes(b"import sys\nprint(sys.argv)\n")
+    commit = _commit_all(origin, "add script")
+
+    lf_checkout = tmp_path / "lf-checkout"
+    _git(tmp_path, "clone", "-q", str(origin), str(lf_checkout))
+    _git(lf_checkout, "config", "core.autocrlf", "false")
+    _git(lf_checkout, "checkout", "-q", "-f", "HEAD")
+
+    crlf_checkout = tmp_path / "crlf-checkout"
+    _git(tmp_path, "clone", "-q", str(origin), str(crlf_checkout))
+    _git(crlf_checkout, "config", "core.autocrlf", "true")
+    _git(crlf_checkout, "checkout", "-q", "-f", "HEAD")
+
+    # An explicit `eol=lf` attribute overrides core.autocrlf entirely, for
+    # both the working-tree checkout and `git archive` output — so, unlike
+    # test_without_gitattributes_archive_output_is_not_platform_independent,
+    # both checkouts below produce identical LF bytes despite the differing
+    # local core.autocrlf config. That is the property .gitattributes exists
+    # to guarantee.
+    lf_bytes = (lf_checkout / "script.py").read_bytes()
+    crlf_bytes = (crlf_checkout / "script.py").read_bytes()
+    assert b"\r\n" not in lf_bytes
+    assert b"\r\n" not in crlf_bytes
+    assert lf_bytes == crlf_bytes
+
+    lf_manifest = create_manifest.build_manifest(commit, lf_checkout)
+    crlf_manifest = create_manifest.build_manifest(commit, crlf_checkout)
+
+    def checksum(manifest: dict, path: str) -> str:
+        return next(entry["sha256"] for entry in manifest["files"] if entry["path"] == path)
+
+    assert checksum(lf_manifest, "script.py") == checksum(crlf_manifest, "script.py")
+    assert lf_manifest["files"] == crlf_manifest["files"]
+    assert lf_manifest["totalBytes"] == crlf_manifest["totalBytes"]
+
+
+def test_without_gitattributes_archive_output_is_not_platform_independent(tmp_path: Path) -> None:
+    """Documents *why* .gitattributes is mandatory, not cosmetic.
+
+    Without an explicit `eol` attribute, `git archive` output for a
+    plain-text file inherits whichever machine's local core.autocrlf ran the
+    archive command — so two clones with different core.autocrlf legitimately
+    produce different manifests for the identical commit. This is exactly
+    the ambiguity the real repository's `.gitattributes` (`* text=auto
+    eol=lf` plus explicit per-extension rules) removes.
+    """
+    origin = tmp_path / "origin"
+    _init_repo(origin)
+    (origin / "script.py").write_bytes(b"import sys\nprint(sys.argv)\n")
+    commit = _commit_all(origin, "add script, no .gitattributes")
+
+    lf_checkout = tmp_path / "lf-checkout"
+    _git(tmp_path, "clone", "-q", str(origin), str(lf_checkout))
+    _git(lf_checkout, "config", "core.autocrlf", "false")
+
+    crlf_checkout = tmp_path / "crlf-checkout"
+    _git(tmp_path, "clone", "-q", str(origin), str(crlf_checkout))
+    _git(crlf_checkout, "config", "core.autocrlf", "true")
+
+    lf_manifest = create_manifest.build_manifest(commit, lf_checkout)
+    crlf_manifest = create_manifest.build_manifest(commit, crlf_checkout)
+
+    def checksum(manifest: dict, path: str) -> str:
+        return next(entry["sha256"] for entry in manifest["files"] if entry["path"] == path)
+
+    assert checksum(lf_manifest, "script.py") != checksum(crlf_manifest, "script.py")
+
+
+def test_explicit_eol_crlf_override_is_honoured_and_verifiable(tmp_path: Path) -> None:
+    """Reproduces the scripts/bootstrap.ps1 case: a file intentionally always
+    CRLF via an explicit `.gitattributes` override, verified against a real
+    `git archive` extraction with no Git present in the extraction directory.
+    """
+    origin = tmp_path / "origin"
+    _init_repo(origin)
+    (origin / ".gitattributes").write_text("*.ps1 text eol=crlf\n", encoding="utf-8")
+    (origin / "bootstrap.ps1").write_bytes(b"Write-Host 'hello'\r\n")
+    _commit_all(origin, "add windows script")
+
+    manifest = create_manifest.build_manifest("HEAD", origin)
+    entry = next(e for e in manifest["files"] if e["path"] == "bootstrap.ps1")
+    (origin / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+
+    archive_dir = tmp_path / "archive-extraction"
+    archive_dir.mkdir()
+    tar_path = tmp_path / "export.tar"
+    with tar_path.open("wb") as handle:
+        subprocess.run(  # noqa: S603
+            ["git", "archive", "--format=tar", "HEAD"],  # noqa: S607
+            cwd=origin,
+            stdout=handle,
+            check=True,
+        )
+    import tarfile
+
+    with tarfile.open(tar_path) as archive:
+        archive.extractall(archive_dir, filter="data")  # noqa: S202
+
+    extracted_bytes = (archive_dir / "bootstrap.ps1").read_bytes()
+    assert b"\r\n" in extracted_bytes
+    assert hashlib.sha256(extracted_bytes).hexdigest() == entry["sha256"]
+
+    tree_errors = validate_pack.validate_manifest_against_directory(
+        archive_dir, manifest_root=origin
+    )
+    assert tree_errors == []
+
+
+def test_missing_tracked_file_fails_validation(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    (repo / "src" / "app.py").unlink()
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    assert errors
+    assert any("uncommitted changes" in e and "src/app.py" in e for e in errors)
+
+
+def test_modified_tracked_file_fails_validation(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    (repo / "src" / "app.py").write_text("print('tampered')\n", encoding="utf-8")
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    assert errors
+    assert any("uncommitted changes" in e and "src/app.py" in e for e in errors)
+
+
+def test_manifest_cannot_silently_validate_stale_data(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    stale_checksum = next(e["sha256"] for e in manifest["files"] if e["path"] == "src/app.py")
+
+    # A brand new commit lands (e.g. a merge) without regenerating the manifest.
+    (repo / "src" / "app.py").write_text("print('new behaviour')\n", encoding="utf-8")
+    _commit_all(repo, "unrelated change, manifest not regenerated")
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    # The manifest still describes the pre-change checksum, and that
+    # checksum no longer matches HEAD's actual content — it is never
+    # silently accepted as still valid.
+    assert any("checksum mismatch" in e and "src/app.py" in e for e in errors)
+    current_checksum = create_manifest.build_manifest("HEAD", repo)
+    new_checksum = next(e["sha256"] for e in current_checksum["files"] if e["path"] == "src/app.py")
+    assert new_checksum != stale_checksum
+
+
+def test_tampered_checksum_is_caught_even_on_a_clean_tree(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    for entry in manifest["files"]:
+        if entry["path"] == "src/app.py":
+            entry["sha256"] = "0" * 64
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    assert any("checksum mismatch" in e and "src/app.py" in e for e in errors)
+
+
+def test_manifest_omitting_a_tracked_file_fails(repo: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    manifest["files"] = [e for e in manifest["files"] if e["path"] != "src/app.py"]
+    manifest["fileCount"] = len(manifest["files"])
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    assert any("omits Git-tracked files" in e and "src/app.py" in e for e in errors)
+
+
+def test_directory_verification_matches_git_archive_extraction(repo: Path, tmp_path: Path) -> None:
+    """A `git archive` extraction, verified with no Git present in that directory."""
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+
+    archive_dir = tmp_path / "archive-extraction"
+    archive_dir.mkdir()
+    tar_path = tmp_path / "export.tar"
+    with tar_path.open("wb") as handle:
+        subprocess.run(  # noqa: S603
+            ["git", "archive", "--format=tar", "HEAD"],  # noqa: S607
+            cwd=repo,
+            stdout=handle,
+            check=True,
+        )
+    import tarfile
+
+    with tarfile.open(tar_path) as archive:
+        archive.extractall(archive_dir, filter="data")  # noqa: S202
+
+    tree_errors = validate_pack.validate_manifest_against_directory(archive_dir, manifest_root=repo)
+    assert tree_errors == []
+
+
+def test_directory_verification_tolerates_manifest_file_committed_and_present(
+    repo: Path, tmp_path: Path
+) -> None:
+    """PACK_MANIFEST.json is itself typically tracked, so `git archive` includes
+    it even though the manifest's own file list always excludes it (see
+    selfExcluded). Its presence in the verified directory must not be
+    reported as "absent from the manifest".
+    """
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    _git(repo, "add", "PACK_MANIFEST.json")
+    _git(repo, "commit", "-q", "-m", "record manifest")
+
+    archive_dir = tmp_path / "archive-with-manifest"
+    archive_dir.mkdir()
+    tar_path = tmp_path / "export-with-manifest.tar"
+    with tar_path.open("wb") as handle:
+        subprocess.run(  # noqa: S603
+            ["git", "archive", "--format=tar", "HEAD"],  # noqa: S607
+            cwd=repo,
+            stdout=handle,
+            check=True,
+        )
+    import tarfile
+
+    with tarfile.open(tar_path) as archive:
+        archive.extractall(archive_dir, filter="data")  # noqa: S202
+    assert (archive_dir / "PACK_MANIFEST.json").is_file()
+
+    tree_errors = validate_pack.validate_manifest_against_directory(archive_dir, manifest_root=repo)
+    assert tree_errors == []
+
+
+def test_directory_verification_detects_tampering_outside_git(repo: Path, tmp_path: Path) -> None:
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+
+    export_dir = tmp_path / "export"
+    export_dir.mkdir()
+    (export_dir / "README.md").write_text("hello\n", encoding="utf-8")
+    (export_dir / "src").mkdir()
+    (export_dir / "src" / "app.py").write_text("print('tampered outside repo')\n", encoding="utf-8")
+
+    tree_errors = validate_pack.validate_manifest_against_directory(export_dir, manifest_root=repo)
+    assert any("checksum mismatch" in e for e in tree_errors)
+
+
+def test_content_changing_merge_is_correctly_stale(repo: Path) -> None:
+    """A merge that DOES change tracked content must still be flagged stale."""
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    _git(repo, "add", "PACK_MANIFEST.json")
+    _git(repo, "commit", "-q", "-m", "record manifest")
+
+    _git(repo, "checkout", "-q", "-b", "feature")
+    (repo / "docs.md").write_text("unrelated new file\n", encoding="utf-8")
+    _commit_all(repo, "feature work")
+    _git(repo, "checkout", "-q", "main")
+    _git(repo, "merge", "-q", "--no-ff", "-m", "merge feature", "feature")
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    assert errors  # docs.md is new tracked content -> correctly stale
+
+
+def test_no_op_merge_commit_does_not_stale_the_manifest(repo: Path) -> None:
+    """Reproduces the exact WP-200 incident (commit 95329ca) as a regression test.
+
+    A PR merge that produces a brand new commit SHA but changes no tracked
+    file content (e.g. an already-up-to-date branch merged with --no-ff)
+    must NOT force a manifest-regeneration commit — only a merge that
+    actually changes tracked content should be treated as stale.
+    """
+    manifest = create_manifest.build_manifest("HEAD", repo)
+    (repo / "PACK_MANIFEST.json").write_text(
+        __import__("json").dumps(manifest, indent=2) + "\n", encoding="utf-8"
+    )
+    _git(repo, "add", "PACK_MANIFEST.json")
+    _git(repo, "commit", "-q", "-m", "record manifest")
+
+    _git(repo, "checkout", "-q", "-b", "empty-branch")
+    # An empty commit: a new SHA, but identical tree to its parent.
+    _git(repo, "commit", "-q", "--allow-empty", "-m", "empty commit, no content change")
+    _git(repo, "checkout", "-q", "main")
+    _git(repo, "merge", "-q", "--no-ff", "-m", "no-op merge", "empty-branch")
+
+    errors: list[str] = []
+    validate_pack.validate_manifest(errors, [], repo_root=repo)
+    assert errors == []
+
+
+def test_regeneration_is_deterministic_across_repeated_runs(repo: Path) -> None:
+    first = create_manifest.build_manifest("HEAD", repo)
+    second = create_manifest.build_manifest("HEAD", repo)
+    assert first["files"] == second["files"]
+    assert first["fileCount"] == second["fileCount"]
+    assert first["totalBytes"] == second["totalBytes"]
+    assert first["source"] == second["source"]
+    # generatedAt is a wall-clock timestamp and is deliberately excluded
+    # from the determinism guarantee — everything else must match exactly.