317 lines
14 KiB
Python
317 lines
14 KiB
Python
"""Fail-closed integration tests for the parentless public source boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
|
EXPORTER = REPOSITORY_ROOT / "scripts" / "export-public-source.mjs"
|
|
VALIDATOR = REPOSITORY_ROOT / "scripts" / "validate-public-source.mjs"
|
|
|
|
|
|
class PublicSourceExportTests(unittest.TestCase):
|
|
def _repository(
|
|
self,
|
|
root: Path,
|
|
*,
|
|
license_present: bool = True,
|
|
managed_validation_workflow: str | None = None,
|
|
) -> Path:
|
|
repository = root / "private-source"
|
|
repository.mkdir()
|
|
files: dict[str, str | bytes] = {
|
|
"README.md": "# Product\n",
|
|
"SECURITY.md": "# Security\n",
|
|
"CONTRIBUTING.md": "# Contributing\n",
|
|
"docker-compose.yml": "services: {}\n",
|
|
"docker-compose.validation-tls.yml": "services: {}\n",
|
|
"deploy/validation/Caddyfile": "https://modelforge.example.test:8443 {}\n",
|
|
"backend/Dockerfile": (
|
|
'LABEL org.opencontainers.image.source="https://git.example.com/'
|
|
'Jens/ITWorx-ModelForge"\n'
|
|
),
|
|
"frontend/Dockerfile": (
|
|
'LABEL org.opencontainers.image.source="https://git.example.com/'
|
|
'Jens/ITWorx-ModelForge"\n'
|
|
),
|
|
"node-agent/Dockerfile": (
|
|
'LABEL org.opencontainers.image.source="https://git.example.com/'
|
|
'Jens/ITWorx-ModelForge"\n'
|
|
),
|
|
"runtime-worker/Dockerfile": (
|
|
'LABEL org.opencontainers.image.source="https://git.example.com/'
|
|
'Jens/ITWorx-ModelForge"\n'
|
|
),
|
|
"VERSION": "1.2.1\n",
|
|
"backend/pyproject.toml": "[project]\nname = 'fixture-api'\n",
|
|
"clients/README.md": "# Client usage\n",
|
|
"clients/python/modelforge_client.py": "# Python client\n",
|
|
"clients/typescript/modelforge-client.mjs": "// JavaScript client\n",
|
|
"clients/typescript/modelforge-client.d.mts": "// TypeScript declarations\n",
|
|
"scripts/m16_chaos.py": "# Chaos scenarios\n",
|
|
"scripts/m16_invariants.py": "# Invariant checks\n",
|
|
"scripts/m16_soak.py": "# Soak checks\n",
|
|
"scripts/operator_identity.py": "# Operator identities\n",
|
|
"scripts/rc_isolation_preflight.py": "# RC isolation\n",
|
|
"scripts/rc_stack_retention.py": "# RC retention\n",
|
|
"frontend/package.json": '{"name":"fixture-web"}\n',
|
|
"node-agent/pyproject.toml": "[project]\nname = 'fixture-agent'\n",
|
|
"runtime-worker/pyproject.toml": "[project]\nname = 'fixture-worker'\n",
|
|
}
|
|
if license_present:
|
|
files["LICENSE"] = (REPOSITORY_ROOT / "LICENSE").read_bytes()
|
|
if managed_validation_workflow is not None:
|
|
files[".gitea/workflows/managed-validation.yml"] = managed_validation_workflow
|
|
for relative_path, contents in files.items():
|
|
destination = repository / relative_path
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
if isinstance(contents, bytes):
|
|
destination.write_bytes(contents)
|
|
else:
|
|
destination.write_text(contents, encoding="utf-8")
|
|
(repository / "public-source.allowlist").write_text(
|
|
"\n".join(sorted(files)) + "\n", encoding="utf-8"
|
|
)
|
|
subprocess.run(["git", "init", "-q"], cwd=repository, check=True)
|
|
subprocess.run(["git", "add", "."], cwd=repository, check=True)
|
|
subprocess.run(
|
|
[
|
|
"git",
|
|
"-c",
|
|
"user.name=ModelForge test",
|
|
"-c",
|
|
"user.email=test@example.invalid",
|
|
"commit",
|
|
"-qm",
|
|
"fixture",
|
|
],
|
|
cwd=repository,
|
|
check=True,
|
|
)
|
|
return repository
|
|
|
|
def _export(self, repository: Path, output: Path) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["node", str(EXPORTER), "--repository", str(repository), "--output", str(output)],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
check=False,
|
|
)
|
|
|
|
def _validate(self, output: Path) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["node", str(VALIDATOR)],
|
|
cwd=output,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
check=False,
|
|
)
|
|
|
|
def test_export_requires_an_explicit_tracked_license(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = self._repository(root, license_present=False)
|
|
result = self._export(repository, root / "public")
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("canonical LICENSE is missing", result.stderr)
|
|
|
|
def test_manifest_detects_content_tampering_and_unexpected_files(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
output = root / "public"
|
|
result = self._export(self._repository(root), output)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
validation = self._validate(output)
|
|
self.assertEqual(validation.returncode, 0, validation.stdout + validation.stderr)
|
|
self.assertTrue((output / "clients/python/modelforge_client.py").is_file())
|
|
self.assertTrue((output / "clients/typescript/modelforge-client.mjs").is_file())
|
|
manifest = json.loads((output / "PUBLIC_SOURCE_MANIFEST.json").read_text())
|
|
self.assertEqual(
|
|
manifest["sourceRepository"], "https://git.example.com/example/modelforge"
|
|
)
|
|
bound_paths = {entry["path"] for entry in manifest["files"]}
|
|
for relative_path in (
|
|
"backend/Dockerfile",
|
|
"frontend/Dockerfile",
|
|
"node-agent/Dockerfile",
|
|
"runtime-worker/Dockerfile",
|
|
):
|
|
self.assertIn(relative_path, bound_paths)
|
|
self.assertIn(
|
|
'org.opencontainers.image.source="https://git.example.com/example/modelforge"',
|
|
(output / relative_path).read_text(),
|
|
)
|
|
|
|
dockerfile_path = output / "backend/Dockerfile"
|
|
original_dockerfile = dockerfile_path.read_bytes()
|
|
dockerfile_entry = next(
|
|
entry for entry in manifest["files"] if entry["path"] == "backend/Dockerfile"
|
|
)
|
|
forged_dockerfile = original_dockerfile.replace(
|
|
b"https://git.example.com/example/modelforge",
|
|
b"https://attacker.invalid/repository",
|
|
)
|
|
dockerfile_path.write_bytes(forged_dockerfile)
|
|
dockerfile_entry["bytes"] = len(forged_dockerfile)
|
|
dockerfile_entry["sha256"] = hashlib.sha256(forged_dockerfile).hexdigest()
|
|
(output / "PUBLIC_SOURCE_MANIFEST.json").write_text(
|
|
json.dumps(manifest), encoding="utf-8"
|
|
)
|
|
identity_tampering = self._validate(output)
|
|
self.assertNotEqual(identity_tampering.returncode, 0)
|
|
self.assertIn("source label does not match", identity_tampering.stderr)
|
|
|
|
dockerfile_path.write_bytes(original_dockerfile)
|
|
original_manifest = json.loads(
|
|
(output / "PUBLIC_SOURCE_MANIFEST.json").read_text(encoding="utf-8")
|
|
)
|
|
original_manifest["files"] = [
|
|
{
|
|
"path": entry["path"],
|
|
"bytes": len((output / entry["path"]).read_bytes()),
|
|
"sha256": hashlib.sha256((output / entry["path"]).read_bytes()).hexdigest(),
|
|
}
|
|
for entry in manifest["files"]
|
|
]
|
|
(output / "PUBLIC_SOURCE_MANIFEST.json").write_text(
|
|
json.dumps(original_manifest), encoding="utf-8"
|
|
)
|
|
|
|
original_readme = (output / "README.md").read_bytes()
|
|
(output / "README.md").write_text("tampered\n", encoding="utf-8")
|
|
tampered = self._validate(output)
|
|
self.assertNotEqual(tampered.returncode, 0)
|
|
self.assertIn("Manifest", tampered.stderr)
|
|
|
|
(output / "README.md").write_bytes(original_readme)
|
|
(output / "not-reviewed.txt").write_text("extra\n", encoding="utf-8")
|
|
unexpected = self._validate(output)
|
|
self.assertNotEqual(unexpected.returncode, 0)
|
|
self.assertIn("Unexpected public source files", unexpected.stderr)
|
|
|
|
def test_dirty_source_tree_cannot_claim_the_head_revision(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = self._repository(root)
|
|
(repository / "README.md").write_text("changed after commit\n", encoding="utf-8")
|
|
result = self._export(repository, root / "public")
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("commit the exact source tree", result.stderr)
|
|
|
|
def test_private_owner_inventory_is_excluded_from_public_source(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = self._repository(root)
|
|
private_files = {
|
|
"scripts/portfolio_gitea_snapshot.py": "ExampleRAG owner inventory\n",
|
|
"scripts/portfolio_gitea_repository_ids.json": '{"private-repo": 12}\n',
|
|
"backend/tests/test_portfolio_gitea_snapshot.py": "192." + "168.1.40\n",
|
|
"docs/operations/PORTFOLIO_GITEA_SNAPSHOT.md": "Private owner audit\n",
|
|
}
|
|
for relative_path, contents in private_files.items():
|
|
destination = repository / relative_path
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(contents, encoding="utf-8")
|
|
public_script = repository / "scripts/public.py"
|
|
public_script.write_text("print('public')\n", encoding="utf-8")
|
|
with (repository / "public-source.allowlist").open("a", encoding="utf-8") as allowlist:
|
|
allowlist.write("scripts/**\nbackend/**\ndocs/operations/**\n")
|
|
subprocess.run(["git", "add", "."], cwd=repository, check=True)
|
|
subprocess.run(
|
|
[
|
|
"git", "-c", "user.name=ModelForge test", "-c", "user.email=test@example.invalid",
|
|
"commit", "-qm", "private inventory fixture",
|
|
],
|
|
cwd=repository,
|
|
check=True,
|
|
)
|
|
output = root / "public"
|
|
result = self._export(repository, output)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertEqual(self._validate(output).returncode, 0)
|
|
self.assertTrue((output / "scripts/public.py").is_file())
|
|
for relative_path in private_files:
|
|
self.assertFalse((output / relative_path).exists())
|
|
|
|
def test_sanitized_destination_collisions_fail_before_writing(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = self._repository(root)
|
|
private_name = "Tow" + "er.txt"
|
|
public_name = "GPU " + "Node.txt"
|
|
(repository / private_name).write_text("one\n", encoding="utf-8")
|
|
(repository / public_name).write_text("two\n", encoding="utf-8")
|
|
with (repository / "public-source.allowlist").open("a", encoding="utf-8") as allowlist:
|
|
allowlist.write(f"{private_name}\n{public_name}\n")
|
|
subprocess.run(["git", "add", "."], cwd=repository, check=True)
|
|
subprocess.run(
|
|
[
|
|
"git",
|
|
"-c",
|
|
"user.name=ModelForge test",
|
|
"-c",
|
|
"user.email=test@example.invalid",
|
|
"commit",
|
|
"-qm",
|
|
"collision fixture",
|
|
],
|
|
cwd=repository,
|
|
check=True,
|
|
)
|
|
result = self._export(repository, root / "public")
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("finding", result.stderr)
|
|
|
|
def test_public_managed_validation_requires_explicit_owner_dispatch(self) -> None:
|
|
workflow = "name: Managed validation\n\non:\n pull_request:\n workflow_dispatch:\n"
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = self._repository(
|
|
root,
|
|
managed_validation_workflow=workflow,
|
|
)
|
|
output = root / "public"
|
|
result = self._export(repository, output)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
canonical = (
|
|
repository / ".gitea/workflows/managed-validation.yml"
|
|
).read_text(encoding="utf-8")
|
|
exported = (
|
|
output / ".gitea/workflows/managed-validation.yml"
|
|
).read_text(encoding="utf-8")
|
|
|
|
self.assertIn(" pull_request:\n", canonical)
|
|
self.assertNotIn(" pull_request:\n", exported)
|
|
self.assertIn(" workflow_dispatch:\n", exported)
|
|
self.assertIn("fork PRs never reach private runners", exported)
|
|
|
|
def test_changed_managed_validation_trigger_fails_closed(self) -> None:
|
|
workflow = "name: Managed validation\n\non:\n workflow_dispatch:\n"
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
repository = self._repository(
|
|
root,
|
|
managed_validation_workflow=workflow,
|
|
)
|
|
result = self._export(repository, root / "public")
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("has no expected pull_request trigger", result.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|