"""Structural safety checks for public-candidate server acceptance.""" from __future__ import annotations import importlib.util import json import os import signal import subprocess import tempfile import unittest from pathlib import Path from unittest import mock ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".gitea" / "workflows" / "public-candidate-acceptance.yml" MANAGED_WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml" ACCEPTANCE = ROOT / "scripts" / "rc_server_acceptance.py" TRIVY_VALIDATOR = ROOT / "scripts" / "validate_trivy_report.py" RUNTIME_DOCKERFILE = ROOT / "runtime-worker" / "Dockerfile" RUNTIME_PYPROJECT = ROOT / "runtime-worker" / "pyproject.toml" class RcAcceptancePolicyTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.workflow = WORKFLOW.read_text(encoding="utf-8") cls.managed = MANAGED_WORKFLOW.read_text(encoding="utf-8") cls.acceptance = ACCEPTANCE.read_text(encoding="utf-8") cls.trivy = TRIVY_VALIDATOR.read_text(encoding="utf-8") cls.runtime_dockerfile = RUNTIME_DOCKERFILE.read_text(encoding="utf-8") cls.runtime_pyproject = RUNTIME_PYPROJECT.read_text(encoding="utf-8") def test_workflow_is_manual_and_never_calls_production_deployment(self) -> None: self.assertIn("workflow_dispatch:", self.workflow) self.assertNotIn("\n push:", self.workflow) self.assertNotIn("pull_request:", self.workflow) self.assertNotIn("deploy.py", self.workflow) self.assertNotIn("DEPLOY_STABLE_TO_PRODUCTION", self.workflow) def test_exact_source_and_docker_are_required(self) -> None: self.assertIn(r"^[0-9a-f]{40}$", self.workflow) self.assertIn("ref: ${{ inputs.source_commit }}", self.workflow) self.assertIn("--canonical-source-commit", self.workflow) self.assertIn('"--canonical-source-commit", canonical_commit', self.acceptance) self.assertIn('"source_commit": canonical_commit', self.acceptance) self.assertIn("docker version", self.workflow) self.assertNotIn("SKIPPED: no docker", self.workflow) def test_optional_candidate_freeze_retains_only_a_verified_transport_bundle(self) -> None: self.assertIn("retain_image_bundle:", self.workflow) self.assertIn("RETAIN_IMAGE_BUNDLE: ${{ inputs.retain_image_bundle }}", self.workflow) self.assertIn("--retain-image-bundle", self.workflow) self.assertIn("--image-bundle", self.acceptance) self.assertIn("acceptance-evidence/release/*.tar", self.workflow) self.assertIn("acceptance-evidence/release/*.tar.gz", self.workflow) self.assertIn("atexit.register(cleanup_candidate_images", self.acceptance) self.assertIn('canonical_commit="${{ inputs.source_commit }}"', self.workflow) self.assertNotIn("candidate_commit=", self.workflow) def test_security_tools_are_downloaded_with_exact_checksums(self) -> None: self.assertIn("GITLEAKS_VERSION: 8.30.1", self.workflow) self.assertIn( "GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb", self.workflow, ) self.assertIn('printf \'GITLEAKS=%s\\n\'', self.workflow) self.assertIn('\"${GITLEAKS}\" dir \"${public_source}\"', self.workflow) self.assertIn("sha256sum --check --strict", self.workflow) self.assertNotIn("command -v gitleaks", self.workflow) def test_unmerged_branch_can_use_the_existing_managed_dispatch_entry(self) -> None: self.assertIn("workflow_call:", self.workflow) self.assertIn("uses: ./.gitea/workflows/public-candidate-acceptance.yml", self.managed) self.assertIn("inputs.profile == 'build'", self.managed) self.assertIn("source_commit: ${{ gitea.sha }}", self.managed) def test_all_four_images_are_built_and_scanned(self) -> None: for image in ( "modelforge-api", "modelforge-web", "modelforge-node-agent", "modelforge-runtime-worker", ): self.assertIn(image, self.acceptance) self.assertIn('"Metadata"', self.trivy) self.assertIn('metadata.get("ImageID") != expected_image_id', self.trivy) self.assertIn("reject_duplicate_keys", self.trivy) self.assertIn("fixable_high_critical + unreviewed_unfixed_high_critical", self.trivy) self.assertIn("public-candidate-unfixed-vulnerabilities.json", self.acceptance) def test_runtime_image_applies_security_updates_and_exact_inference_pins(self) -> None: self.assertIn("apt-get upgrade --yes", self.runtime_dockerfile) self.assertIn("apt-get purge --yes linux-libc-dev", self.runtime_dockerfile) self.assertNotIn("apt-get autoremove", self.runtime_dockerfile) self.assertIn("pip check", self.runtime_dockerfile) self.assertIn("python -m pip uninstall --yes pip setuptools wheel", self.runtime_dockerfile) for dependency in ( "filelock==3.20.3", "h2==4.4.1", "idna==3.15", "lief==0.17.2", "Pillow==12.3.0", "Pygments==2.20.0", "protobuf==5.29.6", "requests==2.33.0", "sentencepiece==0.2.2", "sentence-transformers==6.0.1", "transformers==5.16.1", "urllib3==2.7.0", ): self.assertIn(dependency, self.runtime_pyproject) self.assertNotIn("'filelock==3.20.3'", self.runtime_dockerfile) def test_runtime_worker_has_only_group_read_access_to_the_agent_credential(self) -> None: self.assertIn( "groupadd --system --gid 101 modelforge-agent", self.runtime_dockerfile ) self.assertIn( "--groups modelforge-agent --home /app modelforge-runtime", self.runtime_dockerfile, ) def test_compose_project_is_isolated_and_always_removed(self) -> None: self.assertIn('project = f"modelforge-validation-', self.acceptance) self.assertIn('"MODELFORGE_SOURCE_REFERENCE": "validation-acceptance"', self.acceptance) self.assertIn('project="modelforge-validation-${suffix}"', self.workflow) self.assertNotIn('project = f"modelforge-rc-', self.acceptance) self.assertIn('"MODELFORGE_API_BIND": "127.0.0.1"', self.acceptance) self.assertIn('"MODELFORGE_API_PUBLISHED_PORT": "0"', self.acceptance) self.assertIn('"production_changed": False', self.acceptance) self.assertIn('"compute_identity_created": False', self.acceptance) self.assertIn('"down", "--volumes", "--remove-orphans"', self.acceptance) self.assertIn("atexit.register(cleanup_candidate_images", self.acceptance) self.assertIn('"docker", "image", "rm", "--force", tag', self.acceptance) self.assertNotIn("docker-compose.runtime-worker.yml", self.acceptance) self.assertNotIn("docker-compose.node-agent.yml", self.acceptance) def test_acceptance_override_disables_restarts_for_every_service(self) -> None: spec = importlib.util.spec_from_file_location("rc_server_acceptance", ACCEPTANCE) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) write_override = getattr(module, "write_acceptance_compose_override", None) self.assertIsNotNone( write_override, "acceptance must expose the same override writer used by the RC run", ) with tempfile.TemporaryDirectory() as directory: override = Path(directory) / "acceptance-compose.override.yml" write_override(override) env = os.environ.copy() env.update( { "MODELFORGE_VERSION": "9.9.9-test", "MODELFORGE_COMMIT": "a" * 40, "MODELFORGE_BUILT_AT": "2099-01-01T00:00:00+00:00", "MODELFORGE_POSTGRES_DB": "modelforge_validation_test", "MODELFORGE_POSTGRES_ADMIN_PASSWORD": "admin-test-password", "MODELFORGE_MIGRATION_DB_PASSWORD": "owner-test-password", "MODELFORGE_RUNTIME_DB_PASSWORD": "runtime-test-password", "MODELFORGE_MIGRATION_DATABASE_URL": ( "postgresql+psycopg://modelforge:owner-test-password@" "postgres:5432/modelforge_validation_test" ), "MODELFORGE_RUNTIME_DATABASE_URL": ( "postgresql+psycopg://modelforge_runtime:runtime-test-password@" "postgres:5432/modelforge_validation_test" ), "MODELFORGE_BACKUP_ENCRYPTION_KEY": "acceptance-test-key", "MODELFORGE_CORS_ORIGINS": "https://modelforge.example.test", "VITE_API_BASE_URL": "https://modelforge.example.test", } ) result = subprocess.run( [ "docker", "compose", "-f", str(ROOT / "docker-compose.yml"), "-f", str(ROOT / "docker-compose.production.yml"), "-f", str(override), "config", "--format", "json", ], cwd=ROOT, env=env, text=True, capture_output=True, check=False, ) self.assertEqual(result.returncode, 0, result.stderr) services = json.loads(result.stdout)["services"] self.assertTrue(services) self.assertEqual( {name: service.get("restart") for name, service in services.items()}, {name: "no" for name in services}, ) def test_compose_cleanup_runs_when_acceptance_receives_sigterm(self) -> None: spec = importlib.util.spec_from_file_location("rc_server_acceptance_cleanup", ACCEPTANCE) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) managed_cleanup = getattr(module, "managed_compose_cleanup", None) self.assertIsNotNone( managed_cleanup, "acceptance must guard its RC project with interruption-safe cleanup", ) compose = ("docker", "compose", "-p", "modelforge-validation-test") env = {"COMPOSE_PROJECT_NAME": "modelforge-validation-test"} commands: list[tuple[tuple[str, ...], dict[str, object]]] = [] registered: list[object] = [] unregistered: list[object] = [] handlers: dict[int, object] = {} def fake_run(*args: str, **kwargs: object) -> str: commands.append((args, kwargs)) return "" def fake_signal(signum: int, handler: object) -> object: previous = handlers.get(signum, signal.SIG_DFL) handlers[signum] = handler return previous with ( mock.patch.object(module, "run", side_effect=fake_run), mock.patch.object(module.atexit, "register", side_effect=registered.append), mock.patch.object(module.atexit, "unregister", side_effect=unregistered.append), mock.patch.object(module.signal, "signal", side_effect=fake_signal), self.assertRaises(SystemExit) as raised, managed_cleanup(compose, env), ): handler = handlers[signal.SIGTERM] assert callable(handler) handler(signal.SIGTERM, None) self.assertEqual(raised.exception.code, 128 + signal.SIGTERM) self.assertEqual( commands, [ ( compose + ("down", "--volumes", "--remove-orphans"), {"env": env, "check": False}, ) ], ) self.assertEqual(len(registered), 1) self.assertEqual(unregistered, registered) def test_clean_install_bootstraps_roles_across_a_remote_docker_daemon(self) -> None: self.assertIn("def provision_database_roles(", self.acceptance) self.assertIn('run("docker", "cp", str(bootstrap)', self.acceptance) self.assertIn('"psql",', self.acceptance) self.assertIn('"--file",', self.acceptance) self.assertIn( '"--wait", "postgres", "redis", env=env', self.acceptance ) self.assertLess( self.acceptance.index("provision_database_roles(compose, env, database)"), self.acceptance.index( '"--wait", "api", "web", env=env' ), ) def test_clean_install_seeds_config_across_a_remote_docker_daemon(self) -> None: self.assertIn("def provision_config_volume(project: str)", self.acceptance) self.assertIn('f"{project}_acceptance-config"', self.acceptance) self.assertIn('f"com.docker.compose.project={project}"', self.acceptance) self.assertIn('run("docker", "cp", f"{ROOT / \'config\'}/."', self.acceptance) self.assertIn( 'run("docker", "container", "rm", "--force", seed, check=False)', self.acceptance, ) self.assertIn('" read_only: true\\n"', self.acceptance) self.assertLess( self.acceptance.index("provision_config_volume(project)"), self.acceptance.index('"--wait", "api", "web", env=env'), ) def test_http_contract_runs_inside_the_isolated_compose_network(self) -> None: self.assertIn("def container_http_status(", self.acceptance) self.assertIn("def wait_for_container_status(", self.acceptance) self.assertIn('headers["X-ModelForge-Admin-Token"]', self.acceptance) self.assertIn('os.environ["MODELFORGE_OPERATOR_API_KEY"]', self.acceptance) self.assertIn('wait_for_container_status(api_container, "http://web:3000/", 200)', self.acceptance) self.assertNotIn("urllib.request.urlopen(request, timeout=timeout)", self.acceptance) def test_workflow_has_an_always_run_exact_project_cleanup_fallback(self) -> None: self.assertIn("- name: Always remove acceptance Docker resources", self.workflow) cleanup = self.workflow.split( " - name: Always remove acceptance Docker resources", 1 )[1].split(" - name: Upload acceptance evidence", 1)[0] self.assertIn("if: always()", cleanup) self.assertIn('label=com.docker.compose.project=${project}', cleanup) self.assertIn("docker container rm --force", cleanup) self.assertIn("docker volume rm --force", cleanup) self.assertIn("docker network rm", cleanup) self.assertIn("docker image rm --force", cleanup) self.assertNotIn("docker system prune", cleanup) def test_python_entrypoints_compile(self) -> None: for path in (ACCEPTANCE, TRIVY_VALIDATOR, ROOT / "scripts" / "release_build.py"): result = subprocess.run( ["python", "-m", "py_compile", str(path)], capture_output=True, check=False, ) with self.subTest(path=path.name): self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace")) def test_trivy_validator_binds_image_and_rejects_blockers(self) -> None: spec = importlib.util.spec_from_file_location("trivy_report", TRIVY_VALIDATOR) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) image_id = "sha256:" + "a" * 64 report = { "ArtifactName": image_id, "ArtifactType": "container_image", "Metadata": {"ImageID": image_id}, "Results": [ { "Target": "debian", "Class": "os-pkgs", "Type": "debian", "Vulnerabilities": [ { "VulnerabilityID": "CVE-2099-9999", "PkgName": "blocked-package", "InstalledVersion": "1.0.0", "FixedVersion": "1.0.1", "Severity": "HIGH", } ], } ], } with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "report.json" path.write_text(json.dumps(report), encoding="utf-8") result = module.validate(path, image_id) self.assertEqual(result["release_blockers"], 1) with self.assertRaisesRegex(ValueError, "does not match"): module.validate(path, "sha256:" + "b" * 64) def test_trivy_validator_rejects_duplicate_keys_and_empty_coverage(self) -> None: spec = importlib.util.spec_from_file_location("trivy_report_negative", TRIVY_VALIDATOR) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) image_id = "sha256:" + "a" * 64 with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "report.json" path.write_text('{"ArtifactType":"container_image","ArtifactType":"container_image"}', encoding="utf-8") with self.assertRaisesRegex(ValueError, "duplicate JSON key"): module.validate(path, image_id) path.write_text( json.dumps( { "ArtifactType": "container_image", "Metadata": {"ImageID": image_id}, "Results": [], } ), encoding="utf-8", ) with self.assertRaisesRegex(ValueError, "no package result coverage"): module.validate(path, image_id) def test_trivy_validator_allows_only_exact_reviewed_upstream_unfixed_findings(self) -> None: spec = importlib.util.spec_from_file_location("trivy_report_reviewed", TRIVY_VALIDATOR) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) image_id = "sha256:" + "a" * 64 finding = { "VulnerabilityID": "CVE-2099-0001", "PkgName": "example-package", "InstalledVersion": "1.0.0", "FixedVersion": "", "Severity": "HIGH", } report = { "ArtifactType": "container_image", "Metadata": {"ImageID": image_id}, "Results": [ { "Target": "debian", "Class": "os-pkgs", "Vulnerabilities": [finding], } ], } baseline = { "schema_version": 1, "vulnerabilities": [ { "vulnerability_id": "CVE-2099-0001", "package": "example-package", "installed_version": "1.0.0", "severity": "HIGH", } ], } with tempfile.TemporaryDirectory() as directory: report_path = Path(directory) / "report.json" baseline_path = Path(directory) / "baseline.json" report_path.write_text(json.dumps(report), encoding="utf-8") baseline_path.write_text(json.dumps(baseline), encoding="utf-8") summary = module.validate(report_path, image_id, baseline_path) self.assertEqual(summary["reviewed_unfixed_high_critical"], 1) self.assertEqual(summary["release_blockers"], 0) finding["InstalledVersion"] = "1.0.1" report_path.write_text(json.dumps(report), encoding="utf-8") self.assertEqual(module.validate(report_path, image_id, baseline_path)["release_blockers"], 1) finding["InstalledVersion"] = "1.0.0" finding["FixedVersion"] = "1.0.2" report_path.write_text(json.dumps(report), encoding="utf-8") summary = module.validate(report_path, image_id, baseline_path) self.assertEqual(summary["fixable_high_critical"], 1) self.assertEqual(summary["release_blockers"], 1) if __name__ == "__main__": unittest.main()