"""Build, scan and clean-install a ModelForge candidate in an isolated Compose project.""" from __future__ import annotations import argparse import atexit import base64 import json import os import secrets import shutil import signal import subprocess import sys import time from collections.abc import Iterator from contextlib import contextmanager from datetime import UTC, datetime from pathlib import Path from urllib.parse import urlparse ROOT = Path(__file__).resolve().parents[1] IMAGES = ( "modelforge-api", "modelforge-web", "modelforge-node-agent", "modelforge-runtime-worker", ) ACCEPTANCE_SERVICES = ("postgres", "redis", "api", "migrate", "web") CONFIG_SEED_IMAGE = ( "redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf" ) def write_acceptance_compose_override(path: Path) -> None: """Write the disposable RC service policy used by acceptance.""" services = [] for service in ACCEPTANCE_SERVICES: services.append(f' {service}:\n restart: "no"\n') if service == "api": services.append( " volumes:\n" " - type: volume\n" " source: acceptance-config\n" " target: /app/config\n" " read_only: true\n" ) path.write_text( "services:\n" f"{''.join(services)}" "volumes:\n" " acceptance-config:\n", encoding="utf-8", newline="\n", ) def run( *args: str, cwd: Path = ROOT, env: dict[str, str] | None = None, check: bool = True, capture: bool = False, ) -> str: completed = subprocess.run( list(args), cwd=cwd, env=env, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE if capture else None, stderr=subprocess.STDOUT if capture else None, check=False, ) output = (completed.stdout or "").strip() if check and completed.returncode: raise RuntimeError(f"{' '.join(args)} failed ({completed.returncode}): {output[-2000:]}") return output def container_http_status( container: str, url: str, *, operator: bool = False ) -> tuple[int, str]: probe = """ import json import os import sys import urllib.error import urllib.request headers = {} if sys.argv[2] == "operator": headers["X-ModelForge-Admin-Token"] = os.environ["MODELFORGE_OPERATOR_API_KEY"] request = urllib.request.Request(sys.argv[1], headers=headers) try: with urllib.request.urlopen(request, timeout=10) as response: status = response.status body = response.read().decode("utf-8", "replace") except urllib.error.HTTPError as error: status = error.code body = error.read().decode("utf-8", "replace") except OSError as error: status = 0 body = str(error) print(json.dumps({"status": status, "body": body})) """ output = run( "docker", "exec", container, "python", "-c", probe, url, "operator" if operator else "anonymous", capture=True, ) result = json.loads(output) return int(result["status"]), str(result["body"]) def wait_for_container_status( container: str, url: str, expected: int, timeout: int = 120 ) -> str: deadline = time.monotonic() + timeout last = "no response" while time.monotonic() < deadline: try: status, body = container_http_status(container, url) last = f"HTTP {status}: {body[:200]}" if status == expected: return body except OSError as exc: last = str(exc) time.sleep(2) raise RuntimeError(f"{url} did not reach HTTP {expected}: {last}") def provision_database_roles( compose: tuple[str, ...], env: dict[str, str], database: str ) -> None: """Apply the idempotent role bootstrap through the Docker API. Gitea Actions talks to a sibling Docker daemon. Host bind mounts therefore resolve on the daemon, not in the job container, so the normal init-directory mount is intentionally not relied on by acceptance. ``docker cp`` preserves the exact release SQL while working for both local and remote daemons. """ postgres = run(*compose, "ps", "--quiet", "postgres", env=env, capture=True) if not postgres: raise RuntimeError("acceptance PostgreSQL container was not created") bootstrap = ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql" container_path = "/tmp/001-modelforge-roles.sql" run("docker", "cp", str(bootstrap), f"{postgres}:{container_path}") try: run( "docker", "exec", postgres, "psql", "--username", env.get("MODELFORGE_POSTGRES_ADMIN_USER", "postgres"), "--dbname", database, "--file", container_path, ) finally: run("docker", "exec", postgres, "rm", "--force", container_path, check=False) def provision_config_volume(project: str) -> None: """Populate the read-only API config volume through the Docker API.""" volume = f"{project}_acceptance-config" seed = f"{project}-config-seed" run( "docker", "volume", "create", "--label", f"com.docker.compose.project={project}", "--label", "com.docker.compose.volume=acceptance-config", volume, ) run( "docker", "create", "--name", seed, "--label", f"com.docker.compose.project={project}", "--volume", f"{volume}:/app/config", CONFIG_SEED_IMAGE, "true", ) try: run("docker", "cp", f"{ROOT / 'config'}/.", f"{seed}:/app/config") finally: run("docker", "container", "rm", "--force", seed, check=False) def cleanup_candidate_images(version: str, commit: str) -> None: """Remove only tags stamped by this exact acceptance commit.""" for image in IMAGES: tag = f"{image}:{version}" revision = run( "docker", "inspect", "--format", '{{index .Config.Labels "org.opencontainers.image.revision"}}', tag, check=False, capture=True, ) if revision == commit: print(f"Removing acceptance image {tag}", flush=True) run("docker", "image", "rm", "--force", tag, check=False) def _terminate_after_cleanup(signum: int, _frame: object) -> None: raise SystemExit(128 + signum) @contextmanager def managed_compose_cleanup( compose: tuple[str, ...], env: dict[str, str] ) -> Iterator[None]: """Remove the disposable project on normal, failed, or interrupted exits.""" def cleanup() -> None: run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False) atexit.register(cleanup) previous_handlers = { signum: signal.signal(signum, _terminate_after_cleanup) for signum in (signal.SIGINT, signal.SIGTERM) } try: yield finally: cleanup_complete = False try: cleanup() cleanup_complete = True finally: if cleanup_complete: atexit.unregister(cleanup) for signum, handler in previous_handlers.items(): signal.signal(signum, handler) def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--public-api-origin", required=True) parser.add_argument("--output", type=Path, default=Path("acceptance-evidence")) parser.add_argument("--trivy", default="trivy") parser.add_argument("--project-suffix", default=os.environ.get("GITHUB_RUN_ID", "manual")) parser.add_argument( "--canonical-source-commit", default=None, help=( "exact canonical source SHA for a parentless public export; release labels are bound " "to it only when PUBLIC_SOURCE_MANIFEST.json proves the mapping" ), ) parser.add_argument( "--retain-image-bundle", action="store_true", help="retain the verified four-image Docker transport bundle in the output directory", ) args = parser.parse_args() origin = args.public_api_origin.rstrip("/") parsed = urlparse(origin) if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.path: raise SystemExit("--public-api-origin must be a bare absolute HTTP(S) origin") if shutil.which("docker") is None: raise SystemExit("Docker is required on the acceptance runner") if shutil.which(args.trivy) is None: raise SystemExit(f"Trivy executable not found: {args.trivy}") commit = run("git", "rev-parse", "HEAD", capture=True) if run("git", "status", "--porcelain", capture=True): raise SystemExit("Acceptance must run from a clean, exact source commit") canonical_commit = args.canonical_source_commit or commit if len(canonical_commit) != 40 or any( character not in "0123456789abcdef" for character in canonical_commit ): raise SystemExit("--canonical-source-commit must be an exact lowercase 40-character SHA") version = (ROOT / "VERSION").read_text(encoding="utf-8").strip() atexit.register(cleanup_candidate_images, version, canonical_commit) built_at = datetime.now(UTC).isoformat() output = args.output.resolve() output.mkdir(parents=True, exist_ok=True) project_suffix = "".join(ch for ch in args.project_suffix.lower() if ch.isalnum())[-24:] project = f"modelforge-validation-{project_suffix or secrets.token_hex(6)}" if not project.startswith("modelforge-validation-"): raise SystemExit("refusing a non-validation Compose project name") release_command = [ sys.executable, str(ROOT / "scripts" / "release_build.py"), "--output", str(output / "release"), "--public-api-origin", origin, ] if args.canonical_source_commit is not None: release_command.extend(("--canonical-source-commit", canonical_commit)) if args.retain_image_bundle: release_command.append("--image-bundle") run(*release_command) image_records: list[dict[str, str]] = [] for image in IMAGES: tag = f"{image}:{version}" image_id = run("docker", "inspect", "--format", "{{.Id}}", tag, capture=True) raw_report = output / f"trivy-{image}.json" trivy_summary = output / f"trivy-{image}-summary.json" run( args.trivy, "image", "--scanners", "vuln", "--severity", "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", "--format", "json", "--output", str(raw_report), image_id, ) run( sys.executable, str(ROOT / "scripts" / "validate_trivy_report.py"), "--report", str(raw_report), "--image-id", image_id, "--summary", str(trivy_summary), "--reviewed-unfixed", str(ROOT / "config" / "public-candidate-unfixed-vulnerabilities.json"), ) image_records.append({"name": image, "tag": tag, "image_id": image_id}) admin_password = secrets.token_hex(24) owner_password = secrets.token_hex(24) runtime_password = secrets.token_hex(24) operator_key = secrets.token_urlsafe(48) backup_key = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii") database = "modelforge_validation" env = os.environ.copy() env.update( { "COMPOSE_PROJECT_NAME": project, "MODELFORGE_VERSION": version, "MODELFORGE_COMMIT": canonical_commit, "MODELFORGE_BUILT_AT": built_at, "MODELFORGE_API_IMAGE": f"modelforge-api:{version}", "MODELFORGE_WEB_IMAGE": f"modelforge-web:{version}", "MODELFORGE_NODE_AGENT_IMAGE": f"modelforge-node-agent:{version}", "MODELFORGE_RUNTIME_WORKER_IMAGE": f"modelforge-runtime-worker:{version}", "MODELFORGE_POSTGRES_DB": database, "MODELFORGE_POSTGRES_ADMIN_PASSWORD": admin_password, "MODELFORGE_MIGRATION_DB_PASSWORD": owner_password, "MODELFORGE_RUNTIME_DB_PASSWORD": runtime_password, "MODELFORGE_MIGRATION_DATABASE_URL": ( f"postgresql+psycopg://modelforge:{owner_password}@postgres:5432/{database}" ), "MODELFORGE_RUNTIME_DATABASE_URL": ( f"postgresql+psycopg://modelforge_runtime:{runtime_password}@postgres:5432/{database}" ), "MODELFORGE_OPERATOR_API_KEY": operator_key, "MODELFORGE_BACKUP_ENCRYPTION_KEY": backup_key, "MODELFORGE_CORS_ORIGINS": origin, "VITE_API_BASE_URL": origin, "MODELFORGE_SOURCE_COMMIT": canonical_commit, "MODELFORGE_SOURCE_REFERENCE": "validation-acceptance", "MODELFORGE_SOURCE_REPOSITORY": "public-source-candidate", "MODELFORGE_API_BIND": "127.0.0.1", "MODELFORGE_API_PUBLISHED_PORT": "0", "MODELFORGE_WEB_BIND": "127.0.0.1", "MODELFORGE_WEB_PORT": "0", "MODELFORGE_POSTGRES_BIND": "127.0.0.1", "MODELFORGE_POSTGRES_PORT": "0", "MODELFORGE_REDIS_BIND": "127.0.0.1", "MODELFORGE_REDIS_PORT": "0", "MODELFORGE_ALLOW_REMOTE_CODE": "false", "MODELFORGE_RESTORE_ALLOW_PRODUCTION_TARGET": "false", } ) acceptance_override = output / "acceptance-compose.override.yml" write_acceptance_compose_override(acceptance_override) compose = ( "docker", "compose", "-p", project, "-f", str(ROOT / "docker-compose.yml"), "-f", str(ROOT / "docker-compose.production.yml"), "-f", str(acceptance_override), ) summary: dict[str, object] = { "schema_version": 1, "project": project, "source_commit": canonical_commit, "build_tree_commit": commit, "version": version, "public_api_origin": origin, "image_bundle_retained": args.retain_image_bundle, "production_changed": False, "compute_identity_created": False, "images": image_records, "checks": {}, } with managed_compose_cleanup(compose, env): try: run(*compose, "config", "-q", env=env) run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env) provision_database_roles(compose, env, database) provision_config_volume(project) run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env) api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True) if not api_container: raise RuntimeError("acceptance API container was not created") api = "http://127.0.0.1:8000" wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200) wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200) version_status, version_body = container_http_status( api_container, f"{api}/api/v1/version" ) unauthenticated, _ = container_http_status( api_container, f"{api}/api/v1/admin/recovery/dashboard" ) authenticated, _ = container_http_status( api_container, f"{api}/api/v1/admin/recovery/dashboard", operator=True, ) wait_for_container_status(api_container, "http://web:3000/", 200) if version_status != 200 or unauthenticated != 401 or authenticated != 200: raise RuntimeError( "acceptance boundary mismatch: " f"version={version_status}, unauthenticated={unauthenticated}, " f"authenticated={authenticated}" ) version_payload = json.loads(version_body) if version_payload.get("version") != version: raise RuntimeError(f"running version does not match {version}: {version_payload}") volumes = run( "docker", "volume", "ls", "--filter", f"label=com.docker.compose.project={project}", "--format", "{{.Name}}", capture=True, ).splitlines() if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes): raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}") summary["checks"] = { "live": 200, "ready": 200, "version": version_status, "admin_without_key": unauthenticated, "admin_with_key": authenticated, "console": 200, "database_roles_provisioned": True, "network_probe_container": "api", "isolated_volumes": sorted(volumes), } summary["result"] = "PASS" (output / "compose-ps.json").write_text( run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n", encoding="utf-8", newline="\n", ) except BaseException: summary["result"] = "FAIL" (output / "compose-logs.txt").write_text( run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n", encoding="utf-8", newline="\n", ) raise finally: (output / "acceptance-summary.json").write_text( json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n", ) print(f"Isolated RC acceptance PASS for {canonical_commit} in {project}") return 0 if __name__ == "__main__": raise SystemExit(main())