515 lines
18 KiB
Python
515 lines
18 KiB
Python
"""Build, scan and clean-install a ModelForge candidate in an isolated Compose project."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import atexit
|
|
import base64
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
IMAGES = (
|
|
"modelforge-api",
|
|
"modelforge-web",
|
|
"modelforge-node-agent",
|
|
"modelforge-runtime-worker",
|
|
)
|
|
ACCEPTANCE_SERVICES = ("postgres", "redis", "api", "migrate", "web")
|
|
CONFIG_SEED_IMAGE = (
|
|
"redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"
|
|
)
|
|
|
|
|
|
def write_acceptance_compose_override(path: Path) -> None:
|
|
"""Write the disposable RC service policy used by acceptance."""
|
|
|
|
services = []
|
|
for service in ACCEPTANCE_SERVICES:
|
|
services.append(f' {service}:\n restart: "no"\n')
|
|
if service == "api":
|
|
services.append(
|
|
" volumes:\n"
|
|
" - type: volume\n"
|
|
" source: acceptance-config\n"
|
|
" target: /app/config\n"
|
|
" read_only: true\n"
|
|
)
|
|
path.write_text(
|
|
"services:\n"
|
|
f"{''.join(services)}"
|
|
"volumes:\n"
|
|
" acceptance-config:\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
|
|
|
|
def run(
|
|
*args: str,
|
|
cwd: Path = ROOT,
|
|
env: dict[str, str] | None = None,
|
|
check: bool = True,
|
|
capture: bool = False,
|
|
) -> str:
|
|
completed = subprocess.run(
|
|
list(args),
|
|
cwd=cwd,
|
|
env=env,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
stdout=subprocess.PIPE if capture else None,
|
|
stderr=subprocess.STDOUT if capture else None,
|
|
check=False,
|
|
)
|
|
output = (completed.stdout or "").strip()
|
|
if check and completed.returncode:
|
|
raise RuntimeError(f"{' '.join(args)} failed ({completed.returncode}): {output[-2000:]}")
|
|
return output
|
|
|
|
|
|
def container_http_status(
|
|
container: str, url: str, *, operator: bool = False
|
|
) -> tuple[int, str]:
|
|
probe = """
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
headers = {}
|
|
if sys.argv[2] == "operator":
|
|
headers["X-ModelForge-Admin-Token"] = os.environ["MODELFORGE_OPERATOR_API_KEY"]
|
|
request = urllib.request.Request(sys.argv[1], headers=headers)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
status = response.status
|
|
body = response.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as error:
|
|
status = error.code
|
|
body = error.read().decode("utf-8", "replace")
|
|
except OSError as error:
|
|
status = 0
|
|
body = str(error)
|
|
print(json.dumps({"status": status, "body": body}))
|
|
"""
|
|
output = run(
|
|
"docker",
|
|
"exec",
|
|
container,
|
|
"python",
|
|
"-c",
|
|
probe,
|
|
url,
|
|
"operator" if operator else "anonymous",
|
|
capture=True,
|
|
)
|
|
result = json.loads(output)
|
|
return int(result["status"]), str(result["body"])
|
|
|
|
|
|
def wait_for_container_status(
|
|
container: str, url: str, expected: int, timeout: int = 120
|
|
) -> str:
|
|
deadline = time.monotonic() + timeout
|
|
last = "no response"
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
status, body = container_http_status(container, url)
|
|
last = f"HTTP {status}: {body[:200]}"
|
|
if status == expected:
|
|
return body
|
|
except OSError as exc:
|
|
last = str(exc)
|
|
time.sleep(2)
|
|
raise RuntimeError(f"{url} did not reach HTTP {expected}: {last}")
|
|
|
|
|
|
def provision_database_roles(
|
|
compose: tuple[str, ...], env: dict[str, str], database: str
|
|
) -> None:
|
|
"""Apply the idempotent role bootstrap through the Docker API.
|
|
|
|
Gitea Actions talks to a sibling Docker daemon. Host bind mounts therefore resolve on the
|
|
daemon, not in the job container, so the normal init-directory mount is intentionally not
|
|
relied on by acceptance. ``docker cp`` preserves the exact release SQL while working for both
|
|
local and remote daemons.
|
|
"""
|
|
|
|
postgres = run(*compose, "ps", "--quiet", "postgres", env=env, capture=True)
|
|
if not postgres:
|
|
raise RuntimeError("acceptance PostgreSQL container was not created")
|
|
bootstrap = ROOT / "deploy" / "postgres" / "init" / "001-modelforge-roles.sql"
|
|
container_path = "/tmp/001-modelforge-roles.sql"
|
|
run("docker", "cp", str(bootstrap), f"{postgres}:{container_path}")
|
|
try:
|
|
run(
|
|
"docker",
|
|
"exec",
|
|
postgres,
|
|
"psql",
|
|
"--username",
|
|
env.get("MODELFORGE_POSTGRES_ADMIN_USER", "postgres"),
|
|
"--dbname",
|
|
database,
|
|
"--file",
|
|
container_path,
|
|
)
|
|
finally:
|
|
run("docker", "exec", postgres, "rm", "--force", container_path, check=False)
|
|
|
|
|
|
def provision_config_volume(project: str) -> None:
|
|
"""Populate the read-only API config volume through the Docker API."""
|
|
|
|
volume = f"{project}_acceptance-config"
|
|
seed = f"{project}-config-seed"
|
|
run(
|
|
"docker",
|
|
"volume",
|
|
"create",
|
|
"--label",
|
|
f"com.docker.compose.project={project}",
|
|
"--label",
|
|
"com.docker.compose.volume=acceptance-config",
|
|
volume,
|
|
)
|
|
run(
|
|
"docker",
|
|
"create",
|
|
"--name",
|
|
seed,
|
|
"--label",
|
|
f"com.docker.compose.project={project}",
|
|
"--volume",
|
|
f"{volume}:/app/config",
|
|
CONFIG_SEED_IMAGE,
|
|
"true",
|
|
)
|
|
try:
|
|
run("docker", "cp", f"{ROOT / 'config'}/.", f"{seed}:/app/config")
|
|
finally:
|
|
run("docker", "container", "rm", "--force", seed, check=False)
|
|
|
|
|
|
def cleanup_candidate_images(version: str, commit: str) -> None:
|
|
"""Remove only tags stamped by this exact acceptance commit."""
|
|
|
|
for image in IMAGES:
|
|
tag = f"{image}:{version}"
|
|
revision = run(
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
'{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
|
tag,
|
|
check=False,
|
|
capture=True,
|
|
)
|
|
if revision == commit:
|
|
print(f"Removing acceptance image {tag}", flush=True)
|
|
run("docker", "image", "rm", "--force", tag, check=False)
|
|
|
|
|
|
def _terminate_after_cleanup(signum: int, _frame: object) -> None:
|
|
raise SystemExit(128 + signum)
|
|
|
|
|
|
@contextmanager
|
|
def managed_compose_cleanup(
|
|
compose: tuple[str, ...], env: dict[str, str]
|
|
) -> Iterator[None]:
|
|
"""Remove the disposable project on normal, failed, or interrupted exits."""
|
|
|
|
def cleanup() -> None:
|
|
run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False)
|
|
|
|
atexit.register(cleanup)
|
|
previous_handlers = {
|
|
signum: signal.signal(signum, _terminate_after_cleanup)
|
|
for signum in (signal.SIGINT, signal.SIGTERM)
|
|
}
|
|
try:
|
|
yield
|
|
finally:
|
|
cleanup_complete = False
|
|
try:
|
|
cleanup()
|
|
cleanup_complete = True
|
|
finally:
|
|
if cleanup_complete:
|
|
atexit.unregister(cleanup)
|
|
for signum, handler in previous_handlers.items():
|
|
signal.signal(signum, handler)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--public-api-origin", required=True)
|
|
parser.add_argument("--output", type=Path, default=Path("acceptance-evidence"))
|
|
parser.add_argument("--trivy", default="trivy")
|
|
parser.add_argument("--project-suffix", default=os.environ.get("GITHUB_RUN_ID", "manual"))
|
|
parser.add_argument(
|
|
"--canonical-source-commit",
|
|
default=None,
|
|
help=(
|
|
"exact canonical source SHA for a parentless public export; release labels are bound "
|
|
"to it only when PUBLIC_SOURCE_MANIFEST.json proves the mapping"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--retain-image-bundle",
|
|
action="store_true",
|
|
help="retain the verified four-image Docker transport bundle in the output directory",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
origin = args.public_api_origin.rstrip("/")
|
|
parsed = urlparse(origin)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.path:
|
|
raise SystemExit("--public-api-origin must be a bare absolute HTTP(S) origin")
|
|
if shutil.which("docker") is None:
|
|
raise SystemExit("Docker is required on the acceptance runner")
|
|
if shutil.which(args.trivy) is None:
|
|
raise SystemExit(f"Trivy executable not found: {args.trivy}")
|
|
|
|
commit = run("git", "rev-parse", "HEAD", capture=True)
|
|
if run("git", "status", "--porcelain", capture=True):
|
|
raise SystemExit("Acceptance must run from a clean, exact source commit")
|
|
canonical_commit = args.canonical_source_commit or commit
|
|
if len(canonical_commit) != 40 or any(
|
|
character not in "0123456789abcdef" for character in canonical_commit
|
|
):
|
|
raise SystemExit("--canonical-source-commit must be an exact lowercase 40-character SHA")
|
|
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
|
|
atexit.register(cleanup_candidate_images, version, canonical_commit)
|
|
built_at = datetime.now(UTC).isoformat()
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
|
|
project_suffix = "".join(ch for ch in args.project_suffix.lower() if ch.isalnum())[-24:]
|
|
project = f"modelforge-validation-{project_suffix or secrets.token_hex(6)}"
|
|
if not project.startswith("modelforge-validation-"):
|
|
raise SystemExit("refusing a non-validation Compose project name")
|
|
|
|
release_command = [
|
|
sys.executable,
|
|
str(ROOT / "scripts" / "release_build.py"),
|
|
"--output",
|
|
str(output / "release"),
|
|
"--public-api-origin",
|
|
origin,
|
|
]
|
|
if args.canonical_source_commit is not None:
|
|
release_command.extend(("--canonical-source-commit", canonical_commit))
|
|
if args.retain_image_bundle:
|
|
release_command.append("--image-bundle")
|
|
run(*release_command)
|
|
|
|
image_records: list[dict[str, str]] = []
|
|
for image in IMAGES:
|
|
tag = f"{image}:{version}"
|
|
image_id = run("docker", "inspect", "--format", "{{.Id}}", tag, capture=True)
|
|
raw_report = output / f"trivy-{image}.json"
|
|
trivy_summary = output / f"trivy-{image}-summary.json"
|
|
run(
|
|
args.trivy,
|
|
"image",
|
|
"--scanners",
|
|
"vuln",
|
|
"--severity",
|
|
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
|
|
"--format",
|
|
"json",
|
|
"--output",
|
|
str(raw_report),
|
|
image_id,
|
|
)
|
|
run(
|
|
sys.executable,
|
|
str(ROOT / "scripts" / "validate_trivy_report.py"),
|
|
"--report",
|
|
str(raw_report),
|
|
"--image-id",
|
|
image_id,
|
|
"--summary",
|
|
str(trivy_summary),
|
|
"--reviewed-unfixed",
|
|
str(ROOT / "config" / "public-candidate-unfixed-vulnerabilities.json"),
|
|
)
|
|
image_records.append({"name": image, "tag": tag, "image_id": image_id})
|
|
|
|
admin_password = secrets.token_hex(24)
|
|
owner_password = secrets.token_hex(24)
|
|
runtime_password = secrets.token_hex(24)
|
|
operator_key = secrets.token_urlsafe(48)
|
|
backup_key = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")
|
|
database = "modelforge_validation"
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"COMPOSE_PROJECT_NAME": project,
|
|
"MODELFORGE_VERSION": version,
|
|
"MODELFORGE_COMMIT": canonical_commit,
|
|
"MODELFORGE_BUILT_AT": built_at,
|
|
"MODELFORGE_API_IMAGE": f"modelforge-api:{version}",
|
|
"MODELFORGE_WEB_IMAGE": f"modelforge-web:{version}",
|
|
"MODELFORGE_NODE_AGENT_IMAGE": f"modelforge-node-agent:{version}",
|
|
"MODELFORGE_RUNTIME_WORKER_IMAGE": f"modelforge-runtime-worker:{version}",
|
|
"MODELFORGE_POSTGRES_DB": database,
|
|
"MODELFORGE_POSTGRES_ADMIN_PASSWORD": admin_password,
|
|
"MODELFORGE_MIGRATION_DB_PASSWORD": owner_password,
|
|
"MODELFORGE_RUNTIME_DB_PASSWORD": runtime_password,
|
|
"MODELFORGE_MIGRATION_DATABASE_URL": (
|
|
f"postgresql+psycopg://modelforge:{owner_password}@postgres:5432/{database}"
|
|
),
|
|
"MODELFORGE_RUNTIME_DATABASE_URL": (
|
|
f"postgresql+psycopg://modelforge_runtime:{runtime_password}@postgres:5432/{database}"
|
|
),
|
|
"MODELFORGE_OPERATOR_API_KEY": operator_key,
|
|
"MODELFORGE_BACKUP_ENCRYPTION_KEY": backup_key,
|
|
"MODELFORGE_CORS_ORIGINS": origin,
|
|
"VITE_API_BASE_URL": origin,
|
|
"MODELFORGE_SOURCE_COMMIT": canonical_commit,
|
|
"MODELFORGE_SOURCE_REFERENCE": "validation-acceptance",
|
|
"MODELFORGE_SOURCE_REPOSITORY": "public-source-candidate",
|
|
"MODELFORGE_API_BIND": "127.0.0.1",
|
|
"MODELFORGE_API_PUBLISHED_PORT": "0",
|
|
"MODELFORGE_WEB_BIND": "127.0.0.1",
|
|
"MODELFORGE_WEB_PORT": "0",
|
|
"MODELFORGE_POSTGRES_BIND": "127.0.0.1",
|
|
"MODELFORGE_POSTGRES_PORT": "0",
|
|
"MODELFORGE_REDIS_BIND": "127.0.0.1",
|
|
"MODELFORGE_REDIS_PORT": "0",
|
|
"MODELFORGE_ALLOW_REMOTE_CODE": "false",
|
|
"MODELFORGE_RESTORE_ALLOW_PRODUCTION_TARGET": "false",
|
|
}
|
|
)
|
|
acceptance_override = output / "acceptance-compose.override.yml"
|
|
write_acceptance_compose_override(acceptance_override)
|
|
compose = (
|
|
"docker",
|
|
"compose",
|
|
"-p",
|
|
project,
|
|
"-f",
|
|
str(ROOT / "docker-compose.yml"),
|
|
"-f",
|
|
str(ROOT / "docker-compose.production.yml"),
|
|
"-f",
|
|
str(acceptance_override),
|
|
)
|
|
|
|
summary: dict[str, object] = {
|
|
"schema_version": 1,
|
|
"project": project,
|
|
"source_commit": canonical_commit,
|
|
"build_tree_commit": commit,
|
|
"version": version,
|
|
"public_api_origin": origin,
|
|
"image_bundle_retained": args.retain_image_bundle,
|
|
"production_changed": False,
|
|
"compute_identity_created": False,
|
|
"images": image_records,
|
|
"checks": {},
|
|
}
|
|
with managed_compose_cleanup(compose, env):
|
|
try:
|
|
run(*compose, "config", "-q", env=env)
|
|
run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env)
|
|
provision_database_roles(compose, env, database)
|
|
provision_config_volume(project)
|
|
run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env)
|
|
api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True)
|
|
if not api_container:
|
|
raise RuntimeError("acceptance API container was not created")
|
|
api = "http://127.0.0.1:8000"
|
|
|
|
wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200)
|
|
wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200)
|
|
version_status, version_body = container_http_status(
|
|
api_container, f"{api}/api/v1/version"
|
|
)
|
|
unauthenticated, _ = container_http_status(
|
|
api_container, f"{api}/api/v1/admin/recovery/dashboard"
|
|
)
|
|
authenticated, _ = container_http_status(
|
|
api_container,
|
|
f"{api}/api/v1/admin/recovery/dashboard",
|
|
operator=True,
|
|
)
|
|
wait_for_container_status(api_container, "http://web:3000/", 200)
|
|
if version_status != 200 or unauthenticated != 401 or authenticated != 200:
|
|
raise RuntimeError(
|
|
"acceptance boundary mismatch: "
|
|
f"version={version_status}, unauthenticated={unauthenticated}, "
|
|
f"authenticated={authenticated}"
|
|
)
|
|
version_payload = json.loads(version_body)
|
|
if version_payload.get("version") != version:
|
|
raise RuntimeError(f"running version does not match {version}: {version_payload}")
|
|
|
|
volumes = run(
|
|
"docker",
|
|
"volume",
|
|
"ls",
|
|
"--filter",
|
|
f"label=com.docker.compose.project={project}",
|
|
"--format",
|
|
"{{.Name}}",
|
|
capture=True,
|
|
).splitlines()
|
|
if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes):
|
|
raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}")
|
|
summary["checks"] = {
|
|
"live": 200,
|
|
"ready": 200,
|
|
"version": version_status,
|
|
"admin_without_key": unauthenticated,
|
|
"admin_with_key": authenticated,
|
|
"console": 200,
|
|
"database_roles_provisioned": True,
|
|
"network_probe_container": "api",
|
|
"isolated_volumes": sorted(volumes),
|
|
}
|
|
summary["result"] = "PASS"
|
|
(output / "compose-ps.json").write_text(
|
|
run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
except BaseException:
|
|
summary["result"] = "FAIL"
|
|
(output / "compose-logs.txt").write_text(
|
|
run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
raise
|
|
finally:
|
|
(output / "acceptance-summary.json").write_text(
|
|
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
|
|
print(f"Isolated RC acceptance PASS for {canonical_commit} in {project}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|