1193 lines
56 KiB
YAML
1193 lines
56 KiB
YAML
name: Managed validation
|
|
|
|
# Protected-source validation. The ordinary validation job preserves the existing profile contract.
|
|
# The security job always scans source for pull requests and full runs; only an explicit dispatched
|
|
# `security` profile pays the cost of building and scanning the four exact shipping images.
|
|
|
|
on:
|
|
# Public exports require explicit owner dispatch; fork PRs never reach private runners.
|
|
workflow_dispatch:
|
|
inputs:
|
|
profile:
|
|
description: Allowlisted validation profile
|
|
required: true
|
|
default: full
|
|
type: choice
|
|
options: [test, lint, typecheck, build, security, full]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: managed-validation-${{ gitea.repository }}-${{ gitea.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
full:
|
|
name: full
|
|
runs-on: linux-validation
|
|
timeout-minutes: 45
|
|
env:
|
|
PROFILE: ${{ inputs.profile || 'full' }}
|
|
BACKEND_VENV: /tmp/modelforge-validation-backend-venv
|
|
NODE_AGENT_VENV: /tmp/modelforge-validation-node-agent-venv
|
|
RUNTIME_WORKER_VENV: /tmp/modelforge-validation-runtime-worker-venv
|
|
VALIDATION_COMPOSE_PROJECT: modelforge-validation-${{ gitea.run_id }}-${{ gitea.run_attempt }}
|
|
MODELFORGE_POSTGRES_ADMIN_PASSWORD: managed-validation-admin
|
|
MODELFORGE_MIGRATION_DB_PASSWORD: managed-validation-owner
|
|
MODELFORGE_RUNTIME_DB_PASSWORD: managed-validation-runtime
|
|
# Port zero asks Docker for a collision-free ephemeral loopback port. The boundary step
|
|
# exports the resolved URLs only to later steps in this job.
|
|
MODELFORGE_POSTGRES_PORT: "0"
|
|
steps:
|
|
- name: Check out validation source with full history
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 0
|
|
fetch-tags: true
|
|
persist-credentials: false
|
|
|
|
- name: Production delivery policy
|
|
shell: bash
|
|
run: python3 -m unittest discover -s .gitea/tests -p 'test_*.py' -v
|
|
|
|
- name: Validate the requested profile
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
case "${PROFILE}" in
|
|
test|lint|typecheck|build|security|full) ;;
|
|
*) echo "Profile is not allowlisted: ${PROFILE}" >&2; exit 2 ;;
|
|
esac
|
|
echo "profile=${PROFILE}"
|
|
echo "commit=$(git rev-parse HEAD)"
|
|
|
|
- name: Repository hygiene
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
git diff --check
|
|
python3 scripts/rc_readiness.py
|
|
if git grep -nE '^(<<<<<<< |=======$|>>>>>>> )' -- . ':!*.lock' ':!*.patch'; then
|
|
echo "Unresolved merge markers detected" >&2
|
|
exit 1
|
|
fi
|
|
echo "hygiene: clean"
|
|
|
|
- name: Prepare the report directory
|
|
shell: bash
|
|
run: mkdir -p reports
|
|
|
|
# Each Python component gets its own environment. Sharing one environment made the later
|
|
# component inherit the earlier component's dependency resolution and invalidated isolation.
|
|
- name: Toolchain
|
|
if: ${{ env.PROFILE != 'security' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
command -v python3 >/dev/null || { echo "python3 is not on PATH" >&2; exit 1; }
|
|
command -v node >/dev/null || { echo "node is not on PATH" >&2; exit 1; }
|
|
command -v npm >/dev/null || { echo "npm is not on PATH" >&2; exit 1; }
|
|
echo " python $(python3 --version)"
|
|
echo " node $(node --version)"
|
|
echo " npm $(npm --version)"
|
|
for venv in "${BACKEND_VENV}" "${NODE_AGENT_VENV}" "${RUNTIME_WORKER_VENV}"; do
|
|
python3 -m venv "${venv}"
|
|
"${venv}/bin/python" -m pip install --disable-pip-version-check --quiet \
|
|
--require-hashes --only-binary=:all: --index-url https://pypi.org/simple \
|
|
-r config/security/build-toolchain.lock
|
|
done
|
|
|
|
# Build first so the backend packaging policy inspects the artifact that would ship instead
|
|
# of recording a skip because frontend/dist does not exist yet.
|
|
- name: Console — install frozen, typecheck, test, build
|
|
if: ${{ env.PROFILE != 'security' }}
|
|
id: console
|
|
shell: bash
|
|
working-directory: frontend
|
|
run: |
|
|
set -euo pipefail
|
|
npm ci
|
|
if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then npx tsc --noEmit; fi
|
|
if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
|
|
npx vitest run --reporter=default --reporter=junit --outputFile=../reports/frontend.xml
|
|
node --test ../clients/typescript/modelforge-client.test.mjs
|
|
fi
|
|
if [[ "${PROFILE}" == build || "${PROFILE}" == full ]]; then npm run build; fi
|
|
|
|
- name: Managed PostgreSQL boundary
|
|
if: ${{ env.PROFILE == 'full' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
command -v docker >/dev/null || {
|
|
echo "The full profile requires Docker for its disposable PostgreSQL boundary." >&2
|
|
exit 1
|
|
}
|
|
# Compose validates required interpolation for every declared service even though this
|
|
# step starts PostgreSQL only. Keep these container-internal URLs scoped to this shell;
|
|
# the backend test process receives only the resolved host-side test URLs below.
|
|
export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:managed-validation-owner@postgres:5432/modelforge
|
|
export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:managed-validation-runtime@postgres:5432/modelforge
|
|
docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" \
|
|
up --detach --wait postgres
|
|
docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" ps postgres
|
|
# The Actions job runs in a container, so the Docker host's loopback is not this job's
|
|
# 127.0.0.1. Join the disposable Compose network and use Docker service DNS instead.
|
|
validation_network="${VALIDATION_COMPOSE_PROJECT}_default"
|
|
job_container="$(hostname)"
|
|
docker inspect "${job_container}" >/dev/null
|
|
docker network connect "${validation_network}" "${job_container}"
|
|
# Docker can report PostgreSQL healthy before the newly attached job container
|
|
# can reach its listener. Bound the network readiness check to 30 seconds.
|
|
python3 - <<'PY'
|
|
import socket
|
|
import time
|
|
|
|
for attempt in range(30):
|
|
try:
|
|
with socket.create_connection(("postgres", 5432), timeout=2):
|
|
break
|
|
except (OSError, TimeoutError):
|
|
if attempt == 29:
|
|
raise
|
|
time.sleep(1)
|
|
PY
|
|
{
|
|
echo "MODELFORGE_TEST_POSTGRES_URL=postgresql+psycopg://postgres:managed-validation-admin@postgres:5432/modelforge"
|
|
echo "MODELFORGE_TEST_POSTGRES_OWNER_URL=postgresql+psycopg://modelforge:managed-validation-owner@postgres:5432/modelforge"
|
|
echo "MODELFORGE_TEST_POSTGRES_RUNTIME_URL=postgresql+psycopg://modelforge_runtime:managed-validation-runtime@postgres:5432/modelforge"
|
|
echo "MODELFORGE_TEST_POSTGRES_CONTAINER=$(docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" ps -q postgres)"
|
|
echo "MODELFORGE_TEST_POSTGRES_DATABASE_LIFECYCLE=managed-validation-v1"
|
|
} >> "${GITHUB_ENV}"
|
|
|
|
- name: Backend — install, lint, typecheck, test
|
|
if: ${{ env.PROFILE != 'security' }}
|
|
id: backend
|
|
shell: bash
|
|
working-directory: backend
|
|
run: |
|
|
set -euo pipefail
|
|
"${BACKEND_VENV}/bin/python" -m pip install --disable-pip-version-check --quiet \
|
|
--no-build-isolation -e '.[dev]'
|
|
if [[ "${PROFILE}" == full ]]; then
|
|
MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}" \
|
|
"${BACKEND_VENV}/bin/python" -m alembic upgrade head
|
|
MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}" \
|
|
"${BACKEND_VENV}/bin/python" -m alembic downgrade 20260828_0022
|
|
MODELFORGE_MIGRATION_DATABASE_URL="${MODELFORGE_TEST_POSTGRES_OWNER_URL}" \
|
|
"${BACKEND_VENV}/bin/python" -m alembic upgrade head
|
|
fi
|
|
if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
|
|
"${BACKEND_VENV}/bin/python" -m ruff check src tests
|
|
fi
|
|
if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
|
|
"${BACKEND_VENV}/bin/python" -m mypy src
|
|
fi
|
|
if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
|
|
"${BACKEND_VENV}/bin/python" -m pytest -q --no-header --junitxml=../reports/backend.xml
|
|
fi
|
|
|
|
- name: Node Agent — install, lint, typecheck, test
|
|
if: ${{ env.PROFILE != 'security' }}
|
|
id: node_agent
|
|
shell: bash
|
|
working-directory: node-agent
|
|
run: |
|
|
set -euo pipefail
|
|
"${NODE_AGENT_VENV}/bin/python" -m pip install --disable-pip-version-check --quiet \
|
|
--no-build-isolation -e '../backend' -e '.[dev]'
|
|
if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
|
|
"${NODE_AGENT_VENV}/bin/python" -m ruff check src
|
|
fi
|
|
if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
|
|
"${NODE_AGENT_VENV}/bin/python" -m mypy src
|
|
fi
|
|
if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
|
|
"${NODE_AGENT_VENV}/bin/python" -m pytest -q --no-header \
|
|
--junitxml=../reports/node-agent.xml
|
|
fi
|
|
|
|
- name: Runtime Worker — install, lint, typecheck, test
|
|
if: ${{ env.PROFILE != 'security' }}
|
|
id: runtime_worker
|
|
shell: bash
|
|
working-directory: runtime-worker
|
|
run: |
|
|
set -euo pipefail
|
|
"${RUNTIME_WORKER_VENV}/bin/python" -m pip install \
|
|
--disable-pip-version-check --quiet --no-build-isolation -e '.[dev]'
|
|
if [[ "${PROFILE}" == lint || "${PROFILE}" == full ]]; then
|
|
"${RUNTIME_WORKER_VENV}/bin/python" -m ruff check src
|
|
fi
|
|
if [[ "${PROFILE}" == typecheck || "${PROFILE}" == full ]]; then
|
|
"${RUNTIME_WORKER_VENV}/bin/python" -m mypy src
|
|
fi
|
|
if [[ "${PROFILE}" == test || "${PROFILE}" == full ]]; then
|
|
"${RUNTIME_WORKER_VENV}/bin/python" -m pytest -q --no-header \
|
|
--junitxml=../reports/runtime-worker.xml
|
|
fi
|
|
|
|
- name: Compose projections
|
|
if: ${{ env.PROFILE == 'full' || env.PROFILE == 'build' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
if ! command -v docker >/dev/null; then
|
|
echo "SKIPPED: no docker CLI on this runner; image security has its own fail-closed job."
|
|
exit 0
|
|
fi
|
|
export MODELFORGE_POSTGRES_ADMIN_PASSWORD=compose-validation-admin
|
|
export MODELFORGE_MIGRATION_DB_PASSWORD=compose-validation-owner
|
|
export MODELFORGE_RUNTIME_DB_PASSWORD=compose-validation-runtime
|
|
export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:compose-validation-runtime@postgres:5432/modelforge
|
|
export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:compose-validation-owner@postgres:5432/modelforge
|
|
export MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS=127.0.0.1
|
|
export MODELFORGE_AGENT_CA_CERT_PATH=./config/ca.crt
|
|
failures=0
|
|
check() {
|
|
if docker compose --env-file .env.example "$@" config -q >/dev/null 2>&1; then
|
|
echo " OK $*"
|
|
else
|
|
echo " FAIL $*"; failures=$((failures + 1))
|
|
fi
|
|
}
|
|
check -f docker-compose.yml
|
|
for overlay in backup dr gpu node-agent node-recovery runtime-worker; do
|
|
check -f docker-compose.yml -f "docker-compose.${overlay}.yml"
|
|
done
|
|
check -f docker-compose.yml -f docker-compose.node-agent.yml \
|
|
-f docker-compose.node-agent.private-ca.yml
|
|
check -f docker-compose.yml -f docker-compose.runtime-worker.yml \
|
|
-f docker-compose.runtime-worker.private-ca.yml
|
|
[[ "${failures}" -eq 0 ]] || { echo "${failures} projection(s) invalid" >&2; exit 1; }
|
|
|
|
- name: Configuration documentation is current
|
|
if: ${{ env.PROFILE == 'full' }}
|
|
shell: bash
|
|
run: |
|
|
"${BACKEND_VENV}/bin/python" scripts/generate_configuration_docs.py --check
|
|
|
|
- name: Refuse a validation that silently ran no tests
|
|
if: ${{ env.PROFILE == 'full' || env.PROFILE == 'test' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
"${BACKEND_VENV}/bin/python" - <<'PY'
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
expected = {
|
|
"backend.xml": "backend",
|
|
"node-agent.xml": "node-agent",
|
|
"runtime-worker.xml": "runtime-worker",
|
|
"frontend.xml": "frontend",
|
|
}
|
|
reports = Path("reports")
|
|
failures = []
|
|
total = 0
|
|
for filename, component in expected.items():
|
|
path = reports / filename
|
|
if not path.is_file():
|
|
failures.append(f"{component}: no test report was produced")
|
|
continue
|
|
root = ET.parse(path).getroot()
|
|
suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite"))
|
|
tests = sum(int(suite.get("tests", 0)) for suite in suites)
|
|
errors = sum(int(suite.get("errors", 0)) for suite in suites)
|
|
failed = sum(int(suite.get("failures", 0)) for suite in suites)
|
|
skipped = sum(int(suite.get("skipped", 0)) for suite in suites)
|
|
executed = tests - skipped
|
|
total += tests
|
|
print(f" {component:16} {tests:5} tests, {skipped} skipped, "
|
|
f"{failed} failed, {errors} errors")
|
|
if executed <= 0:
|
|
failures.append(f"{component}: {tests} tests collected, {executed} executed")
|
|
if failed or errors:
|
|
failures.append(f"{component}: {failed} failed, {errors} errors")
|
|
print(f" {'TOTAL':16} {total:5} tests")
|
|
if failures:
|
|
print("\nManaged validation refused:", file=sys.stderr)
|
|
for failure in failures:
|
|
print(f" - {failure}", file=sys.stderr)
|
|
sys.exit(1)
|
|
PY
|
|
|
|
- name: Remove managed PostgreSQL boundary
|
|
if: ${{ always() && env.PROFILE == 'full' }}
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:managed-validation-owner@postgres:5432/modelforge
|
|
export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:managed-validation-runtime@postgres:5432/modelforge
|
|
docker network disconnect "${VALIDATION_COMPOSE_PROJECT}_default" "$(hostname)" \
|
|
>/dev/null 2>&1 || true
|
|
docker compose --env-file .env.example -p "${VALIDATION_COMPOSE_PROJECT}" \
|
|
down --volumes --remove-orphans
|
|
|
|
- name: Validation summary
|
|
if: always()
|
|
shell: bash
|
|
run: |
|
|
echo "commit: $(git rev-parse HEAD)"
|
|
echo "profile: ${PROFILE}"
|
|
ls -l reports/ 2>/dev/null || echo "no reports directory"
|
|
|
|
security:
|
|
name: security
|
|
if: ${{ !inputs.profile || inputs.profile == 'security' || inputs.profile == 'full' }}
|
|
runs-on: linux-validation
|
|
# Explicit worst-case step bounds total 113 minutes for the image profile. The 17-minute job
|
|
# reserve means cleanup, the always-run evaluator and immutable upload retain time after
|
|
# earlier bounds.
|
|
timeout-minutes: 130
|
|
env:
|
|
SECURITY_PROFILE: ${{ gitea.event_name == 'workflow_dispatch' && inputs.profile == 'security' && 'security' || 'full' }}
|
|
PIP_AUDIT_VERSION: 2.10.1
|
|
PIP_AUDIT_SHA256: 99ef3f600a317c1945f1e89e227ef26e1c2d618429b8bd3fa6f4f7c440c4611a
|
|
GITLEAKS_VERSION: 8.30.1
|
|
GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb
|
|
TRIVY_VERSION: 0.74.0
|
|
TRIVY_SHA256: 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a
|
|
steps:
|
|
- name: Prepare fail-closed security evidence
|
|
timeout-minutes: 2
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
umask 077
|
|
: "${GITHUB_RUN_ID:?missing run id}"
|
|
: "${GITHUB_RUN_ATTEMPT:?missing run attempt}"
|
|
: "${GITHUB_SHA:?missing commit SHA}"
|
|
: "${GITHUB_SERVER_URL:?missing server URL}"
|
|
: "${GITHUB_REPOSITORY:?missing repository identity}"
|
|
: "${GITHUB_ENV:?missing runner environment file}"
|
|
run_key="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_SHA}"
|
|
temp_base="${RUNNER_TEMP:-/tmp}"
|
|
[[ -d "${temp_base}" && ! -L "${temp_base}" ]]
|
|
create_private_root() {
|
|
label="$1"
|
|
root="$(mktemp -d "${temp_base%/}/modelforge-${label}-${run_key}.XXXXXXXX")"
|
|
[[ -d "${root}" && ! -L "${root}" ]]
|
|
[[ "$(stat -c '%u' "${root}")" == "$(id -u)" ]]
|
|
[[ "$(stat -c '%a' "${root}")" == 700 ]]
|
|
printf '%s\n' "${root}"
|
|
}
|
|
REPORTS="$(create_private_root security-evidence)"
|
|
SECURITY_TOOLS="$(create_private_root security-tools)"
|
|
SOURCE_REQUIREMENTS="${REPORTS}/source-requirements"
|
|
IMAGE_REQUIREMENTS="${REPORTS}/image-requirements"
|
|
case "${GITHUB_SERVER_URL%/}" in
|
|
https://git.example.com|http://git.example.com|http://192.0.2.10:3000|http://gitea:3000|http://Gitea:3000|http://gitea-runner-internal:3000)
|
|
SOURCE_SERVER_URL="https://git.example.com" ;;
|
|
*) echo "Untrusted Gitea server URL (legacy: Untrusted non-HTTPS Gitea server URL)" >&2; exit 1 ;;
|
|
esac
|
|
[[ "${GITHUB_REPOSITORY}" == "Jens/ITWorx-ModelForge" ]] || {
|
|
echo "Unexpected canonical repository identity: ${GITHUB_REPOSITORY}" >&2
|
|
exit 1
|
|
}
|
|
SOURCE_REPOSITORY="${SOURCE_SERVER_URL}/${GITHUB_REPOSITORY}"
|
|
[[ "${SOURCE_REPOSITORY}" == "https://git.example.com/example/modelforge" ]]
|
|
install -d -m 700 "${REPORTS}/status" "${REPORTS}/versions"
|
|
{
|
|
printf 'REPORTS=%s\n' "${REPORTS}"
|
|
printf 'SECURITY_TOOLS=%s\n' "${SECURITY_TOOLS}"
|
|
printf 'SOURCE_REQUIREMENTS=%s\n' "${SOURCE_REQUIREMENTS}"
|
|
printf 'IMAGE_REQUIREMENTS=%s\n' "${IMAGE_REQUIREMENTS}"
|
|
printf 'SOURCE_REPOSITORY=%s\n' "${SOURCE_REPOSITORY}"
|
|
} >> "${GITHUB_ENV}"
|
|
export REPORTS SECURITY_TOOLS SOURCE_REQUIREMENTS IMAGE_REQUIREMENTS SOURCE_REPOSITORY
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
result = {
|
|
"schema_version": 1,
|
|
"commit": os.environ.get("GITHUB_SHA", "unavailable"),
|
|
"ref": os.environ.get("GITHUB_REF", "unavailable"),
|
|
"run_id": os.environ.get("GITHUB_RUN_ID", "unavailable"),
|
|
"run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", "unavailable"),
|
|
"profile": os.environ.get("SECURITY_PROFILE", "unavailable"),
|
|
"evaluated_at": datetime.now(UTC).isoformat(),
|
|
"blocked_findings": [{
|
|
"scanner": "security-gate",
|
|
"target": "bootstrap",
|
|
"advisory_id": "GATE-INCOMPLETE",
|
|
"severity": "ERROR",
|
|
"reason": "the central evaluator did not complete",
|
|
}],
|
|
"applied_risks": [],
|
|
"unused_or_invalid_risks": [],
|
|
"decision": "FAIL",
|
|
}
|
|
path = Path(os.environ["REPORTS"]) / "security-gate-result.json"
|
|
path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
PY
|
|
|
|
- name: Check out security source with full history
|
|
timeout-minutes: 5
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
fetch-depth: 0
|
|
fetch-tags: true
|
|
persist-credentials: false
|
|
|
|
- name: Prove complete non-shallow history
|
|
timeout-minutes: 6
|
|
shell: bash
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
run: |
|
|
set -euo pipefail
|
|
: "${GITHUB_TOKEN:?missing read-only repository token}"
|
|
cleanup_history_auth() {
|
|
original_code=$?
|
|
trap - EXIT
|
|
git config --local --unset-all http.extraheader >/dev/null 2>&1 || true
|
|
exit "${original_code}"
|
|
}
|
|
trap cleanup_history_auth EXIT
|
|
git remote set-url origin "${SOURCE_REPOSITORY}"
|
|
auth_header="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 -w 0)"
|
|
git config --local http.extraheader "AUTHORIZATION: basic ${auth_header}"
|
|
unset auth_header
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 3m git fetch --force --prune \
|
|
--prune-tags --tags origin \
|
|
'+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*'
|
|
fetch_code=$?
|
|
printf '%s\n' "${fetch_code}" > "${REPORTS}/status/history-fetch.exit"
|
|
timeout --signal=TERM --kill-after=30s 2m python3 scripts/security_gate.py history \
|
|
--repository . --expected-commit "${GITHUB_SHA}" \
|
|
--output "${REPORTS}/git-history.json"
|
|
code=$?
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/history.exit"
|
|
MODELFORGE_VERSION="$(tr -d '\r\n' < VERSION)"
|
|
version_code=$?
|
|
if [[ "${version_code}" == 0 && \
|
|
! "${MODELFORGE_VERSION}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-rc\.(0|[1-9][0-9]*))?$ ]]; then
|
|
version_code=2
|
|
fi
|
|
if [[ "${version_code}" == 0 ]]; then
|
|
printf 'MODELFORGE_VERSION=%s\n' "${MODELFORGE_VERSION}" >> "${GITHUB_ENV}"
|
|
fi
|
|
printf '%s\n' "${version_code}" > "${REPORTS}/status/release-version.exit"
|
|
set -e
|
|
|
|
- name: Install checksum-pinned security scanners
|
|
timeout-minutes: 15
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
[[ -d "${SECURITY_TOOLS}" && ! -L "${SECURITY_TOOLS}" ]]
|
|
install -d -m 700 "${SECURITY_TOOLS}/downloads" "${SECURITY_TOOLS}/bin"
|
|
record_command() {
|
|
name="$1"
|
|
shift
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "$@"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/${name}.exit"
|
|
}
|
|
|
|
record_command pip-audit-download python3 -m pip download \
|
|
--disable-pip-version-check --index-url https://pypi.org/simple \
|
|
--require-hashes --only-binary=:all: --dest "${SECURITY_TOOLS}/downloads" \
|
|
--requirement config/security/pip-audit-toolchain.lock
|
|
printf '%s %s\n' "${PIP_AUDIT_SHA256}" \
|
|
"${SECURITY_TOOLS}/downloads/pip_audit-${PIP_AUDIT_VERSION}-py3-none-any.whl" \
|
|
> "${SECURITY_TOOLS}/pip-audit.sha256"
|
|
record_command pip-audit-hash sha256sum --check --strict \
|
|
"${SECURITY_TOOLS}/pip-audit.sha256"
|
|
set +e
|
|
if [[ "$(<"${REPORTS}/status/pip-audit-download.exit")" == 0 && \
|
|
"$(<"${REPORTS}/status/pip-audit-hash.exit")" == 0 ]]; then
|
|
timeout --signal=TERM --kill-after=30s 10m python3 -m venv \
|
|
"${SECURITY_TOOLS}/pip-audit-venv" && \
|
|
timeout --signal=TERM --kill-after=30s 10m \
|
|
"${SECURITY_TOOLS}/pip-audit-venv/bin/pip" install \
|
|
--disable-pip-version-check --no-index \
|
|
--find-links "${SECURITY_TOOLS}/downloads" --require-hashes \
|
|
--requirement config/security/pip-audit-toolchain.lock
|
|
code=$?
|
|
else
|
|
code=125
|
|
fi
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-audit-install.exit"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 1m \
|
|
"${SECURITY_TOOLS}/pip-audit-venv/bin/pip-audit" --version \
|
|
> "${REPORTS}/versions/pip-audit.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-audit-version.exit"
|
|
|
|
record_command gitleaks-download curl --fail --location --show-error \
|
|
--retry 3 --retry-all-errors --output "${SECURITY_TOOLS}/downloads/gitleaks.tar.gz" \
|
|
"https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
|
|
printf '%s %s\n' "${GITLEAKS_SHA256}" \
|
|
"${SECURITY_TOOLS}/downloads/gitleaks.tar.gz" > "${SECURITY_TOOLS}/gitleaks.sha256"
|
|
record_command gitleaks-hash sha256sum --check --strict \
|
|
"${SECURITY_TOOLS}/gitleaks.sha256"
|
|
if [[ "$(<"${REPORTS}/status/gitleaks-hash.exit")" == 0 ]]; then
|
|
record_command gitleaks-extract tar --extract --gzip \
|
|
--file "${SECURITY_TOOLS}/downloads/gitleaks.tar.gz" \
|
|
--directory "${SECURITY_TOOLS}/bin" gitleaks
|
|
else
|
|
printf '125\n' > "${REPORTS}/status/gitleaks-extract.exit"
|
|
fi
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 1m \
|
|
"${SECURITY_TOOLS}/bin/gitleaks" version > "${REPORTS}/versions/gitleaks.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks-version.exit"
|
|
|
|
gitleaks_self_test() (
|
|
set -euo pipefail
|
|
test_root="$(mktemp -d "${SECURITY_TOOLS}/gitleaks-self-test.XXXXXXXX")"
|
|
cleanup_self_test() {
|
|
original_code=$?
|
|
trap - EXIT
|
|
set +e
|
|
rm -r -- "${test_root}"
|
|
cleanup_code=$?
|
|
set -e
|
|
if [[ "${cleanup_code}" != 0 ]]; then
|
|
exit 125
|
|
fi
|
|
exit "${original_code}"
|
|
}
|
|
trap cleanup_self_test EXIT
|
|
default_trigger="${test_root}/default-trigger"
|
|
control_repository="${test_root}/control"
|
|
custom_formats=(ghp gho ghu ghs ghr github-pat)
|
|
install -d -m 700 "${default_trigger}" "${control_repository}"
|
|
for format in "${custom_formats[@]}"; do
|
|
install -d -m 700 "${test_root}/custom-${format}"
|
|
done
|
|
for repository in "${default_trigger}" "${control_repository}" \
|
|
"${test_root}"/custom-*; do
|
|
git -C "${repository}" init --quiet
|
|
git -C "${repository}" config user.name security-gate
|
|
git -C "${repository}" config user.email security-gate@invalid.example
|
|
done
|
|
python3 - "${test_root}" "${default_trigger}/credential.txt" <<'PY'
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
root = Path(sys.argv[1])
|
|
body = "".join(("aBcDeFgHiJkLmNoPqRsTuVwXyZ", "0123456789"))
|
|
tokens = {
|
|
"ghp": "".join(("gh", "p_", body)),
|
|
"gho": "".join(("gh", "o_", body)),
|
|
"ghu": "".join(("gh", "u_", body)),
|
|
"ghs": "".join(("gh", "s_", body)),
|
|
"ghr": "".join(("gh", "r_", body)),
|
|
"github-pat": "".join(("github", "_pat_", "A" * 82)),
|
|
}
|
|
aws_key = "".join(("AK", "IA", "A1B2C3D4E5F6G7H8"))
|
|
for name, token in tokens.items():
|
|
(root / f"custom-{name}" / "credential.txt").write_text(
|
|
f'token = "{token}"\n', encoding="utf-8"
|
|
)
|
|
Path(sys.argv[2]).write_text(f'key = "{aws_key}"\n', encoding="utf-8")
|
|
PY
|
|
for repository in "${default_trigger}" "${test_root}"/custom-*; do
|
|
git -C "${repository}" add credential.txt
|
|
git -C "${repository}" commit --quiet -m synthetic-trigger
|
|
printf 'credential removed\n' > "${repository}/credential.txt"
|
|
git -C "${repository}" add credential.txt
|
|
git -C "${repository}" commit --quiet -m remove-trigger
|
|
done
|
|
printf 'token = "example-not-a-credential"\n' > "${control_repository}/control.txt"
|
|
git -C "${control_repository}" add control.txt
|
|
git -C "${control_repository}" commit --quiet -m benign-control
|
|
|
|
for format in "${custom_formats[@]}"; do
|
|
set +e
|
|
timeout --signal=TERM --kill-after=10s 1m \
|
|
"${SECURITY_TOOLS}/bin/gitleaks" git \
|
|
--config "${GITHUB_WORKSPACE}/.gitleaks.toml" --log-opts=--all \
|
|
--redact=100 --report-format json \
|
|
--report-path "${test_root}/custom-${format}.json" \
|
|
--no-banner "${test_root}/custom-${format}" >/dev/null 2>&1
|
|
custom_trigger_code=$?
|
|
set -e
|
|
[[ "${custom_trigger_code}" == 1 ]]
|
|
done
|
|
set +e
|
|
timeout --signal=TERM --kill-after=10s 1m \
|
|
"${SECURITY_TOOLS}/bin/gitleaks" git \
|
|
--config "${GITHUB_WORKSPACE}/.gitleaks.toml" --log-opts=--all \
|
|
--redact=100 --report-format json \
|
|
--report-path "${test_root}/custom-control.json" \
|
|
--no-banner "${control_repository}" >/dev/null 2>&1
|
|
custom_control_code=$?
|
|
timeout --signal=TERM --kill-after=10s 1m \
|
|
"${SECURITY_TOOLS}/bin/gitleaks" git --log-opts=--all \
|
|
--redact=100 --report-format json \
|
|
--report-path "${test_root}/default-trigger.json" \
|
|
--no-banner "${default_trigger}" >/dev/null 2>&1
|
|
default_trigger_code=$?
|
|
timeout --signal=TERM --kill-after=10s 1m \
|
|
"${SECURITY_TOOLS}/bin/gitleaks" git --log-opts=--all \
|
|
--redact=100 --report-format json \
|
|
--report-path "${test_root}/default-control.json" \
|
|
--no-banner "${control_repository}" >/dev/null 2>&1
|
|
default_control_code=$?
|
|
set -e
|
|
[[ "${custom_control_code}" == 0 && "${default_trigger_code}" == 1 && \
|
|
"${default_control_code}" == 0 ]]
|
|
)
|
|
set +e
|
|
gitleaks_self_test
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks-self-test.exit"
|
|
|
|
record_command trivy-download curl --fail --location --show-error \
|
|
--retry 3 --retry-all-errors --output "${SECURITY_TOOLS}/downloads/trivy.tar.gz" \
|
|
"https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
|
|
printf '%s %s\n' "${TRIVY_SHA256}" \
|
|
"${SECURITY_TOOLS}/downloads/trivy.tar.gz" > "${SECURITY_TOOLS}/trivy.sha256"
|
|
record_command trivy-hash sha256sum --check --strict \
|
|
"${SECURITY_TOOLS}/trivy.sha256"
|
|
if [[ "$(<"${REPORTS}/status/trivy-hash.exit")" == 0 ]]; then
|
|
record_command trivy-extract tar --extract --gzip \
|
|
--file "${SECURITY_TOOLS}/downloads/trivy.tar.gz" \
|
|
--directory "${SECURITY_TOOLS}/bin" trivy
|
|
else
|
|
printf '125\n' > "${REPORTS}/status/trivy-extract.exit"
|
|
fi
|
|
|
|
- name: Collect source security reports
|
|
timeout-minutes: 30
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
PIP_AUDIT="${SECURITY_TOOLS}/pip-audit-venv/bin/pip-audit"
|
|
GITLEAKS="${SECURITY_TOOLS}/bin/gitleaks"
|
|
TRIVY="${SECURITY_TOOLS}/bin/trivy"
|
|
GITLEAKS_TEMP="$(mktemp -d "${SECURITY_TOOLS}/gitleaks-run.XXXXXXXX")"
|
|
RAW_GITLEAKS_DEFAULT="${GITLEAKS_TEMP}/default.json"
|
|
RAW_GITLEAKS_CUSTOM="${GITLEAKS_TEMP}/custom.json"
|
|
RAW_GITLEAKS="${GITLEAKS_TEMP}/combined.json"
|
|
DEFAULT_GITLEAKS_REPOSITORY="${GITLEAKS_TEMP}/default-worktree"
|
|
cleanup_source_scan() {
|
|
original_code=$?
|
|
trap - EXIT
|
|
set +e
|
|
cleanup_code=0
|
|
if [[ -e "${DEFAULT_GITLEAKS_REPOSITORY}/.git" ]]; then
|
|
git worktree remove --force "${DEFAULT_GITLEAKS_REPOSITORY}" \
|
|
>/dev/null 2>&1
|
|
cleanup_code=$?
|
|
fi
|
|
git worktree prune
|
|
prune_code=$?
|
|
rm -r -- "${GITLEAKS_TEMP}"
|
|
remove_code=$?
|
|
if [[ "${cleanup_code}" != 0 || "${prune_code}" != 0 || \
|
|
"${remove_code}" != 0 ]]; then
|
|
cleanup_code=125
|
|
fi
|
|
printf '%s\n' "${cleanup_code}" > "${REPORTS}/status/gitleaks-cleanup.exit"
|
|
set -e
|
|
if [[ "${cleanup_code}" != 0 ]]; then
|
|
exit 125
|
|
fi
|
|
exit "${original_code}"
|
|
}
|
|
trap cleanup_source_scan EXIT
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 2m python3 scripts/security_gate.py \
|
|
source-inventory --repository . --output "${REPORTS}/source-inventory.json" \
|
|
--requirements-directory "${SOURCE_REQUIREMENTS}"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/source-inventory.exit"
|
|
|
|
# Gitleaks 8.30.1 drops newly added rules when extend.useDefault is used (#2170).
|
|
# Run its built-in defaults on a config-free temporary worktree, run the explicit
|
|
# repository rules separately, then combine only temporary raw reports.
|
|
set +e
|
|
git worktree add --detach "${DEFAULT_GITLEAKS_REPOSITORY}" "${GITHUB_SHA}" \
|
|
>/dev/null 2>&1
|
|
default_code=$?
|
|
if [[ "${default_code}" == 0 ]]; then
|
|
rm -f -- "${DEFAULT_GITLEAKS_REPOSITORY}/.gitleaks.toml"
|
|
remove_config_code=$?
|
|
if [[ "${remove_config_code}" == 0 ]]; then
|
|
timeout --signal=TERM --kill-after=30s 10m "${GITLEAKS}" git \
|
|
--log-opts=--all --redact=100 --report-format json \
|
|
--report-path "${RAW_GITLEAKS_DEFAULT}" --no-banner \
|
|
"${DEFAULT_GITLEAKS_REPOSITORY}"
|
|
default_code=$?
|
|
else
|
|
default_code=125
|
|
fi
|
|
fi
|
|
set -e
|
|
printf '%s\n' "${default_code}" > "${REPORTS}/status/gitleaks-default.exit"
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${GITLEAKS}" git \
|
|
--config .gitleaks.toml --log-opts=--all --redact=100 --report-format json \
|
|
--report-path "${RAW_GITLEAKS_CUSTOM}" --no-banner .
|
|
custom_code=$?
|
|
set -e
|
|
printf '%s\n' "${custom_code}" > "${REPORTS}/status/gitleaks-custom.exit"
|
|
|
|
set +e
|
|
python3 - "${RAW_GITLEAKS_DEFAULT}" "${RAW_GITLEAKS_CUSTOM}" \
|
|
"${RAW_GITLEAKS}" <<'PY'
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
findings = []
|
|
for value in sys.argv[1:3]:
|
|
report = json.loads(Path(value).read_text(encoding="utf-8"))
|
|
if not isinstance(report, list):
|
|
raise TypeError(f"Gitleaks report is not a list: {value}")
|
|
findings.extend(report)
|
|
Path(sys.argv[3]).write_text(
|
|
json.dumps(findings, separators=(",", ":")) + "\n", encoding="utf-8"
|
|
)
|
|
PY
|
|
merge_code=$?
|
|
set -e
|
|
printf '%s\n' "${merge_code}" > "${REPORTS}/status/gitleaks-merge.exit"
|
|
if [[ ( "${default_code}" == 0 || "${default_code}" == 1 ) && \
|
|
( "${custom_code}" == 0 || "${custom_code}" == 1 ) && \
|
|
"${merge_code}" == 0 ]]; then
|
|
if [[ "${default_code}" == 1 || "${custom_code}" == 1 ]]; then
|
|
code=1
|
|
else
|
|
code=0
|
|
fi
|
|
else
|
|
code=125
|
|
fi
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks.exit"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 2m python3 scripts/security_gate.py \
|
|
sanitize-gitleaks --input "${RAW_GITLEAKS}" --output "${REPORTS}/gitleaks.json"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/gitleaks-sanitize.exit"
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict \
|
|
--progress-spinner off --format json \
|
|
--output "${REPORTS}/pip-source-backend.json" \
|
|
--requirement "${SOURCE_REQUIREMENTS}/backend.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-source-backend.exit"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict \
|
|
--progress-spinner off --format json \
|
|
--output "${REPORTS}/pip-source-node-agent.json" \
|
|
--requirement "${SOURCE_REQUIREMENTS}/node-agent.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-source-node-agent.exit"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict \
|
|
--progress-spinner off --format json \
|
|
--output "${REPORTS}/pip-source-runtime-worker.json" \
|
|
--requirement "${SOURCE_REQUIREMENTS}/runtime-worker.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-source-runtime-worker.exit"
|
|
|
|
set +e
|
|
npm --version > "${REPORTS}/versions/npm.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/npm-version.exit"
|
|
set +e
|
|
(cd frontend && timeout --signal=TERM --kill-after=30s 10m npm audit --json \
|
|
> "${REPORTS}/npm-audit-all.json")
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/npm-audit-all.exit"
|
|
set +e
|
|
(cd frontend && timeout --signal=TERM --kill-after=30s 10m npm audit --omit=dev --json \
|
|
> "${REPORTS}/npm-audit-production.json")
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/npm-audit-production.exit"
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 20m "${TRIVY}" fs \
|
|
--scanners vuln,misconfig --severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL \
|
|
--format json --output "${REPORTS}/trivy-fs.json" "${GITHUB_WORKSPACE}"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/trivy-fs.exit"
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 1m "${TRIVY}" version --format json \
|
|
> "${REPORTS}/versions/trivy.json"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/trivy-version.exit"
|
|
|
|
- name: Build and scan exact shipping images
|
|
if: ${{ gitea.event_name == 'workflow_dispatch' && inputs.profile == 'security' }}
|
|
timeout-minutes: 40
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
PIP_AUDIT="${SECURITY_TOOLS}/pip-audit-venv/bin/pip-audit"
|
|
TRIVY="${SECURITY_TOOLS}/bin/trivy"
|
|
API_IMAGE="modelforge-security-api:${GITHUB_SHA}"
|
|
WEB_IMAGE="modelforge-security-web:${GITHUB_SHA}"
|
|
NODE_AGENT_IMAGE="modelforge-security-node-agent:${GITHUB_SHA}"
|
|
RUNTIME_WORKER_IMAGE="modelforge-security-runtime-worker:${GITHUB_SHA}"
|
|
install -d -m 700 "${IMAGE_REQUIREMENTS}"
|
|
|
|
record_command() {
|
|
name="$1"
|
|
shift
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 25m "$@"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/${name}.exit"
|
|
}
|
|
|
|
record_command docker docker version
|
|
record_command image-build-api docker build --pull --no-cache \
|
|
--build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
|
|
--build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
|
|
--label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
|
|
--label "org.opencontainers.image.revision=${GITHUB_SHA}" \
|
|
--label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
|
|
--tag "${API_IMAGE}" \
|
|
--file backend/Dockerfile .
|
|
record_command image-build-web docker build --pull --no-cache \
|
|
--build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
|
|
--build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
|
|
--build-arg "VITE_API_BASE_URL=https://security-scan.invalid" \
|
|
--label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
|
|
--label "org.opencontainers.image.revision=${GITHUB_SHA}" \
|
|
--label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
|
|
--label "io.modelforge.security.bundle-path=/usr/share/nginx/html" \
|
|
--tag "${WEB_IMAGE}" \
|
|
--file frontend/Dockerfile frontend
|
|
record_command image-build-node-agent docker build --pull --no-cache \
|
|
--build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
|
|
--build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
|
|
--label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
|
|
--label "org.opencontainers.image.revision=${GITHUB_SHA}" \
|
|
--label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
|
|
--tag "${NODE_AGENT_IMAGE}" \
|
|
--file node-agent/Dockerfile .
|
|
record_command image-build-runtime-worker docker build --pull --no-cache \
|
|
--build-arg "MODELFORGE_COMMIT=${GITHUB_SHA}" \
|
|
--build-arg "MODELFORGE_VERSION=${MODELFORGE_VERSION}" \
|
|
--label "org.opencontainers.image.source=${SOURCE_REPOSITORY}" \
|
|
--label "org.opencontainers.image.revision=${GITHUB_SHA}" \
|
|
--label "org.opencontainers.image.version=${MODELFORGE_VERSION}" \
|
|
--tag "${RUNTIME_WORKER_IMAGE}" \
|
|
--file runtime-worker/Dockerfile .
|
|
|
|
for target in api web node-agent runtime-worker; do
|
|
case "${target}" in
|
|
api) image="${API_IMAGE}" ;;
|
|
web) image="${WEB_IMAGE}" ;;
|
|
node-agent) image="${NODE_AGENT_IMAGE}" ;;
|
|
runtime-worker) image="${RUNTIME_WORKER_IMAGE}" ;;
|
|
esac
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 3m python3 scripts/security_gate.py \
|
|
image-identity --target "${target}" --tag "${image}" \
|
|
--expected-revision "${GITHUB_SHA}" \
|
|
--expected-source "${SOURCE_REPOSITORY}" \
|
|
--expected-version "${MODELFORGE_VERSION}" \
|
|
--output "${REPORTS}/image-identity-${target}.json"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/image-identity-${target}.exit"
|
|
done
|
|
|
|
image_field() {
|
|
python3 - "$1" "$2" <<'PY'
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
print(json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))[sys.argv[2]])
|
|
PY
|
|
}
|
|
set +e
|
|
API_ID="$(image_field "${REPORTS}/image-identity-api.json" image_id)" \
|
|
|| API_ID=invalid
|
|
WEB_ID="$(image_field "${REPORTS}/image-identity-web.json" image_id)" \
|
|
|| WEB_ID=invalid
|
|
WEB_BUNDLE_PATH="$(image_field "${REPORTS}/image-identity-web.json" bundle_path)" \
|
|
|| WEB_BUNDLE_PATH=invalid
|
|
NODE_AGENT_ID="$(image_field "${REPORTS}/image-identity-node-agent.json" image_id)" \
|
|
|| NODE_AGENT_ID=invalid
|
|
RUNTIME_WORKER_ID="$(image_field \
|
|
"${REPORTS}/image-identity-runtime-worker.json" image_id)" \
|
|
|| RUNTIME_WORKER_ID=invalid
|
|
set -e
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 3m python3 scripts/security_gate.py \
|
|
web-bundle-evidence --image-id "${WEB_ID}" \
|
|
--bundle-path "${WEB_BUNDLE_PATH}" \
|
|
--expected-origin https://security-scan.invalid \
|
|
--output "${REPORTS}/web-bundle-origin.json"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/web-bundle-origin.exit"
|
|
|
|
INVENTORY_PROGRAM='import importlib.metadata as m,json,platform; print(json.dumps({"python_version":platform.python_version(),"packages":[{"name":d.metadata.get("Name",d.name),"version":d.version} for d in m.distributions()]}))'
|
|
runtime_python() {
|
|
image="$1"
|
|
for candidate in python python3; do
|
|
if docker run --rm --entrypoint "${candidate}" "${image}" \
|
|
-c 'import sys; print(sys.executable)' >/dev/null 2>&1; then
|
|
printf '%s\n' "${candidate}"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
collect_inventory() {
|
|
target="$1"
|
|
image="$2"
|
|
raw="${SECURITY_TOOLS}/${target}-inventory.json"
|
|
set +e
|
|
python_command="$(runtime_python "${image}")"
|
|
code=$?
|
|
if [[ "${code}" == 0 ]]; then
|
|
timeout --signal=TERM --kill-after=30s 5m docker run --rm \
|
|
--entrypoint "${python_command}" \
|
|
"${image}" -c "${INVENTORY_PROGRAM}" > "${raw}"
|
|
code=$?
|
|
fi
|
|
if [[ "${code}" == 0 && "${target}" == "runtime-worker" ]]; then
|
|
timeout --signal=TERM --kill-after=30s 5m docker run --rm \
|
|
--entrypoint "${python_command}" \
|
|
"${image}" -c 'import importlib.metadata as m; import modelforge_runtime_worker.main, sentence_transformers, transformers, filelock, h2, idna, lief, pygments, requests; packages={d.metadata.get("Name", d.name).lower() for d in m.distributions()}; assert not ({"pip", "setuptools", "wheel"} & packages), packages'
|
|
code=$?
|
|
fi
|
|
if [[ "${code}" == 0 ]]; then
|
|
python3 scripts/security_gate.py image-inventory --target "${target}" \
|
|
--input "${raw}" --output "${REPORTS}/image-inventory-${target}.json" \
|
|
--requirements-output "${IMAGE_REQUIREMENTS}/${target}.txt"
|
|
code=$?
|
|
fi
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/image-inventory-${target}.exit"
|
|
}
|
|
collect_inventory api "${API_ID}"
|
|
collect_inventory node-agent "${NODE_AGENT_ID}"
|
|
collect_inventory runtime-worker "${RUNTIME_WORKER_ID}"
|
|
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict --no-deps \
|
|
--disable-pip --progress-spinner off --format json \
|
|
--output "${REPORTS}/pip-image-api.json" \
|
|
--requirement "${IMAGE_REQUIREMENTS}/api.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-image-api.exit"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict --no-deps \
|
|
--disable-pip --progress-spinner off --format json \
|
|
--output "${REPORTS}/pip-image-node-agent.json" \
|
|
--requirement "${IMAGE_REQUIREMENTS}/node-agent.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-image-node-agent.exit"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 10m "${PIP_AUDIT}" --strict --no-deps \
|
|
--disable-pip --progress-spinner off --format json \
|
|
--output "${REPORTS}/pip-image-runtime-worker.json" \
|
|
--requirement "${IMAGE_REQUIREMENTS}/runtime-worker.txt"
|
|
code=$?
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/pip-image-runtime-worker.exit"
|
|
|
|
scan_image() {
|
|
target="$1"
|
|
image_id="$2"
|
|
raw_report="${SECURITY_TOOLS}/trivy-image-${target}.raw.json"
|
|
set +e
|
|
timeout --signal=TERM --kill-after=30s 25m "${TRIVY}" image --scanners vuln \
|
|
--severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL --format json \
|
|
--output "${raw_report}" "${image_id}"
|
|
code=$?
|
|
if [[ "${code}" == 0 ]]; then
|
|
python3 scripts/security_gate.py bind-trivy-image \
|
|
--input "${raw_report}" \
|
|
--output "${REPORTS}/trivy-image-${target}.json" \
|
|
--image-id "${image_id}" \
|
|
--expected-tag "modelforge-security-${target}:${GITHUB_SHA}"
|
|
code=$?
|
|
fi
|
|
set -e
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/trivy-image-${target}.exit"
|
|
}
|
|
scan_image api "${API_ID}"
|
|
scan_image web "${WEB_ID}"
|
|
scan_image node-agent "${NODE_AGENT_ID}"
|
|
scan_image runtime-worker "${RUNTIME_WORKER_ID}"
|
|
|
|
- name: Remove exact security images
|
|
if: ${{ always() && gitea.event_name == 'workflow_dispatch' && inputs.profile == 'security' }}
|
|
timeout-minutes: 5
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
code=0
|
|
for target in api web node-agent runtime-worker; do
|
|
image="modelforge-security-${target}:${GITHUB_SHA}"
|
|
if docker image inspect "${image}" >/dev/null 2>&1; then
|
|
docker image rm "${image}" >/dev/null || code=1
|
|
fi
|
|
if docker image inspect "${image}" >/dev/null 2>&1; then
|
|
code=1
|
|
fi
|
|
done
|
|
printf '%s\n' "${code}" > "${REPORTS}/status/image-cleanup.exit"
|
|
exit "${code}"
|
|
|
|
- name: Evaluate the complete security gate
|
|
if: always()
|
|
timeout-minutes: 5
|
|
shell: bash
|
|
run: |
|
|
python3 scripts/security_gate.py evaluate --reports-dir "${REPORTS}" \
|
|
--accepted-risks config/security/accepted-risks.json \
|
|
--gitleaks-false-positives config/security/gitleaks-false-positives.json \
|
|
--repository . \
|
|
--commit "${GITHUB_SHA}" --ref "${GITHUB_REF}" --run-id "${GITHUB_RUN_ID}" \
|
|
--source-repository "${SOURCE_REPOSITORY}" \
|
|
--profile "${SECURITY_PROFILE}" \
|
|
--output "${REPORTS}/security-gate-result.json"
|
|
|
|
- name: Upload immutable security evidence
|
|
if: always()
|
|
timeout-minutes: 5
|
|
uses: https://gitea.com/actions/gitea-upload-artifact@81f940d004763f986ba3582c007fd842dd5cb0d7
|
|
with:
|
|
name: security-gate-${{ gitea.run_id }}-${{ gitea.run_attempt }}-${{ gitea.sha }}
|
|
# Do not upload the report root wholesale. The evaluator accepts only this same fixed
|
|
# evidence manifest, so a scanner/tool side-file can neither be hashed nor exported.
|
|
path: |
|
|
${{ env.REPORTS }}/security-gate-result.json
|
|
${{ env.REPORTS }}/git-history.json
|
|
${{ env.REPORTS }}/source-inventory.json
|
|
${{ env.REPORTS }}/gitleaks.json
|
|
${{ env.REPORTS }}/npm-audit-all.json
|
|
${{ env.REPORTS }}/npm-audit-production.json
|
|
${{ env.REPORTS }}/trivy-fs.json
|
|
${{ env.REPORTS }}/versions/pip-audit.txt
|
|
${{ env.REPORTS }}/versions/gitleaks.txt
|
|
${{ env.REPORTS }}/versions/npm.txt
|
|
${{ env.REPORTS }}/versions/trivy.json
|
|
${{ env.REPORTS }}/source-requirements/backend.txt
|
|
${{ env.REPORTS }}/source-requirements/node-agent.txt
|
|
${{ env.REPORTS }}/source-requirements/runtime-worker.txt
|
|
${{ env.REPORTS }}/pip-source-backend.json
|
|
${{ env.REPORTS }}/pip-source-node-agent.json
|
|
${{ env.REPORTS }}/pip-source-runtime-worker.json
|
|
${{ env.REPORTS }}/image-identity-api.json
|
|
${{ env.REPORTS }}/image-identity-web.json
|
|
${{ env.REPORTS }}/image-identity-node-agent.json
|
|
${{ env.REPORTS }}/image-identity-runtime-worker.json
|
|
${{ env.REPORTS }}/web-bundle-origin.json
|
|
${{ env.REPORTS }}/image-inventory-api.json
|
|
${{ env.REPORTS }}/image-inventory-node-agent.json
|
|
${{ env.REPORTS }}/image-inventory-runtime-worker.json
|
|
${{ env.REPORTS }}/image-requirements/api.txt
|
|
${{ env.REPORTS }}/image-requirements/node-agent.txt
|
|
${{ env.REPORTS }}/image-requirements/runtime-worker.txt
|
|
${{ env.REPORTS }}/pip-image-api.json
|
|
${{ env.REPORTS }}/pip-image-node-agent.json
|
|
${{ env.REPORTS }}/pip-image-runtime-worker.json
|
|
${{ env.REPORTS }}/trivy-image-api.json
|
|
${{ env.REPORTS }}/trivy-image-web.json
|
|
${{ env.REPORTS }}/trivy-image-node-agent.json
|
|
${{ env.REPORTS }}/trivy-image-runtime-worker.json
|
|
${{ env.REPORTS }}/status/history-fetch.exit
|
|
${{ env.REPORTS }}/status/history.exit
|
|
${{ env.REPORTS }}/status/release-version.exit
|
|
${{ env.REPORTS }}/status/pip-audit-download.exit
|
|
${{ env.REPORTS }}/status/pip-audit-hash.exit
|
|
${{ env.REPORTS }}/status/pip-audit-install.exit
|
|
${{ env.REPORTS }}/status/pip-audit-version.exit
|
|
${{ env.REPORTS }}/status/gitleaks-download.exit
|
|
${{ env.REPORTS }}/status/gitleaks-hash.exit
|
|
${{ env.REPORTS }}/status/gitleaks-extract.exit
|
|
${{ env.REPORTS }}/status/gitleaks-version.exit
|
|
${{ env.REPORTS }}/status/gitleaks-self-test.exit
|
|
${{ env.REPORTS }}/status/trivy-download.exit
|
|
${{ env.REPORTS }}/status/trivy-hash.exit
|
|
${{ env.REPORTS }}/status/trivy-extract.exit
|
|
${{ env.REPORTS }}/status/trivy-version.exit
|
|
${{ env.REPORTS }}/status/source-inventory.exit
|
|
${{ env.REPORTS }}/status/gitleaks-default.exit
|
|
${{ env.REPORTS }}/status/gitleaks-custom.exit
|
|
${{ env.REPORTS }}/status/gitleaks-merge.exit
|
|
${{ env.REPORTS }}/status/gitleaks-cleanup.exit
|
|
${{ env.REPORTS }}/status/gitleaks.exit
|
|
${{ env.REPORTS }}/status/gitleaks-sanitize.exit
|
|
${{ env.REPORTS }}/status/pip-source-backend.exit
|
|
${{ env.REPORTS }}/status/pip-source-node-agent.exit
|
|
${{ env.REPORTS }}/status/pip-source-runtime-worker.exit
|
|
${{ env.REPORTS }}/status/npm-version.exit
|
|
${{ env.REPORTS }}/status/npm-audit-all.exit
|
|
${{ env.REPORTS }}/status/npm-audit-production.exit
|
|
${{ env.REPORTS }}/status/trivy-fs.exit
|
|
${{ env.REPORTS }}/status/docker.exit
|
|
${{ env.REPORTS }}/status/image-build-api.exit
|
|
${{ env.REPORTS }}/status/image-build-web.exit
|
|
${{ env.REPORTS }}/status/image-build-node-agent.exit
|
|
${{ env.REPORTS }}/status/image-build-runtime-worker.exit
|
|
${{ env.REPORTS }}/status/image-identity-api.exit
|
|
${{ env.REPORTS }}/status/image-identity-web.exit
|
|
${{ env.REPORTS }}/status/image-identity-node-agent.exit
|
|
${{ env.REPORTS }}/status/image-identity-runtime-worker.exit
|
|
${{ env.REPORTS }}/status/web-bundle-origin.exit
|
|
${{ env.REPORTS }}/status/image-inventory-api.exit
|
|
${{ env.REPORTS }}/status/image-inventory-node-agent.exit
|
|
${{ env.REPORTS }}/status/image-inventory-runtime-worker.exit
|
|
${{ env.REPORTS }}/status/pip-image-api.exit
|
|
${{ env.REPORTS }}/status/pip-image-node-agent.exit
|
|
${{ env.REPORTS }}/status/pip-image-runtime-worker.exit
|
|
${{ env.REPORTS }}/status/trivy-image-api.exit
|
|
${{ env.REPORTS }}/status/trivy-image-web.exit
|
|
${{ env.REPORTS }}/status/trivy-image-node-agent.exit
|
|
${{ env.REPORTS }}/status/trivy-image-runtime-worker.exit
|
|
${{ env.REPORTS }}/status/image-cleanup.exit
|
|
if-no-files-found: error
|
|
retention-days: 90
|
|
|
|
# Gitea only lists dispatchable workflow files from the default branch. Reusing the dedicated
|
|
# workflow from this managed entry point lets an unmerged branch prove its exact public export on
|
|
# the server. Ordinary PR validation and every profile except an explicit `build` dispatch remain
|
|
# unchanged.
|
|
public_candidate_acceptance:
|
|
name: Public candidate server acceptance
|
|
if: ${{ gitea.event_name == 'workflow_dispatch' && inputs.profile == 'build' }}
|
|
uses: ./.gitea/workflows/public-candidate-acceptance.yml
|
|
with:
|
|
source_commit: ${{ gitea.sha }}
|
|
public_api_origin: https://modelforge.example.test
|