Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0
This commit is contained in:
Public Source Publisher committed
1 parent
7082ab955a
commit
848f0d8f8a
434 files changed
+67298
-4315
No files matched your search
+44
-9
@@ -1,6 +1,8 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
$PSNativeCommandUseErrorActionPreference = $true
|
||||
|
||||
python scripts/rc_readiness.py
|
||||
|
||||
Push-Location backend
|
||||
try {
|
||||
python -m compileall -q src tests
|
||||
@@ -19,6 +21,8 @@ try {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
node --test clients/typescript/modelforge-client.test.mjs
|
||||
|
||||
Push-Location node-agent
|
||||
try {
|
||||
python -m compileall -q src tests
|
||||
@@ -39,17 +43,48 @@ try {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
docker compose --env-file .env.example config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.gpu.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.runtime-worker.yml config --quiet
|
||||
$env:MODELFORGE_AGENT_CA_CERT_PATH = "./validation-ca.crt"
|
||||
$env:MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS = "192.0.2.10"
|
||||
$composeValidationEnvironment = [ordered]@{
|
||||
MODELFORGE_POSTGRES_DB = "modelforge"
|
||||
VITE_API_BASE_URL = "http://127.0.0.1:8000"
|
||||
MODELFORGE_CORS_ORIGINS = "http://127.0.0.1:3000"
|
||||
MODELFORGE_VERSION = (Get-Content VERSION -Raw).Trim()
|
||||
MODELFORGE_OPERATOR_API_KEY = "compose-validation-operator-not-for-production"
|
||||
MODELFORGE_BACKUP_ENCRYPTION_KEY = "compose-validation-backup-not-for-production"
|
||||
MODELFORGE_POSTGRES_ADMIN_PASSWORD = "compose-validation-admin"
|
||||
MODELFORGE_MIGRATION_DB_PASSWORD = "compose-validation-owner"
|
||||
MODELFORGE_RUNTIME_DB_PASSWORD = "compose-validation-runtime"
|
||||
MODELFORGE_RUNTIME_DATABASE_URL = "postgresql+psycopg://modelforge_runtime:compose-validation-runtime@postgres:5432/modelforge"
|
||||
MODELFORGE_MIGRATION_DATABASE_URL = "postgresql+psycopg://modelforge:compose-validation-owner@postgres:5432/modelforge"
|
||||
MODELFORGE_AGENT_CA_CERT_PATH = "./validation-ca.crt"
|
||||
MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS = "192.0.2.10"
|
||||
}
|
||||
$previousComposeEnvironment = @{}
|
||||
foreach ($name in $composeValidationEnvironment.Keys) {
|
||||
$previousComposeEnvironment[$name] = [Environment]::GetEnvironmentVariable($name, "Process")
|
||||
[Environment]::SetEnvironmentVariable(
|
||||
$name,
|
||||
$composeValidationEnvironment[$name],
|
||||
"Process"
|
||||
)
|
||||
}
|
||||
try {
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.production.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.backup.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.dr.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.node-recovery.yml config --quiet
|
||||
docker compose --env-file .env.example config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.gpu.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.runtime-worker.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.node-agent.private-ca.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml -f docker-compose.node-agent.private-ca.yml -f docker-compose.runtime-worker.yml -f docker-compose.runtime-worker.private-ca.yml config --quiet
|
||||
} finally {
|
||||
Remove-Item Env:MODELFORGE_AGENT_CA_CERT_PATH
|
||||
Remove-Item Env:MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS
|
||||
foreach ($name in $composeValidationEnvironment.Keys) {
|
||||
[Environment]::SetEnvironmentVariable(
|
||||
$name,
|
||||
$previousComposeEnvironment[$name],
|
||||
"Process"
|
||||
)
|
||||
}
|
||||
}
|
||||
Write-Host "M5 checks passed."
|
||||
Write-Host "ModelForge component and Compose checks passed."
|
||||
+21
-1
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
python scripts/rc_readiness.py
|
||||
|
||||
(cd backend && python -m compileall -q src tests)
|
||||
(cd backend && python -m ruff check src tests)
|
||||
(cd backend && python -m mypy src)
|
||||
@@ -15,6 +17,7 @@ set -eu
|
||||
(cd runtime-worker && python -m pytest -q)
|
||||
(cd frontend && npm test -- --run)
|
||||
(cd frontend && npm run build)
|
||||
node --test clients/typescript/modelforge-client.test.mjs
|
||||
# The console's API origin is compiled into the bundle, so a release image can be wrong while every
|
||||
# source-level check is green. Only assert this when a candidate image already exists locally.
|
||||
if docker image inspect "modelforge-web:$(cat VERSION)" >/dev/null 2>&1; then
|
||||
@@ -25,6 +28,22 @@ if docker image inspect "modelforge-web:$(cat VERSION)" >/dev/null 2>&1; then
|
||||
else
|
||||
printf '%s\n' "console release-image acceptance skipped: no modelforge-web:$(cat VERSION) built"
|
||||
fi
|
||||
# Render only with synthetic inputs, never the operator's live credentials.
|
||||
(
|
||||
export MODELFORGE_POSTGRES_DB=modelforge
|
||||
export VITE_API_BASE_URL=http://127.0.0.1:8000
|
||||
export MODELFORGE_CORS_ORIGINS=http://127.0.0.1:3000
|
||||
export MODELFORGE_VERSION="$(cat VERSION)"
|
||||
export MODELFORGE_OPERATOR_API_KEY=compose-validation-operator-not-for-production
|
||||
export MODELFORGE_BACKUP_ENCRYPTION_KEY=compose-validation-backup-not-for-production
|
||||
export MODELFORGE_POSTGRES_ADMIN_PASSWORD=compose-validation-admin
|
||||
export MODELFORGE_MIGRATION_DB_PASSWORD=compose-validation-owner
|
||||
export MODELFORGE_RUNTIME_DB_PASSWORD=compose-validation-runtime
|
||||
export MODELFORGE_RUNTIME_DATABASE_URL=postgresql+psycopg://modelforge_runtime:compose-validation-runtime@postgres:5432/modelforge
|
||||
export MODELFORGE_MIGRATION_DATABASE_URL=postgresql+psycopg://modelforge:compose-validation-owner@postgres:5432/modelforge
|
||||
for overlay in production backup dr node-recovery; do
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f "docker-compose.${overlay}.yml" config --quiet
|
||||
done
|
||||
docker compose --env-file .env.example config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.yml -f docker-compose.gpu.yml config --quiet
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml config --quiet
|
||||
@@ -39,4 +58,5 @@ MODELFORGE_AGENT_CONTROL_PLANE_HOST_ADDRESS=192.0.2.10 \
|
||||
docker compose --env-file .env.example -f docker-compose.node-agent.yml \
|
||||
-f docker-compose.node-agent.private-ca.yml -f docker-compose.runtime-worker.yml \
|
||||
-f docker-compose.runtime-worker.private-ca.yml config --quiet
|
||||
printf '%s\n' 'M5 checks passed.'
|
||||
)
|
||||
printf '%s\n' 'ModelForge component and Compose checks passed.'
|
||||
@@ -24,6 +24,7 @@ if (!outputArgument) fail("Usage: node scripts/export-public-source.mjs --output
|
||||
|
||||
const output = path.resolve(outputArgument);
|
||||
const reportPath = reportArgument ? path.resolve(reportArgument) : undefined;
|
||||
const publicSourceRepository = "https://git.example.com/example/modelforge";
|
||||
if (output === root || output.startsWith(`${root}${path.sep}`)) {
|
||||
fail("The public export must be outside the canonical repository.");
|
||||
}
|
||||
@@ -55,6 +56,16 @@ const deniedPrefixes = [
|
||||
"reports/"
|
||||
];
|
||||
|
||||
// This owner-specific inventory collector is used only by the private portfolio audit.
|
||||
// It is not part of the product, release archive, or public source contract.
|
||||
const privateOnlyPaths = new Set([
|
||||
".gitea/workflows/public-source-sync.yml",
|
||||
"scripts/portfolio_gitea_snapshot.py",
|
||||
"scripts/portfolio_gitea_repository_ids.json",
|
||||
"backend/tests/test_portfolio_gitea_snapshot.py",
|
||||
"docs/operations/PORTFOLIO_GITEA_SNAPSHOT.md"
|
||||
]);
|
||||
|
||||
// The canonical source contains useful integration examples and operational tests tied to the
|
||||
// private deployment. The public root keeps the behavior while replacing those identifiers with
|
||||
// stable documentation-only examples. Transform before scanning and before hashing the export.
|
||||
@@ -119,7 +130,9 @@ const tracked = execFileSync("git", ["-C", root, "ls-files", "-z"], { encoding:
|
||||
.split("\0")
|
||||
.filter(Boolean)
|
||||
.map((entry) => entry.replaceAll("\\", "/"));
|
||||
const files = tracked.filter(selected).sort((left, right) => left.localeCompare(right, "en"));
|
||||
const files = tracked
|
||||
.filter((relativePath) => selected(relativePath) && !privateOnlyPaths.has(relativePath))
|
||||
.sort((left, right) => left.localeCompare(right, "en"));
|
||||
if (files.length === 0) fail("The allowlist selected no tracked files.");
|
||||
|
||||
const findings = [];
|
||||
@@ -210,5 +223,10 @@ fs.writeFileSync(path.join(output, "PUBLIC_SOURCE_EXPORT.md"),
|
||||
"This parentless candidate excludes private operational history and uses synthetic example identifiers.\n",
|
||||
"utf8");
|
||||
fs.writeFileSync(path.join(output, "PUBLIC_SOURCE_MANIFEST.json"),
|
||||
`${JSON.stringify({ schemaVersion: 1, sourceRevision, files: manifest }, null, 2)}\n`, "utf8");
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
sourceRevision,
|
||||
sourceRepository: publicSourceRepository,
|
||||
files: manifest
|
||||
}, null, 2)}\n`, "utf8");
|
||||
console.log(`Exported ${files.length} reviewed files to ${output}`);
|
||||
@@ -237,8 +237,11 @@ def render_configuration_doc() -> str:
|
||||
"",
|
||||
"## Restarts",
|
||||
"",
|
||||
"Every setting is read at process start. Changing any of them requires restarting the "
|
||||
"control plane; none is re-read from the environment while the process is running.",
|
||||
"Deployment environment settings are read at process start; changing an environment value "
|
||||
"requires a restart. The Console also supports a closed set of numeric serving overrides "
|
||||
"which take effect for new requests after preview and explicit apply. See the "
|
||||
"[configuration workflow](operations/CONFIGURATION_WORKFLOW.md) for scope, acknowledgement, "
|
||||
"rollback and secret-safe transfer. Other settings remain deployment-managed.",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -12,7 +12,8 @@
|
||||
set -eu
|
||||
|
||||
BASE_URL="${MODELFORGE_BASE_URL:-http://api:8000}"
|
||||
TOKEN="${MODELFORGE_OPERATOR_API_KEY:?operator API key is required for scheduled backups}"
|
||||
TOKEN="${MODELFORGE_BACKUP_ADMIN_API_KEY:-${MODELFORGE_OPERATOR_API_KEY:-}}"
|
||||
: "${TOKEN:?MODELFORGE_BACKUP_ADMIN_API_KEY or MODELFORGE_OPERATOR_API_KEY is required for scheduled backups}"
|
||||
PREFIX="${MODELFORGE_BACKUP_ID_PREFIX:-scheduled}"
|
||||
STAMP="$(date -u +%Y%m%d-%H%M)"
|
||||
BACKUP_ID="${PREFIX}-${STAMP}"
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Generate a human key once; deployment configuration contains only its digest.
|
||||
|
||||
Run locally in a private terminal. Do not redirect the complete output into deployment config.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--subject", required=True)
|
||||
parser.add_argument("--role", choices=("viewer", "operator", "admin"), required=True)
|
||||
parser.add_argument("--days", type=int, default=30)
|
||||
args = parser.parse_args()
|
||||
if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._@-]{0,127}", args.subject):
|
||||
parser.error("subject must be 1-128 letters, digits, dots, underscores, @ or hyphens")
|
||||
if args.subject == "legacy-operator" or not 1 <= args.days <= 365:
|
||||
parser.error("reserved subject or expiry outside 1-365 days")
|
||||
token = "mfop_" + secrets.token_urlsafe(32)
|
||||
record = {
|
||||
"credential_id": str(uuid.uuid4()),
|
||||
"subject": args.subject,
|
||||
"role": args.role,
|
||||
"token_sha256": hashlib.sha256(token.encode()).hexdigest(),
|
||||
"expires_at": (datetime.now(UTC) + timedelta(days=args.days)).isoformat(),
|
||||
}
|
||||
print(json.dumps({"one_time_key": token, "deployment_record": record}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Produce a salted password verifier without putting plaintext in command arguments."""
|
||||
|
||||
import getpass
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "backend" / "src"))
|
||||
from modelforge_api.services.operator_password import hash_password
|
||||
|
||||
if __name__ == "__main__":
|
||||
password = (
|
||||
getpass.getpass("Operator password: ")
|
||||
if sys.stdin.isatty()
|
||||
else sys.stdin.readline(4098).rstrip("\r\n")
|
||||
)
|
||||
try:
|
||||
print(hash_password(password))
|
||||
except (ValueError, UnicodeError):
|
||||
print("Invalid password: enter 1 to 1024 characters.", file=sys.stderr)
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,388 @@
|
||||
"""Fail closed when an RC Compose plan can overlap a protected installation.
|
||||
|
||||
The public-candidate workflow proves a disposable API/Web/Database stack. P12 adds a
|
||||
GPU-capable Node Agent and Runtime Worker, which makes an accidental shared volume, host
|
||||
network, port, or resource name materially more dangerous. This helper validates the rendered
|
||||
Compose JSON and the resources observed after startup. It deliberately accepts no implicit
|
||||
production defaults: callers must provide the candidate root and every protected port/name they
|
||||
want to keep outside the validation project namespace.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROJECT_PATTERN = re.compile(
|
||||
r"modelforge-validation-[a-z0-9](?:[a-z0-9-]{0,39}[a-z0-9])?\Z"
|
||||
)
|
||||
VOLUME_NAME_PATTERN = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*")
|
||||
CONFIG_TARGET = "/app/config"
|
||||
CONFIG_MANIFEST = "models/initial-candidates.yaml"
|
||||
|
||||
|
||||
class IsolationError(ValueError):
|
||||
"""A candidate can share state or reach outside its owned RC namespace."""
|
||||
|
||||
|
||||
def _mapping(value: object, label: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping):
|
||||
raise IsolationError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _sequence(value: object, label: str) -> Sequence[Any]:
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)):
|
||||
raise IsolationError(f"{label} must be a list")
|
||||
return value
|
||||
|
||||
|
||||
def validate_project(project: str) -> str:
|
||||
if PROJECT_PATTERN.fullmatch(project) is None:
|
||||
raise IsolationError(
|
||||
"validation project must use a bounded modelforge-validation-* namespace"
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def _is_relative_to(path: Path, root: Path) -> bool:
|
||||
try:
|
||||
path.relative_to(root)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _published_port(port: Mapping[str, Any], service_name: str, public_tls_port: int | None) -> int | None:
|
||||
published = port.get("published")
|
||||
if published in (None, ""):
|
||||
return None
|
||||
try:
|
||||
result = int(str(published))
|
||||
except ValueError as exc:
|
||||
raise IsolationError(f"{service_name} has an invalid published port") from exc
|
||||
if not 0 <= result <= 65535:
|
||||
raise IsolationError(f"{service_name} has an invalid published port")
|
||||
host_ip = str(port.get("host_ip") or "")
|
||||
if host_ip not in {"127.0.0.1", "::1"} and not (
|
||||
service_name == "tls"
|
||||
and public_tls_port == result == 18444
|
||||
and host_ip == "0.0.0.0"
|
||||
and str(port.get("target")) == "8443"
|
||||
and port.get("protocol", "tcp") == "tcp"
|
||||
):
|
||||
raise IsolationError(f"{service_name} publishes a non-loopback port")
|
||||
return result
|
||||
|
||||
|
||||
def _validate_service_mount(
|
||||
mount: Mapping[str, Any], *, service_name: str, candidate_root: Path, declared_volumes: set[str]
|
||||
) -> None:
|
||||
mount_type = mount.get("type")
|
||||
source = mount.get("source")
|
||||
target = str(mount.get("target") or "")
|
||||
if target == "/var/run/docker.sock":
|
||||
raise IsolationError(f"{service_name} mounts the Docker socket")
|
||||
if mount_type == "bind":
|
||||
if not isinstance(source, str) or not source:
|
||||
raise IsolationError(f"{service_name} has an invalid bind source")
|
||||
resolved = Path(source)
|
||||
if not resolved.is_absolute():
|
||||
resolved = candidate_root / resolved
|
||||
try:
|
||||
resolved = resolved.resolve()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
raise IsolationError(f"{service_name} has an invalid bind source") from exc
|
||||
if not _is_relative_to(resolved, candidate_root):
|
||||
raise IsolationError(f"{service_name} bind source escapes the candidate root")
|
||||
try:
|
||||
resolved = resolved.resolve(strict=True)
|
||||
except FileNotFoundError as exc:
|
||||
raise IsolationError(f"{service_name} has a missing bind source") from exc
|
||||
except (OSError, RuntimeError) as exc:
|
||||
raise IsolationError(f"{service_name} has an invalid bind source") from exc
|
||||
if not _is_relative_to(resolved, candidate_root):
|
||||
raise IsolationError(f"{service_name} bind source escapes the candidate root")
|
||||
if mount.get("read_only") is not True:
|
||||
raise IsolationError(f"{service_name} has a writable bind mount")
|
||||
if target == CONFIG_TARGET:
|
||||
manifest = resolved / CONFIG_MANIFEST
|
||||
try:
|
||||
resolved_manifest = manifest.resolve(strict=True)
|
||||
except FileNotFoundError as exc:
|
||||
raise IsolationError(
|
||||
f"{service_name} {CONFIG_TARGET} bind is missing {CONFIG_MANIFEST} under its source"
|
||||
) from exc
|
||||
except (OSError, RuntimeError) as exc:
|
||||
raise IsolationError(
|
||||
f"{service_name} {CONFIG_TARGET} bind has an invalid {CONFIG_MANIFEST}"
|
||||
) from exc
|
||||
if not _is_relative_to(resolved_manifest, resolved) or not resolved_manifest.is_file():
|
||||
raise IsolationError(
|
||||
f"{service_name} {CONFIG_TARGET} bind is missing {CONFIG_MANIFEST} under its source"
|
||||
)
|
||||
return
|
||||
if mount_type == "volume":
|
||||
if not isinstance(source, str) or VOLUME_NAME_PATTERN.fullmatch(source) is None:
|
||||
raise IsolationError(f"{service_name} has an invalid named volume")
|
||||
if source not in declared_volumes:
|
||||
raise IsolationError(f"{service_name} uses an undeclared volume {source}")
|
||||
return
|
||||
if mount_type in {"tmpfs", "npipe"}:
|
||||
return
|
||||
raise IsolationError(f"{service_name} has unsupported mount type {mount_type!r}")
|
||||
|
||||
|
||||
def validate_compose(
|
||||
document: Mapping[str, Any],
|
||||
*,
|
||||
project: str,
|
||||
candidate_root: Path,
|
||||
protected_ports: set[int],
|
||||
protected_resource_names: set[str],
|
||||
public_tls_port: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Validate a rendered Compose document before any RC container starts."""
|
||||
|
||||
validate_project(project)
|
||||
candidate_root = candidate_root.resolve()
|
||||
services = _mapping(document.get("services"), "services")
|
||||
if not services:
|
||||
raise IsolationError("Compose plan has no services")
|
||||
top_volumes = _mapping(document.get("volumes", {}), "volumes")
|
||||
top_networks = _mapping(document.get("networks", {}), "networks")
|
||||
declared_volumes = set(top_volumes)
|
||||
declared_networks = set(top_networks)
|
||||
if not declared_volumes or not declared_networks:
|
||||
raise IsolationError("validation plan requires its own declared volumes and network")
|
||||
if public_tls_port is not None and public_tls_port != 18444:
|
||||
raise IsolationError("the only admitted public TLS test port is 18444")
|
||||
if public_tls_port is not None and "tls" not in services:
|
||||
raise IsolationError("public TLS test port requires the isolated tls service")
|
||||
|
||||
for name, configuration in top_volumes.items():
|
||||
if VOLUME_NAME_PATTERN.fullmatch(name) is None:
|
||||
raise IsolationError(f"invalid declared volume {name!r}")
|
||||
config = _mapping(configuration, f"volume {name}")
|
||||
if config.get("external"):
|
||||
raise IsolationError(f"volume {name} is external")
|
||||
actual_name = config.get("name")
|
||||
if actual_name is not None and actual_name != f"{project}_{name}":
|
||||
raise IsolationError(f"volume {name} has a non-project name")
|
||||
|
||||
for name, configuration in top_networks.items():
|
||||
if VOLUME_NAME_PATTERN.fullmatch(name) is None:
|
||||
raise IsolationError(f"invalid declared network {name!r}")
|
||||
config = _mapping(configuration, f"network {name}")
|
||||
if config.get("external"):
|
||||
raise IsolationError(f"network {name} is external")
|
||||
actual_name = config.get("name")
|
||||
if actual_name is not None and actual_name != f"{project}_{name}":
|
||||
raise IsolationError(f"network {name} has a non-project name")
|
||||
|
||||
published_ports: set[int] = set()
|
||||
for name, configuration in services.items():
|
||||
service = _mapping(configuration, f"service {name}")
|
||||
if service.get("container_name"):
|
||||
raise IsolationError(f"{name} declares a global container name")
|
||||
if service.get("network_mode") == "host":
|
||||
raise IsolationError(f"{name} uses the host network")
|
||||
configured_networks = service.get("networks", [])
|
||||
if isinstance(configured_networks, Mapping):
|
||||
service_networks: Sequence[Any] = list(configured_networks)
|
||||
else:
|
||||
service_networks = _sequence(configured_networks, f"{name}.networks")
|
||||
network_names = []
|
||||
for network in service_networks:
|
||||
network_name = network if isinstance(network, str) else _mapping(network, f"{name}.network").get("target")
|
||||
if network_name not in declared_networks:
|
||||
raise IsolationError(f"{name} uses an undeclared network")
|
||||
network_names.append(network_name)
|
||||
if name == "tls" and public_tls_port is not None:
|
||||
if set(network_names) != {"ingress"}:
|
||||
raise IsolationError("tls service must join only the candidate ingress network")
|
||||
if service.get("image") != "caddy:2.10-alpine@sha256:4c6e91c6ed0e2fa03efd5b44747b625fec79bc9cd06ac5235a779726618e530d":
|
||||
raise IsolationError("tls service must use the pinned Caddy image")
|
||||
if service.get("read_only") is not True or service.get("cap_drop") != ["ALL"]:
|
||||
raise IsolationError("tls service must have read-only root and no capabilities")
|
||||
if service.get("privileged") or service.get("cap_add") or service.get("build"):
|
||||
raise IsolationError("tls service cannot be privileged or locally built")
|
||||
if "no-new-privileges:true" not in _sequence(service.get("security_opt", []), "tls.security_opt"):
|
||||
raise IsolationError("tls service must refuse new privileges")
|
||||
mounts = _sequence(service.get("volumes", []), "tls.volumes")
|
||||
targets = {str(_mapping(mount, "tls.volume").get("target")) for mount in mounts}
|
||||
if targets != {"/etc/caddy/Caddyfile", "/run/tls/fullchain.pem", "/run/tls/privkey.pem"} or len(mounts) != 3:
|
||||
raise IsolationError("tls service must mount only its config and certificate pair")
|
||||
tls_ports = _sequence(service.get("ports", []), "tls.ports")
|
||||
if len(tls_ports) != 1:
|
||||
raise IsolationError("tls service must publish exactly one port")
|
||||
tls_port = _mapping(tls_ports[0], "tls.port")
|
||||
if str(tls_port.get("published")) != "18444" or tls_port.get("host_ip") != "0.0.0.0":
|
||||
raise IsolationError("tls service must bind only public test port 18444")
|
||||
for mount in _sequence(service.get("volumes", []), f"{name}.volumes"):
|
||||
_validate_service_mount(
|
||||
_mapping(mount, f"{name}.volume"),
|
||||
service_name=name,
|
||||
candidate_root=candidate_root,
|
||||
declared_volumes=declared_volumes,
|
||||
)
|
||||
for port in _sequence(service.get("ports", []), f"{name}.ports"):
|
||||
published = _published_port(_mapping(port, f"{name}.port"), name, public_tls_port)
|
||||
if published and published in protected_ports:
|
||||
raise IsolationError(f"{name} reuses protected host port {published}")
|
||||
if published:
|
||||
published_ports.add(published)
|
||||
|
||||
serialized = json.dumps(document, sort_keys=True)
|
||||
referenced = sorted(name for name in protected_resource_names if name and name in serialized)
|
||||
if referenced:
|
||||
raise IsolationError(f"Compose plan references protected resources: {', '.join(referenced)}")
|
||||
return {
|
||||
"project": project,
|
||||
"services": sorted(services),
|
||||
"declared_volumes": sorted(declared_volumes),
|
||||
"declared_networks": sorted(declared_networks),
|
||||
"published_ports": sorted(published_ports),
|
||||
}
|
||||
|
||||
|
||||
def validate_observed_resources(
|
||||
document: Mapping[str, Any], *, project: str
|
||||
) -> dict[str, list[str]]:
|
||||
"""Ensure resources created by Compose retain the exact validation project label and name."""
|
||||
|
||||
validate_project(project)
|
||||
expected_label = project
|
||||
prefixes = {"containers": f"{project}-", "volumes": f"{project}_", "networks": f"{project}_"}
|
||||
result: dict[str, list[str]] = {}
|
||||
for kind, prefix in prefixes.items():
|
||||
items = _sequence(document.get(kind), kind)
|
||||
if not items:
|
||||
raise IsolationError(f"no observed {kind} for {project}")
|
||||
names: list[str] = []
|
||||
for item in items:
|
||||
resource = _mapping(item, kind)
|
||||
name = resource.get("name")
|
||||
labels = _mapping(resource.get("labels"), f"{kind}.labels")
|
||||
if not isinstance(name, str) or not name.startswith(prefix):
|
||||
raise IsolationError(f"{kind} resource has a non-project name")
|
||||
if labels.get("com.docker.compose.project") != expected_label:
|
||||
raise IsolationError(f"{kind} resource has no exact project label")
|
||||
names.append(name)
|
||||
result[kind] = sorted(names)
|
||||
return result
|
||||
|
||||
|
||||
def _docker(*args: str) -> str:
|
||||
completed = subprocess.run(
|
||||
("docker", *args),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
if completed.returncode:
|
||||
raise IsolationError(f"Docker inspection failed: {' '.join(args)}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def collect_observed_resources(project: str) -> dict[str, list[dict[str, object]]]:
|
||||
"""Collect only resources carrying the exact Compose project label.
|
||||
|
||||
``docker inspect`` is used instead of parsing ``docker * ls`` label strings: a label value can
|
||||
itself contain commas or equals signs, and P12 must not mistake a truncated display value for
|
||||
resource ownership evidence.
|
||||
"""
|
||||
|
||||
validate_project(project)
|
||||
kinds = {
|
||||
"containers": ("container", ("ls", "--all", "--quiet")),
|
||||
"volumes": ("volume", ("ls", "--quiet")),
|
||||
"networks": ("network", ("ls", "--quiet")),
|
||||
}
|
||||
result: dict[str, list[dict[str, object]]] = {}
|
||||
for plural, (singular, list_args) in kinds.items():
|
||||
identifiers = [
|
||||
value for value in _docker(
|
||||
singular, *list_args, "--filter", f"label=com.docker.compose.project={project}"
|
||||
).splitlines() if value
|
||||
]
|
||||
resources: list[dict[str, object]] = []
|
||||
for identifier in identifiers:
|
||||
try:
|
||||
inspected = json.loads(_docker(singular, "inspect", identifier))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise IsolationError(f"Docker returned invalid {singular} inspection JSON") from exc
|
||||
if not isinstance(inspected, list) or len(inspected) != 1 or not isinstance(inspected[0], Mapping):
|
||||
raise IsolationError(f"Docker returned ambiguous {singular} inspection data")
|
||||
item = inspected[0]
|
||||
labels = item.get("Labels")
|
||||
if singular == "container":
|
||||
config = _mapping(item.get("Config"), "container.Config")
|
||||
labels = config.get("Labels")
|
||||
resources.append(
|
||||
{
|
||||
"name": str(item.get("Name") or "").lstrip("/"),
|
||||
"labels": dict(_mapping(labels or {}, f"{singular}.Labels")),
|
||||
}
|
||||
)
|
||||
result[plural] = resources
|
||||
return result
|
||||
|
||||
|
||||
def _read_json(path: Path) -> Mapping[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise IsolationError(f"cannot read JSON evidence {path}") from exc
|
||||
return _mapping(value, str(path))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--project", required=True)
|
||||
parser.add_argument("--candidate-root", required=True, type=Path)
|
||||
parser.add_argument("--compose-json", required=True, type=Path)
|
||||
parser.add_argument("--observed-json", type=Path)
|
||||
parser.add_argument("--inspect-docker", action="store_true")
|
||||
parser.add_argument("--protected-port", action="append", default=[], type=int)
|
||||
parser.add_argument("--public-tls-port", type=int)
|
||||
parser.add_argument("--protected-resource", action="append", default=[])
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.observed_json is not None and args.inspect_docker:
|
||||
raise IsolationError("choose observed JSON or Docker inspection, not both")
|
||||
result: dict[str, object] = {
|
||||
"preflight": validate_compose(
|
||||
_read_json(args.compose_json),
|
||||
project=args.project,
|
||||
candidate_root=args.candidate_root,
|
||||
protected_ports=set(args.protected_port),
|
||||
protected_resource_names=set(args.protected_resource),
|
||||
public_tls_port=args.public_tls_port,
|
||||
)
|
||||
}
|
||||
if args.observed_json is not None:
|
||||
result["observed"] = validate_observed_resources(
|
||||
_read_json(args.observed_json), project=args.project
|
||||
)
|
||||
elif args.inspect_docker:
|
||||
result["observed"] = validate_observed_resources(
|
||||
collect_observed_resources(args.project), project=args.project
|
||||
)
|
||||
except IsolationError as exc:
|
||||
print(json.dumps({"result": "FAIL", "error": str(exc)}, sort_keys=True))
|
||||
return 2
|
||||
print(json.dumps({"result": "PASS", **result}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Read-only RC plan/ledger consistency checks; not artifact qualification."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIG = "config/release/rc-program.json"
|
||||
LEDGER = "docs/quality/RC_GAP_MATRIX.md"
|
||||
ROADMAP = "docs/product/RC_EXECUTION_ROADMAP.md"
|
||||
GAP_ID = re.compile(r"RC-[A-Z]+-[0-9]{2}")
|
||||
PACKET_ID = re.compile(r"P[0-9]{2}[A-Z]?")
|
||||
CLASSIFICATIONS = {
|
||||
"COMPLETE", "NEEDS POLISH", "FUNCTIONALLY INCOMPLETE", "UX INCOMPLETE",
|
||||
"OPERATIONALLY INCOMPLETE", "SECURITY BLOCKER", "RC BLOCKER", "POST-RC",
|
||||
}
|
||||
STATUSES = {"OPEN", "IN PROGRESS", "RESOLVED", "VERIFIED", "DEFERRED"}
|
||||
|
||||
|
||||
class InvalidPlan(ValueError):
|
||||
"""The plan cannot safely be interpreted."""
|
||||
|
||||
|
||||
def _object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise InvalidPlan(f"duplicate JSON field: {key}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _read(root: Path, relative: str) -> str:
|
||||
path = (root / relative).resolve()
|
||||
if not path.is_relative_to(root.resolve()):
|
||||
raise InvalidPlan(f"repository path escapes root: {relative}")
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _keys(value: Any, expected: set[str], label: str) -> None:
|
||||
if not isinstance(value, dict) or set(value) != expected:
|
||||
raise InvalidPlan(f"invalid {label} fields")
|
||||
|
||||
|
||||
def _ids(value: Any, pattern: re.Pattern[str], label: str) -> list[str]:
|
||||
if not isinstance(value, list) or any(
|
||||
not isinstance(item, str) or pattern.fullmatch(item) is None for item in value
|
||||
):
|
||||
raise InvalidPlan(f"invalid {label} IDs")
|
||||
if len(set(value)) != len(value):
|
||||
raise InvalidPlan(f"duplicate {label} IDs")
|
||||
return value
|
||||
|
||||
|
||||
def validate(root: Path = ROOT) -> dict[str, Any]:
|
||||
plan = json.loads(_read(root, CONFIG), object_pairs_hook=_object)
|
||||
_keys(plan, {"schema_version", "target_version", "baseline_commit", "ledger", "roadmap",
|
||||
"packets", "deferred_gaps"}, "configuration")
|
||||
if type(plan["schema_version"]) is not int or plan["schema_version"] != 1:
|
||||
raise InvalidPlan("unsupported schema_version")
|
||||
if not isinstance(plan["target_version"], str) or not re.fullmatch(
|
||||
r"[0-9]+\.[0-9]+\.[0-9]+-rc\.[1-9][0-9]*", plan["target_version"]
|
||||
):
|
||||
raise InvalidPlan("invalid target_version")
|
||||
if not isinstance(plan["baseline_commit"], str) or not re.fullmatch(
|
||||
r"[0-9a-f]{40}", plan["baseline_commit"]
|
||||
):
|
||||
raise InvalidPlan("invalid baseline_commit")
|
||||
if plan["ledger"] != LEDGER or plan["roadmap"] != ROADMAP:
|
||||
raise InvalidPlan("ledger and roadmap must use the fixed repository paths")
|
||||
if not _read(root, ROADMAP).strip():
|
||||
raise InvalidPlan("roadmap is empty")
|
||||
rows: dict[str, tuple[str, str]] = {}
|
||||
for line in _read(root, LEDGER).splitlines():
|
||||
if not line.startswith("| RC-"):
|
||||
continue
|
||||
cells = [cell.strip() for cell in re.split(r"(?<!\\)\|", line)[1:-1]]
|
||||
if not line.endswith("|") or len(cells) != 7 or not GAP_ID.fullmatch(cells[0]):
|
||||
raise InvalidPlan("malformed RC ledger row")
|
||||
gap, _, _, classification, _, _, status = cells
|
||||
if gap in rows:
|
||||
raise InvalidPlan(f"duplicate ledger ID: {gap}")
|
||||
if classification not in CLASSIFICATIONS:
|
||||
raise InvalidPlan(f"unknown classification: {gap}")
|
||||
if status.startswith("RESOLVED — ") and status.removeprefix("RESOLVED — ").strip():
|
||||
status = "RESOLVED"
|
||||
if status not in STATUSES:
|
||||
raise InvalidPlan(f"unknown status: {gap}")
|
||||
if (status == "DEFERRED") != (classification == "POST-RC"):
|
||||
raise InvalidPlan(f"invalid deferral: {gap}")
|
||||
rows[gap] = classification, status
|
||||
if not rows:
|
||||
raise InvalidPlan("ledger has no RC issue rows")
|
||||
packets = plan["packets"]
|
||||
if not isinstance(packets, list) or not packets:
|
||||
raise InvalidPlan("packets must be a nonempty list")
|
||||
dependencies: dict[str, list[str]] = {}
|
||||
owners: dict[str, str] = {}
|
||||
for packet in packets:
|
||||
_keys(packet, {"id", "depends_on", "gaps"}, "packet")
|
||||
packet_id = packet["id"]
|
||||
_ids([packet_id], PACKET_ID, "packet")
|
||||
if packet_id in dependencies:
|
||||
raise InvalidPlan(f"duplicate packet ID: {packet_id}")
|
||||
dependencies[packet_id] = _ids(packet["depends_on"], PACKET_ID, "dependency")
|
||||
gaps = _ids(packet["gaps"], GAP_ID, "gap")
|
||||
if not gaps:
|
||||
raise InvalidPlan(f"packet has no gaps: {packet_id}")
|
||||
for gap in gaps:
|
||||
if gap in owners:
|
||||
raise InvalidPlan(f"duplicate gap ownership: {gap}")
|
||||
owners[gap] = packet_id
|
||||
deferred = set(_ids(plan["deferred_gaps"], GAP_ID, "deferred gap"))
|
||||
if deferred & owners.keys():
|
||||
raise InvalidPlan("deferred gaps also assigned to packets")
|
||||
if set(rows) != owners.keys() | deferred:
|
||||
raise InvalidPlan("ledger coverage mismatch: unknown or unmapped gaps")
|
||||
if deferred != {gap for gap, (kind, _) in rows.items() if kind == "POST-RC"}:
|
||||
raise InvalidPlan("only POST-RC rows may be deferred")
|
||||
order: list[str] = []
|
||||
for packet_id, refs in dependencies.items():
|
||||
if set(refs) - dependencies.keys():
|
||||
raise InvalidPlan(f"unknown dependencies: {packet_id}")
|
||||
remaining = dict(dependencies)
|
||||
while remaining:
|
||||
available = sorted(key for key, refs in remaining.items() if set(refs) <= set(order))
|
||||
if not available:
|
||||
raise InvalidPlan("packet dependency cycle")
|
||||
order.extend(available)
|
||||
for key in available:
|
||||
del remaining[key]
|
||||
outstanding = sorted(gap for gap in owners if rows[gap][1] != "VERIFIED")
|
||||
return {
|
||||
"packet_count": len(packets), "required_gap_count": len(owners),
|
||||
"verified_count": len(owners) - len(outstanding), "outstanding_ids": outstanding,
|
||||
"topological_order": order, "all_required_rows_verified": not outstanding,
|
||||
"scope": "ledger consistency only; artifact qualification is separate",
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--require-verified", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
result = validate()
|
||||
except (ValueError, OSError) as exc:
|
||||
print(json.dumps({"error": str(exc), "all_required_rows_verified": False}))
|
||||
return 2
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
return int(args.require_verified and not result["all_required_rows_verified"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+174
-102
@@ -9,14 +9,16 @@ import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
IMAGES = (
|
||||
"modelforge-api",
|
||||
@@ -24,11 +26,36 @@ IMAGES = (
|
||||
"modelforge-node-agent",
|
||||
"modelforge-runtime-worker",
|
||||
)
|
||||
ACCEPTANCE_SERVICES = ("postgres", "redis", "api", "migrate", "web")
|
||||
CONFIG_SEED_IMAGE = (
|
||||
"redis:7-alpine@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"
|
||||
)
|
||||
|
||||
|
||||
def write_acceptance_compose_override(path: Path) -> None:
|
||||
"""Write the disposable RC service policy used by acceptance."""
|
||||
|
||||
services = []
|
||||
for service in ACCEPTANCE_SERVICES:
|
||||
services.append(f' {service}:\n restart: "no"\n')
|
||||
if service == "api":
|
||||
services.append(
|
||||
" volumes:\n"
|
||||
" - type: volume\n"
|
||||
" source: acceptance-config\n"
|
||||
" target: /app/config\n"
|
||||
" read_only: true\n"
|
||||
)
|
||||
path.write_text(
|
||||
"services:\n"
|
||||
f"{''.join(services)}"
|
||||
"volumes:\n"
|
||||
" acceptance-config:\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
|
||||
def run(
|
||||
*args: str,
|
||||
cwd: Path = ROOT,
|
||||
@@ -197,12 +224,57 @@ def cleanup_candidate_images(version: str, commit: str) -> None:
|
||||
run("docker", "image", "rm", "--force", tag, check=False)
|
||||
|
||||
|
||||
def _terminate_after_cleanup(signum: int, _frame: object) -> None:
|
||||
raise SystemExit(128 + signum)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def managed_compose_cleanup(
|
||||
compose: tuple[str, ...], env: dict[str, str]
|
||||
) -> Iterator[None]:
|
||||
"""Remove the disposable project on normal, failed, or interrupted exits."""
|
||||
|
||||
def cleanup() -> None:
|
||||
run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False)
|
||||
|
||||
atexit.register(cleanup)
|
||||
previous_handlers = {
|
||||
signum: signal.signal(signum, _terminate_after_cleanup)
|
||||
for signum in (signal.SIGINT, signal.SIGTERM)
|
||||
}
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
cleanup_complete = False
|
||||
try:
|
||||
cleanup()
|
||||
cleanup_complete = True
|
||||
finally:
|
||||
if cleanup_complete:
|
||||
atexit.unregister(cleanup)
|
||||
for signum, handler in previous_handlers.items():
|
||||
signal.signal(signum, handler)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--public-api-origin", required=True)
|
||||
parser.add_argument("--output", type=Path, default=Path("acceptance-evidence"))
|
||||
parser.add_argument("--trivy", default="trivy")
|
||||
parser.add_argument("--project-suffix", default=os.environ.get("GITHUB_RUN_ID", "manual"))
|
||||
parser.add_argument(
|
||||
"--canonical-source-commit",
|
||||
default=None,
|
||||
help=(
|
||||
"exact canonical source SHA for a parentless public export; release labels are bound "
|
||||
"to it only when PUBLIC_SOURCE_MANIFEST.json proves the mapping"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retain-image-bundle",
|
||||
action="store_true",
|
||||
help="retain the verified four-image Docker transport bundle in the output directory",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
origin = args.public_api_origin.rstrip("/")
|
||||
@@ -217,25 +289,35 @@ def main() -> int:
|
||||
commit = run("git", "rev-parse", "HEAD", capture=True)
|
||||
if run("git", "status", "--porcelain", capture=True):
|
||||
raise SystemExit("Acceptance must run from a clean, exact source commit")
|
||||
canonical_commit = args.canonical_source_commit or commit
|
||||
if len(canonical_commit) != 40 or any(
|
||||
character not in "0123456789abcdef" for character in canonical_commit
|
||||
):
|
||||
raise SystemExit("--canonical-source-commit must be an exact lowercase 40-character SHA")
|
||||
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
|
||||
atexit.register(cleanup_candidate_images, version, commit)
|
||||
atexit.register(cleanup_candidate_images, version, canonical_commit)
|
||||
built_at = datetime.now(UTC).isoformat()
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
project_suffix = "".join(ch for ch in args.project_suffix.lower() if ch.isalnum())[-24:]
|
||||
project = f"modelforge-rc-{project_suffix or secrets.token_hex(6)}"
|
||||
if not project.startswith("modelforge-rc-"):
|
||||
raise SystemExit("refusing a non-RC Compose project name")
|
||||
project = f"modelforge-validation-{project_suffix or secrets.token_hex(6)}"
|
||||
if not project.startswith("modelforge-validation-"):
|
||||
raise SystemExit("refusing a non-validation Compose project name")
|
||||
|
||||
run(
|
||||
release_command = [
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "release_build.py"),
|
||||
"--output",
|
||||
str(output / "release"),
|
||||
"--public-api-origin",
|
||||
origin,
|
||||
)
|
||||
]
|
||||
if args.canonical_source_commit is not None:
|
||||
release_command.extend(("--canonical-source-commit", canonical_commit))
|
||||
if args.retain_image_bundle:
|
||||
release_command.append("--image-bundle")
|
||||
run(*release_command)
|
||||
|
||||
image_records: list[dict[str, str]] = []
|
||||
for image in IMAGES:
|
||||
@@ -275,13 +357,13 @@ def main() -> int:
|
||||
runtime_password = secrets.token_hex(24)
|
||||
operator_key = secrets.token_urlsafe(48)
|
||||
backup_key = base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii")
|
||||
database = "modelforge_rc"
|
||||
database = "modelforge_validation"
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"MODELFORGE_VERSION": version,
|
||||
"MODELFORGE_COMMIT": commit,
|
||||
"MODELFORGE_COMMIT": canonical_commit,
|
||||
"MODELFORGE_BUILT_AT": built_at,
|
||||
"MODELFORGE_API_IMAGE": f"modelforge-api:{version}",
|
||||
"MODELFORGE_WEB_IMAGE": f"modelforge-web:{version}",
|
||||
@@ -301,8 +383,8 @@ def main() -> int:
|
||||
"MODELFORGE_BACKUP_ENCRYPTION_KEY": backup_key,
|
||||
"MODELFORGE_CORS_ORIGINS": origin,
|
||||
"VITE_API_BASE_URL": origin,
|
||||
"MODELFORGE_SOURCE_COMMIT": commit,
|
||||
"MODELFORGE_SOURCE_REFERENCE": "rc-acceptance",
|
||||
"MODELFORGE_SOURCE_COMMIT": canonical_commit,
|
||||
"MODELFORGE_SOURCE_REFERENCE": "validation-acceptance",
|
||||
"MODELFORGE_SOURCE_REPOSITORY": "public-source-candidate",
|
||||
"MODELFORGE_API_BIND": "127.0.0.1",
|
||||
"MODELFORGE_API_PUBLISHED_PORT": "0",
|
||||
@@ -317,19 +399,7 @@ def main() -> int:
|
||||
}
|
||||
)
|
||||
acceptance_override = output / "acceptance-compose.override.yml"
|
||||
acceptance_override.write_text(
|
||||
"services:\n"
|
||||
" api:\n"
|
||||
" volumes:\n"
|
||||
" - type: volume\n"
|
||||
" source: acceptance-config\n"
|
||||
" target: /app/config\n"
|
||||
" read_only: true\n"
|
||||
"volumes:\n"
|
||||
" acceptance-config:\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
write_acceptance_compose_override(acceptance_override)
|
||||
compose = (
|
||||
"docker",
|
||||
"compose",
|
||||
@@ -346,95 +416,97 @@ def main() -> int:
|
||||
summary: dict[str, object] = {
|
||||
"schema_version": 1,
|
||||
"project": project,
|
||||
"source_commit": commit,
|
||||
"source_commit": canonical_commit,
|
||||
"build_tree_commit": commit,
|
||||
"version": version,
|
||||
"public_api_origin": origin,
|
||||
"image_bundle_retained": args.retain_image_bundle,
|
||||
"production_changed": False,
|
||||
"compute_identity_created": False,
|
||||
"images": image_records,
|
||||
"checks": {},
|
||||
}
|
||||
try:
|
||||
run(*compose, "config", "-q", env=env)
|
||||
run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env)
|
||||
provision_database_roles(compose, env, database)
|
||||
provision_config_volume(project)
|
||||
run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env)
|
||||
api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True)
|
||||
if not api_container:
|
||||
raise RuntimeError("acceptance API container was not created")
|
||||
api = "http://127.0.0.1:8000"
|
||||
with managed_compose_cleanup(compose, env):
|
||||
try:
|
||||
run(*compose, "config", "-q", env=env)
|
||||
run(*compose, "up", "-d", "--no-build", "--wait", "postgres", "redis", env=env)
|
||||
provision_database_roles(compose, env, database)
|
||||
provision_config_volume(project)
|
||||
run(*compose, "up", "-d", "--no-build", "--wait", "api", "web", env=env)
|
||||
api_container = run(*compose, "ps", "--quiet", "api", env=env, capture=True)
|
||||
if not api_container:
|
||||
raise RuntimeError("acceptance API container was not created")
|
||||
api = "http://127.0.0.1:8000"
|
||||
|
||||
wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200)
|
||||
wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200)
|
||||
version_status, version_body = container_http_status(
|
||||
api_container, f"{api}/api/v1/version"
|
||||
)
|
||||
unauthenticated, _ = container_http_status(
|
||||
api_container, f"{api}/api/v1/admin/recovery/dashboard"
|
||||
)
|
||||
authenticated, _ = container_http_status(
|
||||
api_container,
|
||||
f"{api}/api/v1/admin/recovery/dashboard",
|
||||
operator=True,
|
||||
)
|
||||
wait_for_container_status(api_container, "http://web:3000/", 200)
|
||||
if version_status != 200 or unauthenticated != 401 or authenticated != 200:
|
||||
raise RuntimeError(
|
||||
"acceptance boundary mismatch: "
|
||||
f"version={version_status}, unauthenticated={unauthenticated}, "
|
||||
f"authenticated={authenticated}"
|
||||
wait_for_container_status(api_container, f"{api}/api/v1/health/live", 200)
|
||||
wait_for_container_status(api_container, f"{api}/api/v1/health/ready", 200)
|
||||
version_status, version_body = container_http_status(
|
||||
api_container, f"{api}/api/v1/version"
|
||||
)
|
||||
version_payload = json.loads(version_body)
|
||||
if version_payload.get("version") != version:
|
||||
raise RuntimeError(f"running version does not match {version}: {version_payload}")
|
||||
unauthenticated, _ = container_http_status(
|
||||
api_container, f"{api}/api/v1/admin/recovery/dashboard"
|
||||
)
|
||||
authenticated, _ = container_http_status(
|
||||
api_container,
|
||||
f"{api}/api/v1/admin/recovery/dashboard",
|
||||
operator=True,
|
||||
)
|
||||
wait_for_container_status(api_container, "http://web:3000/", 200)
|
||||
if version_status != 200 or unauthenticated != 401 or authenticated != 200:
|
||||
raise RuntimeError(
|
||||
"acceptance boundary mismatch: "
|
||||
f"version={version_status}, unauthenticated={unauthenticated}, "
|
||||
f"authenticated={authenticated}"
|
||||
)
|
||||
version_payload = json.loads(version_body)
|
||||
if version_payload.get("version") != version:
|
||||
raise RuntimeError(f"running version does not match {version}: {version_payload}")
|
||||
|
||||
volumes = run(
|
||||
"docker",
|
||||
"volume",
|
||||
"ls",
|
||||
"--filter",
|
||||
f"label=com.docker.compose.project={project}",
|
||||
"--format",
|
||||
"{{.Name}}",
|
||||
capture=True,
|
||||
).splitlines()
|
||||
if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes):
|
||||
raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}")
|
||||
summary["checks"] = {
|
||||
"live": 200,
|
||||
"ready": 200,
|
||||
"version": version_status,
|
||||
"admin_without_key": unauthenticated,
|
||||
"admin_with_key": authenticated,
|
||||
"console": 200,
|
||||
"database_roles_provisioned": True,
|
||||
"network_probe_container": "api",
|
||||
"isolated_volumes": sorted(volumes),
|
||||
}
|
||||
summary["result"] = "PASS"
|
||||
(output / "compose-ps.json").write_text(
|
||||
run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
except Exception:
|
||||
summary["result"] = "FAIL"
|
||||
(output / "compose-logs.txt").write_text(
|
||||
run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
run(*compose, "down", "--volumes", "--remove-orphans", env=env, check=False)
|
||||
(output / "acceptance-summary.json").write_text(
|
||||
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
volumes = run(
|
||||
"docker",
|
||||
"volume",
|
||||
"ls",
|
||||
"--filter",
|
||||
f"label=com.docker.compose.project={project}",
|
||||
"--format",
|
||||
"{{.Name}}",
|
||||
capture=True,
|
||||
).splitlines()
|
||||
if not volumes or any(not volume.startswith(f"{project}_") for volume in volumes):
|
||||
raise RuntimeError(f"Compose volumes are not isolated under {project}: {volumes}")
|
||||
summary["checks"] = {
|
||||
"live": 200,
|
||||
"ready": 200,
|
||||
"version": version_status,
|
||||
"admin_without_key": unauthenticated,
|
||||
"admin_with_key": authenticated,
|
||||
"console": 200,
|
||||
"database_roles_provisioned": True,
|
||||
"network_probe_container": "api",
|
||||
"isolated_volumes": sorted(volumes),
|
||||
}
|
||||
summary["result"] = "PASS"
|
||||
(output / "compose-ps.json").write_text(
|
||||
run(*compose, "ps", "--format", "json", env=env, capture=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
except BaseException:
|
||||
summary["result"] = "FAIL"
|
||||
(output / "compose-logs.txt").write_text(
|
||||
run(*compose, "logs", "--no-color", env=env, check=False, capture=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
(output / "acceptance-summary.json").write_text(
|
||||
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
|
||||
print(f"Isolated RC acceptance PASS for {commit} in {project}")
|
||||
print(f"Isolated RC acceptance PASS for {canonical_commit} in {project}")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
"""Inventory stale candidate Compose stacks and optionally stop one exact, quiescent project.
|
||||
|
||||
The default inventory mode is read-only. The only mutation available is ``docker stop`` for
|
||||
containers inspected under one exact ``modelforge-validation-*`` Compose project label. Volumes and
|
||||
images are never removed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
PROJECT_PREFIX = "modelforge-validation-"
|
||||
PROJECT_PATTERN = re.compile(
|
||||
r"modelforge-validation-[a-z0-9](?:[a-z0-9-]{0,39}[a-z0-9])?\Z"
|
||||
)
|
||||
PROJECT_LABEL = "com.docker.compose.project"
|
||||
SERVICE_LABEL = "com.docker.compose.service"
|
||||
ONEOFF_LABEL = "com.docker.compose.oneoff"
|
||||
CONFIG_TARGET = "/app/config"
|
||||
CONFIG_MANIFEST = "models/initial-candidates.yaml"
|
||||
DEFAULT_STALE_AFTER_HOURS = 24
|
||||
STOPPABLE_RUNNING_SERVICES = {"postgres", "redis", "web"}
|
||||
NEVER_STOP_RUNNING_SERVICES = {"api", "migrate", "node-agent", "runtime-worker"}
|
||||
|
||||
|
||||
class RetentionError(ValueError):
|
||||
"""Docker state or operator input cannot be used safely."""
|
||||
|
||||
|
||||
def validate_project(project: str) -> str:
|
||||
if PROJECT_PATTERN.fullmatch(project) is None:
|
||||
raise RetentionError(
|
||||
"project must be an exact bounded modelforge-validation-* name"
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def _mapping(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise RetentionError(f"Docker returned invalid {label}")
|
||||
return value
|
||||
|
||||
|
||||
def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise RetentionError(f"evidence record has duplicate field: {key}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _docker(*args: str) -> str:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
("docker", *args),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
except OSError as exc:
|
||||
raise RetentionError("Docker CLI could not be started") from exc
|
||||
if completed.returncode:
|
||||
raise RetentionError(f"Docker command failed: {' '.join(args)}")
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def _parse_inspection(raw: str, identifier: str) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RetentionError("Docker returned invalid container inspection JSON") from exc
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise RetentionError("Docker returned ambiguous container inspection data")
|
||||
item = _mapping(value[0], "container inspection")
|
||||
inspected_id = item.get("Id")
|
||||
if not isinstance(inspected_id, str) or inspected_id != identifier:
|
||||
raise RetentionError("Docker returned a different container than requested")
|
||||
return item
|
||||
|
||||
|
||||
def _created_at(value: object) -> datetime:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise RetentionError("container creation time is missing")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise RetentionError("container creation time is invalid") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise RetentionError("container creation time has no timezone")
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _path_stat(path: Path) -> object | None:
|
||||
try:
|
||||
return path.stat()
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return None
|
||||
except OSError as exc:
|
||||
raise RetentionError(f"cannot inspect bind source path: {path}") from exc
|
||||
|
||||
|
||||
def _bind_sources(container: dict[str, Any]) -> list[dict[str, object]]:
|
||||
mounts = container.get("Mounts")
|
||||
if not isinstance(mounts, list):
|
||||
raise RetentionError("container mount data is missing or ambiguous")
|
||||
result: list[dict[str, object]] = []
|
||||
for raw_mount in mounts:
|
||||
mount = _mapping(raw_mount, "mount")
|
||||
if mount.get("Type") != "bind":
|
||||
continue
|
||||
source = mount.get("Source")
|
||||
destination = mount.get("Destination")
|
||||
if not isinstance(source, str) or not source or not Path(source).is_absolute():
|
||||
raise RetentionError("container has an invalid bind source")
|
||||
if not isinstance(destination, str) or not destination.startswith("/"):
|
||||
raise RetentionError("container has an invalid bind destination")
|
||||
source_path = Path(source)
|
||||
source_stat = _path_stat(source_path)
|
||||
source_exists = source_stat is not None
|
||||
entry: dict[str, object] = {
|
||||
"destination": destination,
|
||||
"source": str(source_path),
|
||||
"source_exists": source_exists,
|
||||
}
|
||||
if PurePosixPath(destination).as_posix().rstrip("/") == CONFIG_TARGET:
|
||||
manifest = source_path / CONFIG_MANIFEST
|
||||
manifest_stat = _path_stat(manifest) if source_exists else None
|
||||
entry["manifest"] = str(manifest)
|
||||
entry["manifest_exists"] = (
|
||||
manifest_stat is not None and stat.S_ISREG(manifest_stat.st_mode)
|
||||
)
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
|
||||
def _inspect_container(identifier: str) -> dict[str, Any]:
|
||||
item = _parse_inspection(_docker("container", "inspect", identifier), identifier)
|
||||
config = _mapping(item.get("Config"), "container config")
|
||||
labels = config.get("Labels")
|
||||
if not isinstance(labels, dict):
|
||||
raise RetentionError("container Compose labels are missing or ambiguous")
|
||||
project = labels.get(PROJECT_LABEL)
|
||||
if not isinstance(project, str) or not project:
|
||||
raise RetentionError("container project label is invalid")
|
||||
if not project.startswith(PROJECT_PREFIX):
|
||||
return {"project": project}
|
||||
validate_project(project)
|
||||
|
||||
name = item.get("Name")
|
||||
if not isinstance(name, str) or not name.startswith("/"):
|
||||
raise RetentionError("container project label or name is invalid")
|
||||
state = _mapping(item.get("State"), "container state")
|
||||
status = state.get("Status")
|
||||
if not isinstance(status, str) or not status:
|
||||
raise RetentionError("container state status is missing")
|
||||
service = labels.get(SERVICE_LABEL)
|
||||
if not isinstance(service, str) or not service:
|
||||
raise RetentionError("container Compose service label is missing")
|
||||
return {
|
||||
"id": identifier,
|
||||
"name": name.lstrip("/"),
|
||||
"project": project,
|
||||
"service": service,
|
||||
"oneoff": labels.get(ONEOFF_LABEL),
|
||||
"status": status.lower(),
|
||||
"created_at": _created_at(item.get("Created")),
|
||||
"bind_sources": _bind_sources(item),
|
||||
}
|
||||
|
||||
|
||||
def collect_projects(
|
||||
*, now: datetime | None = None, stale_after_hours: int = DEFAULT_STALE_AFTER_HOURS
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Inventory Compose containers and group only exact validation project labels."""
|
||||
|
||||
if stale_after_hours < 1 or stale_after_hours > 8760:
|
||||
raise RetentionError("stale age must be between 1 and 8760 hours")
|
||||
identifiers = [
|
||||
line.strip()
|
||||
for line in _docker(
|
||||
"container", "ls", "--all", "--quiet", "--no-trunc", "--filter", f"label={PROJECT_LABEL}"
|
||||
).splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if len(set(identifiers)) != len(identifiers):
|
||||
raise RetentionError("Docker returned duplicate container identifiers")
|
||||
|
||||
projects: dict[str, list[dict[str, Any]]] = {}
|
||||
for identifier in identifiers:
|
||||
container = _inspect_container(identifier)
|
||||
project = container["project"]
|
||||
if project.startswith(PROJECT_PREFIX):
|
||||
validate_project(project)
|
||||
if not container["name"].startswith(f"{project}-"):
|
||||
raise RetentionError(f"container name is outside exact project {project}")
|
||||
projects.setdefault(project, []).append(container)
|
||||
|
||||
current = (now or datetime.now(UTC)).astimezone(UTC)
|
||||
result: list[dict[str, Any]] = []
|
||||
for project, containers in sorted(projects.items()):
|
||||
created_values = [item["created_at"] for item in containers]
|
||||
oldest = min(created_values)
|
||||
age_seconds = int((current - oldest).total_seconds())
|
||||
if age_seconds < 0:
|
||||
raise RetentionError(f"container creation time is in the future for {project}")
|
||||
sources = [
|
||||
{"container": item["name"], **source}
|
||||
for item in containers
|
||||
for source in item["bind_sources"]
|
||||
]
|
||||
result.append(
|
||||
{
|
||||
"project": project,
|
||||
"oldest_container_created_at": oldest.isoformat().replace("+00:00", "Z"),
|
||||
"age_hours": round(age_seconds / 3600, 2),
|
||||
"stale": age_seconds >= stale_after_hours * 3600,
|
||||
"missing_bind_sources": [source for source in sources if not source["source_exists"]],
|
||||
"missing_manifests": [
|
||||
source for source in sources if source.get("manifest") and not source["manifest_exists"]
|
||||
],
|
||||
"containers": [
|
||||
{
|
||||
key: (
|
||||
item[key].isoformat().replace("+00:00", "Z")
|
||||
if key == "created_at"
|
||||
else item[key]
|
||||
)
|
||||
for key in ("id", "name", "service", "status", "created_at")
|
||||
}
|
||||
for item in sorted(containers, key=lambda row: row["name"])
|
||||
],
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _evidence_record(path: Path | None, *, project: str) -> Path:
|
||||
if path is None:
|
||||
raise RetentionError("stop requires --evidence-record for retained evidence")
|
||||
try:
|
||||
resolved = path.resolve(strict=True)
|
||||
record_stat = resolved.stat()
|
||||
if not stat.S_ISREG(record_stat.st_mode) or record_stat.st_size == 0:
|
||||
raise RetentionError("evidence record must be a nonempty regular file")
|
||||
record = json.loads(resolved.read_text(encoding="utf-8"), object_pairs_hook=_unique_object)
|
||||
except OSError as exc:
|
||||
raise RetentionError("evidence record cannot be read") from exc
|
||||
except RuntimeError as exc:
|
||||
raise RetentionError("evidence record path is ambiguous") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RetentionError("evidence record must be valid JSON") from exc
|
||||
except UnicodeError as exc:
|
||||
raise RetentionError("evidence record must be valid UTF-8 JSON") from exc
|
||||
if (
|
||||
not isinstance(record, dict)
|
||||
or set(record) != {"schema_version", "project", "retained", "evidence_reference"}
|
||||
or type(record["schema_version"]) is not int
|
||||
or record["schema_version"] != 1
|
||||
or record["project"] != project
|
||||
or record["retained"] is not True
|
||||
or not isinstance(record["evidence_reference"], str)
|
||||
or not record["evidence_reference"].strip()
|
||||
):
|
||||
raise RetentionError("evidence record must identify this project and retained evidence")
|
||||
return resolved
|
||||
|
||||
|
||||
def stop_project(
|
||||
project: str,
|
||||
*,
|
||||
confirm_stop: str | None,
|
||||
confirm_no_active_jobs: bool,
|
||||
evidence_record: Path | None,
|
||||
stale_after_hours: int,
|
||||
now: datetime | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Stop running containers for one exact stale or incomplete validation project."""
|
||||
|
||||
validate_project(project)
|
||||
if confirm_stop != project:
|
||||
raise RetentionError("stop requires --confirm-stop with the exact project name")
|
||||
if not confirm_no_active_jobs:
|
||||
raise RetentionError("stop requires --confirm-no-active-jobs")
|
||||
evidence = _evidence_record(evidence_record, project=project)
|
||||
|
||||
projects = collect_projects(now=now, stale_after_hours=stale_after_hours)
|
||||
selected = [item for item in projects if item["project"] == project]
|
||||
if len(selected) != 1:
|
||||
raise RetentionError(f"exact validation project was not found unambiguously: {project}")
|
||||
details = selected[0]
|
||||
if not details["stale"] and not details["missing_bind_sources"] and not details["missing_manifests"]:
|
||||
raise RetentionError("stop is limited to a stale project or one with missing bind-source files")
|
||||
|
||||
project_containers = [
|
||||
_inspect_container(container["id"])
|
||||
for container in details["containers"]
|
||||
]
|
||||
running: list[dict[str, Any]] = []
|
||||
for container in project_containers:
|
||||
if container["status"] not in {"running", "restarting", "paused"}:
|
||||
continue
|
||||
service = container["service"]
|
||||
if service == "api":
|
||||
raise RetentionError("refusing stop while the project API is running")
|
||||
if container["status"] != "running":
|
||||
raise RetentionError(f"refusing stop while service state is ambiguous: {container['status']}")
|
||||
if str(container["oneoff"]).lower() in {"true", "1", "yes"}:
|
||||
raise RetentionError("refusing stop while a one-off job container is running")
|
||||
if service in NEVER_STOP_RUNNING_SERVICES or service not in STOPPABLE_RUNNING_SERVICES:
|
||||
raise RetentionError(f"refusing stop while service activity is unknown: {service}")
|
||||
running.append(container)
|
||||
|
||||
stopped: list[str] = []
|
||||
if running:
|
||||
identifiers = [item["id"] for item in running]
|
||||
_docker("stop", "--time", "30", *identifiers)
|
||||
stopped = identifiers
|
||||
# Confirm every ID still has the same exact project label and is no longer running.
|
||||
for item in running:
|
||||
identifier = item["id"]
|
||||
after = _inspect_container(identifier)
|
||||
if (
|
||||
after["project"] != project
|
||||
or after["name"] != item["name"]
|
||||
or after["service"] != item["service"]
|
||||
):
|
||||
raise RetentionError("container project label changed during stop")
|
||||
if after["status"] in {"running", "restarting", "paused"}:
|
||||
raise RetentionError("Docker reported success but a selected container is still running")
|
||||
return {
|
||||
"action": "stop",
|
||||
"project": project,
|
||||
"stopped_container_ids": stopped,
|
||||
"evidence_record": str(evidence),
|
||||
"volumes_or_images_removed": False,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--stale-after-hours", type=int, default=DEFAULT_STALE_AFTER_HOURS)
|
||||
parser.add_argument(
|
||||
"--stop-project", help="stop eligible containers for one exact validation project"
|
||||
)
|
||||
parser.add_argument("--confirm-stop", help="must exactly match --stop-project")
|
||||
parser.add_argument("--confirm-no-active-jobs", action="store_true")
|
||||
parser.add_argument("--evidence-record", type=Path, help="nonempty file proving evidence was retained")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.stop_project is None:
|
||||
if args.confirm_stop or args.confirm_no_active_jobs or args.evidence_record is not None:
|
||||
raise RetentionError("stop confirmation options require --stop-project")
|
||||
result: dict[str, object] = {
|
||||
"result": "PASS",
|
||||
"mode": "read-only",
|
||||
"stale_after_hours": args.stale_after_hours,
|
||||
"projects": collect_projects(stale_after_hours=args.stale_after_hours),
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
"result": "PASS",
|
||||
**stop_project(
|
||||
args.stop_project,
|
||||
confirm_stop=args.confirm_stop,
|
||||
confirm_no_active_jobs=args.confirm_no_active_jobs,
|
||||
evidence_record=args.evidence_record,
|
||||
stale_after_hours=args.stale_after_hours,
|
||||
),
|
||||
}
|
||||
except (RetentionError, OSError) as exc:
|
||||
print(json.dumps({"result": "FAIL", "error": str(exc)}, sort_keys=True))
|
||||
return 2
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+348
-12
@@ -17,6 +17,7 @@ import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -30,7 +31,7 @@ from urllib.parse import urlparse
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "backend" / "src"))
|
||||
|
||||
from modelforge_api.domain.release import (
|
||||
from modelforge_api.domain.release import ( # noqa: E402
|
||||
CURRENT_AGENT_PROTOCOL_VERSION,
|
||||
MINIMUM_POSTGRES_MAJOR,
|
||||
MINIMUM_UPGRADE_SOURCE,
|
||||
@@ -43,22 +44,33 @@ from modelforge_api.domain.release import (
|
||||
TARGET_SCHEMA_REVISION,
|
||||
)
|
||||
|
||||
#: image name -> (build context, Dockerfile path), both relative to the repository root. The Node
|
||||
#: Agent and Runtime Worker build from the repository root because their Dockerfiles copy a
|
||||
#: component directory from that shared context. Keeping the four distributable images in one
|
||||
#: inventory makes it impossible for the private runtime boundary to disappear from a release.
|
||||
#: image name -> (build context, Dockerfile path), both relative to the repository root. Python
|
||||
#: images build from the repository root so they can consume the shared hash-locked toolchain.
|
||||
#: Keeping the four distributable images in one inventory makes it impossible for the private
|
||||
#: runtime boundary to disappear from a release.
|
||||
IMAGES = {
|
||||
"modelforge-api": ("backend", "backend/Dockerfile"),
|
||||
"modelforge-api": (".", "backend/Dockerfile"),
|
||||
"modelforge-web": ("frontend", "frontend/Dockerfile"),
|
||||
"modelforge-node-agent": (".", "node-agent/Dockerfile"),
|
||||
"modelforge-runtime-worker": (".", "runtime-worker/Dockerfile"),
|
||||
}
|
||||
CANONICAL_SOURCE_REPOSITORY = "https://git.example.com/example/modelforge"
|
||||
PUBLIC_SOURCE_REPOSITORY = "https://git.example.com/example/modelforge"
|
||||
OCI_SOURCE_LABEL = "org.opencontainers.image.source"
|
||||
SOURCE_DOCKERFILES = (
|
||||
"backend/Dockerfile",
|
||||
"frontend/Dockerfile",
|
||||
"node-agent/Dockerfile",
|
||||
"runtime-worker/Dockerfile",
|
||||
)
|
||||
|
||||
#: What a release tarball contains. Deliberately no model weights, no .env, no database dump and no
|
||||
#: credential of any kind — see the packaging test, which fails if that changes.
|
||||
ARTIFACT_PATHS = (
|
||||
"docker-compose.yml",
|
||||
"docker-compose.production.yml",
|
||||
"docker-compose.rc-ingress.yml",
|
||||
"docker-compose.validation-tls.yml",
|
||||
"docker-compose.node-agent.yml",
|
||||
"docker-compose.runtime-worker.yml",
|
||||
"docker-compose.gpu.yml",
|
||||
@@ -68,9 +80,25 @@ ARTIFACT_PATHS = (
|
||||
"VERSION",
|
||||
"README.md",
|
||||
"config",
|
||||
"clients",
|
||||
"deploy/postgres",
|
||||
"deploy/validation",
|
||||
"docs",
|
||||
"backend/alembic",
|
||||
"backend/alembic.ini",
|
||||
"backend/pyproject.toml",
|
||||
"backend/src",
|
||||
"scripts/bootstrap.py",
|
||||
"scripts/m16_chaos.py",
|
||||
"scripts/m16_invariants.py",
|
||||
"scripts/m16_soak.py",
|
||||
"scripts/operator_identity.py",
|
||||
"scripts/operator_password.py",
|
||||
"scripts/modelforge_scheduled_backup.sh",
|
||||
"scripts/preflight.py",
|
||||
"scripts/rc_isolation_preflight.py",
|
||||
"scripts/rc_stack_retention.py",
|
||||
"scripts/upgrade.py",
|
||||
)
|
||||
|
||||
|
||||
@@ -85,7 +113,10 @@ def run(*args: str, check: bool = True, cwd: Path | None = None) -> str:
|
||||
check=False,
|
||||
)
|
||||
if check and completed.returncode != 0:
|
||||
raise RuntimeError(f"{' '.join(args)} failed: {completed.stderr.strip()[:500]}")
|
||||
detail = (completed.stderr or completed.stdout or "").strip()
|
||||
raise RuntimeError(
|
||||
f"{' '.join(args)} failed ({completed.returncode}): {detail[-2000:]}"
|
||||
)
|
||||
return (completed.stdout or "").strip()
|
||||
|
||||
|
||||
@@ -97,8 +128,104 @@ def sha256_file(path: Path) -> str:
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def source_identity(allow_dirty: bool) -> dict[str, Any]:
|
||||
commit = run("git", "rev-parse", "HEAD")
|
||||
def _is_exact_commit(value: str) -> bool:
|
||||
return len(value) == 40 and all(character in "0123456789abcdef" for character in value)
|
||||
|
||||
|
||||
def _load_json_object(path: Path) -> dict[str, Any]:
|
||||
"""Load one manifest object while refusing duplicate fields that could obscure provenance."""
|
||||
|
||||
def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON key {key!r}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicates)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise SystemExit(f"cannot read provenance manifest {path}: {exc}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit(f"provenance manifest {path} must contain a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def _dockerfile_source_repository(root: Path, relative_path: str) -> str:
|
||||
path = root / relative_path
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise SystemExit(f"cannot read release Dockerfile {relative_path}: {exc}") from exc
|
||||
labels = re.findall(
|
||||
rf'(?m)^LABEL\s+{re.escape(OCI_SOURCE_LABEL)}="([^"]+)"\s*$', text
|
||||
)
|
||||
if len(labels) != 1:
|
||||
raise SystemExit(f"{relative_path} must declare exactly one {OCI_SOURCE_LABEL} label")
|
||||
return labels[0]
|
||||
|
||||
|
||||
def _public_source_repository(root: Path, manifest: dict[str, Any]) -> str:
|
||||
source_repository = manifest.get("sourceRepository")
|
||||
if source_repository != PUBLIC_SOURCE_REPOSITORY:
|
||||
raise SystemExit(
|
||||
"PUBLIC_SOURCE_MANIFEST.json sourceRepository is not the approved sanitized identity"
|
||||
)
|
||||
files = manifest.get("files")
|
||||
if not isinstance(files, list) or not files:
|
||||
raise SystemExit("PUBLIC_SOURCE_MANIFEST.json has no reviewed file inventory")
|
||||
|
||||
entries: dict[str, dict[str, Any]] = {}
|
||||
for entry in files:
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get("path"), str):
|
||||
raise SystemExit("PUBLIC_SOURCE_MANIFEST.json has a malformed file entry")
|
||||
relative_path = entry["path"]
|
||||
if relative_path in entries:
|
||||
raise SystemExit(f"PUBLIC_SOURCE_MANIFEST.json repeats {relative_path!r}")
|
||||
entries[relative_path] = entry
|
||||
|
||||
for relative_path in SOURCE_DOCKERFILES:
|
||||
entry = entries.get(relative_path)
|
||||
if entry is None:
|
||||
raise SystemExit(
|
||||
f"PUBLIC_SOURCE_MANIFEST.json does not bind {relative_path}"
|
||||
)
|
||||
path = root / relative_path
|
||||
try:
|
||||
payload = path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise SystemExit(f"cannot read public source file {relative_path}: {exc}") from exc
|
||||
if (
|
||||
entry.get("bytes") != len(payload)
|
||||
or entry.get("sha256") != hashlib.sha256(payload).hexdigest()
|
||||
):
|
||||
raise SystemExit(
|
||||
f"PUBLIC_SOURCE_MANIFEST.json does not bind the current {relative_path}"
|
||||
)
|
||||
if _dockerfile_source_repository(root, relative_path) != source_repository:
|
||||
raise SystemExit(
|
||||
f"{relative_path} source label does not match the manifest sourceRepository"
|
||||
)
|
||||
return source_repository
|
||||
|
||||
|
||||
def _assert_image_source_identity(
|
||||
images: dict[str, dict[str, Any]], source_repository: str
|
||||
) -> None:
|
||||
for image, record in images.items():
|
||||
labels = record.get("labels")
|
||||
if not isinstance(labels, dict) or labels.get(OCI_SOURCE_LABEL) != source_repository:
|
||||
raise SystemExit(
|
||||
f"built image {image} source label does not match source identity "
|
||||
f"{source_repository!r}"
|
||||
)
|
||||
|
||||
|
||||
def source_identity(
|
||||
allow_dirty: bool, canonical_source_commit: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
build_commit = run("git", "rev-parse", "HEAD")
|
||||
dirty = bool(run("git", "status", "--porcelain"))
|
||||
if dirty and not allow_dirty:
|
||||
raise SystemExit(
|
||||
@@ -107,13 +234,73 @@ def source_identity(allow_dirty: bool) -> dict[str, Any]:
|
||||
"rehearsal build."
|
||||
)
|
||||
describe = run("git", "describe", "--tags", "--always", check=False)
|
||||
return {
|
||||
"commit": commit,
|
||||
identity: dict[str, Any] = {
|
||||
"commit": build_commit,
|
||||
"describe": describe or None,
|
||||
"dirty": dirty,
|
||||
"repository": run("git", "remote", "get-url", "origin", check=False) or None,
|
||||
"branch": run("git", "rev-parse", "--abbrev-ref", "HEAD", check=False) or None,
|
||||
}
|
||||
public_manifest = ROOT / "PUBLIC_SOURCE_MANIFEST.json"
|
||||
if public_manifest.exists():
|
||||
if not public_manifest.is_file() or public_manifest.is_symlink():
|
||||
raise SystemExit("PUBLIC_SOURCE_MANIFEST.json must be a regular file")
|
||||
if canonical_source_commit is None:
|
||||
raise SystemExit(
|
||||
"a public-source build requires --canonical-source-commit so the export manifest "
|
||||
"can bind it to the canonical source"
|
||||
)
|
||||
if not _is_exact_commit(canonical_source_commit):
|
||||
raise SystemExit(
|
||||
"--canonical-source-commit must be an exact lowercase 40-character SHA"
|
||||
)
|
||||
manifest = _load_json_object(public_manifest)
|
||||
if (
|
||||
manifest.get("schemaVersion") != 1
|
||||
or manifest.get("sourceRevision") != canonical_source_commit
|
||||
):
|
||||
raise SystemExit(
|
||||
"PUBLIC_SOURCE_MANIFEST.json does not bind this build tree to the requested "
|
||||
"canonical source commit"
|
||||
)
|
||||
source_repository = _public_source_repository(ROOT, manifest)
|
||||
identity.update(
|
||||
{
|
||||
"commit": canonical_source_commit,
|
||||
"build_commit": build_commit,
|
||||
"build_describe": describe or None,
|
||||
"build_repository": identity["repository"],
|
||||
"build_branch": identity["branch"],
|
||||
"public_source_manifest_sha256": sha256_file(public_manifest),
|
||||
"source_repository": source_repository,
|
||||
}
|
||||
)
|
||||
# ``describe`` and ``branch`` describe the parentless build tree, not the canonical revision.
|
||||
# Do not present either as canonical provenance or expose the private build remote.
|
||||
identity["describe"] = None
|
||||
identity["repository"] = None
|
||||
identity["branch"] = None
|
||||
return identity
|
||||
|
||||
if canonical_source_commit is not None:
|
||||
if not _is_exact_commit(canonical_source_commit):
|
||||
raise SystemExit("--canonical-source-commit must be an exact lowercase 40-character SHA")
|
||||
if canonical_source_commit != build_commit:
|
||||
raise SystemExit(
|
||||
"canonical source differs from the build tree, but PUBLIC_SOURCE_MANIFEST.json is "
|
||||
"absent; refusing an unbound source-label override"
|
||||
)
|
||||
|
||||
source_labels = {
|
||||
relative_path: _dockerfile_source_repository(ROOT, relative_path)
|
||||
for relative_path in SOURCE_DOCKERFILES
|
||||
}
|
||||
if any(value != CANONICAL_SOURCE_REPOSITORY for value in source_labels.values()):
|
||||
raise SystemExit(
|
||||
"private release Dockerfiles do not use the canonical source repository label"
|
||||
)
|
||||
identity["source_repository"] = CANONICAL_SOURCE_REPOSITORY
|
||||
return identity
|
||||
|
||||
|
||||
def source_date_epoch(built_at: str) -> int:
|
||||
@@ -189,6 +376,11 @@ def build_images(
|
||||
run(
|
||||
"docker",
|
||||
"build",
|
||||
# A release must execute current package-manager upgrade layers. Reusing a layer
|
||||
# that predates an upstream security fix would make the packaged image differ
|
||||
# from the cache-free image validated by the managed security gate.
|
||||
"--pull",
|
||||
"--no-cache",
|
||||
# BuildKit attestations carry an exporter invocation timestamp and make the local
|
||||
# manifest-list ID vary. ModelForge emits its own commit-bound CycloneDX and image
|
||||
# provenance below, so suppress the duplicate nondeterministic attestations.
|
||||
@@ -221,6 +413,15 @@ def build_images(
|
||||
run("docker", "inspect", "--format", "{{index .RepoDigests 0}}", tag, check=False)
|
||||
or None
|
||||
)
|
||||
rootfs_layers = json.loads(
|
||||
run("docker", "inspect", "--format", "{{json .RootFS.Layers}}", tag) or "[]"
|
||||
)
|
||||
if not (
|
||||
isinstance(rootfs_layers, list)
|
||||
and rootfs_layers
|
||||
and all(isinstance(layer, str) and layer.startswith("sha256:") for layer in rootfs_layers)
|
||||
):
|
||||
raise RuntimeError(f"{tag} does not expose a content-addressable RootFS layer set")
|
||||
built[image] = {
|
||||
"tag": tag,
|
||||
"image_id": image_id,
|
||||
@@ -233,10 +434,121 @@ def build_images(
|
||||
for key, value in (labels or {}).items()
|
||||
if key.startswith("org.opencontainers.image.")
|
||||
},
|
||||
# Docker config IDs are daemon-local after ``docker load``. Diff IDs instead bind the
|
||||
# transport bundle to the actual uncompressed filesystem layer content on every host.
|
||||
"rootfs_layers": rootfs_layers,
|
||||
}
|
||||
return built
|
||||
|
||||
|
||||
def _bundle_member(tar: tarfile.TarFile, name: str) -> tarfile.TarInfo:
|
||||
if not name or name.startswith("/") or ".." in Path(name).parts:
|
||||
raise ValueError(f"unsafe image-bundle member {name!r}")
|
||||
try:
|
||||
member = tar.getmember(name)
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"image bundle is missing {name!r}") from exc
|
||||
if not member.isfile():
|
||||
raise ValueError(f"image-bundle member {name!r} is not a regular file")
|
||||
return member
|
||||
|
||||
|
||||
def _bundle_json(tar: tarfile.TarFile, name: str) -> Any:
|
||||
member = _bundle_member(tar, name)
|
||||
stream = tar.extractfile(member)
|
||||
if stream is None: # pragma: no cover - tarfile guarantees a stream for regular files
|
||||
raise ValueError(f"cannot read image-bundle member {name!r}")
|
||||
try:
|
||||
return json.loads(stream.read().decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"image-bundle member {name!r} is not valid JSON") from exc
|
||||
|
||||
|
||||
def verify_image_bundle(bundle: Path, images: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Verify the saved images retain exact tags, OCI labels and filesystem layer sets.
|
||||
|
||||
This deliberately does not compare the Docker image/config ID after a later ``docker load``:
|
||||
Docker daemons can rewrite it. The bundle itself contains the stable config and the rootfs diff
|
||||
IDs, which are content-addressable and can be rechecked after transport.
|
||||
"""
|
||||
|
||||
expected_tags = {str(record["tag"]): name for name, record in images.items()}
|
||||
if set(images) != set(IMAGES) or len(expected_tags) != len(IMAGES):
|
||||
raise ValueError("an RC bundle must contain exactly the four declared release images")
|
||||
|
||||
with tarfile.open(bundle, mode="r") as archive:
|
||||
manifest = _bundle_json(archive, "manifest.json")
|
||||
if not isinstance(manifest, list) or len(manifest) != len(IMAGES):
|
||||
raise ValueError("image bundle must have one manifest entry for each release image")
|
||||
observed: dict[str, dict[str, Any]] = {}
|
||||
for entry in manifest:
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError("image bundle manifest entry is not an object")
|
||||
tags = entry.get("RepoTags")
|
||||
if not isinstance(tags, list) or len(tags) != 1 or tags[0] not in expected_tags:
|
||||
raise ValueError(f"image bundle has an unexpected tag set: {tags!r}")
|
||||
tag = tags[0]
|
||||
name = expected_tags[tag]
|
||||
if name in observed:
|
||||
raise ValueError(f"image bundle repeats {tag}")
|
||||
config_name = entry.get("Config")
|
||||
if not isinstance(config_name, str):
|
||||
raise ValueError(f"image bundle has no config for {tag}")
|
||||
config = _bundle_json(archive, config_name)
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError(f"image bundle config for {tag} is not an object")
|
||||
layers = entry.get("Layers")
|
||||
if not isinstance(layers, list) or not layers:
|
||||
raise ValueError(f"image bundle has no filesystem layers for {tag}")
|
||||
for layer in layers:
|
||||
if not isinstance(layer, str):
|
||||
raise ValueError(f"image bundle layer entry for {tag} is not a path")
|
||||
_bundle_member(archive, layer)
|
||||
rootfs = config.get("rootfs")
|
||||
config_layers = rootfs.get("diff_ids") if isinstance(rootfs, dict) else None
|
||||
expected = images[name]
|
||||
if config_layers != expected.get("rootfs_layers"):
|
||||
raise ValueError(f"image bundle layer set does not match built {tag}")
|
||||
config_section = config.get("config")
|
||||
actual_labels = config_section.get("Labels") if isinstance(config_section, dict) else None
|
||||
actual_labels = actual_labels or {}
|
||||
if not isinstance(actual_labels, dict) or any(
|
||||
actual_labels.get(key) != value for key, value in expected["labels"].items()
|
||||
):
|
||||
raise ValueError(f"image bundle OCI labels do not match built {tag}")
|
||||
if config.get("architecture") != expected.get("architecture"):
|
||||
raise ValueError(f"image bundle architecture does not match built {tag}")
|
||||
observed[name] = {
|
||||
"tag": tag,
|
||||
"config": config_name,
|
||||
"rootfs_layers": config_layers,
|
||||
"layer_members": layers,
|
||||
}
|
||||
|
||||
if set(observed) != set(IMAGES):
|
||||
raise ValueError("image bundle did not contain every declared release image")
|
||||
return observed
|
||||
|
||||
|
||||
def export_image_bundle(
|
||||
output: Path, version: str, images: dict[str, dict[str, Any]]
|
||||
) -> dict[str, Any]:
|
||||
"""Save and verify a transport bundle without rebuilding or re-tagging any image."""
|
||||
|
||||
bundle = output / f"modelforge-{version}-images.tar"
|
||||
if bundle.exists():
|
||||
bundle.unlink()
|
||||
tags = [str(images[name]["tag"]) for name in IMAGES]
|
||||
run("docker", "image", "save", "--output", str(bundle), *tags)
|
||||
verified = verify_image_bundle(bundle, images)
|
||||
return {
|
||||
"path": bundle.name,
|
||||
"sha256": sha256_file(bundle),
|
||||
"bytes": bundle.stat().st_size,
|
||||
"images": verified,
|
||||
}
|
||||
|
||||
|
||||
def package(output: Path, version: str, sbom_source: Path | None = None) -> Path:
|
||||
"""Assemble the release tarball from tracked paths only."""
|
||||
|
||||
@@ -300,6 +612,19 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", default="dist")
|
||||
parser.add_argument("--skip-images", action="store_true", help="package without building")
|
||||
parser.add_argument(
|
||||
"--image-bundle",
|
||||
action="store_true",
|
||||
help="save and verify the four exact built images as a Docker transport bundle",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--canonical-source-commit",
|
||||
default=None,
|
||||
help=(
|
||||
"canonical source SHA bound by PUBLIC_SOURCE_MANIFEST.json when building a curated "
|
||||
"parentless export; ordinary checkouts may only use their own HEAD"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--allow-dirty", action="store_true")
|
||||
parser.add_argument(
|
||||
"--built-at",
|
||||
@@ -317,6 +642,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
),
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
if args.image_bundle and args.skip_images:
|
||||
raise SystemExit("--image-bundle requires built images; do not combine it with --skip-images")
|
||||
|
||||
output = (ROOT / args.output).resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
@@ -329,7 +656,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
|
||||
print(f"{PRODUCT_NAME} {version} — release build", flush=True)
|
||||
source = source_identity(args.allow_dirty)
|
||||
source = source_identity(args.allow_dirty, args.canonical_source_commit)
|
||||
print(f" source commit {source['commit']}"
|
||||
f"{' (DIRTY)' if source['dirty'] else ''}", flush=True)
|
||||
|
||||
@@ -338,6 +665,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
if args.skip_images
|
||||
else build_images(version, source["commit"], built_at, public_api_origin)
|
||||
)
|
||||
if images:
|
||||
_assert_image_source_identity(images, source["source_repository"])
|
||||
|
||||
sbom_output = output / "sbom"
|
||||
run(
|
||||
@@ -386,6 +715,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"refusing to package a release that cannot reach its own API."
|
||||
)
|
||||
|
||||
image_bundle = None
|
||||
if args.image_bundle:
|
||||
print(" saving and verifying the exact four-image transport bundle", flush=True)
|
||||
image_bundle = export_image_bundle(output, version, images)
|
||||
|
||||
archive = package(output, version, sbom_output)
|
||||
print(f" packaged {archive.name} ({archive.stat().st_size} bytes)", flush=True)
|
||||
|
||||
@@ -420,6 +754,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"console": {"public_api_origin": public_api_origin or None},
|
||||
"artifacts": {},
|
||||
}
|
||||
if image_bundle is not None:
|
||||
manifest["image_bundle"] = image_bundle
|
||||
|
||||
manifest["sbom"] = {
|
||||
"format": "CycloneDX 1.5",
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
HELPER="$ROOT/scripts/modelforge_scheduled_backup.sh"
|
||||
TEMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TEMP_DIR"' EXIT HUP INT TERM
|
||||
mkdir -p "$TEMP_DIR/bin"
|
||||
|
||||
cat >"$TEMP_DIR/bin/curl" <<'CURL'
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
expected="${MODELFORGE_TEST_EXPECTED_KEY:?test did not provide an expected credential}"
|
||||
found=false
|
||||
for argument do
|
||||
if [ "$argument" = "X-ModelForge-Admin-Token: $expected" ]; then
|
||||
found=true
|
||||
fi
|
||||
done
|
||||
if [ "$found" != true ]; then
|
||||
printf '%s\n' 'scheduled backup sent the wrong admin credential' >&2
|
||||
exit 97
|
||||
fi
|
||||
|
||||
case " $* " in
|
||||
*"/verify "*)
|
||||
printf '%s\n' verify >>"$MODELFORGE_TEST_CURL_CALLS"
|
||||
printf '%s' '{"state":"VERIFIED"}'
|
||||
;;
|
||||
*"/retention/run "*)
|
||||
printf '%s\n' retention >>"$MODELFORGE_TEST_CURL_CALLS"
|
||||
;;
|
||||
*"/recovery/backups "*)
|
||||
printf '%s\n' create >>"$MODELFORGE_TEST_CURL_CALLS"
|
||||
printf '%s' '{"id":"00000000-0000-0000-0000-000000000001","state":"CREATED"}'
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' 'scheduled backup sent an unexpected request' >&2
|
||||
exit 98
|
||||
;;
|
||||
esac
|
||||
CURL
|
||||
chmod +x "$TEMP_DIR/bin/curl"
|
||||
|
||||
assert_calls() {
|
||||
expected_calls="$1"
|
||||
actual_calls="$(cat "$TEMP_DIR/calls")"
|
||||
if [ "$actual_calls" != "$expected_calls" ]; then
|
||||
printf 'expected API calls:\n%s\nactual API calls:\n%s\n' "$expected_calls" "$actual_calls" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_secret_not_logged() {
|
||||
secret="$1"
|
||||
if grep -F "$secret" "$TEMP_DIR/stdout" "$TEMP_DIR/stderr" >/dev/null 2>&1; then
|
||||
printf '%s\n' 'scheduled backup printed a credential to its output' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
: >"$TEMP_DIR/calls"
|
||||
if ! MODELFORGE_BACKUP_ADMIN_API_KEY=dedicated-named-admin-secret \
|
||||
MODELFORGE_OPERATOR_API_KEY=legacy-global-secret \
|
||||
MODELFORGE_TEST_EXPECTED_KEY=dedicated-named-admin-secret \
|
||||
MODELFORGE_TEST_CURL_CALLS="$TEMP_DIR/calls" \
|
||||
PATH="$TEMP_DIR/bin:$PATH" sh "$HELPER" >"$TEMP_DIR/stdout" 2>"$TEMP_DIR/stderr"; then
|
||||
cat "$TEMP_DIR/stdout" "$TEMP_DIR/stderr" >&2
|
||||
printf '%s\n' 'scheduled backup rejected its dedicated named administrator key' >&2
|
||||
exit 1
|
||||
fi
|
||||
assert_calls 'create
|
||||
verify
|
||||
retention'
|
||||
assert_secret_not_logged dedicated-named-admin-secret
|
||||
|
||||
: >"$TEMP_DIR/calls"
|
||||
if ! MODELFORGE_OPERATOR_API_KEY=legacy-global-secret \
|
||||
MODELFORGE_TEST_EXPECTED_KEY=legacy-global-secret \
|
||||
MODELFORGE_TEST_CURL_CALLS="$TEMP_DIR/calls" \
|
||||
PATH="$TEMP_DIR/bin:$PATH" sh "$HELPER" >"$TEMP_DIR/stdout" 2>"$TEMP_DIR/stderr"; then
|
||||
cat "$TEMP_DIR/stdout" "$TEMP_DIR/stderr" >&2
|
||||
printf '%s\n' 'scheduled backup no longer accepts the legacy administrator key' >&2
|
||||
exit 1
|
||||
fi
|
||||
assert_calls 'create
|
||||
verify
|
||||
retention'
|
||||
assert_secret_not_logged legacy-global-secret
|
||||
|
||||
: >"$TEMP_DIR/calls"
|
||||
set +e
|
||||
env -u MODELFORGE_BACKUP_ADMIN_API_KEY -u MODELFORGE_OPERATOR_API_KEY \
|
||||
MODELFORGE_TEST_CURL_CALLS="$TEMP_DIR/calls" PATH="$TEMP_DIR/bin:$PATH" \
|
||||
sh "$HELPER" >"$TEMP_DIR/stdout" 2>"$TEMP_DIR/stderr"
|
||||
missing_key_status=$?
|
||||
set -e
|
||||
if [ "$missing_key_status" -eq 0 ] || [ -s "$TEMP_DIR/calls" ]; then
|
||||
printf '%s\n' 'scheduled backup did not fail closed before making an API call' >&2
|
||||
cat "$TEMP_DIR/stdout" "$TEMP_DIR/stderr" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -F 'MODELFORGE_BACKUP_ADMIN_API_KEY' "$TEMP_DIR/stderr" >/dev/null 2>&1; then
|
||||
printf '%s\n' 'missing-key diagnostic did not name the dedicated setting' >&2
|
||||
cat "$TEMP_DIR/stderr" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' 'scheduled backup authentication tests passed'
|
||||
@@ -27,8 +27,14 @@ function walk(directory, prefix = "") {
|
||||
const required = [
|
||||
"README.md", "SECURITY.md", "CONTRIBUTING.md", "LICENSE", "VERSION",
|
||||
"PUBLIC_SOURCE_EXPORT.md", "PUBLIC_SOURCE_MANIFEST.json", "docker-compose.yml",
|
||||
"backend/Dockerfile", "frontend/Dockerfile", "node-agent/Dockerfile", "runtime-worker/Dockerfile",
|
||||
"docker-compose.validation-tls.yml", "deploy/validation/Caddyfile",
|
||||
"backend/pyproject.toml", "frontend/package.json", "node-agent/pyproject.toml",
|
||||
"runtime-worker/pyproject.toml"
|
||||
"runtime-worker/pyproject.toml", "clients/README.md",
|
||||
"clients/python/modelforge_client.py", "clients/typescript/modelforge-client.mjs",
|
||||
"clients/typescript/modelforge-client.d.mts", "scripts/m16_chaos.py",
|
||||
"scripts/m16_invariants.py", "scripts/m16_soak.py", "scripts/operator_identity.py",
|
||||
"scripts/rc_isolation_preflight.py", "scripts/rc_stack_retention.py"
|
||||
];
|
||||
const missing = required.filter((entry) => !fs.existsSync(path.join(process.cwd(), entry)));
|
||||
if (missing.length > 0) {
|
||||
@@ -46,6 +52,7 @@ if (
|
||||
manifest.schemaVersion !== 1 ||
|
||||
typeof manifest.sourceRevision !== "string" ||
|
||||
!/^[0-9a-f]{40}$/u.test(manifest.sourceRevision) ||
|
||||
manifest.sourceRepository !== "https://git.example.com/example/modelforge" ||
|
||||
!Array.isArray(manifest.files) ||
|
||||
manifest.files.length === 0
|
||||
) {
|
||||
@@ -53,6 +60,7 @@ if (
|
||||
}
|
||||
|
||||
const declared = new Set();
|
||||
const manifestEntries = new Map();
|
||||
for (const entry of manifest.files) {
|
||||
if (
|
||||
!entry ||
|
||||
@@ -71,6 +79,7 @@ for (const entry of manifest.files) {
|
||||
const collisionKey = entry.path.toLocaleLowerCase("en-US");
|
||||
if (declared.has(collisionKey)) fail(`Duplicate manifest path: ${entry.path}`);
|
||||
declared.add(collisionKey);
|
||||
manifestEntries.set(entry.path, entry);
|
||||
|
||||
const absolutePath = path.join(process.cwd(), ...entry.path.split("/"));
|
||||
if (!fs.existsSync(absolutePath)) fail(`Manifest file is missing: ${entry.path}`);
|
||||
@@ -84,6 +93,20 @@ for (const entry of manifest.files) {
|
||||
if (digest !== entry.sha256) fail(`Manifest hash mismatch: ${entry.path}`);
|
||||
}
|
||||
|
||||
for (const relativePath of [
|
||||
"backend/Dockerfile", "frontend/Dockerfile", "node-agent/Dockerfile", "runtime-worker/Dockerfile"
|
||||
]) {
|
||||
const entry = manifestEntries.get(relativePath);
|
||||
if (!entry) fail(`Manifest does not bind ${relativePath}`);
|
||||
const text = fs.readFileSync(relativePath, "utf8");
|
||||
const labels = [...text.matchAll(
|
||||
/^LABEL\s+org\.opencontainers\.image\.source="([^"]+)"\s*$/gmu
|
||||
)];
|
||||
if (labels.length !== 1 || labels[0][1] !== manifest.sourceRepository) {
|
||||
fail(`${relativePath} source label does not match the manifest sourceRepository`);
|
||||
}
|
||||
}
|
||||
|
||||
const generated = new Set(["public_source_export.md", "public_source_manifest.json"]);
|
||||
const unexpected = walk(process.cwd()).filter(
|
||||
(entry) => !declared.has(entry.toLocaleLowerCase("en-US")) &&
|
||||
|
||||
Reference in new issue
Block a user