1036 lines
41 KiB
Python
1036 lines
41 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import update
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlmodel import Session, select
|
|
|
|
from ..core.config import settings
|
|
from ..core.time import utc_now
|
|
from ..db.models import DeploymentJob, Policy
|
|
from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT
|
|
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
|
|
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
|
from ..schemas.forge_runtime_readiness import (
|
|
ForgeRuntimeReadinessResultV1,
|
|
ForgeRuntimeReadinessV1,
|
|
)
|
|
from .audit_logger import log_event
|
|
from .deployment_broker import DeploymentBrokerClient
|
|
from .enrollment_plane import EnrollmentPlane
|
|
from .policy_engine import evaluate_policy
|
|
from .secret_store import get_secret
|
|
|
|
_API_VERSION = "itworx.forge-first-install/v1"
|
|
_PREFLIGHT_API_VERSION = "itworx.forge-bootstrap-preflight/v1"
|
|
_APPLICATION_ID = "itworx-forge"
|
|
_REPOSITORY = "jens/itworx-forge"
|
|
_OPERATION = "forge_first_install"
|
|
_SECRET_REF = "FORGE_DEPLOYMENT_BROKER"
|
|
_FORGE_SOURCE_URL = "https://gitea.example.com/Jens/itworx-forge"
|
|
_FORGE_PRIVATE_PACKAGE_OWNER_SETTING = "FORGE_PRIVATE_PACKAGE_OWNER"
|
|
_FORGE_PRIVATE_PACKAGE_OWNER_PATTERN = re.compile(r"[a-z0-9][a-z0-9_-]{0,38}\Z")
|
|
_FORGE_OIDC_ISSUER = "https://auth.example-host.com/application/o/itworx-forge/"
|
|
_FORGE_OIDC_CLIENT_ID = "itworx-forge"
|
|
_FORGE_OIDC_CALLBACK = "https://forge.example.com/auth/callback"
|
|
_FORGE_OIDC_GROUPS = ["forge-owners", "forge-operators", "forge-viewers"]
|
|
_FORGE_OIDC_SCOPES = ["email", "openid", "profile"]
|
|
_FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256 = {
|
|
"openid": "38e58da331cc5378d4f269389242aecb80adf30093c405524dc1ab5aab9d452f",
|
|
"email": "d1200eabeffc3b7f379e9c1636367b69befb0b1ac2b5d82b7be20e78667f2b94",
|
|
"profile": "648b3f6dbf2212940309a581b239c9996562ca017a89be22bcfa93f9aa2c26ae",
|
|
}
|
|
_FORGE_OIDC_PROFILE_MAPPING_NAME = "ITWorx Forge profile and groups claim v1"
|
|
_FORGE_OIDC_OPENID_MAPPING_MANAGED = "goauthentik.io/providers/oauth2/scope-openid"
|
|
_FORGE_OIDC_EMAIL_MAPPING_MANAGED = "goauthentik.io/providers/oauth2/scope-email"
|
|
_OIDC_ATTESTATION_FIELDS = {
|
|
"issuer", "audience", "clientId", "clientType", "callback", "applicationId",
|
|
"providerId", "clientSecretMatched", "authorizationFlowId", "invalidationFlowId",
|
|
"roleGroups", "scopeMappingIds", "scopeMappingExpressionSha256",
|
|
"profileMappingName", "standardMappingsManaged", "supportedScopes",
|
|
"authorizationCodeFlowSupported", "signingKeyId", "signingKeyType",
|
|
"signingKeyPrivateAvailable", "signingKeyFingerprintSha256", "signingKeyKid",
|
|
"tokenSigningAlgorithm", "jwksKid",
|
|
}
|
|
_AUTHENTIK_FORGE_OIDC_BINDING_VALID_SETTING = "AUTHENTIK_FORGE_OIDC_BINDING_VALID"
|
|
_AUTHENTIK_FORGE_OIDC_PROVIDER_ID_SETTING = "AUTHENTIK_FORGE_OIDC_PROVIDER_ID"
|
|
_AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS = {
|
|
"openid": "AUTHENTIK_FORGE_OIDC_OPENID_CLAIM_MAPPING",
|
|
"email": "AUTHENTIK_FORGE_OIDC_EMAIL_CLAIM_MAPPING",
|
|
"profile": "AUTHENTIK_FORGE_OIDC_PROFILE_CLAIM_MAPPING",
|
|
}
|
|
_AUTHENTIK_SIGNING_KEY_ID_SETTING = "AUTHENTIK_FORGE_OIDC_SIGNING_KEY_ID"
|
|
_AUTHENTIK_AUTHORIZATION_FLOW_SETTING = "AUTHENTIK_OIDC_AUTHORIZATION_FLOW"
|
|
_AUTHENTIK_INVALIDATION_FLOW_SETTING = "AUTHENTIK_OIDC_INVALIDATION_FLOW"
|
|
_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
|
_HEALTHY_SERVICES = ["api", "dashboard", "itworx-forge", "postgres", "worker"]
|
|
_PROFILE_CHECKS = {
|
|
"deployment_apply_policy",
|
|
"controller_trust",
|
|
"managed_appdata_root_selection",
|
|
"exact_main",
|
|
"source_manifest",
|
|
"managed_appdata",
|
|
"proxy_network",
|
|
"resource_collisions",
|
|
"push_validation",
|
|
"image_registry",
|
|
"oidc",
|
|
"secret_delivery",
|
|
}
|
|
_FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"}
|
|
_ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"}
|
|
_TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"}
|
|
_RUNTIME_READINESS_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
|
|
|
|
|
|
class ForgeBootstrapApplyError(ValueError):
|
|
"""A fail-closed error in the fixed Forge first-install contract."""
|
|
|
|
|
|
def _canonical_json(value: Any) -> bytes:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def _sha256(value: Any) -> str:
|
|
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
|
|
|
|
|
def canonical_preflight_receipt(job: DeploymentJob) -> str:
|
|
"""Hash the immutable preflight receipt as stored in AppOps, not caller data."""
|
|
receipt = {
|
|
"apiVersion": _PREFLIGHT_API_VERSION,
|
|
"jobId": job.id,
|
|
"applicationId": job.application_id,
|
|
"repository": job.repository,
|
|
"revision": job.revision,
|
|
"jobStatus": job.status,
|
|
"planSha256": job.plan_sha256,
|
|
"profile": dict(job.plan_json or {}),
|
|
"preflight": dict(job.preflight_json or {}),
|
|
"evidence": dict(job.evidence_json or {}),
|
|
}
|
|
return _sha256(receipt)
|
|
|
|
|
|
def _idempotency_key(value: str) -> str:
|
|
normalized = value.strip()
|
|
if not _IDEMPOTENCY.fullmatch(normalized):
|
|
raise ForgeBootstrapApplyError("Idempotency-Key must contain 8 to 128 safe characters.")
|
|
return normalized
|
|
|
|
|
|
def _unknown_summary() -> str:
|
|
return "The fixed Forge controller result is unknown; reconcile its status before retrying."
|
|
|
|
|
|
def _preflight_ready(
|
|
session: Session, profile: ForgeBootstrapApplyV1
|
|
) -> tuple[DeploymentJob, str]:
|
|
job = session.get(DeploymentJob, profile.preflight_job_id)
|
|
if (
|
|
not job
|
|
or job.application_id != _APPLICATION_ID
|
|
or job.repository != _REPOSITORY
|
|
or job.revision != profile.revision
|
|
or job.status != "preflight_succeeded"
|
|
or not job.completed_at
|
|
or job.plan_sha256 != _sha256(dict(job.plan_json or {}))
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"A matching passed Forge preflight receipt is required before first install."
|
|
)
|
|
try:
|
|
stored_profile = ForgeBootstrapPreflightV1.model_validate(job.plan_json)
|
|
except ValueError as exc:
|
|
raise ForgeBootstrapApplyError("The stored Forge preflight receipt is invalid.") from exc
|
|
if (
|
|
stored_profile.revision != profile.revision
|
|
or stored_profile.workspace_image_digest != profile.workspace_image_digest
|
|
or stored_profile.frontdoor_image_digest != profile.frontdoor_image_digest
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"The stored Forge preflight receipt does not match this profile."
|
|
)
|
|
|
|
preflight = dict(job.preflight_json or {})
|
|
if (
|
|
set(preflight) != {"apiVersion", "status", "blockers", "unknowns", "checks"}
|
|
or preflight.get("apiVersion") != _PREFLIGHT_API_VERSION
|
|
or preflight.get("status") != "passed"
|
|
or preflight.get("blockers") != []
|
|
or preflight.get("unknowns") != []
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"Forge first install requires a passed preflight with no unknown checks."
|
|
)
|
|
checks = preflight.get("checks")
|
|
if not isinstance(checks, list) or len(checks) != len(_PROFILE_CHECKS):
|
|
raise ForgeBootstrapApplyError("The stored Forge preflight checks are incomplete.")
|
|
seen: set[str] = set()
|
|
for check in checks:
|
|
if (
|
|
not isinstance(check, dict)
|
|
or set(check) != {"name", "status", "detail"}
|
|
or not isinstance(check.get("name"), str)
|
|
or check["name"] not in _PROFILE_CHECKS
|
|
or check["name"] in seen
|
|
or check.get("status") != "passed"
|
|
or not isinstance(check.get("detail"), str)
|
|
or len(check["detail"]) > 256
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"Forge first install requires every fixed preflight check to pass."
|
|
)
|
|
seen.add(check["name"])
|
|
if seen != _PROFILE_CHECKS:
|
|
raise ForgeBootstrapApplyError("The stored Forge preflight checks are incomplete.")
|
|
|
|
evidence = dict(job.evidence_json or {})
|
|
if (
|
|
set(evidence)
|
|
!= {"profile", "policyId", "workspaceImageDigest", "frontdoorImageDigest", "controller"}
|
|
or evidence.get("profile") != "forge-production-bootstrap"
|
|
or not isinstance(evidence.get("policyId"), str)
|
|
or evidence.get("workspaceImageDigest") != profile.workspace_image_digest
|
|
or evidence.get("frontdoorImageDigest") != profile.frontdoor_image_digest
|
|
or not isinstance(evidence.get("controller"), dict)
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"The stored Forge preflight evidence does not match this profile."
|
|
)
|
|
current_policy_id = _allowed_policy_id(session)
|
|
if current_policy_id is None or current_policy_id != evidence["policyId"]:
|
|
raise ForgeBootstrapApplyError(
|
|
"The exact-target Forge deployment policy changed after preflight."
|
|
)
|
|
if not _attestations_ready(evidence["controller"], profile):
|
|
raise ForgeBootstrapApplyError(
|
|
"The stored Forge preflight lacks verified controller attestations."
|
|
)
|
|
receipt_sha = canonical_preflight_receipt(job)
|
|
if profile.preflight_sha256 != receipt_sha:
|
|
raise ForgeBootstrapApplyError(
|
|
"The supplied preflight receipt digest does not match the stored preflight receipt."
|
|
)
|
|
return job, receipt_sha
|
|
|
|
|
|
def _allowed_policy_id(session: Session) -> str | None:
|
|
policies = session.exec(
|
|
select(Policy).where(
|
|
(Policy.action == "deployment.apply") & (Policy.target == _APPLICATION_ID)
|
|
)
|
|
).all()
|
|
if len(policies) != 1:
|
|
return None
|
|
policy = policies[0]
|
|
decision = evaluate_policy(
|
|
session, action="deployment.apply", target=_APPLICATION_ID, source="manual"
|
|
)
|
|
allowed = bool(
|
|
policy.manual == "allowed"
|
|
and policy.voice == "blocked"
|
|
and policy.risk == "high"
|
|
and decision.get("decision") == "allowed"
|
|
and decision.get("policy_id") == policy.id
|
|
)
|
|
return policy.id if allowed else None
|
|
|
|
|
|
def _attestations_ready(controller: dict[str, Any], profile: ForgeBootstrapApplyV1) -> bool:
|
|
attestations = controller.get("attestations")
|
|
if not isinstance(attestations, dict) or set(attestations) != {
|
|
"push_validation", "image_registry", "oidc", "secret_delivery",
|
|
}:
|
|
return False
|
|
push = attestations.get("push_validation")
|
|
registry = attestations.get("image_registry")
|
|
oidc = attestations.get("oidc")
|
|
secret_delivery = attestations.get("secret_delivery")
|
|
private_owner = _protected_forge_private_package_owner()
|
|
return bool(
|
|
isinstance(push, dict)
|
|
and set(push) == {"context", "conclusion", "revision"}
|
|
and push == {
|
|
"context": "Managed validation / full (push)",
|
|
"conclusion": "success",
|
|
"revision": profile.revision,
|
|
}
|
|
and isinstance(registry, dict)
|
|
and private_owner is not None
|
|
and registry
|
|
== {
|
|
"owner": private_owner,
|
|
"ownerVisibility": "private",
|
|
"workspaceDigest": profile.workspace_image_digest,
|
|
"frontdoorDigest": profile.frontdoor_image_digest,
|
|
"workspacePullable": True,
|
|
"frontdoorPullable": True,
|
|
"workspaceSource": _FORGE_SOURCE_URL,
|
|
"workspaceRevision": profile.revision,
|
|
"frontdoorSource": _FORGE_SOURCE_URL,
|
|
"frontdoorRevision": profile.revision,
|
|
}
|
|
and forge_oidc_attestation_matches(oidc)
|
|
and isinstance(secret_delivery, dict)
|
|
and set(secret_delivery) == {"requiredFilesReady", "metadataSha256"}
|
|
and secret_delivery.get("requiredFilesReady") is True
|
|
and isinstance(secret_delivery.get("metadataSha256"), str)
|
|
and re.fullmatch(r"[0-9a-f]{64}", secret_delivery["metadataSha256"])
|
|
)
|
|
|
|
|
|
def _protected_forge_private_package_owner() -> str | None:
|
|
owner = settings.apis.get(_FORGE_PRIVATE_PACKAGE_OWNER_SETTING)
|
|
if not isinstance(owner, str) or _FORGE_PRIVATE_PACKAGE_OWNER_PATTERN.fullmatch(owner) is None:
|
|
return None
|
|
return owner
|
|
|
|
|
|
def forge_oidc_attestation_matches(value: object) -> bool:
|
|
"""Validate the fixed controller proof against protected AppOps Authentik pins."""
|
|
if not isinstance(value, dict) or set(value) != _OIDC_ATTESTATION_FIELDS:
|
|
return False
|
|
if settings.apis.get(_AUTHENTIK_FORGE_OIDC_BINDING_VALID_SETTING) is not True:
|
|
return False
|
|
provider_id = settings.apis.get(_AUTHENTIK_FORGE_OIDC_PROVIDER_ID_SETTING)
|
|
mapping_ids = {
|
|
scope: settings.apis.get(setting)
|
|
for scope, setting in _AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS.items()
|
|
}
|
|
signing_key_id = settings.apis.get(_AUTHENTIK_SIGNING_KEY_ID_SETTING)
|
|
authorization_flow = settings.apis.get(_AUTHENTIK_AUTHORIZATION_FLOW_SETTING)
|
|
invalidation_flow = settings.apis.get(_AUTHENTIK_INVALIDATION_FLOW_SETTING)
|
|
uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
|
if (
|
|
not isinstance(provider_id, str)
|
|
or not re.fullmatch(r"[1-9][0-9]{0,19}", provider_id)
|
|
or any(not isinstance(item, str) or not re.fullmatch(uuid_pattern, item) for item in mapping_ids.values())
|
|
or len(set(mapping_ids.values())) != len(_AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS)
|
|
or not isinstance(signing_key_id, str)
|
|
or not re.fullmatch(uuid_pattern, signing_key_id)
|
|
or not isinstance(authorization_flow, str)
|
|
or not authorization_flow.strip()
|
|
or not isinstance(invalidation_flow, str)
|
|
or not invalidation_flow.strip()
|
|
or value.get("providerId") != provider_id
|
|
or value.get("scopeMappingIds") != mapping_ids
|
|
or value.get("signingKeyId") != signing_key_id
|
|
or value.get("authorizationFlowId") != authorization_flow
|
|
or value.get("invalidationFlowId") != invalidation_flow
|
|
):
|
|
return False
|
|
application_id = value.get("applicationId")
|
|
if (
|
|
not isinstance(application_id, str)
|
|
or not re.fullmatch(uuid_pattern, application_id)
|
|
or value.get("providerId") != provider_id
|
|
):
|
|
return False
|
|
fingerprint = value.get("signingKeyFingerprintSha256")
|
|
if not isinstance(fingerprint, str) or not re.fullmatch(r"[0-9a-f]{64}", fingerprint):
|
|
return False
|
|
jwks_kid = value.get("jwksKid")
|
|
keypair_kid = value.get("signingKeyKid")
|
|
return bool(
|
|
value.get("issuer") == _FORGE_OIDC_ISSUER
|
|
and value.get("audience") == _FORGE_OIDC_CLIENT_ID
|
|
and value.get("clientId") == _FORGE_OIDC_CLIENT_ID
|
|
and value.get("clientType") == "confidential"
|
|
and value.get("callback") == _FORGE_OIDC_CALLBACK
|
|
and value.get("clientSecretMatched") is True
|
|
and isinstance(value.get("authorizationFlowId"), str)
|
|
and isinstance(value.get("invalidationFlowId"), str)
|
|
and value.get("roleGroups") == _FORGE_OIDC_GROUPS
|
|
and value.get("profileMappingName") == _FORGE_OIDC_PROFILE_MAPPING_NAME
|
|
and isinstance(value.get("scopeMappingExpressionSha256"), dict)
|
|
and value.get("scopeMappingExpressionSha256") == _FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256
|
|
and value.get("standardMappingsManaged") is True
|
|
and value.get("supportedScopes") == _FORGE_OIDC_SCOPES
|
|
and value.get("authorizationCodeFlowSupported") is True
|
|
and value.get("signingKeyType") == "RSA"
|
|
and value.get("signingKeyPrivateAvailable") is True
|
|
and value.get("tokenSigningAlgorithm") == "RS256"
|
|
and _forge_oidc_kid_is_valid(keypair_kid)
|
|
and jwks_kid == keypair_kid
|
|
)
|
|
|
|
|
|
def _forge_oidc_kid_is_valid(value: object) -> bool:
|
|
if not isinstance(value, str):
|
|
return False
|
|
if re.fullmatch(r"[0-9a-f]{32}", value):
|
|
return True
|
|
if re.fullmatch(r"[A-Za-z0-9_-]{86}", value) is None:
|
|
return False
|
|
try:
|
|
decoded = base64.b64decode(value + "==", altchars=b"-_", validate=True)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
return (
|
|
len(decoded) == 64
|
|
and base64.urlsafe_b64encode(decoded).decode("ascii").rstrip("=") == value
|
|
)
|
|
|
|
|
|
def _controller_secret(session: Session) -> str | None:
|
|
trust = EnrollmentPlane.get_trust_config(session)
|
|
if (
|
|
trust is None
|
|
# The fixed Tower controller shares the existing enrollment identity
|
|
# with RepoPulse; changing global trust would invalidate that binding.
|
|
or trust.controller_identity != "unraid-controller"
|
|
or trust.trusted_appdata_root != MANAGED_DOCKER_APPDATA_ROOT
|
|
or trust.managed_appdata_backup_acknowledged is not True
|
|
):
|
|
return None
|
|
try:
|
|
secret = get_secret(session, _SECRET_REF, touch=False)
|
|
except Exception: # noqa: BLE001 - never expose secret store errors.
|
|
return None
|
|
if (
|
|
not isinstance(secret, str)
|
|
or len(secret) < 32
|
|
or not secret.isascii()
|
|
or not secret.isprintable()
|
|
):
|
|
return None
|
|
return secret
|
|
|
|
|
|
def _response_identity_valid(
|
|
result: dict[str, Any], *, request_id: str, profile: ForgeBootstrapApplyV1
|
|
) -> bool:
|
|
return bool(
|
|
result.get("apiVersion") == _API_VERSION
|
|
and result.get("request") == request_id
|
|
and result.get("repository") == _REPOSITORY
|
|
and result.get("revision") == profile.revision
|
|
and result.get("operation") == _OPERATION
|
|
and result.get("workspaceImageDigest") == profile.workspace_image_digest
|
|
and result.get("frontdoorImageDigest") == profile.frontdoor_image_digest
|
|
)
|
|
|
|
|
|
def _classify_controller_result(
|
|
result: object,
|
|
*,
|
|
request_id: str,
|
|
profile: ForgeBootstrapApplyV1,
|
|
) -> tuple[str, str, str, dict[str, Any]]:
|
|
"""Return status/stage/summary and strictly projected controller evidence."""
|
|
if not isinstance(result, dict) or not _response_identity_valid(
|
|
result, request_id=request_id, profile=profile
|
|
):
|
|
return (
|
|
"outcome_unknown",
|
|
"controller_response_unverified",
|
|
_unknown_summary(),
|
|
{},
|
|
)
|
|
status = result.get("status")
|
|
base_fields = {
|
|
"apiVersion",
|
|
"status",
|
|
"request",
|
|
"repository",
|
|
"revision",
|
|
"operation",
|
|
"workspaceImageDigest",
|
|
"frontdoorImageDigest",
|
|
}
|
|
identity = {key: result[key] for key in sorted(base_fields)}
|
|
if status in {"accepted", "running"} and set(result) == base_fields:
|
|
return (
|
|
"applying",
|
|
"controller_accepted" if status == "accepted" else "controller_running",
|
|
"Forge first install is running on the fixed controller.",
|
|
identity,
|
|
)
|
|
if status == "needs_database_credentials" and set(result) == base_fields:
|
|
return (
|
|
"needs_database_credentials",
|
|
"controller_needs_database_credentials",
|
|
"Forge first install is paused before migration until fixed database credentials are provisioned.",
|
|
identity,
|
|
)
|
|
if (
|
|
status == "preflight_changed"
|
|
and set(result) == base_fields | {"noMutation"}
|
|
and result.get("noMutation") is True
|
|
):
|
|
return (
|
|
"preflight_changed",
|
|
"controller_preflight_changed",
|
|
"Controller rejected changed Forge first-install preflight before mutation.",
|
|
{**identity, "noMutation": True},
|
|
)
|
|
if status == "deployed" and set(result) == base_fields | {"receipt"}:
|
|
receipt = result.get("receipt")
|
|
if not isinstance(receipt, dict) or set(receipt) != {
|
|
"revision",
|
|
"workspaceImageDigest",
|
|
"frontdoorImageDigest",
|
|
"migration",
|
|
"health",
|
|
}:
|
|
return (
|
|
"outcome_unknown",
|
|
"controller_receipt_unverified",
|
|
_unknown_summary(),
|
|
{},
|
|
)
|
|
migration = receipt.get("migration")
|
|
health = receipt.get("health")
|
|
if (
|
|
receipt.get("revision") != profile.revision
|
|
or receipt.get("workspaceImageDigest") != profile.workspace_image_digest
|
|
or receipt.get("frontdoorImageDigest") != profile.frontdoor_image_digest
|
|
or not isinstance(migration, dict)
|
|
or set(migration) != {"service", "status", "result"}
|
|
or migration
|
|
!= {"service": "migrate", "status": "completed", "result": "succeeded"}
|
|
or not isinstance(health, dict)
|
|
or set(health) != {"status", "services"}
|
|
or health.get("status") != "healthy"
|
|
or health.get("services") != _HEALTHY_SERVICES
|
|
):
|
|
return (
|
|
"outcome_unknown",
|
|
"controller_receipt_unverified",
|
|
_unknown_summary(),
|
|
{},
|
|
)
|
|
safe_receipt = {
|
|
"revision": profile.revision,
|
|
"workspaceImageDigest": profile.workspace_image_digest,
|
|
"frontdoorImageDigest": profile.frontdoor_image_digest,
|
|
"migration": dict(migration),
|
|
"health": {"status": "healthy", "services": list(_HEALTHY_SERVICES)},
|
|
}
|
|
return (
|
|
"deployed",
|
|
"controller_deployed",
|
|
"Forge first install completed with exact migration and health receipts.",
|
|
{**identity, "receipt": safe_receipt},
|
|
)
|
|
if status == "failed" and set(result) == base_fields | {"failure"}:
|
|
failure = result.get("failure")
|
|
if (
|
|
isinstance(failure, dict)
|
|
and set(failure) == {"code", "preservedData"}
|
|
and failure.get("code") in _FAILED_CODES
|
|
and failure.get("preservedData") is True
|
|
):
|
|
return (
|
|
"failed",
|
|
"controller_failed_preserving_data",
|
|
"Controller reported a first-install failure and confirmed PostgreSQL data preservation.",
|
|
{
|
|
**identity,
|
|
"failure": {
|
|
"code": failure["code"],
|
|
"preservedData": True,
|
|
},
|
|
},
|
|
)
|
|
return (
|
|
"outcome_unknown",
|
|
"controller_response_unverified",
|
|
_unknown_summary(),
|
|
{},
|
|
)
|
|
|
|
|
|
def _serialize(job: DeploymentJob, *, replay: bool = False) -> dict[str, Any]:
|
|
return {
|
|
"id": job.id,
|
|
"applicationId": job.application_id,
|
|
"repository": job.repository,
|
|
"revision": job.revision,
|
|
"status": job.status,
|
|
"stage": job.stage,
|
|
"preflight": dict(job.preflight_json or {}),
|
|
"evidence": dict(job.evidence_json or {}),
|
|
"result": job.result_summary,
|
|
"request": job.broker_request_id,
|
|
"createdAt": job.created_at.isoformat(),
|
|
"updatedAt": job.updated_at.isoformat(),
|
|
"completedAt": job.completed_at.isoformat() if job.completed_at else None,
|
|
"idempotentReplay": replay,
|
|
}
|
|
|
|
|
|
class ForgeBootstrapApplyService:
|
|
"""One fixed Forge first-install operation with durable, non-replaying apply."""
|
|
|
|
def __init__(self, *, broker: DeploymentBrokerClient | None = None):
|
|
self.broker = broker or DeploymentBrokerClient()
|
|
|
|
@staticmethod
|
|
def request_id(profile: ForgeBootstrapApplyV1, idempotency_key: str) -> str:
|
|
normalized = _idempotency_key(idempotency_key)
|
|
payload = {
|
|
"repository": _REPOSITORY,
|
|
"operation": _OPERATION,
|
|
"profile": profile.model_dump(mode="json"),
|
|
"idempotencyKey": normalized,
|
|
}
|
|
return _sha256(payload)
|
|
|
|
def apply(
|
|
self,
|
|
session: Session,
|
|
profile: ForgeBootstrapApplyV1,
|
|
*,
|
|
idempotency_key: str,
|
|
) -> dict[str, Any]:
|
|
normalized_key = _idempotency_key(idempotency_key)
|
|
idempotency_digest = hashlib.sha256(
|
|
f"forge-first-install:{normalized_key}".encode()
|
|
).hexdigest()
|
|
profile_json = profile.model_dump(mode="json")
|
|
plan_sha = _sha256(profile_json)
|
|
existing = session.exec(
|
|
select(DeploymentJob).where(
|
|
DeploymentJob.idempotency_sha256 == idempotency_digest
|
|
)
|
|
).first()
|
|
if existing:
|
|
if (
|
|
existing.application_id != _APPLICATION_ID
|
|
or existing.repository != _REPOSITORY
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"Idempotency-Key already belongs to a different application or repository."
|
|
)
|
|
if existing.plan_sha256 != plan_sha:
|
|
raise ForgeBootstrapApplyError(
|
|
"Idempotency-Key was already used for a different Forge first-install profile."
|
|
)
|
|
return self._refresh_job(session, existing, replay=True)
|
|
|
|
preflight_job, receipt_sha = _preflight_ready(session, profile)
|
|
if _allowed_policy_id(session) is None:
|
|
raise ForgeBootstrapApplyError(
|
|
"An allowed exact-target deployment.apply policy is required for Forge first install."
|
|
)
|
|
secret = _controller_secret(session)
|
|
if secret is None:
|
|
raise ForgeBootstrapApplyError(
|
|
"Stored managed-root trust and the fixed Forge deployment secret are required."
|
|
)
|
|
|
|
request_id = self.request_id(profile, normalized_key)
|
|
job_id = f"forge-install-{idempotency_digest}"
|
|
now = utc_now()
|
|
job = DeploymentJob(
|
|
id=job_id,
|
|
application_id=_APPLICATION_ID,
|
|
repository=_REPOSITORY,
|
|
revision=profile.revision,
|
|
plan_sha256=plan_sha,
|
|
idempotency_sha256=idempotency_digest,
|
|
status="applying",
|
|
stage="controller_request_pending",
|
|
plan_json=profile_json,
|
|
preflight_json={
|
|
"apiVersion": _PREFLIGHT_API_VERSION,
|
|
"jobId": preflight_job.id,
|
|
"sha256": receipt_sha,
|
|
},
|
|
evidence_json={"profile": "forge-first-install", "controller": {}},
|
|
broker_request_id=request_id,
|
|
result_summary="Forge first install is awaiting the fixed controller response.",
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
session.add(job)
|
|
try:
|
|
# The durable request intent must exist before the only POST attempt.
|
|
session.commit()
|
|
session.refresh(job)
|
|
except IntegrityError:
|
|
session.rollback()
|
|
raced = session.get(DeploymentJob, job_id)
|
|
if raced and (
|
|
raced.application_id != _APPLICATION_ID
|
|
or raced.repository != _REPOSITORY
|
|
):
|
|
raise ForgeBootstrapApplyError(
|
|
"Idempotency-Key already belongs to a different application or repository."
|
|
) from None
|
|
if raced and raced.plan_sha256 == plan_sha:
|
|
return self._refresh_job(session, raced, replay=True)
|
|
raise ForgeBootstrapApplyError(
|
|
"The Forge first-install request could not be durably claimed."
|
|
) from None
|
|
|
|
log_event(
|
|
session,
|
|
source="deployment-control-plane",
|
|
identity="operator",
|
|
action_id="deployment.apply",
|
|
target=_APPLICATION_ID,
|
|
target_type="deployment",
|
|
risk="high",
|
|
policy_decision="allowed",
|
|
reason="Submitted one fixed Forge first-install request after exact preflight validation.",
|
|
result="submitted",
|
|
execution_plan={
|
|
"request": request_id,
|
|
"planSha256": plan_sha,
|
|
"preflightSha256": receipt_sha,
|
|
},
|
|
redacted_metadata={"profile": "forge-first-install", "jobId": job_id},
|
|
)
|
|
try:
|
|
response = self.broker.forge_bootstrap_apply(
|
|
profile,
|
|
idempotency_key=normalized_key,
|
|
secret=secret,
|
|
)
|
|
except Exception: # noqa: BLE001 - durable intent is reconciled with GET only.
|
|
response = {"status": "unreachable"}
|
|
return self._record_response(
|
|
session,
|
|
job,
|
|
response,
|
|
profile,
|
|
expected_updated_at=job.updated_at,
|
|
)
|
|
|
|
def status(self, session: Session, job_id: str) -> dict[str, Any] | None:
|
|
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
|
return None
|
|
job = session.get(DeploymentJob, job_id)
|
|
if (
|
|
not job
|
|
or job.application_id != _APPLICATION_ID
|
|
or job.repository != _REPOSITORY
|
|
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
|
|
):
|
|
return None
|
|
return self._refresh_job(session, job)
|
|
|
|
@staticmethod
|
|
def _runtime_readiness_unavailable() -> dict[str, Any]:
|
|
return ForgeRuntimeReadinessResultV1.model_validate(
|
|
{
|
|
"status": "unavailable",
|
|
"message": "forge_runtime_readiness_unavailable",
|
|
"health": {
|
|
"status": "unavailable",
|
|
"services": [
|
|
{"name": name, "status": "unavailable"}
|
|
for name in _RUNTIME_READINESS_SERVICES
|
|
],
|
|
},
|
|
"migrationCompleted": False,
|
|
}
|
|
).model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
|
|
def current_readiness(self, session: Session, job_id: str) -> dict[str, Any]:
|
|
"""Read current controller state only for a verified successful Forge job."""
|
|
unavailable = self._runtime_readiness_unavailable()
|
|
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
|
return unavailable
|
|
try:
|
|
job = session.get(DeploymentJob, job_id)
|
|
except Exception: # noqa: BLE001 - do not expose protected store/DB failures.
|
|
return unavailable
|
|
if (
|
|
job is None
|
|
or job.application_id != _APPLICATION_ID
|
|
or job.repository != _REPOSITORY
|
|
or job.status != "deployed"
|
|
or job.stage not in {"controller_deployed", "resume_controller_deployed"}
|
|
or job.completed_at is None
|
|
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
|
|
):
|
|
return unavailable
|
|
try:
|
|
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
|
|
except (TypeError, ValueError):
|
|
return unavailable
|
|
if (
|
|
job.revision != profile.revision
|
|
or not isinstance(job.plan_json, dict)
|
|
or _sha256(job.plan_json) != job.plan_sha256
|
|
):
|
|
return unavailable
|
|
evidence = job.evidence_json
|
|
if (
|
|
not isinstance(evidence, dict)
|
|
or set(evidence) != {"profile", "controller"}
|
|
or evidence.get("profile") != "forge-first-install"
|
|
):
|
|
return unavailable
|
|
historical_status, _stage, _summary, _safe_evidence = _classify_controller_result(
|
|
evidence.get("controller"),
|
|
request_id=job.broker_request_id or "",
|
|
profile=profile,
|
|
)
|
|
if historical_status != "deployed":
|
|
return unavailable
|
|
|
|
# This protected per-target secret lookup is deliberately after every
|
|
# durable job/profile/receipt validation above.
|
|
secret = _controller_secret(session)
|
|
if secret is None:
|
|
return unavailable
|
|
try:
|
|
response = self.broker.forge_bootstrap_readiness(
|
|
job.broker_request_id or "", secret=secret
|
|
)
|
|
current = ForgeRuntimeReadinessV1.model_validate(response)
|
|
except Exception: # noqa: BLE001 - broker and schema errors are safe unavailable.
|
|
return unavailable
|
|
now = int(datetime.now(UTC).timestamp())
|
|
if (
|
|
current.request != job.broker_request_id
|
|
or current.repository != _REPOSITORY
|
|
or current.operation != _OPERATION
|
|
or current.profile.revision != profile.revision
|
|
or current.profile.workspace_image_digest != profile.workspace_image_digest
|
|
or current.profile.frontdoor_image_digest != profile.frontdoor_image_digest
|
|
or current.observed_at < now - 30
|
|
or current.observed_at > now + 5
|
|
):
|
|
return unavailable
|
|
message = {
|
|
"ready": "forge_runtime_ready",
|
|
"not_ready": "forge_runtime_not_ready",
|
|
"unavailable": "forge_runtime_readiness_unavailable",
|
|
}[current.status]
|
|
return ForgeRuntimeReadinessResultV1.model_validate(
|
|
{
|
|
"status": current.status,
|
|
"message": message,
|
|
"jobId": job.id,
|
|
"revision": current.profile.revision,
|
|
"workspaceImageDigest": current.profile.workspace_image_digest,
|
|
"frontdoorImageDigest": current.profile.frontdoor_image_digest,
|
|
"observedAt": current.observed_at,
|
|
"health": current.health.model_dump(mode="python"),
|
|
"migrationCompleted": current.migration_completed,
|
|
"controllerArtifact": current.controller_artifact.model_dump(
|
|
mode="python", by_alias=True
|
|
),
|
|
}
|
|
).model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
|
|
def resume(self, session: Session, job_id: str) -> dict[str, Any] | None:
|
|
"""Explicitly resume only after a signed read proves pre-migration pause."""
|
|
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
|
return None
|
|
job = session.get(DeploymentJob, job_id)
|
|
if (
|
|
not job
|
|
or job.application_id != _APPLICATION_ID
|
|
or job.repository != _REPOSITORY
|
|
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
|
|
):
|
|
return None
|
|
|
|
# A caller cannot turn an old pending/unknown record into another POST.
|
|
# First reconcile with the controller, then proceed only from its exact
|
|
# safe pre-migration credential state.
|
|
refreshed = self._refresh_job(session, job)
|
|
if refreshed.get("status") != "needs_database_credentials":
|
|
return refreshed
|
|
job = session.get(DeploymentJob, job_id)
|
|
if job is None:
|
|
return refreshed
|
|
session.refresh(job)
|
|
try:
|
|
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
|
|
except ValueError:
|
|
return refreshed
|
|
preflight_receipt = dict(job.preflight_json or {})
|
|
if (
|
|
set(preflight_receipt) != {"apiVersion", "jobId", "sha256"}
|
|
or preflight_receipt.get("apiVersion") != _PREFLIGHT_API_VERSION
|
|
or preflight_receipt.get("jobId") != profile.preflight_job_id
|
|
or preflight_receipt.get("sha256") != profile.preflight_sha256
|
|
):
|
|
return refreshed
|
|
try:
|
|
_preflight_ready(session, profile)
|
|
except ForgeBootstrapApplyError:
|
|
return refreshed
|
|
secret = _controller_secret(session)
|
|
if not secret:
|
|
return refreshed
|
|
|
|
expected_updated_at = job.updated_at
|
|
resume_pending = "Forge database credentials passed the signed pre-migration gate; resume is pending."
|
|
claimed_at = utc_now()
|
|
# Compare-and-swap the observed, signed pre-migration state. Concurrent
|
|
# callers can both reconcile it, but only one can persist the resume claim.
|
|
claim = session.exec(
|
|
update(DeploymentJob)
|
|
.where(
|
|
DeploymentJob.id == job.id,
|
|
DeploymentJob.application_id == _APPLICATION_ID,
|
|
DeploymentJob.repository == _REPOSITORY,
|
|
DeploymentJob.status == "needs_database_credentials",
|
|
DeploymentJob.stage == "controller_needs_database_credentials",
|
|
DeploymentJob.updated_at == expected_updated_at,
|
|
)
|
|
.values(
|
|
status="applying",
|
|
stage="resume_request_pending",
|
|
result_summary=resume_pending,
|
|
updated_at=claimed_at,
|
|
)
|
|
)
|
|
if claim.rowcount != 1:
|
|
session.rollback()
|
|
current = session.get(DeploymentJob, job_id)
|
|
if current is None:
|
|
return refreshed
|
|
session.refresh(current)
|
|
return _serialize(current)
|
|
# The one-shot intent is durable before the only resume POST attempt.
|
|
session.commit()
|
|
session.refresh(job)
|
|
resume_intent_version = job.updated_at
|
|
log_event(
|
|
session,
|
|
source="deployment-control-plane",
|
|
identity="operator",
|
|
action_id="deployment.apply",
|
|
target=_APPLICATION_ID,
|
|
target_type="deployment",
|
|
risk="high",
|
|
policy_decision="allowed",
|
|
reason="Requested one explicit Forge first-install resume after signed pre-migration credential status.",
|
|
result="resume_submitted",
|
|
execution_plan={"request": job.broker_request_id, "jobId": job.id},
|
|
redacted_metadata={"profile": "forge-first-install"},
|
|
)
|
|
try:
|
|
response = self.broker.forge_bootstrap_resume(
|
|
job.broker_request_id or "", secret=secret
|
|
)
|
|
except Exception: # noqa: BLE001 - durable resume intent is reconciled with GET only.
|
|
response = {"status": "unreachable"}
|
|
return self._record_response(
|
|
session,
|
|
job,
|
|
response,
|
|
profile,
|
|
expected_updated_at=resume_intent_version,
|
|
resume_claimed=True,
|
|
)
|
|
|
|
def _refresh_job(
|
|
self,
|
|
session: Session,
|
|
job: DeploymentJob,
|
|
*,
|
|
replay: bool = False,
|
|
) -> dict[str, Any]:
|
|
if job.status not in _ACTIVE_STATUSES:
|
|
return _serialize(job, replay=replay)
|
|
try:
|
|
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
|
|
except ValueError:
|
|
return _serialize(job, replay=replay)
|
|
secret = _controller_secret(session)
|
|
expected_updated_at = job.updated_at
|
|
resume_claimed = job.stage.startswith("resume")
|
|
if not secret:
|
|
return self._record_response(
|
|
session,
|
|
job,
|
|
{"status": "unreachable"},
|
|
profile,
|
|
replay=replay,
|
|
expected_updated_at=expected_updated_at,
|
|
resume_claimed=resume_claimed,
|
|
)
|
|
try:
|
|
response = self.broker.forge_bootstrap_status(
|
|
job.broker_request_id or "", secret=secret
|
|
)
|
|
except Exception: # noqa: BLE001 - reconciliation failures remain unknown.
|
|
response = {"status": "unreachable"}
|
|
return self._record_response(
|
|
session,
|
|
job,
|
|
response,
|
|
profile,
|
|
replay=replay,
|
|
expected_updated_at=expected_updated_at,
|
|
resume_claimed=resume_claimed,
|
|
)
|
|
|
|
@staticmethod
|
|
def _record_response(
|
|
session: Session,
|
|
job: DeploymentJob,
|
|
response: object,
|
|
profile: ForgeBootstrapApplyV1,
|
|
*,
|
|
replay: bool = False,
|
|
expected_updated_at: datetime | None = None,
|
|
resume_claimed: bool = False,
|
|
) -> dict[str, Any]:
|
|
expected_updated_at = expected_updated_at or job.updated_at
|
|
status, stage, result, evidence = _classify_controller_result(
|
|
response,
|
|
request_id=job.broker_request_id or "",
|
|
profile=profile,
|
|
)
|
|
resume_claimed = resume_claimed or job.stage.startswith("resume")
|
|
if resume_claimed:
|
|
if status == "needs_database_credentials":
|
|
status = "outcome_unknown"
|
|
stage = "resume_status_unresolved"
|
|
result = (
|
|
"Resume was already claimed but the controller still reports its pre-migration pause; "
|
|
"reconcile by status without submitting another resume."
|
|
)
|
|
else:
|
|
stage = f"resume_{stage}"
|
|
next_updated_at = utc_now()
|
|
next_completed_at = (
|
|
utc_now() if status in _TERMINAL_STATUSES else job.completed_at
|
|
)
|
|
values = {
|
|
"status": status,
|
|
"stage": stage,
|
|
"result_summary": result,
|
|
"evidence_json": {"profile": "forge-first-install", "controller": evidence},
|
|
"updated_at": next_updated_at,
|
|
"completed_at": next_completed_at,
|
|
}
|
|
statement = update(DeploymentJob).where(
|
|
DeploymentJob.id == job.id,
|
|
DeploymentJob.updated_at == expected_updated_at,
|
|
)
|
|
if status == "needs_database_credentials" and not resume_claimed:
|
|
# A status GET started after the local resume claim may still race a
|
|
# controller that has not processed the POST. It cannot reopen the gate.
|
|
statement = statement.where(DeploymentJob.stage.not_like("resume%"))
|
|
result_update = session.exec(statement.values(**values))
|
|
if result_update.rowcount != 1:
|
|
session.rollback()
|
|
current = session.get(DeploymentJob, job.id)
|
|
if current is None:
|
|
return _serialize(job, replay=replay)
|
|
session.refresh(current)
|
|
return _serialize(current, replay=replay)
|
|
session.commit()
|
|
session.refresh(job)
|
|
return _serialize(job, replay=replay)
|