Publish curated Unraid-AppOps-Gateway source from 451646fee806a47e3256731766980a05a262dcdd
This commit is contained in:
Public Source Publisher committed
1 parent
532f3d9f26
commit
0a82153d95
43 files changed
+17722
-93
No files matched your search
@@ -0,0 +1,923 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import DeploymentJob, Policy
|
||||
from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT
|
||||
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
|
||||
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
||||
from .audit_logger import log_event
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
from .enrollment_plane import EnrollmentPlane
|
||||
from .policy_engine import evaluate_policy
|
||||
from .secret_store import get_secret
|
||||
|
||||
_API_VERSION = "itworx.forge-first-install/v1"
|
||||
_PREFLIGHT_API_VERSION = "itworx.forge-bootstrap-preflight/v1"
|
||||
_APPLICATION_ID = "itworx-forge"
|
||||
_REPOSITORY = "jens/itworx-forge"
|
||||
_OPERATION = "forge_first_install"
|
||||
_SECRET_REF = "FORGE_DEPLOYMENT_BROKER"
|
||||
_FORGE_SOURCE_URL = "https://gitea.example.com/Jens/itworx-forge"
|
||||
_FORGE_PRIVATE_PACKAGE_OWNER_SETTING = "FORGE_PRIVATE_PACKAGE_OWNER"
|
||||
_FORGE_PRIVATE_PACKAGE_OWNER_PATTERN = re.compile(r"[a-z0-9][a-z0-9_-]{0,38}\Z")
|
||||
_FORGE_OIDC_ISSUER = "https://auth.example-host.com/application/o/itworx-forge/"
|
||||
_FORGE_OIDC_CLIENT_ID = "itworx-forge"
|
||||
_FORGE_OIDC_CALLBACK = "https://forge.example.com/auth/callback"
|
||||
_FORGE_OIDC_GROUPS = ["forge-owners", "forge-operators", "forge-viewers"]
|
||||
_FORGE_OIDC_SCOPES = ["email", "openid", "profile"]
|
||||
_FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256 = {
|
||||
"openid": "38e58da331cc5378d4f269389242aecb80adf30093c405524dc1ab5aab9d452f",
|
||||
"email": "d1200eabeffc3b7f379e9c1636367b69befb0b1ac2b5d82b7be20e78667f2b94",
|
||||
"profile": "648b3f6dbf2212940309a581b239c9996562ca017a89be22bcfa93f9aa2c26ae",
|
||||
}
|
||||
_FORGE_OIDC_PROFILE_MAPPING_NAME = "ITWorx Forge profile and groups claim v1"
|
||||
_FORGE_OIDC_OPENID_MAPPING_MANAGED = "goauthentik.io/providers/oauth2/scope-openid"
|
||||
_FORGE_OIDC_EMAIL_MAPPING_MANAGED = "goauthentik.io/providers/oauth2/scope-email"
|
||||
_OIDC_ATTESTATION_FIELDS = {
|
||||
"issuer", "audience", "clientId", "clientType", "callback", "applicationId",
|
||||
"providerId", "clientSecretMatched", "authorizationFlowId", "invalidationFlowId",
|
||||
"roleGroups", "scopeMappingIds", "scopeMappingExpressionSha256",
|
||||
"profileMappingName", "standardMappingsManaged", "supportedScopes",
|
||||
"authorizationCodeFlowSupported", "signingKeyId", "signingKeyType",
|
||||
"signingKeyPrivateAvailable", "signingKeyFingerprintSha256", "signingKeyKid",
|
||||
"tokenSigningAlgorithm", "jwksKid",
|
||||
}
|
||||
_AUTHENTIK_FORGE_OIDC_BINDING_VALID_SETTING = "AUTHENTIK_FORGE_OIDC_BINDING_VALID"
|
||||
_AUTHENTIK_FORGE_OIDC_PROVIDER_ID_SETTING = "AUTHENTIK_FORGE_OIDC_PROVIDER_ID"
|
||||
_AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS = {
|
||||
"openid": "AUTHENTIK_FORGE_OIDC_OPENID_CLAIM_MAPPING",
|
||||
"email": "AUTHENTIK_FORGE_OIDC_EMAIL_CLAIM_MAPPING",
|
||||
"profile": "AUTHENTIK_FORGE_OIDC_PROFILE_CLAIM_MAPPING",
|
||||
}
|
||||
_AUTHENTIK_SIGNING_KEY_ID_SETTING = "AUTHENTIK_FORGE_OIDC_SIGNING_KEY_ID"
|
||||
_AUTHENTIK_AUTHORIZATION_FLOW_SETTING = "AUTHENTIK_OIDC_AUTHORIZATION_FLOW"
|
||||
_AUTHENTIK_INVALIDATION_FLOW_SETTING = "AUTHENTIK_OIDC_INVALIDATION_FLOW"
|
||||
_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
_HEALTHY_SERVICES = ["api", "dashboard", "itworx-forge", "postgres", "worker"]
|
||||
_PROFILE_CHECKS = {
|
||||
"deployment_apply_policy",
|
||||
"controller_trust",
|
||||
"managed_appdata_root_selection",
|
||||
"exact_main",
|
||||
"source_manifest",
|
||||
"managed_appdata",
|
||||
"proxy_network",
|
||||
"resource_collisions",
|
||||
"push_validation",
|
||||
"image_registry",
|
||||
"oidc",
|
||||
"secret_delivery",
|
||||
}
|
||||
_FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"}
|
||||
_ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"}
|
||||
_TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"}
|
||||
|
||||
|
||||
class ForgeBootstrapApplyError(ValueError):
|
||||
"""A fail-closed error in the fixed Forge first-install contract."""
|
||||
|
||||
|
||||
def _canonical_json(value: Any) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(value: Any) -> str:
|
||||
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
||||
|
||||
|
||||
def canonical_preflight_receipt(job: DeploymentJob) -> str:
|
||||
"""Hash the immutable preflight receipt as stored in AppOps, not caller data."""
|
||||
receipt = {
|
||||
"apiVersion": _PREFLIGHT_API_VERSION,
|
||||
"jobId": job.id,
|
||||
"applicationId": job.application_id,
|
||||
"repository": job.repository,
|
||||
"revision": job.revision,
|
||||
"jobStatus": job.status,
|
||||
"planSha256": job.plan_sha256,
|
||||
"profile": dict(job.plan_json or {}),
|
||||
"preflight": dict(job.preflight_json or {}),
|
||||
"evidence": dict(job.evidence_json or {}),
|
||||
}
|
||||
return _sha256(receipt)
|
||||
|
||||
|
||||
def _idempotency_key(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not _IDEMPOTENCY.fullmatch(normalized):
|
||||
raise ForgeBootstrapApplyError("Idempotency-Key must contain 8 to 128 safe characters.")
|
||||
return normalized
|
||||
|
||||
|
||||
def _unknown_summary() -> str:
|
||||
return "The fixed Forge controller result is unknown; reconcile its status before retrying."
|
||||
|
||||
|
||||
def _preflight_ready(
|
||||
session: Session, profile: ForgeBootstrapApplyV1
|
||||
) -> tuple[DeploymentJob, str]:
|
||||
job = session.get(DeploymentJob, profile.preflight_job_id)
|
||||
if (
|
||||
not job
|
||||
or job.application_id != _APPLICATION_ID
|
||||
or job.repository != _REPOSITORY
|
||||
or job.revision != profile.revision
|
||||
or job.status != "preflight_succeeded"
|
||||
or not job.completed_at
|
||||
or job.plan_sha256 != _sha256(dict(job.plan_json or {}))
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"A matching passed Forge preflight receipt is required before first install."
|
||||
)
|
||||
try:
|
||||
stored_profile = ForgeBootstrapPreflightV1.model_validate(job.plan_json)
|
||||
except ValueError as exc:
|
||||
raise ForgeBootstrapApplyError("The stored Forge preflight receipt is invalid.") from exc
|
||||
if (
|
||||
stored_profile.revision != profile.revision
|
||||
or stored_profile.workspace_image_digest != profile.workspace_image_digest
|
||||
or stored_profile.frontdoor_image_digest != profile.frontdoor_image_digest
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"The stored Forge preflight receipt does not match this profile."
|
||||
)
|
||||
|
||||
preflight = dict(job.preflight_json or {})
|
||||
if (
|
||||
set(preflight) != {"apiVersion", "status", "blockers", "unknowns", "checks"}
|
||||
or preflight.get("apiVersion") != _PREFLIGHT_API_VERSION
|
||||
or preflight.get("status") != "passed"
|
||||
or preflight.get("blockers") != []
|
||||
or preflight.get("unknowns") != []
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"Forge first install requires a passed preflight with no unknown checks."
|
||||
)
|
||||
checks = preflight.get("checks")
|
||||
if not isinstance(checks, list) or len(checks) != len(_PROFILE_CHECKS):
|
||||
raise ForgeBootstrapApplyError("The stored Forge preflight checks are incomplete.")
|
||||
seen: set[str] = set()
|
||||
for check in checks:
|
||||
if (
|
||||
not isinstance(check, dict)
|
||||
or set(check) != {"name", "status", "detail"}
|
||||
or not isinstance(check.get("name"), str)
|
||||
or check["name"] not in _PROFILE_CHECKS
|
||||
or check["name"] in seen
|
||||
or check.get("status") != "passed"
|
||||
or not isinstance(check.get("detail"), str)
|
||||
or len(check["detail"]) > 256
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"Forge first install requires every fixed preflight check to pass."
|
||||
)
|
||||
seen.add(check["name"])
|
||||
if seen != _PROFILE_CHECKS:
|
||||
raise ForgeBootstrapApplyError("The stored Forge preflight checks are incomplete.")
|
||||
|
||||
evidence = dict(job.evidence_json or {})
|
||||
if (
|
||||
set(evidence)
|
||||
!= {"profile", "policyId", "workspaceImageDigest", "frontdoorImageDigest", "controller"}
|
||||
or evidence.get("profile") != "forge-production-bootstrap"
|
||||
or not isinstance(evidence.get("policyId"), str)
|
||||
or evidence.get("workspaceImageDigest") != profile.workspace_image_digest
|
||||
or evidence.get("frontdoorImageDigest") != profile.frontdoor_image_digest
|
||||
or not isinstance(evidence.get("controller"), dict)
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"The stored Forge preflight evidence does not match this profile."
|
||||
)
|
||||
current_policy_id = _allowed_policy_id(session)
|
||||
if current_policy_id is None or current_policy_id != evidence["policyId"]:
|
||||
raise ForgeBootstrapApplyError(
|
||||
"The exact-target Forge deployment policy changed after preflight."
|
||||
)
|
||||
if not _attestations_ready(evidence["controller"], profile):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"The stored Forge preflight lacks verified controller attestations."
|
||||
)
|
||||
receipt_sha = canonical_preflight_receipt(job)
|
||||
if profile.preflight_sha256 != receipt_sha:
|
||||
raise ForgeBootstrapApplyError(
|
||||
"The supplied preflight receipt digest does not match the stored preflight receipt."
|
||||
)
|
||||
return job, receipt_sha
|
||||
|
||||
|
||||
def _allowed_policy_id(session: Session) -> str | None:
|
||||
policies = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == "deployment.apply") & (Policy.target == _APPLICATION_ID)
|
||||
)
|
||||
).all()
|
||||
if len(policies) != 1:
|
||||
return None
|
||||
policy = policies[0]
|
||||
decision = evaluate_policy(
|
||||
session, action="deployment.apply", target=_APPLICATION_ID, source="manual"
|
||||
)
|
||||
allowed = bool(
|
||||
policy.manual == "allowed"
|
||||
and policy.voice == "blocked"
|
||||
and policy.risk == "high"
|
||||
and decision.get("decision") == "allowed"
|
||||
and decision.get("policy_id") == policy.id
|
||||
)
|
||||
return policy.id if allowed else None
|
||||
|
||||
|
||||
def _attestations_ready(controller: dict[str, Any], profile: ForgeBootstrapApplyV1) -> bool:
|
||||
attestations = controller.get("attestations")
|
||||
if not isinstance(attestations, dict) or set(attestations) != {
|
||||
"push_validation", "image_registry", "oidc", "secret_delivery",
|
||||
}:
|
||||
return False
|
||||
push = attestations.get("push_validation")
|
||||
registry = attestations.get("image_registry")
|
||||
oidc = attestations.get("oidc")
|
||||
secret_delivery = attestations.get("secret_delivery")
|
||||
private_owner = _protected_forge_private_package_owner()
|
||||
return bool(
|
||||
isinstance(push, dict)
|
||||
and set(push) == {"context", "conclusion", "revision"}
|
||||
and push == {
|
||||
"context": "Managed validation / full (push)",
|
||||
"conclusion": "success",
|
||||
"revision": profile.revision,
|
||||
}
|
||||
and isinstance(registry, dict)
|
||||
and private_owner is not None
|
||||
and registry
|
||||
== {
|
||||
"owner": private_owner,
|
||||
"ownerVisibility": "private",
|
||||
"workspaceDigest": profile.workspace_image_digest,
|
||||
"frontdoorDigest": profile.frontdoor_image_digest,
|
||||
"workspacePullable": True,
|
||||
"frontdoorPullable": True,
|
||||
"workspaceSource": _FORGE_SOURCE_URL,
|
||||
"workspaceRevision": profile.revision,
|
||||
"frontdoorSource": _FORGE_SOURCE_URL,
|
||||
"frontdoorRevision": profile.revision,
|
||||
}
|
||||
and forge_oidc_attestation_matches(oidc)
|
||||
and isinstance(secret_delivery, dict)
|
||||
and set(secret_delivery) == {"requiredFilesReady", "metadataSha256"}
|
||||
and secret_delivery.get("requiredFilesReady") is True
|
||||
and isinstance(secret_delivery.get("metadataSha256"), str)
|
||||
and re.fullmatch(r"[0-9a-f]{64}", secret_delivery["metadataSha256"])
|
||||
)
|
||||
|
||||
|
||||
def _protected_forge_private_package_owner() -> str | None:
|
||||
owner = settings.apis.get(_FORGE_PRIVATE_PACKAGE_OWNER_SETTING)
|
||||
if not isinstance(owner, str) or _FORGE_PRIVATE_PACKAGE_OWNER_PATTERN.fullmatch(owner) is None:
|
||||
return None
|
||||
return owner
|
||||
|
||||
|
||||
def forge_oidc_attestation_matches(value: object) -> bool:
|
||||
"""Validate the fixed controller proof against protected AppOps Authentik pins."""
|
||||
if not isinstance(value, dict) or set(value) != _OIDC_ATTESTATION_FIELDS:
|
||||
return False
|
||||
if settings.apis.get(_AUTHENTIK_FORGE_OIDC_BINDING_VALID_SETTING) is not True:
|
||||
return False
|
||||
provider_id = settings.apis.get(_AUTHENTIK_FORGE_OIDC_PROVIDER_ID_SETTING)
|
||||
mapping_ids = {
|
||||
scope: settings.apis.get(setting)
|
||||
for scope, setting in _AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS.items()
|
||||
}
|
||||
signing_key_id = settings.apis.get(_AUTHENTIK_SIGNING_KEY_ID_SETTING)
|
||||
authorization_flow = settings.apis.get(_AUTHENTIK_AUTHORIZATION_FLOW_SETTING)
|
||||
invalidation_flow = settings.apis.get(_AUTHENTIK_INVALIDATION_FLOW_SETTING)
|
||||
uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
||||
if (
|
||||
not isinstance(provider_id, str)
|
||||
or not re.fullmatch(r"[1-9][0-9]{0,19}", provider_id)
|
||||
or any(not isinstance(item, str) or not re.fullmatch(uuid_pattern, item) for item in mapping_ids.values())
|
||||
or len(set(mapping_ids.values())) != len(_AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS)
|
||||
or not isinstance(signing_key_id, str)
|
||||
or not re.fullmatch(uuid_pattern, signing_key_id)
|
||||
or not isinstance(authorization_flow, str)
|
||||
or not authorization_flow.strip()
|
||||
or not isinstance(invalidation_flow, str)
|
||||
or not invalidation_flow.strip()
|
||||
or value.get("providerId") != provider_id
|
||||
or value.get("scopeMappingIds") != mapping_ids
|
||||
or value.get("signingKeyId") != signing_key_id
|
||||
or value.get("authorizationFlowId") != authorization_flow
|
||||
or value.get("invalidationFlowId") != invalidation_flow
|
||||
):
|
||||
return False
|
||||
application_id = value.get("applicationId")
|
||||
if (
|
||||
not isinstance(application_id, str)
|
||||
or not re.fullmatch(uuid_pattern, application_id)
|
||||
or value.get("providerId") != provider_id
|
||||
):
|
||||
return False
|
||||
fingerprint = value.get("signingKeyFingerprintSha256")
|
||||
if not isinstance(fingerprint, str) or not re.fullmatch(r"[0-9a-f]{64}", fingerprint):
|
||||
return False
|
||||
jwks_kid = value.get("jwksKid")
|
||||
keypair_kid = value.get("signingKeyKid")
|
||||
return bool(
|
||||
value.get("issuer") == _FORGE_OIDC_ISSUER
|
||||
and value.get("audience") == _FORGE_OIDC_CLIENT_ID
|
||||
and value.get("clientId") == _FORGE_OIDC_CLIENT_ID
|
||||
and value.get("clientType") == "confidential"
|
||||
and value.get("callback") == _FORGE_OIDC_CALLBACK
|
||||
and value.get("clientSecretMatched") is True
|
||||
and isinstance(value.get("authorizationFlowId"), str)
|
||||
and isinstance(value.get("invalidationFlowId"), str)
|
||||
and value.get("roleGroups") == _FORGE_OIDC_GROUPS
|
||||
and value.get("profileMappingName") == _FORGE_OIDC_PROFILE_MAPPING_NAME
|
||||
and isinstance(value.get("scopeMappingExpressionSha256"), dict)
|
||||
and value.get("scopeMappingExpressionSha256") == _FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256
|
||||
and value.get("standardMappingsManaged") is True
|
||||
and value.get("supportedScopes") == _FORGE_OIDC_SCOPES
|
||||
and value.get("authorizationCodeFlowSupported") is True
|
||||
and value.get("signingKeyType") == "RSA"
|
||||
and value.get("signingKeyPrivateAvailable") is True
|
||||
and value.get("tokenSigningAlgorithm") == "RS256"
|
||||
and _forge_oidc_kid_is_valid(keypair_kid)
|
||||
and jwks_kid == keypair_kid
|
||||
)
|
||||
|
||||
|
||||
def _forge_oidc_kid_is_valid(value: object) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
if re.fullmatch(r"[0-9a-f]{32}", value):
|
||||
return True
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{86}", value) is None:
|
||||
return False
|
||||
try:
|
||||
decoded = base64.b64decode(value + "==", altchars=b"-_", validate=True)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
len(decoded) == 64
|
||||
and base64.urlsafe_b64encode(decoded).decode("ascii").rstrip("=") == value
|
||||
)
|
||||
|
||||
|
||||
def _controller_secret(session: Session) -> str | None:
|
||||
trust = EnrollmentPlane.get_trust_config(session)
|
||||
if (
|
||||
trust is None
|
||||
# The fixed Tower controller shares the existing enrollment identity
|
||||
# with RepoPulse; changing global trust would invalidate that binding.
|
||||
or trust.controller_identity != "unraid-controller"
|
||||
or trust.trusted_appdata_root != MANAGED_DOCKER_APPDATA_ROOT
|
||||
or trust.managed_appdata_backup_acknowledged is not True
|
||||
):
|
||||
return None
|
||||
try:
|
||||
secret = get_secret(session, _SECRET_REF, touch=False)
|
||||
except Exception: # noqa: BLE001 - never expose secret store errors.
|
||||
return None
|
||||
if (
|
||||
not isinstance(secret, str)
|
||||
or len(secret) < 32
|
||||
or not secret.isascii()
|
||||
or not secret.isprintable()
|
||||
):
|
||||
return None
|
||||
return secret
|
||||
|
||||
|
||||
def _response_identity_valid(
|
||||
result: dict[str, Any], *, request_id: str, profile: ForgeBootstrapApplyV1
|
||||
) -> bool:
|
||||
return bool(
|
||||
result.get("apiVersion") == _API_VERSION
|
||||
and result.get("request") == request_id
|
||||
and result.get("repository") == _REPOSITORY
|
||||
and result.get("revision") == profile.revision
|
||||
and result.get("operation") == _OPERATION
|
||||
and result.get("workspaceImageDigest") == profile.workspace_image_digest
|
||||
and result.get("frontdoorImageDigest") == profile.frontdoor_image_digest
|
||||
)
|
||||
|
||||
|
||||
def _classify_controller_result(
|
||||
result: object,
|
||||
*,
|
||||
request_id: str,
|
||||
profile: ForgeBootstrapApplyV1,
|
||||
) -> tuple[str, str, str, dict[str, Any]]:
|
||||
"""Return status/stage/summary and strictly projected controller evidence."""
|
||||
if not isinstance(result, dict) or not _response_identity_valid(
|
||||
result, request_id=request_id, profile=profile
|
||||
):
|
||||
return (
|
||||
"outcome_unknown",
|
||||
"controller_response_unverified",
|
||||
_unknown_summary(),
|
||||
{},
|
||||
)
|
||||
status = result.get("status")
|
||||
base_fields = {
|
||||
"apiVersion",
|
||||
"status",
|
||||
"request",
|
||||
"repository",
|
||||
"revision",
|
||||
"operation",
|
||||
"workspaceImageDigest",
|
||||
"frontdoorImageDigest",
|
||||
}
|
||||
identity = {key: result[key] for key in sorted(base_fields)}
|
||||
if status in {"accepted", "running"} and set(result) == base_fields:
|
||||
return (
|
||||
"applying",
|
||||
"controller_accepted" if status == "accepted" else "controller_running",
|
||||
"Forge first install is running on the fixed controller.",
|
||||
identity,
|
||||
)
|
||||
if status == "needs_database_credentials" and set(result) == base_fields:
|
||||
return (
|
||||
"needs_database_credentials",
|
||||
"controller_needs_database_credentials",
|
||||
"Forge first install is paused before migration until fixed database credentials are provisioned.",
|
||||
identity,
|
||||
)
|
||||
if (
|
||||
status == "preflight_changed"
|
||||
and set(result) == base_fields | {"noMutation"}
|
||||
and result.get("noMutation") is True
|
||||
):
|
||||
return (
|
||||
"preflight_changed",
|
||||
"controller_preflight_changed",
|
||||
"Controller rejected changed Forge first-install preflight before mutation.",
|
||||
{**identity, "noMutation": True},
|
||||
)
|
||||
if status == "deployed" and set(result) == base_fields | {"receipt"}:
|
||||
receipt = result.get("receipt")
|
||||
if not isinstance(receipt, dict) or set(receipt) != {
|
||||
"revision",
|
||||
"workspaceImageDigest",
|
||||
"frontdoorImageDigest",
|
||||
"migration",
|
||||
"health",
|
||||
}:
|
||||
return (
|
||||
"outcome_unknown",
|
||||
"controller_receipt_unverified",
|
||||
_unknown_summary(),
|
||||
{},
|
||||
)
|
||||
migration = receipt.get("migration")
|
||||
health = receipt.get("health")
|
||||
if (
|
||||
receipt.get("revision") != profile.revision
|
||||
or receipt.get("workspaceImageDigest") != profile.workspace_image_digest
|
||||
or receipt.get("frontdoorImageDigest") != profile.frontdoor_image_digest
|
||||
or not isinstance(migration, dict)
|
||||
or set(migration) != {"service", "status", "result"}
|
||||
or migration
|
||||
!= {"service": "migrate", "status": "completed", "result": "succeeded"}
|
||||
or not isinstance(health, dict)
|
||||
or set(health) != {"status", "services"}
|
||||
or health.get("status") != "healthy"
|
||||
or health.get("services") != _HEALTHY_SERVICES
|
||||
):
|
||||
return (
|
||||
"outcome_unknown",
|
||||
"controller_receipt_unverified",
|
||||
_unknown_summary(),
|
||||
{},
|
||||
)
|
||||
safe_receipt = {
|
||||
"revision": profile.revision,
|
||||
"workspaceImageDigest": profile.workspace_image_digest,
|
||||
"frontdoorImageDigest": profile.frontdoor_image_digest,
|
||||
"migration": dict(migration),
|
||||
"health": {"status": "healthy", "services": list(_HEALTHY_SERVICES)},
|
||||
}
|
||||
return (
|
||||
"deployed",
|
||||
"controller_deployed",
|
||||
"Forge first install completed with exact migration and health receipts.",
|
||||
{**identity, "receipt": safe_receipt},
|
||||
)
|
||||
if status == "failed" and set(result) == base_fields | {"failure"}:
|
||||
failure = result.get("failure")
|
||||
if (
|
||||
isinstance(failure, dict)
|
||||
and set(failure) == {"code", "preservedData"}
|
||||
and failure.get("code") in _FAILED_CODES
|
||||
and failure.get("preservedData") is True
|
||||
):
|
||||
return (
|
||||
"failed",
|
||||
"controller_failed_preserving_data",
|
||||
"Controller reported a first-install failure and confirmed PostgreSQL data preservation.",
|
||||
{
|
||||
**identity,
|
||||
"failure": {
|
||||
"code": failure["code"],
|
||||
"preservedData": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
return (
|
||||
"outcome_unknown",
|
||||
"controller_response_unverified",
|
||||
_unknown_summary(),
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
def _serialize(job: DeploymentJob, *, replay: bool = False) -> dict[str, Any]:
|
||||
return {
|
||||
"id": job.id,
|
||||
"applicationId": job.application_id,
|
||||
"repository": job.repository,
|
||||
"revision": job.revision,
|
||||
"status": job.status,
|
||||
"stage": job.stage,
|
||||
"preflight": dict(job.preflight_json or {}),
|
||||
"evidence": dict(job.evidence_json or {}),
|
||||
"result": job.result_summary,
|
||||
"request": job.broker_request_id,
|
||||
"createdAt": job.created_at.isoformat(),
|
||||
"updatedAt": job.updated_at.isoformat(),
|
||||
"completedAt": job.completed_at.isoformat() if job.completed_at else None,
|
||||
"idempotentReplay": replay,
|
||||
}
|
||||
|
||||
|
||||
class ForgeBootstrapApplyService:
|
||||
"""One fixed Forge first-install operation with durable, non-replaying apply."""
|
||||
|
||||
def __init__(self, *, broker: DeploymentBrokerClient | None = None):
|
||||
self.broker = broker or DeploymentBrokerClient()
|
||||
|
||||
@staticmethod
|
||||
def request_id(profile: ForgeBootstrapApplyV1, idempotency_key: str) -> str:
|
||||
normalized = _idempotency_key(idempotency_key)
|
||||
payload = {
|
||||
"repository": _REPOSITORY,
|
||||
"operation": _OPERATION,
|
||||
"profile": profile.model_dump(mode="json"),
|
||||
"idempotencyKey": normalized,
|
||||
}
|
||||
return _sha256(payload)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
session: Session,
|
||||
profile: ForgeBootstrapApplyV1,
|
||||
*,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
normalized_key = _idempotency_key(idempotency_key)
|
||||
idempotency_digest = hashlib.sha256(
|
||||
f"forge-first-install:{normalized_key}".encode()
|
||||
).hexdigest()
|
||||
profile_json = profile.model_dump(mode="json")
|
||||
plan_sha = _sha256(profile_json)
|
||||
existing = session.exec(
|
||||
select(DeploymentJob).where(
|
||||
DeploymentJob.idempotency_sha256 == idempotency_digest
|
||||
)
|
||||
).first()
|
||||
if existing:
|
||||
if (
|
||||
existing.application_id != _APPLICATION_ID
|
||||
or existing.repository != _REPOSITORY
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"Idempotency-Key already belongs to a different application or repository."
|
||||
)
|
||||
if existing.plan_sha256 != plan_sha:
|
||||
raise ForgeBootstrapApplyError(
|
||||
"Idempotency-Key was already used for a different Forge first-install profile."
|
||||
)
|
||||
return self._refresh_job(session, existing, replay=True)
|
||||
|
||||
preflight_job, receipt_sha = _preflight_ready(session, profile)
|
||||
if _allowed_policy_id(session) is None:
|
||||
raise ForgeBootstrapApplyError(
|
||||
"An allowed exact-target deployment.apply policy is required for Forge first install."
|
||||
)
|
||||
secret = _controller_secret(session)
|
||||
if secret is None:
|
||||
raise ForgeBootstrapApplyError(
|
||||
"Stored managed-root trust and the fixed Forge deployment secret are required."
|
||||
)
|
||||
|
||||
request_id = self.request_id(profile, normalized_key)
|
||||
job_id = f"forge-install-{idempotency_digest}"
|
||||
now = utc_now()
|
||||
job = DeploymentJob(
|
||||
id=job_id,
|
||||
application_id=_APPLICATION_ID,
|
||||
repository=_REPOSITORY,
|
||||
revision=profile.revision,
|
||||
plan_sha256=plan_sha,
|
||||
idempotency_sha256=idempotency_digest,
|
||||
status="applying",
|
||||
stage="controller_request_pending",
|
||||
plan_json=profile_json,
|
||||
preflight_json={
|
||||
"apiVersion": _PREFLIGHT_API_VERSION,
|
||||
"jobId": preflight_job.id,
|
||||
"sha256": receipt_sha,
|
||||
},
|
||||
evidence_json={"profile": "forge-first-install", "controller": {}},
|
||||
broker_request_id=request_id,
|
||||
result_summary="Forge first install is awaiting the fixed controller response.",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(job)
|
||||
try:
|
||||
# The durable request intent must exist before the only POST attempt.
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
except IntegrityError:
|
||||
session.rollback()
|
||||
raced = session.get(DeploymentJob, job_id)
|
||||
if raced and (
|
||||
raced.application_id != _APPLICATION_ID
|
||||
or raced.repository != _REPOSITORY
|
||||
):
|
||||
raise ForgeBootstrapApplyError(
|
||||
"Idempotency-Key already belongs to a different application or repository."
|
||||
) from None
|
||||
if raced and raced.plan_sha256 == plan_sha:
|
||||
return self._refresh_job(session, raced, replay=True)
|
||||
raise ForgeBootstrapApplyError(
|
||||
"The Forge first-install request could not be durably claimed."
|
||||
) from None
|
||||
|
||||
log_event(
|
||||
session,
|
||||
source="deployment-control-plane",
|
||||
identity="operator",
|
||||
action_id="deployment.apply",
|
||||
target=_APPLICATION_ID,
|
||||
target_type="deployment",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
reason="Submitted one fixed Forge first-install request after exact preflight validation.",
|
||||
result="submitted",
|
||||
execution_plan={
|
||||
"request": request_id,
|
||||
"planSha256": plan_sha,
|
||||
"preflightSha256": receipt_sha,
|
||||
},
|
||||
redacted_metadata={"profile": "forge-first-install", "jobId": job_id},
|
||||
)
|
||||
try:
|
||||
response = self.broker.forge_bootstrap_apply(
|
||||
profile,
|
||||
idempotency_key=normalized_key,
|
||||
secret=secret,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - durable intent is reconciled with GET only.
|
||||
response = {"status": "unreachable"}
|
||||
return self._record_response(
|
||||
session,
|
||||
job,
|
||||
response,
|
||||
profile,
|
||||
expected_updated_at=job.updated_at,
|
||||
)
|
||||
|
||||
def status(self, session: Session, job_id: str) -> dict[str, Any] | None:
|
||||
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
||||
return None
|
||||
job = session.get(DeploymentJob, job_id)
|
||||
if (
|
||||
not job
|
||||
or job.application_id != _APPLICATION_ID
|
||||
or job.repository != _REPOSITORY
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
|
||||
):
|
||||
return None
|
||||
return self._refresh_job(session, job)
|
||||
|
||||
def resume(self, session: Session, job_id: str) -> dict[str, Any] | None:
|
||||
"""Explicitly resume only after a signed read proves pre-migration pause."""
|
||||
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
||||
return None
|
||||
job = session.get(DeploymentJob, job_id)
|
||||
if (
|
||||
not job
|
||||
or job.application_id != _APPLICATION_ID
|
||||
or job.repository != _REPOSITORY
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
|
||||
):
|
||||
return None
|
||||
|
||||
# A caller cannot turn an old pending/unknown record into another POST.
|
||||
# First reconcile with the controller, then proceed only from its exact
|
||||
# safe pre-migration credential state.
|
||||
refreshed = self._refresh_job(session, job)
|
||||
if refreshed.get("status") != "needs_database_credentials":
|
||||
return refreshed
|
||||
job = session.get(DeploymentJob, job_id)
|
||||
if job is None:
|
||||
return refreshed
|
||||
session.refresh(job)
|
||||
try:
|
||||
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
|
||||
except ValueError:
|
||||
return refreshed
|
||||
preflight_receipt = dict(job.preflight_json or {})
|
||||
if (
|
||||
set(preflight_receipt) != {"apiVersion", "jobId", "sha256"}
|
||||
or preflight_receipt.get("apiVersion") != _PREFLIGHT_API_VERSION
|
||||
or preflight_receipt.get("jobId") != profile.preflight_job_id
|
||||
or preflight_receipt.get("sha256") != profile.preflight_sha256
|
||||
):
|
||||
return refreshed
|
||||
try:
|
||||
_preflight_ready(session, profile)
|
||||
except ForgeBootstrapApplyError:
|
||||
return refreshed
|
||||
secret = _controller_secret(session)
|
||||
if not secret:
|
||||
return refreshed
|
||||
|
||||
expected_updated_at = job.updated_at
|
||||
resume_pending = "Forge database credentials passed the signed pre-migration gate; resume is pending."
|
||||
claimed_at = utc_now()
|
||||
# Compare-and-swap the observed, signed pre-migration state. Concurrent
|
||||
# callers can both reconcile it, but only one can persist the resume claim.
|
||||
claim = session.exec(
|
||||
update(DeploymentJob)
|
||||
.where(
|
||||
DeploymentJob.id == job.id,
|
||||
DeploymentJob.application_id == _APPLICATION_ID,
|
||||
DeploymentJob.repository == _REPOSITORY,
|
||||
DeploymentJob.status == "needs_database_credentials",
|
||||
DeploymentJob.stage == "controller_needs_database_credentials",
|
||||
DeploymentJob.updated_at == expected_updated_at,
|
||||
)
|
||||
.values(
|
||||
status="applying",
|
||||
stage="resume_request_pending",
|
||||
result_summary=resume_pending,
|
||||
updated_at=claimed_at,
|
||||
)
|
||||
)
|
||||
if claim.rowcount != 1:
|
||||
session.rollback()
|
||||
current = session.get(DeploymentJob, job_id)
|
||||
if current is None:
|
||||
return refreshed
|
||||
session.refresh(current)
|
||||
return _serialize(current)
|
||||
# The one-shot intent is durable before the only resume POST attempt.
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
resume_intent_version = job.updated_at
|
||||
log_event(
|
||||
session,
|
||||
source="deployment-control-plane",
|
||||
identity="operator",
|
||||
action_id="deployment.apply",
|
||||
target=_APPLICATION_ID,
|
||||
target_type="deployment",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
reason="Requested one explicit Forge first-install resume after signed pre-migration credential status.",
|
||||
result="resume_submitted",
|
||||
execution_plan={"request": job.broker_request_id, "jobId": job.id},
|
||||
redacted_metadata={"profile": "forge-first-install"},
|
||||
)
|
||||
try:
|
||||
response = self.broker.forge_bootstrap_resume(
|
||||
job.broker_request_id or "", secret=secret
|
||||
)
|
||||
except Exception: # noqa: BLE001 - durable resume intent is reconciled with GET only.
|
||||
response = {"status": "unreachable"}
|
||||
return self._record_response(
|
||||
session,
|
||||
job,
|
||||
response,
|
||||
profile,
|
||||
expected_updated_at=resume_intent_version,
|
||||
resume_claimed=True,
|
||||
)
|
||||
|
||||
def _refresh_job(
|
||||
self,
|
||||
session: Session,
|
||||
job: DeploymentJob,
|
||||
*,
|
||||
replay: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if job.status not in _ACTIVE_STATUSES:
|
||||
return _serialize(job, replay=replay)
|
||||
try:
|
||||
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
|
||||
except ValueError:
|
||||
return _serialize(job, replay=replay)
|
||||
secret = _controller_secret(session)
|
||||
expected_updated_at = job.updated_at
|
||||
resume_claimed = job.stage.startswith("resume")
|
||||
if not secret:
|
||||
return self._record_response(
|
||||
session,
|
||||
job,
|
||||
{"status": "unreachable"},
|
||||
profile,
|
||||
replay=replay,
|
||||
expected_updated_at=expected_updated_at,
|
||||
resume_claimed=resume_claimed,
|
||||
)
|
||||
try:
|
||||
response = self.broker.forge_bootstrap_status(
|
||||
job.broker_request_id or "", secret=secret
|
||||
)
|
||||
except Exception: # noqa: BLE001 - reconciliation failures remain unknown.
|
||||
response = {"status": "unreachable"}
|
||||
return self._record_response(
|
||||
session,
|
||||
job,
|
||||
response,
|
||||
profile,
|
||||
replay=replay,
|
||||
expected_updated_at=expected_updated_at,
|
||||
resume_claimed=resume_claimed,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _record_response(
|
||||
session: Session,
|
||||
job: DeploymentJob,
|
||||
response: object,
|
||||
profile: ForgeBootstrapApplyV1,
|
||||
*,
|
||||
replay: bool = False,
|
||||
expected_updated_at: datetime | None = None,
|
||||
resume_claimed: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
expected_updated_at = expected_updated_at or job.updated_at
|
||||
status, stage, result, evidence = _classify_controller_result(
|
||||
response,
|
||||
request_id=job.broker_request_id or "",
|
||||
profile=profile,
|
||||
)
|
||||
resume_claimed = resume_claimed or job.stage.startswith("resume")
|
||||
if resume_claimed:
|
||||
if status == "needs_database_credentials":
|
||||
status = "outcome_unknown"
|
||||
stage = "resume_status_unresolved"
|
||||
result = (
|
||||
"Resume was already claimed but the controller still reports its pre-migration pause; "
|
||||
"reconcile by status without submitting another resume."
|
||||
)
|
||||
else:
|
||||
stage = f"resume_{stage}"
|
||||
next_updated_at = utc_now()
|
||||
next_completed_at = (
|
||||
utc_now() if status in _TERMINAL_STATUSES else job.completed_at
|
||||
)
|
||||
values = {
|
||||
"status": status,
|
||||
"stage": stage,
|
||||
"result_summary": result,
|
||||
"evidence_json": {"profile": "forge-first-install", "controller": evidence},
|
||||
"updated_at": next_updated_at,
|
||||
"completed_at": next_completed_at,
|
||||
}
|
||||
statement = update(DeploymentJob).where(
|
||||
DeploymentJob.id == job.id,
|
||||
DeploymentJob.updated_at == expected_updated_at,
|
||||
)
|
||||
if status == "needs_database_credentials" and not resume_claimed:
|
||||
# A status GET started after the local resume claim may still race a
|
||||
# controller that has not processed the POST. It cannot reopen the gate.
|
||||
statement = statement.where(DeploymentJob.stage.not_like("resume%"))
|
||||
result_update = session.exec(statement.values(**values))
|
||||
if result_update.rowcount != 1:
|
||||
session.rollback()
|
||||
current = session.get(DeploymentJob, job.id)
|
||||
if current is None:
|
||||
return _serialize(job, replay=replay)
|
||||
session.refresh(current)
|
||||
return _serialize(current, replay=replay)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
return _serialize(job, replay=replay)
|
||||
Reference in new issue
Block a user