from __future__ import annotations import base64 import hashlib import json import re from datetime import UTC, datetime from typing import Any from sqlalchemy import update from sqlalchemy.exc import IntegrityError from sqlmodel import Session, select from ..core.config import settings from ..core.time import utc_now from ..db.models import DeploymentJob, Policy from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1 from ..schemas.forge_runtime_readiness import ( ForgeRuntimeReadinessResultV1, ForgeRuntimeReadinessV1, ) from .audit_logger import log_event from .deployment_broker import DeploymentBrokerClient from .enrollment_plane import EnrollmentPlane from .policy_engine import evaluate_policy from .secret_store import get_secret _API_VERSION = "itworx.forge-first-install/v1" _PREFLIGHT_API_VERSION = "itworx.forge-bootstrap-preflight/v1" _APPLICATION_ID = "itworx-forge" _REPOSITORY = "jens/itworx-forge" _OPERATION = "forge_first_install" _SECRET_REF = "FORGE_DEPLOYMENT_BROKER" _FORGE_SOURCE_URL = "https://gitea.example.com/Jens/itworx-forge" _FORGE_PRIVATE_PACKAGE_OWNER_SETTING = "FORGE_PRIVATE_PACKAGE_OWNER" _FORGE_PRIVATE_PACKAGE_OWNER_PATTERN = re.compile(r"[a-z0-9][a-z0-9_-]{0,38}\Z") _FORGE_OIDC_ISSUER = "https://auth.example-host.com/application/o/itworx-forge/" _FORGE_OIDC_CLIENT_ID = "itworx-forge" _FORGE_OIDC_CALLBACK = "https://forge.example.com/auth/callback" _FORGE_OIDC_GROUPS = ["forge-owners", "forge-operators", "forge-viewers"] _FORGE_OIDC_SCOPES = ["email", "openid", "profile"] _FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256 = { "openid": "38e58da331cc5378d4f269389242aecb80adf30093c405524dc1ab5aab9d452f", "email": "d1200eabeffc3b7f379e9c1636367b69befb0b1ac2b5d82b7be20e78667f2b94", "profile": "648b3f6dbf2212940309a581b239c9996562ca017a89be22bcfa93f9aa2c26ae", } _FORGE_OIDC_PROFILE_MAPPING_NAME = "ITWorx Forge profile and groups claim v1" _FORGE_OIDC_OPENID_MAPPING_MANAGED = "goauthentik.io/providers/oauth2/scope-openid" _FORGE_OIDC_EMAIL_MAPPING_MANAGED = "goauthentik.io/providers/oauth2/scope-email" _OIDC_ATTESTATION_FIELDS = { "issuer", "audience", "clientId", "clientType", "callback", "applicationId", "providerId", "clientSecretMatched", "authorizationFlowId", "invalidationFlowId", "roleGroups", "scopeMappingIds", "scopeMappingExpressionSha256", "profileMappingName", "standardMappingsManaged", "supportedScopes", "authorizationCodeFlowSupported", "signingKeyId", "signingKeyType", "signingKeyPrivateAvailable", "signingKeyFingerprintSha256", "signingKeyKid", "tokenSigningAlgorithm", "jwksKid", } _AUTHENTIK_FORGE_OIDC_BINDING_VALID_SETTING = "AUTHENTIK_FORGE_OIDC_BINDING_VALID" _AUTHENTIK_FORGE_OIDC_PROVIDER_ID_SETTING = "AUTHENTIK_FORGE_OIDC_PROVIDER_ID" _AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS = { "openid": "AUTHENTIK_FORGE_OIDC_OPENID_CLAIM_MAPPING", "email": "AUTHENTIK_FORGE_OIDC_EMAIL_CLAIM_MAPPING", "profile": "AUTHENTIK_FORGE_OIDC_PROFILE_CLAIM_MAPPING", } _AUTHENTIK_SIGNING_KEY_ID_SETTING = "AUTHENTIK_FORGE_OIDC_SIGNING_KEY_ID" _AUTHENTIK_AUTHORIZATION_FLOW_SETTING = "AUTHENTIK_OIDC_AUTHORIZATION_FLOW" _AUTHENTIK_INVALIDATION_FLOW_SETTING = "AUTHENTIK_OIDC_INVALIDATION_FLOW" _IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") _HEALTHY_SERVICES = ["api", "dashboard", "itworx-forge", "postgres", "worker"] _PROFILE_CHECKS = { "deployment_apply_policy", "controller_trust", "managed_appdata_root_selection", "exact_main", "source_manifest", "managed_appdata", "proxy_network", "resource_collisions", "push_validation", "image_registry", "oidc", "secret_delivery", } _FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"} _ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"} _TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"} _RUNTIME_READINESS_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker") class ForgeBootstrapApplyError(ValueError): """A fail-closed error in the fixed Forge first-install contract.""" def _canonical_json(value: Any) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") def _sha256(value: Any) -> str: return hashlib.sha256(_canonical_json(value)).hexdigest() def canonical_preflight_receipt(job: DeploymentJob) -> str: """Hash the immutable preflight receipt as stored in AppOps, not caller data.""" receipt = { "apiVersion": _PREFLIGHT_API_VERSION, "jobId": job.id, "applicationId": job.application_id, "repository": job.repository, "revision": job.revision, "jobStatus": job.status, "planSha256": job.plan_sha256, "profile": dict(job.plan_json or {}), "preflight": dict(job.preflight_json or {}), "evidence": dict(job.evidence_json or {}), } return _sha256(receipt) def _idempotency_key(value: str) -> str: normalized = value.strip() if not _IDEMPOTENCY.fullmatch(normalized): raise ForgeBootstrapApplyError("Idempotency-Key must contain 8 to 128 safe characters.") return normalized def _unknown_summary() -> str: return "The fixed Forge controller result is unknown; reconcile its status before retrying." def _preflight_ready( session: Session, profile: ForgeBootstrapApplyV1 ) -> tuple[DeploymentJob, str]: job = session.get(DeploymentJob, profile.preflight_job_id) if ( not job or job.application_id != _APPLICATION_ID or job.repository != _REPOSITORY or job.revision != profile.revision or job.status != "preflight_succeeded" or not job.completed_at or job.plan_sha256 != _sha256(dict(job.plan_json or {})) ): raise ForgeBootstrapApplyError( "A matching passed Forge preflight receipt is required before first install." ) try: stored_profile = ForgeBootstrapPreflightV1.model_validate(job.plan_json) except ValueError as exc: raise ForgeBootstrapApplyError("The stored Forge preflight receipt is invalid.") from exc if ( stored_profile.revision != profile.revision or stored_profile.workspace_image_digest != profile.workspace_image_digest or stored_profile.frontdoor_image_digest != profile.frontdoor_image_digest ): raise ForgeBootstrapApplyError( "The stored Forge preflight receipt does not match this profile." ) preflight = dict(job.preflight_json or {}) if ( set(preflight) != {"apiVersion", "status", "blockers", "unknowns", "checks"} or preflight.get("apiVersion") != _PREFLIGHT_API_VERSION or preflight.get("status") != "passed" or preflight.get("blockers") != [] or preflight.get("unknowns") != [] ): raise ForgeBootstrapApplyError( "Forge first install requires a passed preflight with no unknown checks." ) checks = preflight.get("checks") if not isinstance(checks, list) or len(checks) != len(_PROFILE_CHECKS): raise ForgeBootstrapApplyError("The stored Forge preflight checks are incomplete.") seen: set[str] = set() for check in checks: if ( not isinstance(check, dict) or set(check) != {"name", "status", "detail"} or not isinstance(check.get("name"), str) or check["name"] not in _PROFILE_CHECKS or check["name"] in seen or check.get("status") != "passed" or not isinstance(check.get("detail"), str) or len(check["detail"]) > 256 ): raise ForgeBootstrapApplyError( "Forge first install requires every fixed preflight check to pass." ) seen.add(check["name"]) if seen != _PROFILE_CHECKS: raise ForgeBootstrapApplyError("The stored Forge preflight checks are incomplete.") evidence = dict(job.evidence_json or {}) if ( set(evidence) != {"profile", "policyId", "workspaceImageDigest", "frontdoorImageDigest", "controller"} or evidence.get("profile") != "forge-production-bootstrap" or not isinstance(evidence.get("policyId"), str) or evidence.get("workspaceImageDigest") != profile.workspace_image_digest or evidence.get("frontdoorImageDigest") != profile.frontdoor_image_digest or not isinstance(evidence.get("controller"), dict) ): raise ForgeBootstrapApplyError( "The stored Forge preflight evidence does not match this profile." ) current_policy_id = _allowed_policy_id(session) if current_policy_id is None or current_policy_id != evidence["policyId"]: raise ForgeBootstrapApplyError( "The exact-target Forge deployment policy changed after preflight." ) if not _attestations_ready(evidence["controller"], profile): raise ForgeBootstrapApplyError( "The stored Forge preflight lacks verified controller attestations." ) receipt_sha = canonical_preflight_receipt(job) if profile.preflight_sha256 != receipt_sha: raise ForgeBootstrapApplyError( "The supplied preflight receipt digest does not match the stored preflight receipt." ) return job, receipt_sha def _allowed_policy_id(session: Session) -> str | None: policies = session.exec( select(Policy).where( (Policy.action == "deployment.apply") & (Policy.target == _APPLICATION_ID) ) ).all() if len(policies) != 1: return None policy = policies[0] decision = evaluate_policy( session, action="deployment.apply", target=_APPLICATION_ID, source="manual" ) allowed = bool( policy.manual == "allowed" and policy.voice == "blocked" and policy.risk == "high" and decision.get("decision") == "allowed" and decision.get("policy_id") == policy.id ) return policy.id if allowed else None def _attestations_ready(controller: dict[str, Any], profile: ForgeBootstrapApplyV1) -> bool: attestations = controller.get("attestations") if not isinstance(attestations, dict) or set(attestations) != { "push_validation", "image_registry", "oidc", "secret_delivery", }: return False push = attestations.get("push_validation") registry = attestations.get("image_registry") oidc = attestations.get("oidc") secret_delivery = attestations.get("secret_delivery") private_owner = _protected_forge_private_package_owner() return bool( isinstance(push, dict) and set(push) == {"context", "conclusion", "revision"} and push == { "context": "Managed validation / full (push)", "conclusion": "success", "revision": profile.revision, } and isinstance(registry, dict) and private_owner is not None and registry == { "owner": private_owner, "ownerVisibility": "private", "workspaceDigest": profile.workspace_image_digest, "frontdoorDigest": profile.frontdoor_image_digest, "workspacePullable": True, "frontdoorPullable": True, "workspaceSource": _FORGE_SOURCE_URL, "workspaceRevision": profile.revision, "frontdoorSource": _FORGE_SOURCE_URL, "frontdoorRevision": profile.revision, } and forge_oidc_attestation_matches(oidc) and isinstance(secret_delivery, dict) and set(secret_delivery) == {"requiredFilesReady", "metadataSha256"} and secret_delivery.get("requiredFilesReady") is True and isinstance(secret_delivery.get("metadataSha256"), str) and re.fullmatch(r"[0-9a-f]{64}", secret_delivery["metadataSha256"]) ) def _protected_forge_private_package_owner() -> str | None: owner = settings.apis.get(_FORGE_PRIVATE_PACKAGE_OWNER_SETTING) if not isinstance(owner, str) or _FORGE_PRIVATE_PACKAGE_OWNER_PATTERN.fullmatch(owner) is None: return None return owner def forge_oidc_attestation_matches(value: object) -> bool: """Validate the fixed controller proof against protected AppOps Authentik pins.""" if not isinstance(value, dict) or set(value) != _OIDC_ATTESTATION_FIELDS: return False if settings.apis.get(_AUTHENTIK_FORGE_OIDC_BINDING_VALID_SETTING) is not True: return False provider_id = settings.apis.get(_AUTHENTIK_FORGE_OIDC_PROVIDER_ID_SETTING) mapping_ids = { scope: settings.apis.get(setting) for scope, setting in _AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS.items() } signing_key_id = settings.apis.get(_AUTHENTIK_SIGNING_KEY_ID_SETTING) authorization_flow = settings.apis.get(_AUTHENTIK_AUTHORIZATION_FLOW_SETTING) invalidation_flow = settings.apis.get(_AUTHENTIK_INVALIDATION_FLOW_SETTING) uuid_pattern = r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" if ( not isinstance(provider_id, str) or not re.fullmatch(r"[1-9][0-9]{0,19}", provider_id) or any(not isinstance(item, str) or not re.fullmatch(uuid_pattern, item) for item in mapping_ids.values()) or len(set(mapping_ids.values())) != len(_AUTHENTIK_FORGE_OIDC_SCOPE_MAPPING_SETTINGS) or not isinstance(signing_key_id, str) or not re.fullmatch(uuid_pattern, signing_key_id) or not isinstance(authorization_flow, str) or not authorization_flow.strip() or not isinstance(invalidation_flow, str) or not invalidation_flow.strip() or value.get("providerId") != provider_id or value.get("scopeMappingIds") != mapping_ids or value.get("signingKeyId") != signing_key_id or value.get("authorizationFlowId") != authorization_flow or value.get("invalidationFlowId") != invalidation_flow ): return False application_id = value.get("applicationId") if ( not isinstance(application_id, str) or not re.fullmatch(uuid_pattern, application_id) or value.get("providerId") != provider_id ): return False fingerprint = value.get("signingKeyFingerprintSha256") if not isinstance(fingerprint, str) or not re.fullmatch(r"[0-9a-f]{64}", fingerprint): return False jwks_kid = value.get("jwksKid") keypair_kid = value.get("signingKeyKid") return bool( value.get("issuer") == _FORGE_OIDC_ISSUER and value.get("audience") == _FORGE_OIDC_CLIENT_ID and value.get("clientId") == _FORGE_OIDC_CLIENT_ID and value.get("clientType") == "confidential" and value.get("callback") == _FORGE_OIDC_CALLBACK and value.get("clientSecretMatched") is True and isinstance(value.get("authorizationFlowId"), str) and isinstance(value.get("invalidationFlowId"), str) and value.get("roleGroups") == _FORGE_OIDC_GROUPS and value.get("profileMappingName") == _FORGE_OIDC_PROFILE_MAPPING_NAME and isinstance(value.get("scopeMappingExpressionSha256"), dict) and value.get("scopeMappingExpressionSha256") == _FORGE_OIDC_SCOPE_MAPPING_EXPRESSION_SHA256 and value.get("standardMappingsManaged") is True and value.get("supportedScopes") == _FORGE_OIDC_SCOPES and value.get("authorizationCodeFlowSupported") is True and value.get("signingKeyType") == "RSA" and value.get("signingKeyPrivateAvailable") is True and value.get("tokenSigningAlgorithm") == "RS256" and _forge_oidc_kid_is_valid(keypair_kid) and jwks_kid == keypair_kid ) def _forge_oidc_kid_is_valid(value: object) -> bool: if not isinstance(value, str): return False if re.fullmatch(r"[0-9a-f]{32}", value): return True if re.fullmatch(r"[A-Za-z0-9_-]{86}", value) is None: return False try: decoded = base64.b64decode(value + "==", altchars=b"-_", validate=True) except (TypeError, ValueError): return False return ( len(decoded) == 64 and base64.urlsafe_b64encode(decoded).decode("ascii").rstrip("=") == value ) def _controller_secret(session: Session) -> str | None: trust = EnrollmentPlane.get_trust_config(session) if ( trust is None # The fixed Tower controller shares the existing enrollment identity # with RepoPulse; changing global trust would invalidate that binding. or trust.controller_identity != "unraid-controller" or trust.trusted_appdata_root != MANAGED_DOCKER_APPDATA_ROOT or trust.managed_appdata_backup_acknowledged is not True ): return None try: secret = get_secret(session, _SECRET_REF, touch=False) except Exception: # noqa: BLE001 - never expose secret store errors. return None if ( not isinstance(secret, str) or len(secret) < 32 or not secret.isascii() or not secret.isprintable() ): return None return secret def _response_identity_valid( result: dict[str, Any], *, request_id: str, profile: ForgeBootstrapApplyV1 ) -> bool: return bool( result.get("apiVersion") == _API_VERSION and result.get("request") == request_id and result.get("repository") == _REPOSITORY and result.get("revision") == profile.revision and result.get("operation") == _OPERATION and result.get("workspaceImageDigest") == profile.workspace_image_digest and result.get("frontdoorImageDigest") == profile.frontdoor_image_digest ) def _classify_controller_result( result: object, *, request_id: str, profile: ForgeBootstrapApplyV1, ) -> tuple[str, str, str, dict[str, Any]]: """Return status/stage/summary and strictly projected controller evidence.""" if not isinstance(result, dict) or not _response_identity_valid( result, request_id=request_id, profile=profile ): return ( "outcome_unknown", "controller_response_unverified", _unknown_summary(), {}, ) status = result.get("status") base_fields = { "apiVersion", "status", "request", "repository", "revision", "operation", "workspaceImageDigest", "frontdoorImageDigest", } identity = {key: result[key] for key in sorted(base_fields)} if status in {"accepted", "running"} and set(result) == base_fields: return ( "applying", "controller_accepted" if status == "accepted" else "controller_running", "Forge first install is running on the fixed controller.", identity, ) if status == "needs_database_credentials" and set(result) == base_fields: return ( "needs_database_credentials", "controller_needs_database_credentials", "Forge first install is paused before migration until fixed database credentials are provisioned.", identity, ) if ( status == "preflight_changed" and set(result) == base_fields | {"noMutation"} and result.get("noMutation") is True ): return ( "preflight_changed", "controller_preflight_changed", "Controller rejected changed Forge first-install preflight before mutation.", {**identity, "noMutation": True}, ) if status == "deployed" and set(result) == base_fields | {"receipt"}: receipt = result.get("receipt") if not isinstance(receipt, dict) or set(receipt) != { "revision", "workspaceImageDigest", "frontdoorImageDigest", "migration", "health", }: return ( "outcome_unknown", "controller_receipt_unverified", _unknown_summary(), {}, ) migration = receipt.get("migration") health = receipt.get("health") if ( receipt.get("revision") != profile.revision or receipt.get("workspaceImageDigest") != profile.workspace_image_digest or receipt.get("frontdoorImageDigest") != profile.frontdoor_image_digest or not isinstance(migration, dict) or set(migration) != {"service", "status", "result"} or migration != {"service": "migrate", "status": "completed", "result": "succeeded"} or not isinstance(health, dict) or set(health) != {"status", "services"} or health.get("status") != "healthy" or health.get("services") != _HEALTHY_SERVICES ): return ( "outcome_unknown", "controller_receipt_unverified", _unknown_summary(), {}, ) safe_receipt = { "revision": profile.revision, "workspaceImageDigest": profile.workspace_image_digest, "frontdoorImageDigest": profile.frontdoor_image_digest, "migration": dict(migration), "health": {"status": "healthy", "services": list(_HEALTHY_SERVICES)}, } return ( "deployed", "controller_deployed", "Forge first install completed with exact migration and health receipts.", {**identity, "receipt": safe_receipt}, ) if status == "failed" and set(result) == base_fields | {"failure"}: failure = result.get("failure") if ( isinstance(failure, dict) and set(failure) == {"code", "preservedData"} and failure.get("code") in _FAILED_CODES and failure.get("preservedData") is True ): return ( "failed", "controller_failed_preserving_data", "Controller reported a first-install failure and confirmed PostgreSQL data preservation.", { **identity, "failure": { "code": failure["code"], "preservedData": True, }, }, ) return ( "outcome_unknown", "controller_response_unverified", _unknown_summary(), {}, ) def _serialize(job: DeploymentJob, *, replay: bool = False) -> dict[str, Any]: return { "id": job.id, "applicationId": job.application_id, "repository": job.repository, "revision": job.revision, "status": job.status, "stage": job.stage, "preflight": dict(job.preflight_json or {}), "evidence": dict(job.evidence_json or {}), "result": job.result_summary, "request": job.broker_request_id, "createdAt": job.created_at.isoformat(), "updatedAt": job.updated_at.isoformat(), "completedAt": job.completed_at.isoformat() if job.completed_at else None, "idempotentReplay": replay, } class ForgeBootstrapApplyService: """One fixed Forge first-install operation with durable, non-replaying apply.""" def __init__(self, *, broker: DeploymentBrokerClient | None = None): self.broker = broker or DeploymentBrokerClient() @staticmethod def request_id(profile: ForgeBootstrapApplyV1, idempotency_key: str) -> str: normalized = _idempotency_key(idempotency_key) payload = { "repository": _REPOSITORY, "operation": _OPERATION, "profile": profile.model_dump(mode="json"), "idempotencyKey": normalized, } return _sha256(payload) def apply( self, session: Session, profile: ForgeBootstrapApplyV1, *, idempotency_key: str, ) -> dict[str, Any]: normalized_key = _idempotency_key(idempotency_key) idempotency_digest = hashlib.sha256( f"forge-first-install:{normalized_key}".encode() ).hexdigest() profile_json = profile.model_dump(mode="json") plan_sha = _sha256(profile_json) existing = session.exec( select(DeploymentJob).where( DeploymentJob.idempotency_sha256 == idempotency_digest ) ).first() if existing: if ( existing.application_id != _APPLICATION_ID or existing.repository != _REPOSITORY ): raise ForgeBootstrapApplyError( "Idempotency-Key already belongs to a different application or repository." ) if existing.plan_sha256 != plan_sha: raise ForgeBootstrapApplyError( "Idempotency-Key was already used for a different Forge first-install profile." ) return self._refresh_job(session, existing, replay=True) preflight_job, receipt_sha = _preflight_ready(session, profile) if _allowed_policy_id(session) is None: raise ForgeBootstrapApplyError( "An allowed exact-target deployment.apply policy is required for Forge first install." ) secret = _controller_secret(session) if secret is None: raise ForgeBootstrapApplyError( "Stored managed-root trust and the fixed Forge deployment secret are required." ) request_id = self.request_id(profile, normalized_key) job_id = f"forge-install-{idempotency_digest}" now = utc_now() job = DeploymentJob( id=job_id, application_id=_APPLICATION_ID, repository=_REPOSITORY, revision=profile.revision, plan_sha256=plan_sha, idempotency_sha256=idempotency_digest, status="applying", stage="controller_request_pending", plan_json=profile_json, preflight_json={ "apiVersion": _PREFLIGHT_API_VERSION, "jobId": preflight_job.id, "sha256": receipt_sha, }, evidence_json={"profile": "forge-first-install", "controller": {}}, broker_request_id=request_id, result_summary="Forge first install is awaiting the fixed controller response.", created_at=now, updated_at=now, ) session.add(job) try: # The durable request intent must exist before the only POST attempt. session.commit() session.refresh(job) except IntegrityError: session.rollback() raced = session.get(DeploymentJob, job_id) if raced and ( raced.application_id != _APPLICATION_ID or raced.repository != _REPOSITORY ): raise ForgeBootstrapApplyError( "Idempotency-Key already belongs to a different application or repository." ) from None if raced and raced.plan_sha256 == plan_sha: return self._refresh_job(session, raced, replay=True) raise ForgeBootstrapApplyError( "The Forge first-install request could not be durably claimed." ) from None log_event( session, source="deployment-control-plane", identity="operator", action_id="deployment.apply", target=_APPLICATION_ID, target_type="deployment", risk="high", policy_decision="allowed", reason="Submitted one fixed Forge first-install request after exact preflight validation.", result="submitted", execution_plan={ "request": request_id, "planSha256": plan_sha, "preflightSha256": receipt_sha, }, redacted_metadata={"profile": "forge-first-install", "jobId": job_id}, ) try: response = self.broker.forge_bootstrap_apply( profile, idempotency_key=normalized_key, secret=secret, ) except Exception: # noqa: BLE001 - durable intent is reconciled with GET only. response = {"status": "unreachable"} return self._record_response( session, job, response, profile, expected_updated_at=job.updated_at, ) def status(self, session: Session, job_id: str) -> dict[str, Any] | None: if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id): return None job = session.get(DeploymentJob, job_id) if ( not job or job.application_id != _APPLICATION_ID or job.repository != _REPOSITORY or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "") ): return None return self._refresh_job(session, job) @staticmethod def _runtime_readiness_unavailable() -> dict[str, Any]: return ForgeRuntimeReadinessResultV1.model_validate( { "status": "unavailable", "message": "forge_runtime_readiness_unavailable", "health": { "status": "unavailable", "services": [ {"name": name, "status": "unavailable"} for name in _RUNTIME_READINESS_SERVICES ], }, "migrationCompleted": False, } ).model_dump(mode="json", by_alias=True, exclude_none=True) def current_readiness(self, session: Session, job_id: str) -> dict[str, Any]: """Read current controller state only for a verified successful Forge job.""" unavailable = self._runtime_readiness_unavailable() if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id): return unavailable try: job = session.get(DeploymentJob, job_id) except Exception: # noqa: BLE001 - do not expose protected store/DB failures. return unavailable if ( job is None or job.application_id != _APPLICATION_ID or job.repository != _REPOSITORY or job.status != "deployed" or job.stage not in {"controller_deployed", "resume_controller_deployed"} or job.completed_at is None or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "") ): return unavailable try: profile = ForgeBootstrapApplyV1.model_validate(job.plan_json) except (TypeError, ValueError): return unavailable if ( job.revision != profile.revision or not isinstance(job.plan_json, dict) or _sha256(job.plan_json) != job.plan_sha256 ): return unavailable evidence = job.evidence_json if ( not isinstance(evidence, dict) or set(evidence) != {"profile", "controller"} or evidence.get("profile") != "forge-first-install" ): return unavailable historical_status, _stage, _summary, _safe_evidence = _classify_controller_result( evidence.get("controller"), request_id=job.broker_request_id or "", profile=profile, ) if historical_status != "deployed": return unavailable # This protected per-target secret lookup is deliberately after every # durable job/profile/receipt validation above. secret = _controller_secret(session) if secret is None: return unavailable try: response = self.broker.forge_bootstrap_readiness( job.broker_request_id or "", secret=secret ) current = ForgeRuntimeReadinessV1.model_validate(response) except Exception: # noqa: BLE001 - broker and schema errors are safe unavailable. return unavailable now = int(datetime.now(UTC).timestamp()) if ( current.request != job.broker_request_id or current.repository != _REPOSITORY or current.operation != _OPERATION or current.profile.revision != profile.revision or current.profile.workspace_image_digest != profile.workspace_image_digest or current.profile.frontdoor_image_digest != profile.frontdoor_image_digest or current.observed_at < now - 30 or current.observed_at > now + 5 ): return unavailable message = { "ready": "forge_runtime_ready", "not_ready": "forge_runtime_not_ready", "unavailable": "forge_runtime_readiness_unavailable", }[current.status] return ForgeRuntimeReadinessResultV1.model_validate( { "status": current.status, "message": message, "jobId": job.id, "revision": current.profile.revision, "workspaceImageDigest": current.profile.workspace_image_digest, "frontdoorImageDigest": current.profile.frontdoor_image_digest, "observedAt": current.observed_at, "health": current.health.model_dump(mode="python"), "migrationCompleted": current.migration_completed, "controllerArtifact": current.controller_artifact.model_dump( mode="python", by_alias=True ), } ).model_dump(mode="json", by_alias=True, exclude_none=True) def resume(self, session: Session, job_id: str) -> dict[str, Any] | None: """Explicitly resume only after a signed read proves pre-migration pause.""" if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id): return None job = session.get(DeploymentJob, job_id) if ( not job or job.application_id != _APPLICATION_ID or job.repository != _REPOSITORY or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "") ): return None # A caller cannot turn an old pending/unknown record into another POST. # First reconcile with the controller, then proceed only from its exact # safe pre-migration credential state. refreshed = self._refresh_job(session, job) if refreshed.get("status") != "needs_database_credentials": return refreshed job = session.get(DeploymentJob, job_id) if job is None: return refreshed session.refresh(job) try: profile = ForgeBootstrapApplyV1.model_validate(job.plan_json) except ValueError: return refreshed preflight_receipt = dict(job.preflight_json or {}) if ( set(preflight_receipt) != {"apiVersion", "jobId", "sha256"} or preflight_receipt.get("apiVersion") != _PREFLIGHT_API_VERSION or preflight_receipt.get("jobId") != profile.preflight_job_id or preflight_receipt.get("sha256") != profile.preflight_sha256 ): return refreshed try: _preflight_ready(session, profile) except ForgeBootstrapApplyError: return refreshed secret = _controller_secret(session) if not secret: return refreshed expected_updated_at = job.updated_at resume_pending = "Forge database credentials passed the signed pre-migration gate; resume is pending." claimed_at = utc_now() # Compare-and-swap the observed, signed pre-migration state. Concurrent # callers can both reconcile it, but only one can persist the resume claim. claim = session.exec( update(DeploymentJob) .where( DeploymentJob.id == job.id, DeploymentJob.application_id == _APPLICATION_ID, DeploymentJob.repository == _REPOSITORY, DeploymentJob.status == "needs_database_credentials", DeploymentJob.stage == "controller_needs_database_credentials", DeploymentJob.updated_at == expected_updated_at, ) .values( status="applying", stage="resume_request_pending", result_summary=resume_pending, updated_at=claimed_at, ) ) if claim.rowcount != 1: session.rollback() current = session.get(DeploymentJob, job_id) if current is None: return refreshed session.refresh(current) return _serialize(current) # The one-shot intent is durable before the only resume POST attempt. session.commit() session.refresh(job) resume_intent_version = job.updated_at log_event( session, source="deployment-control-plane", identity="operator", action_id="deployment.apply", target=_APPLICATION_ID, target_type="deployment", risk="high", policy_decision="allowed", reason="Requested one explicit Forge first-install resume after signed pre-migration credential status.", result="resume_submitted", execution_plan={"request": job.broker_request_id, "jobId": job.id}, redacted_metadata={"profile": "forge-first-install"}, ) try: response = self.broker.forge_bootstrap_resume( job.broker_request_id or "", secret=secret ) except Exception: # noqa: BLE001 - durable resume intent is reconciled with GET only. response = {"status": "unreachable"} return self._record_response( session, job, response, profile, expected_updated_at=resume_intent_version, resume_claimed=True, ) def _refresh_job( self, session: Session, job: DeploymentJob, *, replay: bool = False, ) -> dict[str, Any]: if job.status not in _ACTIVE_STATUSES: return _serialize(job, replay=replay) try: profile = ForgeBootstrapApplyV1.model_validate(job.plan_json) except ValueError: return _serialize(job, replay=replay) secret = _controller_secret(session) expected_updated_at = job.updated_at resume_claimed = job.stage.startswith("resume") if not secret: return self._record_response( session, job, {"status": "unreachable"}, profile, replay=replay, expected_updated_at=expected_updated_at, resume_claimed=resume_claimed, ) try: response = self.broker.forge_bootstrap_status( job.broker_request_id or "", secret=secret ) except Exception: # noqa: BLE001 - reconciliation failures remain unknown. response = {"status": "unreachable"} return self._record_response( session, job, response, profile, replay=replay, expected_updated_at=expected_updated_at, resume_claimed=resume_claimed, ) @staticmethod def _record_response( session: Session, job: DeploymentJob, response: object, profile: ForgeBootstrapApplyV1, *, replay: bool = False, expected_updated_at: datetime | None = None, resume_claimed: bool = False, ) -> dict[str, Any]: expected_updated_at = expected_updated_at or job.updated_at status, stage, result, evidence = _classify_controller_result( response, request_id=job.broker_request_id or "", profile=profile, ) resume_claimed = resume_claimed or job.stage.startswith("resume") if resume_claimed: if status == "needs_database_credentials": status = "outcome_unknown" stage = "resume_status_unresolved" result = ( "Resume was already claimed but the controller still reports its pre-migration pause; " "reconcile by status without submitting another resume." ) else: stage = f"resume_{stage}" next_updated_at = utc_now() next_completed_at = ( utc_now() if status in _TERMINAL_STATUSES else job.completed_at ) values = { "status": status, "stage": stage, "result_summary": result, "evidence_json": {"profile": "forge-first-install", "controller": evidence}, "updated_at": next_updated_at, "completed_at": next_completed_at, } statement = update(DeploymentJob).where( DeploymentJob.id == job.id, DeploymentJob.updated_at == expected_updated_at, ) if status == "needs_database_credentials" and not resume_claimed: # A status GET started after the local resume claim may still race a # controller that has not processed the POST. It cannot reopen the gate. statement = statement.where(DeploymentJob.stage.not_like("resume%")) result_update = session.exec(statement.values(**values)) if result_update.rowcount != 1: session.rollback() current = session.get(DeploymentJob, job.id) if current is None: return _serialize(job, replay=replay) session.refresh(current) return _serialize(current, replay=replay) session.commit() session.refresh(job) return _serialize(job, replay=replay)