Publish curated Unraid-AppOps-Gateway source from a480f280f26f4aa48c61394fa841d2233067c208

backend/app/services/deployment_broker.py

@@ -29,6 +29,7 @@ from ..core.redaction import redact_message
 from ..schemas.deployment import DeploymentPlanV1
 from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
+from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessV1
 
 _REQUEST_ID = re.compile(r"^[0-9a-f]{64}$")
 _MAX_DELIVERY_STATUS_BYTES = 512 * 1024
@@ -40,6 +41,10 @@ _WORKMESH_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
 _FORGE_SECRET_DELIVERY_PATH = "/v1/appops/forge/bootstrap/secrets"
 _MAX_FORGE_SECRET_DELIVERY_BODY_BYTES = 16 * 1024
 _MAX_FORGE_SECRET_DELIVERY_RESPONSE_BYTES = 64 * 1024
+_FORGE_RUNTIME_READINESS_PATH = "/v1/appops/forge/bootstrap/readiness/"
+_MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES = 32 * 1024
+_FORGE_RUNTIME_READINESS_TOTAL_SECONDS = 35.0
+_FORGE_RUNTIME_READINESS_READ_SECONDS = 20.0
 
 
 def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
@@ -466,7 +471,7 @@ class DeploymentBrokerClient:
                 bytes(response_body).decode("utf-8"),
                 object_pairs_hook=_unique_json_object,
             )
-        except (UnicodeError, ValueError, TypeError):
+        except (UnicodeError, ValueError, TypeError, RecursionError):
             response_body.clear()
             return {
                 "status": "outcome_unknown",
@@ -508,6 +513,106 @@ class DeploymentBrokerClient:
             secret=secret,
         )
 
+    def forge_bootstrap_readiness(
+        self, request_id: str, *, secret: str
+    ) -> dict[str, Any]:
+        """Fetch one fresh fixed-profile runtime snapshot with no receipt fallback."""
+        if not _REQUEST_ID.fullmatch(request_id):
+            return {"status": "unavailable"}
+        if (
+            not self.is_configured()
+            or not isinstance(secret, str)
+            or len(secret) < 32
+            or not secret.isascii()
+            or not secret.isprintable()
+        ):
+            return {"status": "unavailable"}
+
+        timestamp = str(int(time.time()))
+        nonce = uuid.uuid4().hex
+        path = f"{_FORGE_RUNTIME_READINESS_PATH}{request_id}"
+        headers = {
+            "Accept": "application/json",
+            "Accept-Encoding": "identity",
+            "Cache-Control": "no-store",
+            "Pragma": "no-cache",
+            "X-ITWorx-Deployment-Timestamp": timestamp,
+            "X-ITWorx-Deployment-Nonce": nonce,
+            "X-ITWorx-Deployment-Signature": self._signature(
+                secret, "GET", path, timestamp, nonce, b""
+            ),
+        }
+        response_body = bytearray()
+        deadline = time.monotonic() + _FORGE_RUNTIME_READINESS_TOTAL_SECONDS
+        client_timeout = httpx.Timeout(
+            connect=min(self.timeout_seconds, 5.0),
+            read=_FORGE_RUNTIME_READINESS_READ_SECONDS,
+            write=min(self.timeout_seconds, 5.0),
+            pool=min(self.timeout_seconds, 5.0),
+        )
+        try:
+            with httpx.Client(
+                timeout=client_timeout,
+                verify=self._verify_option(),
+                follow_redirects=False,
+            ) as client, client.stream(
+                "GET",
+                f"{self.base_url}{path}",
+                content=b"",
+                headers=headers,
+            ) as response:
+                if response.status_code != 200:
+                    return {"status": "unavailable"}
+                content_lengths = response.headers.get_list("content-length")
+                if len(content_lengths) > 1:
+                    return {"status": "unavailable"}
+                content_length = content_lengths[0] if content_lengths else None
+                if content_length is not None and (
+                    not content_length.isascii()
+                    or not content_length.isdecimal()
+                    or len(content_length) > 10
+                    or int(content_length) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES
+                ):
+                    return {"status": "unavailable"}
+                content_encoding = response.headers.get("content-encoding", "identity").lower()
+                content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
+                if content_encoding != "identity" or content_type != "application/json":
+                    return {"status": "unavailable"}
+                for chunk in response.iter_raw():
+                    if time.monotonic() > deadline:
+                        response_body.clear()
+                        return {"status": "unavailable"}
+                    response_body.extend(chunk)
+                    if len(response_body) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES:
+                        response_body.clear()
+                        return {"status": "unavailable"}
+        except Exception:  # noqa: BLE001 - transport diagnostics may contain held credentials.
+            response_body.clear()
+            return {"status": "unavailable"}
+        if time.monotonic() > deadline:
+            response_body.clear()
+            return {"status": "unavailable"}
+
+        try:
+            decoded = json.loads(
+                bytes(response_body).decode("utf-8", errors="strict"),
+                object_pairs_hook=_unique_json_object,
+            )
+            validated = ForgeRuntimeReadinessV1.model_validate(decoded)
+        except (UnicodeError, ValueError, TypeError, RecursionError):
+            response_body.clear()
+            return {"status": "unavailable"}
+        response_body.clear()
+        now = int(time.time())
+        if (
+            validated.request != request_id
+            or validated.nonce != nonce
+            or validated.observed_at < now - 30
+            or validated.observed_at > now + 5
+        ):
+            return {"status": "unavailable"}
+        return validated.model_dump(mode="json", by_alias=True)
+
     def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]:
         """Read the bootstrap-protected enrollment controller capability."""
         return self._request(

backend/app/services/forge_bootstrap_apply.py

@@ -4,7 +4,7 @@ import base64
 import hashlib
 import json
 import re
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any
 
 from sqlalchemy import update
@@ -17,6 +17,10 @@ from ..db.models import DeploymentJob, Policy
 from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT
 from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
+from ..schemas.forge_runtime_readiness import (
+    ForgeRuntimeReadinessResultV1,
+    ForgeRuntimeReadinessV1,
+)
 from .audit_logger import log_event
 from .deployment_broker import DeploymentBrokerClient
 from .enrollment_plane import EnrollmentPlane
@@ -83,6 +87,7 @@ _PROFILE_CHECKS = {
 _FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"}
 _ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"}
 _TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"}
+_RUNTIME_READINESS_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
 
 
 class ForgeBootstrapApplyError(ValueError):
@@ -715,6 +720,113 @@ class ForgeBootstrapApplyService:
             return None
         return self._refresh_job(session, job)
 
+    @staticmethod
+    def _runtime_readiness_unavailable() -> dict[str, Any]:
+        return ForgeRuntimeReadinessResultV1.model_validate(
+            {
+                "status": "unavailable",
+                "message": "forge_runtime_readiness_unavailable",
+                "health": {
+                    "status": "unavailable",
+                    "services": [
+                        {"name": name, "status": "unavailable"}
+                        for name in _RUNTIME_READINESS_SERVICES
+                    ],
+                },
+                "migrationCompleted": False,
+            }
+        ).model_dump(mode="json", by_alias=True, exclude_none=True)
+
+    def current_readiness(self, session: Session, job_id: str) -> dict[str, Any]:
+        """Read current controller state only for a verified successful Forge job."""
+        unavailable = self._runtime_readiness_unavailable()
+        if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
+            return unavailable
+        try:
+            job = session.get(DeploymentJob, job_id)
+        except Exception:  # noqa: BLE001 - do not expose protected store/DB failures.
+            return unavailable
+        if (
+            job is None
+            or job.application_id != _APPLICATION_ID
+            or job.repository != _REPOSITORY
+            or job.status != "deployed"
+            or job.stage not in {"controller_deployed", "resume_controller_deployed"}
+            or job.completed_at is None
+            or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
+        ):
+            return unavailable
+        try:
+            profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
+        except (TypeError, ValueError):
+            return unavailable
+        if (
+            job.revision != profile.revision
+            or not isinstance(job.plan_json, dict)
+            or _sha256(job.plan_json) != job.plan_sha256
+        ):
+            return unavailable
+        evidence = job.evidence_json
+        if (
+            not isinstance(evidence, dict)
+            or set(evidence) != {"profile", "controller"}
+            or evidence.get("profile") != "forge-first-install"
+        ):
+            return unavailable
+        historical_status, _stage, _summary, _safe_evidence = _classify_controller_result(
+            evidence.get("controller"),
+            request_id=job.broker_request_id or "",
+            profile=profile,
+        )
+        if historical_status != "deployed":
+            return unavailable
+
+        # This protected per-target secret lookup is deliberately after every
+        # durable job/profile/receipt validation above.
+        secret = _controller_secret(session)
+        if secret is None:
+            return unavailable
+        try:
+            response = self.broker.forge_bootstrap_readiness(
+                job.broker_request_id or "", secret=secret
+            )
+            current = ForgeRuntimeReadinessV1.model_validate(response)
+        except Exception:  # noqa: BLE001 - broker and schema errors are safe unavailable.
+            return unavailable
+        now = int(datetime.now(UTC).timestamp())
+        if (
+            current.request != job.broker_request_id
+            or current.repository != _REPOSITORY
+            or current.operation != _OPERATION
+            or current.profile.revision != profile.revision
+            or current.profile.workspace_image_digest != profile.workspace_image_digest
+            or current.profile.frontdoor_image_digest != profile.frontdoor_image_digest
+            or current.observed_at < now - 30
+            or current.observed_at > now + 5
+        ):
+            return unavailable
+        message = {
+            "ready": "forge_runtime_ready",
+            "not_ready": "forge_runtime_not_ready",
+            "unavailable": "forge_runtime_readiness_unavailable",
+        }[current.status]
+        return ForgeRuntimeReadinessResultV1.model_validate(
+            {
+                "status": current.status,
+                "message": message,
+                "jobId": job.id,
+                "revision": current.profile.revision,
+                "workspaceImageDigest": current.profile.workspace_image_digest,
+                "frontdoorImageDigest": current.profile.frontdoor_image_digest,
+                "observedAt": current.observed_at,
+                "health": current.health.model_dump(mode="python"),
+                "migrationCompleted": current.migration_completed,
+                "controllerArtifact": current.controller_artifact.model_dump(
+                    mode="python", by_alias=True
+                ),
+            }
+        ).model_dump(mode="json", by_alias=True, exclude_none=True)
+
     def resume(self, session: Session, job_id: str) -> dict[str, Any] | None:
         """Explicitly resume only after a signed read proves pre-migration pause."""
         if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):