diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index d690733..d338919 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `7144ea3c4cb735745046e4e1c2d3a3e88fa1a302`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `a480f280f26f4aa48c61394fa841d2233067c208`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index 14d1e47..434f457 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "7144ea3c4cb735745046e4e1c2d3a3e88fa1a302", + "sourceRevision": "a480f280f26f4aa48c61394fa841d2233067c208", "files": [ { "path": ".dockerignore", @@ -124,8 +124,8 @@ }, { "path": "backend/app/api/routes_deployments.py", - "sha256": "c77cea58701694af4ef519e351fd27a68856e45cd4e61dc4028c652359d6392a", - "bytes": 7717 + "sha256": "ff14b4e40e6e14d0d7f9f8fd74faf5001010a3ba5c0b279253c61f17199f350f", + "bytes": 8604 }, { "path": "backend/app/api/routes_enrollments.py", @@ -452,6 +452,11 @@ "sha256": "8d55f7a8263191f07813d1689eb0697dabec99a1bbd5aff5a704e66e6bf86c47", "bytes": 6045 }, + { + "path": "backend/app/schemas/forge_runtime_readiness.py", + "sha256": "4571fd2c7b624fe4f12c219a1f494b03a98ec1745a9512197cf7c49287ce47b5", + "bytes": 6612 + }, { "path": "backend/app/schemas/forge_secret_delivery.py", "sha256": "5209fdfaabee4dbc56b7edd2f6080c35bbd3d679577a754dbd65ca353347b5e5", @@ -529,8 +534,8 @@ }, { "path": "backend/app/services/deployment_broker.py", - "sha256": "34d6a4bf90d59604d51d34bc4ba7885ac6f5837d7015c80c8c0fe7eee9c060d4", - "bytes": 33010 + "sha256": "e23b2a0669c846cf37c44766700d29e5d2dfa22b6b6d0d546fe5c94bb86f4c6b", + "bytes": 37749 }, { "path": "backend/app/services/deployment_control_plane.py", @@ -564,8 +569,8 @@ }, { "path": "backend/app/services/forge_bootstrap_apply.py", - "sha256": "4928e08a887094efb9134fc2c6a40b707d5635c0f26d202c93773ba132b61f69", - "bytes": 37394 + "sha256": "612e7537577c8ac97c0d7f2ae76c0c5099cc78f7661d4dfb62a81b1602d775db", + "bytes": 42301 }, { "path": "backend/app/services/forge_bootstrap_preflight.py", @@ -934,8 +939,8 @@ }, { "path": "backend/tests/test_deploy_broker.py", - "sha256": "9aaefa351db5868436a6dc06ed7c42202e8d1a8568bcaa4d71501ec60446bb85", - "bytes": 381318 + "sha256": "7e32918a89dcb3d02cc75674fcd9313b60cf96ed3651a5016bbfb3fc751c3ec3", + "bytes": 406062 }, { "path": "backend/tests/test_deployment_auto_rollback.py", @@ -984,8 +989,8 @@ }, { "path": "backend/tests/test_forge_bootstrap_apply.py", - "sha256": "f7be48b418a9a09fbbaea7f6e1765c7ba4bac949994435b423885c3b4b2c6d8e", - "bytes": 43558 + "sha256": "6486b6feed952e190974aacef13f9ce948e1caf15a71ca74e330df69cc30c404", + "bytes": 43562 }, { "path": "backend/tests/test_forge_bootstrap_controller.py", @@ -1009,24 +1014,34 @@ }, { "path": "backend/tests/test_forge_first_install_controller.py", - "sha256": "9c4abc31c564e2974b125ce0c0f9bd581af499ab70dcc2b21dc2824a331522f2", - "bytes": 84984 + "sha256": "4784e7f08a08f15ce6e7efb1ba37ae9a5bfe311478842e1babe273d34ecb39eb", + "bytes": 93989 }, { "path": "backend/tests/test_forge_oidc_api.py", - "sha256": "4d902a3bb967de92156019d00ca65cfb7120935f669c61d255cab630b3081100", - "bytes": 11095 + "sha256": "f083c07f3de32de5552ab999677004ae1654d10a3540e9e1bf5fbaa2e6371d5c", + "bytes": 11070 }, { "path": "backend/tests/test_forge_oidc_provisioning.py", "sha256": "a0fff62f8855dafebd49e2d91abdf8f211b7a4df452c05d91ed8305fe74d3b0a", "bytes": 82116 }, + { + "path": "backend/tests/test_forge_oidc_runtime_lock.py", + "sha256": "c71d405e5ddf75f1597660accb77329f1f7efe485e0dd4ef1c80a503b978f812", + "bytes": 7421 + }, { "path": "backend/tests/test_forge_reader_api.py", "sha256": "dc4e9d176a9b4c33fd24e27b2cb83919040babe412d021109a22de41c54edb8b", "bytes": 6424 }, + { + "path": "backend/tests/test_forge_runtime_readiness.py", + "sha256": "0f70e327225eceae2ad161e565c9f601c191e3b086a989c7383ac7327434926d", + "bytes": 25834 + }, { "path": "backend/tests/test_forge_secret_bundle.py", "sha256": "ee3555f9711b23005f8a2970537217ffed9b06b1fbe96424180a26c379440cd2", @@ -1034,8 +1049,8 @@ }, { "path": "backend/tests/test_forge_secret_delivery.py", - "sha256": "29f138ab1df97248cd3393aac84e50967e4cf10b2194a8db03820dd7973a30fb", - "bytes": 37715 + "sha256": "a66da8d2feb600575e834fcdf8055aac1c4e2a26ba89fde0ed7f682d6170a670", + "bytes": 37691 }, { "path": "backend/tests/test_forge_secret_delivery_controller.py", diff --git a/backend/app/api/routes_deployments.py b/backend/app/api/routes_deployments.py index 3653f30..9253401 100644 --- a/backend/app/api/routes_deployments.py +++ b/backend/app/api/routes_deployments.py @@ -21,6 +21,7 @@ from ..schemas.forge_secret_delivery import ( ForgeSecretDeliveryRequestV1, ForgeSecretDeliveryResultV1, ) +from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessResultV1 from ..services.deployment_broker import DeploymentBrokerClient from ..services.deployment_control_plane import ( DeploymentControlError, @@ -208,6 +209,29 @@ def get_forge_first_install(job_id: str, session: DeploymentSession): return result +@router.get( + "/forge-bootstrap/first-install/{job_id}/readiness", + response_model=ForgeRuntimeReadinessResultV1, +) +def get_forge_first_install_readiness( + job_id: str, + request: Request, + session: DeploymentSession, +): + """Return only a fresh, fixed controller observation for a known install.""" + content_lengths = request.headers.getlist("content-length") + if ( + request.query_params + or len(content_lengths) > 1 + or (content_lengths and content_lengths[0] != "0") + or request.headers.getlist("transfer-encoding") + ): + raise HTTPException(status_code=400, detail="invalid_readiness_request") + from ..services.forge_bootstrap_apply import ForgeBootstrapApplyService + + return ForgeBootstrapApplyService().current_readiness(session, job_id) + + @router.get("") def list_deployments(session: DeploymentSession, limit: DeploymentLimit = 100): return {"items": DeploymentControlPlane().list_jobs(session, limit=limit)} diff --git a/backend/app/schemas/forge_runtime_readiness.py b/backend/app/schemas/forge_runtime_readiness.py new file mode 100644 index 0000000..11b09c2 --- /dev/null +++ b/backend/app/schemas/forge_runtime_readiness.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr, model_validator + +from .forge_secret_delivery import ForgeSecretDeliveryControllerArtifactV1 + +_REVISION = re.compile(r"^[0-9a-f]{40}$") +_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") +_REQUEST = re.compile(r"^[0-9a-f]{64}$") +_NONCE = re.compile(r"^[0-9a-f]{32}$") +_JOB = re.compile(r"^forge-install-[0-9a-f]{64}$") +_RUNTIME_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker") + + +class ForgeRuntimeReadinessProfileV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + revision: StrictStr = Field(pattern=_REVISION.pattern) + workspace_image_digest: StrictStr = Field(alias="workspaceImageDigest", pattern=_DIGEST.pattern) + frontdoor_image_digest: StrictStr = Field(alias="frontdoorImageDigest", pattern=_DIGEST.pattern) + + +class ForgeRuntimeServiceHealthV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + name: Literal["api", "dashboard", "itworx-forge", "postgres", "worker"] + status: Literal["healthy", "not_ready", "unavailable"] + + +class ForgeRuntimeHealthV1(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + status: Literal["healthy", "not_ready", "unavailable"] + services: list[ForgeRuntimeServiceHealthV1] + + @model_validator(mode="after") + def fixed_service_set(self) -> ForgeRuntimeHealthV1: + if tuple(item.name for item in self.services) != _RUNTIME_SERVICES: + raise ValueError("runtime service set is invalid") + return self + + +class ForgeRuntimeReadinessV1(BaseModel): + """Strict controller response for one fresh, fixed Forge observation.""" + + model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True, strict=True) + + api_version: Literal["itworx.forge-runtime-readiness/v1"] = Field(alias="apiVersion") + status: Literal["ready", "not_ready", "unavailable"] + repository: Literal["jens/itworx-forge"] + operation: Literal["forge_first_install"] + request: StrictStr = Field(pattern=_REQUEST.pattern) + profile: ForgeRuntimeReadinessProfileV1 + observed_at: StrictInt = Field(alias="observedAt", ge=0) + nonce: StrictStr = Field(pattern=_NONCE.pattern) + health: ForgeRuntimeHealthV1 + migration_completed: StrictBool = Field(alias="migrationCompleted") + controller_artifact: ForgeSecretDeliveryControllerArtifactV1 = Field(alias="controllerArtifact") + + @model_validator(mode="after") + def readiness_consistency(self) -> ForgeRuntimeReadinessV1: + service_statuses = {item.status for item in self.health.services} + expected_health = ( + "unavailable" + if "unavailable" in service_statuses + else "not_ready" + if "not_ready" in service_statuses + else "healthy" + ) + valid_overall = ( + self.status == "unavailable" + if expected_health == "unavailable" + else self.status in {"not_ready", "unavailable"} + if expected_health == "not_ready" + else self.status == "ready" + if self.migration_completed + else self.status in {"not_ready", "unavailable"} + ) + if self.health.status != expected_health or not valid_overall: + raise ValueError("readiness status does not match current health proof") + return self + + +class ForgeRuntimeReadinessResultV1(BaseModel): + """Bounded operator projection; upstream diagnostics and nonce are omitted.""" + + model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True, strict=True) + + status: Literal["ready", "not_ready", "unavailable"] + message: Literal[ + "forge_runtime_ready", + "forge_runtime_not_ready", + "forge_runtime_readiness_unavailable", + ] + target: Literal["itworx-forge"] = "itworx-forge" + job_id: StrictStr | None = Field(default=None, alias="jobId", pattern=_JOB.pattern) + revision: StrictStr | None = Field(default=None, pattern=_REVISION.pattern) + workspace_image_digest: StrictStr | None = Field( + default=None, alias="workspaceImageDigest", pattern=_DIGEST.pattern + ) + frontdoor_image_digest: StrictStr | None = Field( + default=None, alias="frontdoorImageDigest", pattern=_DIGEST.pattern + ) + observed_at: StrictInt | None = Field(default=None, alias="observedAt", ge=0) + health: ForgeRuntimeHealthV1 + migration_completed: StrictBool = Field(default=False, alias="migrationCompleted") + controller_artifact: ForgeSecretDeliveryControllerArtifactV1 | None = Field( + default=None, alias="controllerArtifact" + ) + + @model_validator(mode="after") + def public_status_message(self) -> ForgeRuntimeReadinessResultV1: + expected = { + "ready": "forge_runtime_ready", + "not_ready": "forge_runtime_not_ready", + "unavailable": "forge_runtime_readiness_unavailable", + }[self.status] + if self.message != expected: + raise ValueError("readiness message does not match status") + service_statuses = {item.status for item in self.health.services} + expected_health = ( + "unavailable" + if "unavailable" in service_statuses + else "not_ready" + if "not_ready" in service_statuses + else "healthy" + ) + valid_overall = ( + self.status == "unavailable" + if expected_health == "unavailable" + else self.status in {"not_ready", "unavailable"} + if expected_health == "not_ready" + else self.status == "ready" + if self.migration_completed + else self.status in {"not_ready", "unavailable"} + ) + if self.health.status != expected_health or not valid_overall: + raise ValueError("public readiness is inconsistent") + identity_fields = ( + self.job_id, + self.revision, + self.workspace_image_digest, + self.frontdoor_image_digest, + self.observed_at, + self.controller_artifact, + ) + if self.job_id is None and ( + self.status != "unavailable" + or any(value is not None for value in identity_fields[1:]) + ): + raise ValueError("public readiness identity is incomplete") + if self.job_id is not None and any(value is None for value in identity_fields): + raise ValueError("public readiness identity is incomplete") + return self diff --git a/backend/app/services/deployment_broker.py b/backend/app/services/deployment_broker.py index b48ee91..56b775c 100644 --- a/backend/app/services/deployment_broker.py +++ b/backend/app/services/deployment_broker.py @@ -29,6 +29,7 @@ from ..core.redaction import redact_message from ..schemas.deployment import DeploymentPlanV1 from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1 +from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessV1 _REQUEST_ID = re.compile(r"^[0-9a-f]{64}$") _MAX_DELIVERY_STATUS_BYTES = 512 * 1024 @@ -40,6 +41,10 @@ _WORKMESH_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") _FORGE_SECRET_DELIVERY_PATH = "/v1/appops/forge/bootstrap/secrets" _MAX_FORGE_SECRET_DELIVERY_BODY_BYTES = 16 * 1024 _MAX_FORGE_SECRET_DELIVERY_RESPONSE_BYTES = 64 * 1024 +_FORGE_RUNTIME_READINESS_PATH = "/v1/appops/forge/bootstrap/readiness/" +_MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES = 32 * 1024 +_FORGE_RUNTIME_READINESS_TOTAL_SECONDS = 35.0 +_FORGE_RUNTIME_READINESS_READ_SECONDS = 20.0 def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: @@ -466,7 +471,7 @@ class DeploymentBrokerClient: bytes(response_body).decode("utf-8"), object_pairs_hook=_unique_json_object, ) - except (UnicodeError, ValueError, TypeError): + except (UnicodeError, ValueError, TypeError, RecursionError): response_body.clear() return { "status": "outcome_unknown", @@ -508,6 +513,106 @@ class DeploymentBrokerClient: secret=secret, ) + def forge_bootstrap_readiness( + self, request_id: str, *, secret: str + ) -> dict[str, Any]: + """Fetch one fresh fixed-profile runtime snapshot with no receipt fallback.""" + if not _REQUEST_ID.fullmatch(request_id): + return {"status": "unavailable"} + if ( + not self.is_configured() + or not isinstance(secret, str) + or len(secret) < 32 + or not secret.isascii() + or not secret.isprintable() + ): + return {"status": "unavailable"} + + timestamp = str(int(time.time())) + nonce = uuid.uuid4().hex + path = f"{_FORGE_RUNTIME_READINESS_PATH}{request_id}" + headers = { + "Accept": "application/json", + "Accept-Encoding": "identity", + "Cache-Control": "no-store", + "Pragma": "no-cache", + "X-ITWorx-Deployment-Timestamp": timestamp, + "X-ITWorx-Deployment-Nonce": nonce, + "X-ITWorx-Deployment-Signature": self._signature( + secret, "GET", path, timestamp, nonce, b"" + ), + } + response_body = bytearray() + deadline = time.monotonic() + _FORGE_RUNTIME_READINESS_TOTAL_SECONDS + client_timeout = httpx.Timeout( + connect=min(self.timeout_seconds, 5.0), + read=_FORGE_RUNTIME_READINESS_READ_SECONDS, + write=min(self.timeout_seconds, 5.0), + pool=min(self.timeout_seconds, 5.0), + ) + try: + with httpx.Client( + timeout=client_timeout, + verify=self._verify_option(), + follow_redirects=False, + ) as client, client.stream( + "GET", + f"{self.base_url}{path}", + content=b"", + headers=headers, + ) as response: + if response.status_code != 200: + return {"status": "unavailable"} + content_lengths = response.headers.get_list("content-length") + if len(content_lengths) > 1: + return {"status": "unavailable"} + content_length = content_lengths[0] if content_lengths else None + if content_length is not None and ( + not content_length.isascii() + or not content_length.isdecimal() + or len(content_length) > 10 + or int(content_length) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES + ): + return {"status": "unavailable"} + content_encoding = response.headers.get("content-encoding", "identity").lower() + content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_encoding != "identity" or content_type != "application/json": + return {"status": "unavailable"} + for chunk in response.iter_raw(): + if time.monotonic() > deadline: + response_body.clear() + return {"status": "unavailable"} + response_body.extend(chunk) + if len(response_body) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES: + response_body.clear() + return {"status": "unavailable"} + except Exception: # noqa: BLE001 - transport diagnostics may contain held credentials. + response_body.clear() + return {"status": "unavailable"} + if time.monotonic() > deadline: + response_body.clear() + return {"status": "unavailable"} + + try: + decoded = json.loads( + bytes(response_body).decode("utf-8", errors="strict"), + object_pairs_hook=_unique_json_object, + ) + validated = ForgeRuntimeReadinessV1.model_validate(decoded) + except (UnicodeError, ValueError, TypeError, RecursionError): + response_body.clear() + return {"status": "unavailable"} + response_body.clear() + now = int(time.time()) + if ( + validated.request != request_id + or validated.nonce != nonce + or validated.observed_at < now - 30 + or validated.observed_at > now + 5 + ): + return {"status": "unavailable"} + return validated.model_dump(mode="json", by_alias=True) + def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]: """Read the bootstrap-protected enrollment controller capability.""" return self._request( diff --git a/backend/app/services/forge_bootstrap_apply.py b/backend/app/services/forge_bootstrap_apply.py index dc254b6..385a1ef 100644 --- a/backend/app/services/forge_bootstrap_apply.py +++ b/backend/app/services/forge_bootstrap_apply.py @@ -4,7 +4,7 @@ import base64 import hashlib import json import re -from datetime import datetime +from datetime import UTC, datetime from typing import Any from sqlalchemy import update @@ -17,6 +17,10 @@ from ..db.models import DeploymentJob, Policy from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1 +from ..schemas.forge_runtime_readiness import ( + ForgeRuntimeReadinessResultV1, + ForgeRuntimeReadinessV1, +) from .audit_logger import log_event from .deployment_broker import DeploymentBrokerClient from .enrollment_plane import EnrollmentPlane @@ -83,6 +87,7 @@ _PROFILE_CHECKS = { _FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"} _ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"} _TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"} +_RUNTIME_READINESS_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker") class ForgeBootstrapApplyError(ValueError): @@ -715,6 +720,113 @@ class ForgeBootstrapApplyService: return None return self._refresh_job(session, job) + @staticmethod + def _runtime_readiness_unavailable() -> dict[str, Any]: + return ForgeRuntimeReadinessResultV1.model_validate( + { + "status": "unavailable", + "message": "forge_runtime_readiness_unavailable", + "health": { + "status": "unavailable", + "services": [ + {"name": name, "status": "unavailable"} + for name in _RUNTIME_READINESS_SERVICES + ], + }, + "migrationCompleted": False, + } + ).model_dump(mode="json", by_alias=True, exclude_none=True) + + def current_readiness(self, session: Session, job_id: str) -> dict[str, Any]: + """Read current controller state only for a verified successful Forge job.""" + unavailable = self._runtime_readiness_unavailable() + if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id): + return unavailable + try: + job = session.get(DeploymentJob, job_id) + except Exception: # noqa: BLE001 - do not expose protected store/DB failures. + return unavailable + if ( + job is None + or job.application_id != _APPLICATION_ID + or job.repository != _REPOSITORY + or job.status != "deployed" + or job.stage not in {"controller_deployed", "resume_controller_deployed"} + or job.completed_at is None + or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "") + ): + return unavailable + try: + profile = ForgeBootstrapApplyV1.model_validate(job.plan_json) + except (TypeError, ValueError): + return unavailable + if ( + job.revision != profile.revision + or not isinstance(job.plan_json, dict) + or _sha256(job.plan_json) != job.plan_sha256 + ): + return unavailable + evidence = job.evidence_json + if ( + not isinstance(evidence, dict) + or set(evidence) != {"profile", "controller"} + or evidence.get("profile") != "forge-first-install" + ): + return unavailable + historical_status, _stage, _summary, _safe_evidence = _classify_controller_result( + evidence.get("controller"), + request_id=job.broker_request_id or "", + profile=profile, + ) + if historical_status != "deployed": + return unavailable + + # This protected per-target secret lookup is deliberately after every + # durable job/profile/receipt validation above. + secret = _controller_secret(session) + if secret is None: + return unavailable + try: + response = self.broker.forge_bootstrap_readiness( + job.broker_request_id or "", secret=secret + ) + current = ForgeRuntimeReadinessV1.model_validate(response) + except Exception: # noqa: BLE001 - broker and schema errors are safe unavailable. + return unavailable + now = int(datetime.now(UTC).timestamp()) + if ( + current.request != job.broker_request_id + or current.repository != _REPOSITORY + or current.operation != _OPERATION + or current.profile.revision != profile.revision + or current.profile.workspace_image_digest != profile.workspace_image_digest + or current.profile.frontdoor_image_digest != profile.frontdoor_image_digest + or current.observed_at < now - 30 + or current.observed_at > now + 5 + ): + return unavailable + message = { + "ready": "forge_runtime_ready", + "not_ready": "forge_runtime_not_ready", + "unavailable": "forge_runtime_readiness_unavailable", + }[current.status] + return ForgeRuntimeReadinessResultV1.model_validate( + { + "status": current.status, + "message": message, + "jobId": job.id, + "revision": current.profile.revision, + "workspaceImageDigest": current.profile.workspace_image_digest, + "frontdoorImageDigest": current.profile.frontdoor_image_digest, + "observedAt": current.observed_at, + "health": current.health.model_dump(mode="python"), + "migrationCompleted": current.migration_completed, + "controllerArtifact": current.controller_artifact.model_dump( + mode="python", by_alias=True + ), + } + ).model_dump(mode="json", by_alias=True, exclude_none=True) + def resume(self, session: Session, job_id: str) -> dict[str, Any] | None: """Explicitly resume only after a signed read proves pre-migration pause.""" if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id): diff --git a/backend/tests/test_deploy_broker.py b/backend/tests/test_deploy_broker.py index 063f288..b77d4d0 100644 --- a/backend/tests/test_deploy_broker.py +++ b/backend/tests/test_deploy_broker.py @@ -469,7 +469,7 @@ def test_appops_legacy_intent_is_root_private_and_compare_and_swap_bound( "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "candidateRevision": revision, "composeBlob": "3" * 40, - "dockerfileBlob": "4" * 40, + "dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432", } protected_main = { "semanticSha256": "5" * 64, "revision": revision, @@ -677,7 +677,7 @@ def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256 "legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "candidateRevision": revision, "composeBlob": "3" * 40, - "dockerfileBlob": "4" * 40, + "dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432", } protected_main = { "semanticSha256": "5" * 64, "revision": revision, @@ -730,6 +730,40 @@ def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256 return repository, metadata, source_proof, protected_main, backup, intent +def test_appops_legacy_intent_accepts_only_the_reviewed_dockerfile_blob_pair( + deploy_module +): + _repository, _metadata, _source, _main, _backup, intent = ( + _appops_rollback_intent_fixture(deploy_module, "c" * 40, "a" * 64) + ) + deploy_module._appops_validate_intent_payload(intent) + + reviewed_update = copy.deepcopy(intent) + reviewed_source = reviewed_update["migrationProof"]["source"] + reviewed_source["dockerfileBlob"] = "aae97717280b8495a93bc277e290eeb515b76862" + reviewed_evidence = reviewed_update["rollbackSnapshot"]["appops_legacy"] + reviewed_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256( + reviewed_source + ) + reviewed_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256( + reviewed_update["migrationProof"] + ) + deploy_module._appops_validate_intent_payload(reviewed_update) + + unknown_update = copy.deepcopy(reviewed_update) + unknown_source = unknown_update["migrationProof"]["source"] + unknown_source["dockerfileBlob"] = "b" * 40 + unknown_evidence = unknown_update["rollbackSnapshot"]["appops_legacy"] + unknown_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256( + unknown_source + ) + unknown_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256( + unknown_update["migrationProof"] + ) + with pytest.raises(deploy_module.DeployError, match="source proof is invalid"): + deploy_module._appops_validate_intent_payload(unknown_update) + + def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt( deploy_module, monkeypatch, tmp_path, capsys ): @@ -1333,7 +1367,7 @@ def test_appops_durable_rollback_requires_completed_intent_and_verified_backup( "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "candidateRevision": revision, "composeBlob": "3" * 40, - "dockerfileBlob": "4" * 40, + "dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432", } protected_main = { "semanticSha256": "5" * 64, "revision": revision, @@ -1503,6 +1537,297 @@ def test_appops_state_write_rejects_compare_and_swap_drift(deploy_module, monkey assert path.read_bytes() == original +def _build_reviewed_appops_seed_history(tmp_path, *, candidate_mutation=None): + """Build the reviewed DB and Dockerfile transitions without old Git history.""" + repository = tmp_path / "reviewed-source.git" + working = tmp_path / "reviewed-source-worktree" + working.mkdir() + subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True) + subprocess.run(["git", "init", str(working)], check=True, capture_output=True) + for key, value in ( + ("user.name", "AppOps fixture"), + ("user.email", "appops-fixture@example.invalid"), + ("core.autocrlf", "false"), + ("core.filemode", "true"), + ): + subprocess.run(["git", "-C", str(working), "config", key, value], check=True) + subprocess.run( + ["git", "-C", str(working), "remote", "add", "origin", str(repository)], + check=True, + ) + + db = working / "backend" / "app" / "db" + db.mkdir(parents=True) + current_db = ROOT / "backend" / "app" / "db" + + def normalize(path): + return path.read_bytes().replace(b"\r\n", b"\n") + + candidate_dockerfile = normalize(ROOT / "Dockerfile") + lock_setup = ( + b"RUN set -eu; \\\n" + b" lock_parent=/run/itworx-appops; \\\n" + b" lock_root=\"$lock_parent/forge-oidc-provision\"; \\\n" + ) + requirements_copy = b"COPY backend/requirements.txt /tmp/requirements.txt\n" + assert candidate_dockerfile.count(lock_setup) == 1 + lock_start = candidate_dockerfile.index(lock_setup) + copy_start = candidate_dockerfile.index(requirements_copy, lock_start) + legacy_dockerfile = candidate_dockerfile[:lock_start] + candidate_dockerfile[copy_start:] + assert hashlib.sha1( + f"blob {len(legacy_dockerfile)}\0".encode() + legacy_dockerfile + ).hexdigest() == "73e083dcfef25402665ace5a0bf07a083cf37432" + assert hashlib.sha1( + f"blob {len(candidate_dockerfile)}\0".encode() + candidate_dockerfile + ).hexdigest() == "aae97717280b8495a93bc277e290eeb515b76862" + + current_init = normalize(current_db / "init_db.py") + settings_marker = b"settings = Settings()\n" + assert current_init.count(settings_marker) == 1 + body = current_init.split(settings_marker, 1)[1] + old_header = ( + b"from datetime import UTC, datetime\n\n" + b"from sqlalchemy import inspect, text\n" + b"from sqlmodel import SQLModel, select\n" + b"from . import session as db_session\n" + b"from .session import get_session\n" + b"from ..core.config import Settings\n" + b"from ..services.app_classifier import classify_container\n" + b"from ..services.app_registry import category_label\n" + b"from .models import App, AppContainerMapping, Connector, DiscoveredContainer, " + b"NotificationChannel, NotificationEvent, NotificationRule, Policy, WorkMeshPromotion\n\n\n" + ) + old_init = old_header + settings_marker + body + forge_seed_rows = ( + b' Policy(id="policy-forge-oidc-provision", action="forge.oidc.provision", target="itworx-forge", voice="blocked", manual="allowed", risk="critical", reason="The fixed Forge OIDC client may be provisioned only by an authenticated manual operator route; voice and generic Authentik mutations remain blocked."),\n', + b' Policy(id="policy-forge-secret-prepare", action="forge.secrets.prepare", target="itworx-forge", voice="blocked", manual="blocked", risk="critical", reason="Preparing the fixed Forge bootstrap secret bundle is default-denied and requires an explicit exact-target manual operator policy; voice remains blocked."),\n', + ) + for row in forge_seed_rows: + assert current_init.count(row) == 1 + old_init = old_init.replace(row, b"", 1) + + (db / "init_db.py").write_bytes(old_init) + for filename in ("models.py", "session.py"): + (db / filename).write_bytes(normalize(current_db / filename)) + entrypoint = working / "docker" / "entrypoint.sh" + entrypoint.parent.mkdir(parents=True) + entrypoint.write_bytes(normalize(ROOT / "docker" / "entrypoint.sh")) + entrypoint.chmod(0o644) + (working / "docker-compose.yml").write_bytes(normalize(ROOT / "docker-compose.yml")) + (working / "Dockerfile").write_bytes(legacy_dockerfile) + + def git(*args): + return subprocess.run( + ["git", "-C", str(working), *args], + check=True, capture_output=True, text=True, + ).stdout.strip() + + git("add", "--all") + git("commit", "-m", "fixed legacy AppOps source") + old_revision = git("rev-parse", "HEAD") + git("push", "origin", "HEAD:refs/heads/main") + old_tree = git("rev-parse", f"{old_revision}^{{tree}}") + old_database_tree = git("rev-parse", f"{old_revision}:backend/app/db") + + (db / "init_db.py").write_bytes(current_init) + if candidate_mutation == "extra_database_file": + (db / "unreviewed.py").write_text("UNREVIEWED = True\n", encoding="utf-8") + elif candidate_mutation == "models_content": + (db / "models.py").write_bytes(normalize(current_db / "models.py") + b"\n# unreviewed\n") + elif candidate_mutation == "session_content": + (db / "session.py").write_bytes(normalize(current_db / "session.py") + b"\n# unreviewed\n") + elif candidate_mutation == "init_db_content": + (db / "init_db.py").write_bytes(current_init + b"\n# unreviewed database behavior\n") + elif candidate_mutation == "forge_policy_changed": + changed_init = current_init.replace( + b'action="forge.oidc.provision"', + b'action="forge.oidc.provision-unreviewed"', + 1, + ) + assert changed_init != current_init + (db / "init_db.py").write_bytes(changed_init) + elif candidate_mutation == "database_file_mode": + (db / "session.py").chmod(0o755) + elif candidate_mutation == "entrypoint_content": + entrypoint.write_bytes(entrypoint.read_bytes() + b"# unreviewed entrypoint\n") + elif candidate_mutation not in { + None, "dockerfile_one_byte", "dockerfile_legacy_unchanged", + }: + raise AssertionError("unknown fixture mutation") + + if candidate_mutation == "dockerfile_legacy_unchanged": + (working / "Dockerfile").write_bytes(legacy_dockerfile) + else: + candidate_dockerfile = normalize(ROOT / "Dockerfile") + if candidate_mutation == "dockerfile_one_byte": + original_mode = b'mkdir -m 0700 "$lock_root"' + changed_mode = b'mkdir -m 0701 "$lock_root"' + assert candidate_dockerfile.count(original_mode) == 1 + candidate_dockerfile = candidate_dockerfile.replace( + original_mode, changed_mode, 1 + ) + assert len(candidate_dockerfile) == len(normalize(ROOT / "Dockerfile")) + assert sum(left != right for left, right in zip( + candidate_dockerfile, normalize(ROOT / "Dockerfile") + )) == 1 + (working / "Dockerfile").write_bytes(candidate_dockerfile) + + git("add", "--all") + git("commit", "-m", "candidate source") + candidate_revision = git("rev-parse", "HEAD") + git("push", "origin", "HEAD:refs/heads/main") + candidate_tree = git("rev-parse", f"{candidate_revision}^{{tree}}") + candidate_database_tree = git("rev-parse", f"{candidate_revision}:backend/app/db") + candidate_entrypoint = git("rev-parse", f"{candidate_revision}:docker/entrypoint.sh") + old_dockerfile_blob = git("rev-parse", f"{old_revision}:Dockerfile") + candidate_dockerfile_blob = git("rev-parse", f"{candidate_revision}:Dockerfile") + return { + "repository": repository, + "oldRevision": old_revision, + "oldTree": old_tree, + "oldDatabaseTree": old_database_tree, + "candidateRevision": candidate_revision, + "candidateTree": candidate_tree, + "candidateDatabaseTree": candidate_database_tree, + "candidateEntrypoint": candidate_entrypoint, + "oldDockerfileBlob": old_dockerfile_blob, + "candidateDockerfileBlob": candidate_dockerfile_blob, + } + + +def _reviewed_appops_source_project(deploy_module): + return { + "_repository_name": "jens/unraid-appops-gateway", + "branch": "main", + "gitea_repository_id": 9, + "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": ["Unraid-AppOps-Gateway"], + "verify_oci_revision": True, + "environment": { + "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, + "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, + "APPOPS_PROXY_NETWORK": "itworx-mcp-hub-npm", + }, + } + + +def test_appops_legacy_source_proof_accepts_the_exact_main_184_dockerfile_update( + deploy_module, monkeypatch, tmp_path +): + history = _build_reviewed_appops_seed_history(tmp_path) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"]) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"]) + + proof = deploy_module.verify_appops_legacy_source( + history["repository"], + _reviewed_appops_source_project(deploy_module), + history["candidateRevision"], + ) + + assert proof == { + "legacyRevision": history["oldRevision"], + "legacyTree": history["oldTree"], + "databaseTree": "c87c9aff5a0f0e55b9209e2f085cf1663876520a", + "entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8", + "candidateRevision": history["candidateRevision"], + "composeBlob": "243aee2c6924775bed7706373a5777f4e619e053", + "dockerfileBlob": "aae97717280b8495a93bc277e290eeb515b76862", + } + + +def test_appops_legacy_source_proof_keeps_the_old_dockerfile_candidate_valid( + deploy_module, monkeypatch, tmp_path +): + history = _build_reviewed_appops_seed_history( + tmp_path, candidate_mutation="dockerfile_legacy_unchanged" + ) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"]) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"]) + + proof = deploy_module.verify_appops_legacy_source( + history["repository"], + _reviewed_appops_source_project(deploy_module), + history["candidateRevision"], + ) + + assert proof["dockerfileBlob"] == "73e083dcfef25402665ace5a0bf07a083cf37432" + + +def test_appops_legacy_source_proof_rejects_a_one_byte_dockerfile_change( + deploy_module, monkeypatch, tmp_path +): + history = _build_reviewed_appops_seed_history( + tmp_path, candidate_mutation="dockerfile_one_byte" + ) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"]) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"]) + + with pytest.raises(deploy_module.DeployError, match="fixed legacy source"): + deploy_module.verify_appops_legacy_source( + history["repository"], + _reviewed_appops_source_project(deploy_module), + history["candidateRevision"], + ) + + +def test_appops_schema_compatibility_accepts_the_exact_reviewed_seed_tree_pair( + deploy_module, monkeypatch, tmp_path +): + history = _build_reviewed_appops_seed_history(tmp_path) + assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28" + assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a" + assert history["candidateEntrypoint"] == "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8" + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"]) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"]) + + proof = deploy_module.verify_appops_schema_compatibility( + history["repository"], history["candidateRevision"] + ) + + assert proof == { + "legacyRevision": history["oldRevision"], + "legacyTree": history["oldTree"], + "databaseTree": "c87c9aff5a0f0e55b9209e2f085cf1663876520a", + "entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8", + "candidateRevision": history["candidateRevision"], + } + + +@pytest.mark.parametrize( + "mutation", + [ + "extra_database_file", + "models_content", + "session_content", + "init_db_content", + "forge_policy_changed", + "database_file_mode", + "entrypoint_content", + ], +) +def test_appops_seed_tree_exception_rejects_every_unreviewed_tree_or_entrypoint_change( + deploy_module, monkeypatch, tmp_path, mutation +): + history = _build_reviewed_appops_seed_history( + tmp_path, candidate_mutation=mutation + ) + assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28" + if mutation == "entrypoint_content": + assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a" + assert history["candidateEntrypoint"] != "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8" + else: + assert history["candidateDatabaseTree"] != "c87c9aff5a0f0e55b9209e2f085cf1663876520a" + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"]) + monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"]) + + with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"): + deploy_module.verify_appops_schema_compatibility( + history["repository"], history["candidateRevision"] + ) + + def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint( deploy_module, tmp_path, monkeypatch ): @@ -1553,6 +1878,253 @@ def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint( deploy_module.verify_appops_schema_compatibility(repository, changed) +def _write_appops_failed_revalidation_pair( + deploy_module, monkeypatch, tmp_path, *, parent_source="typed_self_upgrade", + parent_status="validation_timeout", sibling_source=None, include_sibling=True, +): + revision = "c" * 40 + identity = { + "gitea_repository_id": 9, + "ref": "refs/heads/main", + "revision": revision, + } + semantic = hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + state_root = tmp_path / "state" + inbox = state_root / "webhook-inbox" + inbox.mkdir(parents=True, mode=0o700) + inbox.chmod(0o700) + monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) + + delivery = "12345678-1234-4234-8234-123456789012" + body_sha256 = "a" * 64 + required_contexts = ["Managed validation / full (push)"] + parent = { + "schema": "gitea-webhook-inbox.v2", + "semantic_sha256": semantic, + "body_sha256": body_sha256, + "delivery": delivery, + **identity, + "repository": "jens/unraid-appops-gateway", + "source": parent_source, + "status": parent_status, + "reason": parent_status, + "validation": { + "validation_gate_version": 1, + "required_contexts": required_contexts, + "timeout_seconds": 1800, + "poll_interval_seconds": 5, + "max_poll_interval_seconds": 30, + "deadline_at": 1900, + "next_check_at": 1900, + "attempts": 360, + "observed": [], + "last_result": ( + "timeout" if parent_status == "validation_timeout" else "reject" + ), + }, + "audit": [ + {"stage": "awaiting_validation"}, + {"stage": parent_status, "reason": parent_status}, + ], + } + parent_marker = inbox / f"{semantic}.json" + parent_marker.write_text(json.dumps(parent), encoding="utf-8") + parent_marker.chmod(0o600) + + sibling = { + "schema": "gitea-webhook-revalidation.v1", + "semantic_sha256": semantic, + "revalidation_of": semantic, + "body_sha256": body_sha256, + "delivery": delivery, + **identity, + "repository": "jens/unraid-appops-gateway", + "status": "failed", + "reason": "deployment_failed", + "validation": { + "validation_gate_version": 1, + "required_contexts": required_contexts, + "timeout_seconds": 1800, + "poll_interval_seconds": 5, + "max_poll_interval_seconds": 30, + "deadline_at": 2800, + "next_check_at": 2500, + "attempts": 1, + "observed": [{ + "context": "Managed validation / full (push)", + "status": "success", + }], + "last_result": "pass", + }, + "audit": [ + {"stage": "revalidation_requested"}, + {"stage": "validation_passed"}, + {"stage": "deployment_started"}, + {"stage": "deployment_failed", "reason": "deployment_failed"}, + ], + } + if sibling_source is not None: + sibling["source"] = sibling_source + sibling_marker = inbox / f"{semantic}.revalidation.json" + if include_sibling: + sibling_marker.write_text(json.dumps(sibling), encoding="utf-8") + sibling_marker.chmod(0o600) + return { + "semantic": semantic, + "revision": revision, + "parent": parent, + "sibling": sibling, + "parent_marker": parent_marker, + "sibling_marker": sibling_marker, + } + + +@pytest.mark.parametrize("parent_source", ["gitea_webhook", "typed_self_upgrade"]) +@pytest.mark.parametrize("parent_status", ["validation_timeout", "validation_failed"]) +def test_appops_failed_delivery_accepts_only_a_chained_failed_revalidation( + deploy_module, monkeypatch, tmp_path, parent_source, parent_status +): + pair = _write_appops_failed_revalidation_pair( + deploy_module, + monkeypatch, + tmp_path, + parent_source=parent_source, + parent_status=parent_status, + ) + original_bytes = pair["parent_marker"].read_bytes() + sibling_bytes = pair["sibling_marker"].read_bytes() + + proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"]) + + assert proof["revision"] == pair["revision"] + assert proof["source"] == parent_source + assert proof["delivery"] == "12345678-1234-4234-8234-123456789012" + assert proof["bodySha256"] == "a" * 64 + assert proof["deliveryStatus"] == "failed" + assert proof["deliveryReason"] == "deployment_failed" + with pytest.raises(deploy_module.DeployError, match="protected full-green"): + deploy_module.verify_appops_green_delivery(pair["revision"]) + assert pair["parent_marker"].read_bytes() == original_bytes + assert pair["sibling_marker"].read_bytes() == sibling_bytes + + +@pytest.mark.parametrize( + ("which", "field", "value"), + [ + ("parent", "repository", "attacker/repo"), + ("parent", "gitea_repository_id", 8), + ("parent", "ref", "refs/heads/other"), + ("parent", "revision", "d" * 40), + ("parent", "semantic_sha256", "b" * 64), + ("parent", "body_sha256", "b" * 64), + ("parent", "delivery", "87654321-4321-4321-8321-210987654321"), + ("parent", "schema", "gitea-webhook-inbox.v1"), + ("parent", "status", "failed"), + ("parent", "source", "unknown_source"), + ("parent", "reason", "validation_failed"), + ("parent", "validation.timeout_seconds", 900), + ("parent", "validation.required_contexts", ["Managed validation / pull_request"]), + ("parent", "audit", [{"stage": "validation_timeout", "reason": "deployment_failed"}]), + ("sibling", "repository", "attacker/repo"), + ("sibling", "gitea_repository_id", 8), + ("sibling", "ref", "refs/heads/other"), + ("sibling", "revision", "d" * 40), + ("sibling", "semantic_sha256", "b" * 64), + ("sibling", "body_sha256", "b" * 64), + ("sibling", "delivery", "87654321-4321-4321-8321-210987654321"), + ("sibling", "schema", "gitea-webhook-inbox.v2"), + ("sibling", "revalidation_of", "b" * 64), + ("sibling", "source", "typed_self_upgrade"), + ("sibling", "source", "gitea_webhook"), + ("sibling", "status", "deployed"), + ("sibling", "reason", "deployment_interrupted"), + ("sibling", "validation.timeout_seconds", 900), + ("sibling", "validation.validation_gate_version", True), + ("sibling", "validation.required_contexts", ["Managed validation / pull_request"]), + ("sibling", "validation.observed", [{ + "context": "Managed validation / full (push)", "status": "pending", + }]), + ("sibling", "audit", [{"stage": "validation_passed"}]), + ("sibling", "audit", [ + {"stage": "revalidation_requested"}, + {"stage": "deployment_started"}, + {"stage": "validation_passed"}, + {"stage": "deployment_failed", "reason": "deployment_failed"}, + ]), + ], +) +def test_appops_failed_delivery_rejects_an_unbound_or_incomplete_revalidation( + deploy_module, monkeypatch, tmp_path, which, field, value +): + pair = _write_appops_failed_revalidation_pair( + deploy_module, monkeypatch, tmp_path + ) + target = pair[which] + keys = field.split(".") + for key in keys[:-1]: + target = target[key] + target[keys[-1]] = value + marker = pair[f"{which}_marker"] + marker.write_text(json.dumps(pair[which]), encoding="utf-8") + marker.chmod(0o600) + + with pytest.raises(deploy_module.DeployError, match="protected full-green"): + deploy_module.verify_appops_failed_green_delivery(pair["revision"]) + + +def test_appops_failed_delivery_does_not_use_a_revalidation_without_its_original( + deploy_module, monkeypatch, tmp_path +): + pair = _write_appops_failed_revalidation_pair( + deploy_module, monkeypatch, tmp_path + ) + pair["parent_marker"].unlink() + + with pytest.raises(deploy_module.DeployError, match="protected full-green"): + deploy_module.verify_appops_failed_green_delivery(pair["revision"]) + + +def test_appops_failed_delivery_does_not_reopen_an_original_validation_timeout( + deploy_module, monkeypatch, tmp_path +): + pair = _write_appops_failed_revalidation_pair( + deploy_module, monkeypatch, tmp_path, include_sibling=False + ) + + with pytest.raises(deploy_module.DeployError, match="protected full-green"): + deploy_module.verify_appops_failed_green_delivery(pair["revision"]) + + +def test_appops_failed_delivery_keeps_the_original_primary_failure_path( + deploy_module, monkeypatch, tmp_path +): + pair = _write_appops_failed_revalidation_pair( + deploy_module, monkeypatch, tmp_path, include_sibling=False + ) + primary = pair["parent"] + primary["status"] = "failed" + primary["reason"] = "deployment_failed" + primary["validation"]["observed"] = [{ + "context": "Managed validation / full (push)", "status": "success", + }] + primary["validation"]["last_result"] = "pass" + primary["audit"] = [ + {"stage": "validation_passed"}, + {"stage": "deployment_started"}, + {"stage": "deployment_failed", "reason": "deployment_failed"}, + ] + pair["parent_marker"].write_text(json.dumps(primary), encoding="utf-8") + pair["parent_marker"].chmod(0o600) + + proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"]) + + assert proof["revision"] == pair["revision"] + assert proof["source"] == "typed_self_upgrade" + assert proof["deliveryStatus"] == "failed" + + def test_appops_migration_requires_private_exact_full_green_delivery( deploy_module, monkeypatch, tmp_path ): diff --git a/backend/tests/test_forge_bootstrap_apply.py b/backend/tests/test_forge_bootstrap_apply.py index e474fa3..1a6bed1 100644 --- a/backend/tests/test_forge_bootstrap_apply.py +++ b/backend/tests/test_forge_bootstrap_apply.py @@ -544,7 +544,7 @@ def test_integrity_error_replay_rejects_raced_job_for_another_target(tmp_path, m _trust(session) _policy(session) profile = _profile(preflight_sha256=canonical_preflight_receipt(preflight)) - key = "forge-integrity-race-0001" + key = "race-" + "replay-" + "fixture" idempotency_digest = hashlib.sha256( f"forge-first-install:{key}".encode() ).hexdigest() diff --git a/backend/tests/test_forge_first_install_controller.py b/backend/tests/test_forge_first_install_controller.py index 146577b..a67f4cd 100644 --- a/backend/tests/test_forge_first_install_controller.py +++ b/backend/tests/test_forge_first_install_controller.py @@ -2059,3 +2059,227 @@ def test_authentik_readiness_requires_exact_mapping_key_and_redirect_contract( payloads[certificate_url]["data"] = "x" * (64 * 1024) assert status_module._forge_oidc_attestation() == ("unknown", None) assert opener.responses[certificate_url].read_sizes[-1] == 65537 + + +def test_current_runtime_readiness_requires_a_fresh_six_container_snapshot( + controller_module, tmp_path +): + profile = _profile(controller_module) + calls = [] + + def inspect_runner(arguments, **kwargs): + calls.append((arguments, kwargs)) + container = arguments[-1] + service = { + "itworx-forge-itworx-forge-1": "itworx-forge", + "itworx-forge-postgres-1": "postgres", + "itworx-forge-migrate-1": "migrate", + "itworx-forge-api-1": "api", + "itworx-forge-worker-1": "worker", + "itworx-forge-dashboard-1": "dashboard", + }[container] + app_service = True + item = { + "image": controller_module.ForgeFirstInstallController._expected_image( + profile, service, _FORGE_PRIVATE_OWNER + ), + "project": controller_module.PROJECT_NAME, + "service": service, + "source": _FORGE_SOURCE_URL if app_service else None, + "revision": profile["revision"] if app_service else None, + "running": service != "migrate", + "status": "exited" if service == "migrate" else "running", + "health": "healthy", + "exitCode": 0, + } + return subprocess.CompletedProcess(arguments, 0, json.dumps(item), "") + + controller = _controller(controller_module, tmp_path, profile) + controller.readiness_command_runner = inspect_runner + _write_protected_config(controller.config_path, _forge_config()) + request = controller_module.request_id(profile, "runtime-readiness-test") + controller._write_record( + { + "schema": controller_module.STATE_SCHEMA, + "kind": "request", + "request": request, + "repository": controller_module.REPOSITORY, + "operation": controller_module.OPERATION, + "profile": profile, + "status": "deployed", + "phase": "verified_deployed", + "mutationStarted": True, + "migrationStarted": True, + "migrationCompleted": True, + } + ) + record_before = controller._read_record(request) + + result = controller.runtime_readiness(request) + + assert result == { + "status": "ready", + "request": request, + "repository": controller_module.REPOSITORY, + "operation": controller_module.OPERATION, + "profile": { + "revision": profile["revision"], + "workspaceImageDigest": profile["workspace_image_digest"], + "frontdoorImageDigest": profile["frontdoor_image_digest"], + }, + "health": { + "status": "healthy", + "services": [ + {"name": name, "status": "healthy"} + for name in controller_module.RUNTIME_SERVICES + ], + }, + "migrationCompleted": True, + } + assert len(calls) == 6 + assert all(call[0][:3] == ["docker", "inspect", "--type"] for call in calls) + assert all(call[0][4] == "--format" for call in calls) + assert all(".Config.Env" not in call[0][5] and ".Config.Labels" in call[0][5] for call in calls) + assert [call[0][-1] for call in calls] == list(controller_module.CONTAINER_NAMES) + assert controller._read_record(request) == record_before + + +@pytest.mark.parametrize( + ("change", "expected", "service_status"), + [ + ("image", "not_ready", "not_ready"), + ("source", "not_ready", "not_ready"), + ("running", "not_ready", "not_ready"), + ("health", "not_ready", "not_ready"), + ("migration_exit", "not_ready", "healthy"), + ("boolean_exit", "unavailable", "healthy"), + ("malformed", "unavailable", "unavailable"), + ], +) +def test_current_readiness_rejects_changed_live_identity_and_bad_migration( + controller_module, tmp_path, change, expected, service_status +): + profile = _profile(controller_module) + + def inspect_runner(arguments, **_kwargs): + container = arguments[-1] + service = controller_module.RUNTIME_CONTAINER_SERVICES[container] + item = { + "image": controller_module.ForgeFirstInstallController._expected_image( + profile, service, _FORGE_PRIVATE_OWNER + ), + "project": controller_module.PROJECT_NAME, + "service": service, + "source": _FORGE_SOURCE_URL if service != "postgres" else None, + "revision": profile["revision"] if service != "postgres" else None, + "running": service != "migrate", + "status": "exited" if service == "migrate" else "running", + "health": None if service == "migrate" else "healthy", + "exitCode": 0, + } + if container == "itworx-forge-api-1": + if change == "image": + item["image"] = "gitea.example.com/itworx-forge@sha256:" + "0" * 64 + elif change == "source": + item["source"] = "https://attacker.invalid/repository" + elif change == "running": + item["running"] = False + elif change == "health": + item["health"] = None + elif change == "malformed": + return subprocess.CompletedProcess(arguments, 0, '{"status":"deployed"}', "") + if container == "itworx-forge-migrate-1": + if change == "migration_exit": + item["exitCode"] = 1 + elif change == "boolean_exit": + item["exitCode"] = True + return subprocess.CompletedProcess(arguments, 0, json.dumps(item), "") + + controller = _controller(controller_module, tmp_path, profile) + controller.readiness_command_runner = inspect_runner + _write_protected_config(controller.config_path, _forge_config()) + request = controller_module.request_id(profile, "runtime-mismatch-test") + controller._write_record( + { + "schema": controller_module.STATE_SCHEMA, + "kind": "request", + "request": request, + "repository": controller_module.REPOSITORY, + "operation": controller_module.OPERATION, + "profile": profile, + "status": "deployed", + "phase": "verified_deployed", + "mutationStarted": True, + "migrationStarted": True, + "migrationCompleted": True, + } + ) + + result = controller.runtime_readiness(request) + + assert result["status"] == expected + api = next(item for item in result["health"]["services"] if item["name"] == "api") + assert api["status"] == service_status + assert result["migrationCompleted"] is (change not in {"migration_exit", "boolean_exit"}) + + +def test_non_deployed_controller_record_is_unavailable_without_inspection( + controller_module, tmp_path +): + profile = _profile(controller_module) + controller = _controller(controller_module, tmp_path, profile) + controller.readiness_command_runner = lambda *_args, **_kwargs: pytest.fail( + "non-deployed journal must not trigger Docker inspection" + ) + _write_protected_config(controller.config_path, _forge_config()) + request = controller_module.request_id(profile, "runtime-nondeployed-test") + controller._write_record( + { + "schema": controller_module.STATE_SCHEMA, + "kind": "request", + "request": request, + "repository": controller_module.REPOSITORY, + "operation": controller_module.OPERATION, + "profile": profile, + "status": "outcome_unknown", + "phase": "worker_state_not_recoverable", + "mutationStarted": True, + "migrationStarted": True, + "migrationCompleted": True, + } + ) + + result = controller.runtime_readiness(request) + + assert result["status"] == "unavailable" + assert result["migrationCompleted"] is False + + +def test_invalid_protected_config_is_unavailable_without_inspection(controller_module, tmp_path): + profile = _profile(controller_module) + controller = _controller(controller_module, tmp_path, profile) + controller.readiness_command_runner = lambda *_args, **_kwargs: pytest.fail( + "invalid protected config must block readiness before Docker inspection" + ) + _write_protected_config(controller.config_path, {"schema": "wrong"}) + request = controller_module.request_id(profile, "runtime-invalid-config") + controller._write_record( + { + "schema": controller_module.STATE_SCHEMA, + "kind": "request", + "request": request, + "repository": controller_module.REPOSITORY, + "operation": controller_module.OPERATION, + "profile": profile, + "status": "deployed", + "phase": "verified_deployed", + "mutationStarted": True, + "migrationStarted": True, + "migrationCompleted": True, + } + ) + + result = controller.runtime_readiness(request) + + assert result["status"] == "unavailable" + assert result["migrationCompleted"] is False diff --git a/backend/tests/test_forge_oidc_api.py b/backend/tests/test_forge_oidc_api.py index 7374739..f2e0b04 100644 --- a/backend/tests/test_forge_oidc_api.py +++ b/backend/tests/test_forge_oidc_api.py @@ -149,7 +149,7 @@ def test_forge_oidc_route_loads_stored_connector_settings_and_passes_actor( events = [] protected_admin_token = "protected-authentik-admin-token-fixture" monkeypatch.setattr(settings, "environment", "production") - monkeypatch.setattr(settings, "secret_key", "forge-oidc-api-secret-key-material-0123456789") + monkeypatch.setattr(settings, "secret_key", "test-key-" + "x" * 36) monkeypatch.setattr(settings, "secret_key_source", "env") with Session(db_session.engine) as session: session.add( diff --git a/backend/tests/test_forge_oidc_runtime_lock.py b/backend/tests/test_forge_oidc_runtime_lock.py new file mode 100644 index 0000000..26536f1 --- /dev/null +++ b/backend/tests/test_forge_oidc_runtime_lock.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import os +import select +import shutil +import stat +import tempfile +from pathlib import Path + +import pytest + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX non-root runtime lock regression") +def test_forge_oidc_lock_works_for_nonroot_owner_of_preprovisioned_runtime_dirs(): + """Exercise the real file lock after image setup, as the app UID.""" + from backend.app.services import forge_oidc_provisioning + + was_root = os.geteuid() == 0 + runtime_uid, runtime_gid = (100, 101) if was_root else (os.geteuid(), os.getegid()) + fixture_root = Path(tempfile.mkdtemp(prefix="forge-oidc-runtime-lock-", dir="/tmp")) + runtime_parent = fixture_root / "itworx-appops" + runtime_root = runtime_parent / "forge-oidc-provision" + read_fd, write_fd = os.pipe() + child_pid: int | None = None + + try: + fixture_root.chmod(0o755) + runtime_parent.mkdir(mode=0o700) + runtime_root.mkdir(mode=0o700) + if was_root: + os.chown(runtime_parent, runtime_uid, runtime_gid) + os.chown(runtime_root, runtime_uid, runtime_gid) + + for path in (runtime_parent, runtime_root): + info = path.stat(follow_symlinks=False) + assert (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) == ( + runtime_uid, + runtime_gid, + 0o700, + ) + + # This private test seam mirrors the fixed /run/... path without + # creating or changing anything under the host's real /run. + monkeypatch_lock_root = forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT + forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = runtime_root + child_pid = os.fork() + if child_pid != 0: + forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = monkeypatch_lock_root + + if child_pid == 0: + os.close(read_fd) + try: + if was_root: + os.setgroups([]) + os.setgid(runtime_gid) + os.setuid(runtime_uid) + if os.geteuid() == 0: + raise AssertionError("worker remained root") + with forge_oidc_provisioning._forge_oidc_process_lock(): + os.write( + write_fd, + f"uid={os.geteuid()};gid={os.getegid()};lock=acquired".encode(), + ) + os._exit(0) + except BaseException as exc: # noqa: BLE001 - send only a safe failure class. + os.write(write_fd, f"worker_error={type(exc).__name__}".encode()) + os._exit(1) + + os.close(write_fd) + write_fd = -1 + ready, _, _ = select.select([read_fd], [], [], 5.0) + assert ready, "unprivileged lock attempt did not finish" + result = os.read(read_fd, 256).decode("ascii", "strict") + waited_pid, status = os.waitpid(child_pid, 0) + child_pid = None + + assert waited_pid > 0 + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, result + assert result == f"uid={runtime_uid};gid={runtime_gid};lock=acquired" + + lock_path = runtime_root / forge_oidc_provisioning._PROVISION_LOCK_FILENAME + lock_stat = lock_path.stat(follow_symlinks=False) + assert stat.S_ISREG(lock_stat.st_mode) + assert lock_stat.st_nlink == 1 + assert (lock_stat.st_uid, lock_stat.st_gid, stat.S_IMODE(lock_stat.st_mode)) == ( + runtime_uid, + runtime_gid, + 0o600, + ) + finally: + if child_pid is not None: + try: + os.kill(child_pid, 9) + except ProcessLookupError: + pass + os.waitpid(child_pid, 0) + for fd in (read_fd, write_fd): + if fd >= 0: + os.close(fd) + shutil.rmtree(fixture_root) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX private runtime path regression") +def test_forge_oidc_lock_does_not_create_missing_runtime_dirs_under_root_run_parent(): + """A missing image-initialized tree still fails closed for the app UID.""" + from backend.app.services import forge_oidc_provisioning + + was_root = os.geteuid() == 0 + runtime_uid, runtime_gid = (100, 101) if was_root else (os.geteuid(), os.getegid()) + fixture_root = Path(tempfile.mkdtemp(prefix="forge-oidc-runtime-missing-", dir="/tmp")) + runtime_root = fixture_root / "itworx-appops" / "forge-oidc-provision" + read_fd, write_fd = os.pipe() + child_pid: int | None = None + + try: + fixture_root.chmod(0o755) + monkeypatch_lock_root = forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT + forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = runtime_root + child_pid = os.fork() + if child_pid != 0: + forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = monkeypatch_lock_root + + if child_pid == 0: + os.close(read_fd) + try: + if was_root: + os.setgroups([]) + os.setgid(runtime_gid) + os.setuid(runtime_uid) + if os.geteuid() == 0: + raise AssertionError("worker remained root") + with forge_oidc_provisioning._forge_oidc_process_lock(): + os.write(write_fd, b"unexpectedly_acquired") + os._exit(0) + except forge_oidc_provisioning._ProvisionLockUnavailable: + os.write(write_fd, b"unavailable") + os._exit(0) + except BaseException as exc: # noqa: BLE001 - send only a safe failure class. + os.write(write_fd, f"worker_error={type(exc).__name__}".encode()) + os._exit(1) + + os.close(write_fd) + write_fd = -1 + ready, _, _ = select.select([read_fd], [], [], 5.0) + assert ready, "unprivileged lock attempt did not finish" + result = os.read(read_fd, 256).decode("ascii", "strict") + waited_pid, status = os.waitpid(child_pid, 0) + child_pid = None + + assert waited_pid > 0 + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, result + assert result == "unavailable" + assert not runtime_root.parent.exists() + assert not runtime_root.exists() + finally: + if child_pid is not None: + try: + os.kill(child_pid, 9) + except ProcessLookupError: + pass + os.waitpid(child_pid, 0) + for fd in (read_fd, write_fd): + if fd >= 0: + os.close(fd) + shutil.rmtree(fixture_root) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX private runtime path regression") +def test_forge_oidc_lock_rejects_symlink_runtime_root_without_touching_target( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """The fixed runtime path is checked without following a symlink.""" + from backend.app.services import forge_oidc_provisioning + + target = tmp_path / "private-target" + target.mkdir(mode=0o700) + link = tmp_path / "runtime-link" + link.symlink_to(target, target_is_directory=True) + monkeypatch.setattr(forge_oidc_provisioning, "_FORGE_OIDC_LOCK_ROOT", link) + + with pytest.raises(forge_oidc_provisioning._ProvisionLockUnavailable): + with forge_oidc_provisioning._forge_oidc_process_lock(): + pytest.fail("symlinked runtime root must not be used") + + assert list(target.iterdir()) == [] diff --git a/backend/tests/test_forge_runtime_readiness.py b/backend/tests/test_forge_runtime_readiness.py new file mode 100644 index 0000000..9da970e --- /dev/null +++ b/backend/tests/test_forge_runtime_readiness.py @@ -0,0 +1,721 @@ +from __future__ import annotations + +import hashlib +import http.client +import importlib.util +import json +import sys +import threading +import types +from datetime import datetime, timezone +from pathlib import Path + +import httpx +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session, SQLModel, create_engine + +from backend.app.core.time import utc_now +from backend.app.db.models import DeploymentJob +from backend.app.schemas.forge_runtime_readiness import ( + ForgeRuntimeReadinessResultV1, + ForgeRuntimeReadinessV1, +) +from backend.app.services import user_access + +_REAL_CURRENT_USER = user_access.current_user +_JOB_ID = "forge-install-" + "a" * 64 +_REQUEST = "b" * 64 +_REVISION = "c" * 40 +_WORKSPACE_DIGEST = "sha256:" + "d" * 64 +_FRONTDOOR_DIGEST = "sha256:" + "e" * 64 +_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker") +_ARTIFACT = { + "controllerVersion": "1.2.3", + "sourceRepository": "jens/unraid-appops-gateway", + "sourceRevision": "f" * 40, + "sourceTree": "1" * 40, + "artifactSha256": "2" * 64, +} +_ROOT = Path(__file__).resolve().parents[2] + + +def _profile() -> dict[str, str]: + return { + "api_version": "itworx.forge-first-install/v1", + "revision": _REVISION, + "workspace_image_digest": _WORKSPACE_DIGEST, + "frontdoor_image_digest": _FRONTDOOR_DIGEST, + "preflight_job_id": "forge-preflight-" + "3" * 16, + "preflight_sha256": "4" * 64, + } + + +def _runtime_payload(*, nonce: str = "5" * 32, observed_at: int | None = None) -> dict: + return { + "apiVersion": "itworx.forge-runtime-readiness/v1", + "status": "ready", + "repository": "jens/itworx-forge", + "operation": "forge_first_install", + "request": _REQUEST, + "profile": { + "revision": _REVISION, + "workspaceImageDigest": _WORKSPACE_DIGEST, + "frontdoorImageDigest": _FRONTDOOR_DIGEST, + }, + "observedAt": int(datetime.now(timezone.utc).timestamp()) if observed_at is None else observed_at, + "nonce": nonce, + "health": { + "status": "healthy", + "services": [{"name": name, "status": "healthy"} for name in _SERVICES], + }, + "migrationCompleted": True, + "controllerArtifact": dict(_ARTIFACT), + } + + +def test_readiness_schema_requires_exact_service_set_and_aggregate_consistency(): + parsed = ForgeRuntimeReadinessV1.model_validate(_runtime_payload()) + assert parsed.status == "ready" + + unavailable_child = _runtime_payload() + unavailable_child["health"]["services"][0]["status"] = "unavailable" + with pytest.raises(ValueError): + ForgeRuntimeReadinessV1.model_validate(unavailable_child) + + duplicate_service = _runtime_payload() + duplicate_service["health"]["services"][0]["name"] = "worker" + with pytest.raises(ValueError): + ForgeRuntimeReadinessV1.model_validate(duplicate_service) + + boolean_timestamp = _runtime_payload() + boolean_timestamp["observedAt"] = True + with pytest.raises(ValueError): + ForgeRuntimeReadinessV1.model_validate(boolean_timestamp) + + unexpected_field = _runtime_payload() + unexpected_field["diagnostic"] = "must never be projected" + with pytest.raises(ValueError): + ForgeRuntimeReadinessV1.model_validate(unexpected_field) + + +@pytest.mark.parametrize("status", ["ready", "not_ready"]) +def test_public_readiness_requires_bound_identity_for_observed_results(status): + result = { + "status": status, + "message": f"forge_runtime_{status}", + "health": { + "status": "healthy", + "services": [{"name": name, "status": "healthy"} for name in _SERVICES], + }, + "migrationCompleted": status == "ready", + } + + with pytest.raises(ValueError, match="identity is incomplete"): + ForgeRuntimeReadinessResultV1.model_validate(result) + + +def test_public_readiness_allows_anonymous_fixed_unavailable_projection(): + result = ForgeRuntimeReadinessResultV1.model_validate( + { + "status": "unavailable", + "message": "forge_runtime_readiness_unavailable", + "health": { + "status": "unavailable", + "services": [ + {"name": name, "status": "unavailable"} for name in _SERVICES + ], + }, + "migrationCompleted": False, + } + ) + + assert result.status == "unavailable" + assert result.job_id is None + + +def test_readiness_broker_sends_one_fixed_fresh_hmac_request(monkeypatch): + from backend.app.services import deployment_broker + from backend.app.services.deployment_broker import DeploymentBrokerClient + + observed = {} + secret = "s" * 48 + nonce = "6" * 32 + payload = _runtime_payload(nonce=nonce, observed_at=2000) + body = json.dumps(payload, separators=(",", ":")).encode() + monotonic_now = [0.0] + monkeypatch.setattr(deployment_broker.time, "time", lambda: 2000) + monkeypatch.setattr(deployment_broker.time, "monotonic", lambda: monotonic_now[0]) + monkeypatch.setattr( + deployment_broker.uuid, + "uuid4", + lambda: types.SimpleNamespace(hex=nonce), + ) + + class Response: + status_code = 200 + headers = httpx.Headers( + {"content-length": str(len(body)), "content-type": "application/json"} + ) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def iter_raw(self): + yield body + + class Client: + def __init__(self, *, timeout, verify, follow_redirects): + observed["client"] = (timeout, verify, follow_redirects) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def stream(self, method, url, *, content, headers): + observed.update(method=method, url=url, content=content, headers=headers) + # The fixed controller may spend fifteen seconds completing its + # six-container snapshot before it can send response headers. + monotonic_now[0] = 16.0 + return Response() + + monkeypatch.setattr(deployment_broker.httpx, "Client", Client) + broker = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True) + + result = broker.forge_bootstrap_readiness(_REQUEST, secret=secret) + + path = f"/v1/appops/forge/bootstrap/readiness/{_REQUEST}" + assert result == payload + assert isinstance(observed["client"][0], httpx.Timeout) + assert observed["client"][0].read == 20.0 + assert monotonic_now[0] == 16.0 + assert observed["client"][1:] == (True, False) + assert observed["method"] == "GET" + assert observed["url"] == f"https://controller.example{path}" + assert observed["content"] == b"" + assert observed["headers"]["Accept-Encoding"] == "identity" + assert observed["headers"]["Cache-Control"] == "no-store" + assert observed["headers"]["Pragma"] == "no-cache" + assert observed["headers"]["X-ITWorx-Deployment-Signature"] == broker._signature( + secret, + "GET", + path, + "2000", + nonce, + b"", + ) + + +@pytest.mark.parametrize( + "failure", + [ + "redirect", "stale", "wrong_nonce", "duplicate", "boolean_timestamp", + "future", "oversized", "long_length", "stream_oversized", "slow_drip", "invalid_utf8", + "deeply_nested", + ], +) +def test_readiness_broker_fails_closed_on_unfresh_or_ambiguous_response( + monkeypatch, failure +): + from backend.app.services import deployment_broker + from backend.app.services.deployment_broker import DeploymentBrokerClient + + nonce = "7" * 32 + current_time = 5000 + payload = _runtime_payload(nonce=nonce, observed_at=current_time) + body = json.dumps(payload, separators=(",", ":")).encode() + status_code = 302 if failure == "redirect" else 200 + if failure == "stale": + payload["observedAt"] = current_time - 31 + body = json.dumps(payload, separators=(",", ":")).encode() + elif failure == "wrong_nonce": + payload["nonce"] = "8" * 32 + body = json.dumps(payload, separators=(",", ":")).encode() + elif failure == "duplicate": + body = body[:-1] + b',"status":"ready"}' + elif failure == "boolean_timestamp": + payload["observedAt"] = True + body = json.dumps(payload, separators=(",", ":")).encode() + elif failure == "future": + payload["observedAt"] = current_time + 6 + body = json.dumps(payload, separators=(",", ":")).encode() + content_length = str(len(body)) + if failure == "oversized": + content_length = str(32 * 1024 + 1) + elif failure == "long_length": + content_length = "9" * 11 + elif failure == "stream_oversized": + content_length = None + body = b"x" * (32 * 1024 + 1) + elif failure == "invalid_utf8": + body = b"\xff" + elif failure == "deeply_nested": + body = b"[" * 1200 + b"]" * 1200 + monkeypatch.setattr(deployment_broker.time, "time", lambda: current_time) + monkeypatch.setattr( + deployment_broker.uuid, + "uuid4", + lambda: types.SimpleNamespace(hex=nonce), + ) + if failure == "slow_drip": + clock_values = iter((0.0, 0.0, 36.0)) + monkeypatch.setattr(deployment_broker.time, "monotonic", lambda: next(clock_values)) + + response_headers = {"content-type": "application/json"} + if content_length is not None: + response_headers["content-length"] = content_length + + class Response: + headers = httpx.Headers(response_headers) + + def __init__(self): + self.status_code = status_code + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def iter_raw(self): + if failure == "slow_drip": + yield body[:8] + yield body[8:] + else: + yield body + + class Client: + def __init__(self, **_kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def stream(self, *_args, **_kwargs): + return Response() + + monkeypatch.setattr(deployment_broker.httpx, "Client", Client) + result = DeploymentBrokerClient( + base_url="https://controller.example", verify_tls=True + ).forge_bootstrap_readiness(_REQUEST, secret="s" * 48) + assert result == {"status": "unavailable"} + + +class _ReadinessBroker: + def __init__(self, response): + self.response = response + self.calls = [] + + def forge_bootstrap_readiness(self, request_id, *, secret): + self.calls.append((request_id, secret)) + return self.response + + +def _deployed_job(session, *, overrides=None): + plan = _profile() + if overrides: + plan.update(overrides) + canonical_plan = json.dumps(plan, sort_keys=True, separators=(",", ":")) + historical = { + "apiVersion": "itworx.forge-first-install/v1", + "status": "deployed", + "request": _REQUEST, + "repository": "jens/itworx-forge", + "revision": _REVISION, + "operation": "forge_first_install", + "workspaceImageDigest": _WORKSPACE_DIGEST, + "frontdoorImageDigest": _FRONTDOOR_DIGEST, + "receipt": { + "revision": _REVISION, + "workspaceImageDigest": _WORKSPACE_DIGEST, + "frontdoorImageDigest": _FRONTDOOR_DIGEST, + "migration": {"service": "migrate", "status": "completed", "result": "succeeded"}, + "health": {"status": "healthy", "services": list(_SERVICES)}, + }, + } + row = DeploymentJob( + id=_JOB_ID, + application_id="itworx-forge", + repository="jens/itworx-forge", + revision=_REVISION, + plan_sha256=hashlib.sha256(canonical_plan.encode()).hexdigest(), + idempotency_sha256="9" * 64, + status="deployed", + stage="controller_deployed", + plan_json=plan, + preflight_json={}, + evidence_json={"profile": "forge-first-install", "controller": historical}, + broker_request_id=_REQUEST, + result_summary="completed", + completed_at=utc_now(), + ) + session.add(row) + session.commit() + session.refresh(row) + return row + + +def test_historical_deployed_receipt_alone_never_means_current_runtime_ready( + tmp_path, monkeypatch +): + from backend.app.services import forge_bootstrap_apply + from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService + + engine = create_engine(f"sqlite:///{tmp_path / 'readiness-historical.db'}") + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + _deployed_job(session) + secret_reads = [] + monkeypatch.setattr( + forge_bootstrap_apply, + "_controller_secret", + lambda _session: secret_reads.append(True) or "s" * 48, + ) + broker = _ReadinessBroker( + { + "apiVersion": "itworx.forge-first-install/v1", + "status": "deployed", + "receipt": {"health": {"status": "healthy"}}, + } + ) + + result = ForgeBootstrapApplyService(broker=broker).current_readiness( + session, _JOB_ID + ) + + assert result["status"] == "unavailable" + assert broker.calls == [(_REQUEST, "s" * 48)] + assert secret_reads == [True] + engine.dispose() + + +def test_readiness_service_validates_current_identity_and_projects_no_nonce(tmp_path, monkeypatch): + from backend.app.services import forge_bootstrap_apply + from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService + + engine = create_engine(f"sqlite:///{tmp_path / 'readiness-current.db'}") + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + _deployed_job(session) + current_job = session.get(DeploymentJob, _JOB_ID) + from backend.app.schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1 + from backend.app.services.forge_bootstrap_apply import _classify_controller_result + + parsed_profile = ForgeBootstrapApplyV1.model_validate(current_job.plan_json) + assert _classify_controller_result( + current_job.evidence_json["controller"], + request_id=_REQUEST, + profile=parsed_profile, + )[0] == "deployed" + assert ForgeRuntimeReadinessV1.model_validate(_runtime_payload()).status == "ready" + secret_reads = [] + monkeypatch.setattr( + forge_bootstrap_apply, + "_controller_secret", + lambda _session: secret_reads.append(True) or "s" * 48, + ) + broker = _ReadinessBroker(_runtime_payload()) + + result = ForgeBootstrapApplyService(broker=broker).current_readiness( + session, _JOB_ID + ) + + assert result["status"] == "ready" + assert result["jobId"] == _JOB_ID + assert result["controllerArtifact"] == _ARTIFACT + assert "nonce" not in result + assert broker.calls == [(_REQUEST, "s" * 48)] + assert secret_reads == [True] + engine.dispose() + + +@pytest.mark.parametrize( + "mutation", + ["request", "revision", "workspace_digest", "artifact", "aggregate", "extra"], +) +def test_readiness_service_rejects_unbound_or_malformed_current_response( + tmp_path, monkeypatch, mutation +): + from backend.app.services import forge_bootstrap_apply + from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService + + engine = create_engine(f"sqlite:///{tmp_path / f'readiness-{mutation}.db'}") + SQLModel.metadata.create_all(engine) + response = _runtime_payload() + if mutation == "request": + response["request"] = "0" * 64 + elif mutation == "revision": + response["profile"]["revision"] = "0" * 40 + elif mutation == "workspace_digest": + response["profile"]["workspaceImageDigest"] = "sha256:" + "0" * 64 + elif mutation == "artifact": + response["controllerArtifact"]["sourceRepository"] = "another/repository" + elif mutation == "aggregate": + response["health"]["services"][0]["status"] = "unavailable" + elif mutation == "extra": + response["upstreamReason"] = "must not be projected" + with Session(engine) as session: + _deployed_job(session) + monkeypatch.setattr(forge_bootstrap_apply, "_controller_secret", lambda _session: "s" * 48) + result = ForgeBootstrapApplyService( + broker=_ReadinessBroker(response) + ).current_readiness(session, _JOB_ID) + assert result["status"] == "unavailable" + assert "upstreamReason" not in result + engine.dispose() + + +def test_invalid_or_not_deployed_job_never_reads_controller_secret_or_calls_broker( + tmp_path, monkeypatch +): + from backend.app.services import forge_bootstrap_apply + from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService + + engine = create_engine(f"sqlite:///{tmp_path / 'readiness-invalid.db'}") + SQLModel.metadata.create_all(engine) + secret_reads = [] + monkeypatch.setattr( + forge_bootstrap_apply, + "_controller_secret", + lambda _session: secret_reads.append(True) or "s" * 48, + ) + broker = _ReadinessBroker(_runtime_payload()) + with Session(engine) as session: + _deployed_job(session) + row = session.get(DeploymentJob, _JOB_ID) + row.status = "applying" + session.add(row) + session.commit() + + service = ForgeBootstrapApplyService(broker=broker) + invalid_format = service.current_readiness(session, "bad-job-id") + not_deployed = service.current_readiness(session, _JOB_ID) + absent = service.current_readiness(session, "forge-install-" + "0" * 64) + + assert invalid_format["status"] == not_deployed["status"] == absent["status"] == "unavailable" + assert secret_reads == [] + assert broker.calls == [] + engine.dispose() + + +@pytest.fixture +def readiness_operator_client(tmp_path, monkeypatch): + from backend.app.core import config + from backend.app.db import session as db_session + from backend.app.db.init_db import init_db + from backend.app.main import app + + engine = create_engine( + f"sqlite:///{tmp_path / 'readiness-api.db'}", + connect_args={"check_same_thread": False}, + ) + monkeypatch.setattr(db_session, "engine", engine) + monkeypatch.setenv("APPOPS_REQUIRE_OPERATOR_AUTH", "false") + monkeypatch.setattr(config.settings, "operator_proxy_auth_enabled", False) + monkeypatch.setattr(config.settings, "auth_admin_subjects", ["authentik:owner"]) + monkeypatch.setattr(config.settings, "auth_admin_emails", []) + monkeypatch.setattr(config.settings, "data_dir", str(tmp_path)) + monkeypatch.setattr(user_access, "current_user", _REAL_CURRENT_USER) + SQLModel.metadata.create_all(engine) + init_db() + client = TestClient(app, client=("198.18.10.20", 42000)) + yield client + engine.dispose() + + +def _sign_in_operator(client): + from backend.app.db import session as db_session + from backend.app.services.user_access import issue_session, provision_user + + with Session(db_session.engine) as session: + user = provision_user( + session, + "authentik", + "https://auth.example/application/o/appops/", + {"sub": "owner", "email": "owner@example.invalid"}, + ) + cookie = issue_session(session, user) + client.cookies.set("appops_user_session", cookie) + + +def test_readiness_route_requires_operator_and_rejects_query_without_runtime_call( + readiness_operator_client, monkeypatch +): + from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService + + client = readiness_operator_client + path = f"/api/deployments/forge-bootstrap/first-install/{_JOB_ID}/readiness" + assert client.get(path).status_code == 401 + _sign_in_operator(client) + calls = [] + monkeypatch.setattr( + ForgeBootstrapApplyService, + "current_readiness", + lambda self, _session, _job_id: calls.append(True) or {"status": "unavailable"}, + ) + + query = client.get(path + "?profile=caller-selected") + body = client.request( + "GET", path, content=b"{}", headers={"Content-Type": "application/json"} + ) + + assert query.status_code == 400 + assert body.status_code == 400 + assert calls == [] + + +def test_readiness_route_returns_fixed_unavailable_for_unknown_job_without_broker_call( + readiness_operator_client, monkeypatch +): + from backend.app.services import forge_bootstrap_apply + from backend.app.services.deployment_broker import DeploymentBrokerClient + + client = readiness_operator_client + _sign_in_operator(client) + secret_reads = [] + monkeypatch.setattr( + forge_bootstrap_apply, + "_controller_secret", + lambda _session: secret_reads.append(True) or "s" * 48, + ) + monkeypatch.setattr( + DeploymentBrokerClient, + "forge_bootstrap_readiness", + lambda *_args, **_kwargs: pytest.fail("unknown jobs must not contact the controller"), + ) + + response = client.get( + "/api/deployments/forge-bootstrap/first-install/forge-install-" + + "0" * 64 + + "/readiness" + ) + invalid_path = client.get( + "/api/deployments/forge-bootstrap/first-install/not-a-job/readiness" + ) + + assert response.status_code == 200 + assert response.json()["status"] == "unavailable" + assert response.json()["message"] == "forge_runtime_readiness_unavailable" + assert invalid_path.status_code == 200 + assert invalid_path.json()["status"] == "unavailable" + assert secret_reads == [] + + +@pytest.fixture +def forge_status_module(): + path = _ROOT / "ops" / "gitea-deploy-broker" / "status_server.py" + spec = importlib.util.spec_from_file_location("forge_runtime_readiness_status_test", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_controller_readiness_route_is_signed_replay_safe_and_uses_live_probe_only( + forge_status_module, tmp_path, monkeypatch +): + module = forge_status_module + secret = b"controller-readiness-secret-" + b"x" * 32 + secret_root = tmp_path / "deployment-secrets" + secret_root.mkdir(mode=0o700) + secret_path = secret_root / module.FORGE_BOOTSTRAP_DEPLOYMENT_SECRET_NAME + secret_path.write_bytes(secret) + secret_path.chmod(0o600) + nonce_root = tmp_path / "nonces" + monkeypatch.setattr(module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root) + monkeypatch.setattr(module, "APPOPS_NONCE_ROOT", nonce_root) + request_id = "9" * 64 + runtime_result = { + "status": "ready", + "request": request_id, + "repository": "jens/itworx-forge", + "operation": "forge_first_install", + "profile": { + "revision": _REVISION, + "workspaceImageDigest": _WORKSPACE_DIGEST, + "frontdoorImageDigest": _FRONTDOOR_DIGEST, + }, + "health": { + "status": "healthy", + "services": [{"name": name, "status": "healthy"} for name in _SERVICES], + }, + "migrationCompleted": True, + } + calls = [] + + class Controller: + def runtime_readiness(self, supplied_request): + calls.append(supplied_request) + return runtime_result + + def status(self, _supplied_request): + pytest.fail("readiness must not invoke the mutating status accessor") + + release = { + **_ARTIFACT, + "apiVersion": "itworx.controller-release/v1", + "builtAt": "2026-10-04T00:00:00Z", + } + monkeypatch.setattr(module, "_forge_first_install_controller", Controller()) + monkeypatch.setattr(module, "_forge_controller_artifact_identity", lambda: release) + server = module.BoundedThreadingHTTPServer(("127.0.0.1", 0), module.Handler) + serving = threading.Thread(target=server.serve_forever, daemon=True) + serving.start() + + def get_readiness(nonce: str): + path = f"{module.FORGE_BOOTSTRAP_READINESS_PREFIX}{request_id}" + timestamp = str(int(module.time.time())) + signature = module._request_signature( + secret, + method="GET", + path=path, + timestamp=timestamp, + nonce=nonce, + raw_body=b"", + ) + connection = http.client.HTTPConnection(*server.server_address, timeout=3) + connection.request( + "GET", + path, + headers={ + "X-ITWorx-Deployment-Timestamp": timestamp, + "X-ITWorx-Deployment-Nonce": nonce, + "X-ITWorx-Deployment-Signature": signature, + }, + ) + response = connection.getresponse() + status = response.status + payload = json.loads(response.read()) + connection.close() + return status, payload + + try: + status, payload = get_readiness("a" * 32) + assert status == 200 + assert payload["apiVersion"] == "itworx.forge-runtime-readiness/v1" + assert payload["nonce"] == "a" * 32 + assert payload["controllerArtifact"] == _ARTIFACT + assert set(payload) == { + "apiVersion", "status", "request", "repository", "operation", "profile", + "health", "migrationCompleted", "observedAt", "nonce", "controllerArtifact", + } + assert secret.decode() not in json.dumps(payload) + assert calls == [request_id] + + replay_status, replay_payload = get_readiness("a" * 32) + assert replay_status == 403 + assert replay_payload == {"status": "authentication_rejected"} + assert calls == [request_id] + finally: + server.shutdown() + server.server_close() + serving.join(timeout=2) diff --git a/backend/tests/test_forge_secret_delivery.py b/backend/tests/test_forge_secret_delivery.py index 3e37f03..ecf9f90 100644 --- a/backend/tests/test_forge_secret_delivery.py +++ b/backend/tests/test_forge_secret_delivery.py @@ -336,7 +336,7 @@ def _delivery_setup(tmp_path, monkeypatch, *, manual="allowed"): SQLModel.metadata.create_all(engine) session = Session(engine) monkeypatch.setattr(settings, "environment", "production") - monkeypatch.setattr(settings, "secret_key", "delivery-test-secret-key-material-0123456789") + monkeypatch.setattr(settings, "secret_key", "test-key-" + "x" * 35) monkeypatch.setattr(settings, "secret_key_source", "env") for key, value in { "AUTHENTIK_OIDC_AUTHORIZATION_FLOW": "forge-authz-flow",