Publish curated Unraid-AppOps-Gateway source from a480f280f26f4aa48c61394fa841d2233067c208
This commit is contained in:
1 parent
e73772cdf2
commit
820244dbb3
13 files changed
+2141
-26
No files matched your search
@@ -1,3 +1,3 @@
|
|||||||
# Public source export
|
# Public source export
|
||||||
|
|
||||||
This source snapshot was generated from the private canonical repository at revision `7144ea3c4cb735745046e4e1c2d3a3e88fa1a302`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
This source snapshot was generated from the private canonical repository at revision `a480f280f26f4aa48c61394fa841d2233067c208`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||||
+32
-17
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"schemaVersion": 1,
|
"schemaVersion": 1,
|
||||||
"sourceRevision": "7144ea3c4cb735745046e4e1c2d3a3e88fa1a302",
|
"sourceRevision": "a480f280f26f4aa48c61394fa841d2233067c208",
|
||||||
"files": [
|
"files": [
|
||||||
{
|
{
|
||||||
"path": ".dockerignore",
|
"path": ".dockerignore",
|
||||||
@@ -124,8 +124,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/api/routes_deployments.py",
|
"path": "backend/app/api/routes_deployments.py",
|
||||||
"sha256": "c77cea58701694af4ef519e351fd27a68856e45cd4e61dc4028c652359d6392a",
|
"sha256": "ff14b4e40e6e14d0d7f9f8fd74faf5001010a3ba5c0b279253c61f17199f350f",
|
||||||
"bytes": 7717
|
"bytes": 8604
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/api/routes_enrollments.py",
|
"path": "backend/app/api/routes_enrollments.py",
|
||||||
@@ -452,6 +452,11 @@
|
|||||||
"sha256": "8d55f7a8263191f07813d1689eb0697dabec99a1bbd5aff5a704e66e6bf86c47",
|
"sha256": "8d55f7a8263191f07813d1689eb0697dabec99a1bbd5aff5a704e66e6bf86c47",
|
||||||
"bytes": 6045
|
"bytes": 6045
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/app/schemas/forge_runtime_readiness.py",
|
||||||
|
"sha256": "4571fd2c7b624fe4f12c219a1f494b03a98ec1745a9512197cf7c49287ce47b5",
|
||||||
|
"bytes": 6612
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/schemas/forge_secret_delivery.py",
|
"path": "backend/app/schemas/forge_secret_delivery.py",
|
||||||
"sha256": "5209fdfaabee4dbc56b7edd2f6080c35bbd3d679577a754dbd65ca353347b5e5",
|
"sha256": "5209fdfaabee4dbc56b7edd2f6080c35bbd3d679577a754dbd65ca353347b5e5",
|
||||||
@@ -529,8 +534,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/services/deployment_broker.py",
|
"path": "backend/app/services/deployment_broker.py",
|
||||||
"sha256": "34d6a4bf90d59604d51d34bc4ba7885ac6f5837d7015c80c8c0fe7eee9c060d4",
|
"sha256": "e23b2a0669c846cf37c44766700d29e5d2dfa22b6b6d0d546fe5c94bb86f4c6b",
|
||||||
"bytes": 33010
|
"bytes": 37749
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/services/deployment_control_plane.py",
|
"path": "backend/app/services/deployment_control_plane.py",
|
||||||
@@ -564,8 +569,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/services/forge_bootstrap_apply.py",
|
"path": "backend/app/services/forge_bootstrap_apply.py",
|
||||||
"sha256": "4928e08a887094efb9134fc2c6a40b707d5635c0f26d202c93773ba132b61f69",
|
"sha256": "612e7537577c8ac97c0d7f2ae76c0c5099cc78f7661d4dfb62a81b1602d775db",
|
||||||
"bytes": 37394
|
"bytes": 42301
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/app/services/forge_bootstrap_preflight.py",
|
"path": "backend/app/services/forge_bootstrap_preflight.py",
|
||||||
@@ -934,8 +939,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_deploy_broker.py",
|
"path": "backend/tests/test_deploy_broker.py",
|
||||||
"sha256": "9aaefa351db5868436a6dc06ed7c42202e8d1a8568bcaa4d71501ec60446bb85",
|
"sha256": "7e32918a89dcb3d02cc75674fcd9313b60cf96ed3651a5016bbfb3fc751c3ec3",
|
||||||
"bytes": 381318
|
"bytes": 406062
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_deployment_auto_rollback.py",
|
"path": "backend/tests/test_deployment_auto_rollback.py",
|
||||||
@@ -984,8 +989,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_bootstrap_apply.py",
|
"path": "backend/tests/test_forge_bootstrap_apply.py",
|
||||||
"sha256": "f7be48b418a9a09fbbaea7f6e1765c7ba4bac949994435b423885c3b4b2c6d8e",
|
"sha256": "6486b6feed952e190974aacef13f9ce948e1caf15a71ca74e330df69cc30c404",
|
||||||
"bytes": 43558
|
"bytes": 43562
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_bootstrap_controller.py",
|
"path": "backend/tests/test_forge_bootstrap_controller.py",
|
||||||
@@ -1009,24 +1014,34 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_first_install_controller.py",
|
"path": "backend/tests/test_forge_first_install_controller.py",
|
||||||
"sha256": "9c4abc31c564e2974b125ce0c0f9bd581af499ab70dcc2b21dc2824a331522f2",
|
"sha256": "4784e7f08a08f15ce6e7efb1ba37ae9a5bfe311478842e1babe273d34ecb39eb",
|
||||||
"bytes": 84984
|
"bytes": 93989
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_oidc_api.py",
|
"path": "backend/tests/test_forge_oidc_api.py",
|
||||||
"sha256": "4d902a3bb967de92156019d00ca65cfb7120935f669c61d255cab630b3081100",
|
"sha256": "f083c07f3de32de5552ab999677004ae1654d10a3540e9e1bf5fbaa2e6371d5c",
|
||||||
"bytes": 11095
|
"bytes": 11070
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_oidc_provisioning.py",
|
"path": "backend/tests/test_forge_oidc_provisioning.py",
|
||||||
"sha256": "a0fff62f8855dafebd49e2d91abdf8f211b7a4df452c05d91ed8305fe74d3b0a",
|
"sha256": "a0fff62f8855dafebd49e2d91abdf8f211b7a4df452c05d91ed8305fe74d3b0a",
|
||||||
"bytes": 82116
|
"bytes": 82116
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_forge_oidc_runtime_lock.py",
|
||||||
|
"sha256": "c71d405e5ddf75f1597660accb77329f1f7efe485e0dd4ef1c80a503b978f812",
|
||||||
|
"bytes": 7421
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_reader_api.py",
|
"path": "backend/tests/test_forge_reader_api.py",
|
||||||
"sha256": "dc4e9d176a9b4c33fd24e27b2cb83919040babe412d021109a22de41c54edb8b",
|
"sha256": "dc4e9d176a9b4c33fd24e27b2cb83919040babe412d021109a22de41c54edb8b",
|
||||||
"bytes": 6424
|
"bytes": 6424
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/tests/test_forge_runtime_readiness.py",
|
||||||
|
"sha256": "0f70e327225eceae2ad161e565c9f601c191e3b086a989c7383ac7327434926d",
|
||||||
|
"bytes": 25834
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_secret_bundle.py",
|
"path": "backend/tests/test_forge_secret_bundle.py",
|
||||||
"sha256": "ee3555f9711b23005f8a2970537217ffed9b06b1fbe96424180a26c379440cd2",
|
"sha256": "ee3555f9711b23005f8a2970537217ffed9b06b1fbe96424180a26c379440cd2",
|
||||||
@@ -1034,8 +1049,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_secret_delivery.py",
|
"path": "backend/tests/test_forge_secret_delivery.py",
|
||||||
"sha256": "29f138ab1df97248cd3393aac84e50967e4cf10b2194a8db03820dd7973a30fb",
|
"sha256": "a66da8d2feb600575e834fcdf8055aac1c4e2a26ba89fde0ed7f682d6170a670",
|
||||||
"bytes": 37715
|
"bytes": 37691
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/tests/test_forge_secret_delivery_controller.py",
|
"path": "backend/tests/test_forge_secret_delivery_controller.py",
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from ..schemas.forge_secret_delivery import (
|
|||||||
ForgeSecretDeliveryRequestV1,
|
ForgeSecretDeliveryRequestV1,
|
||||||
ForgeSecretDeliveryResultV1,
|
ForgeSecretDeliveryResultV1,
|
||||||
)
|
)
|
||||||
|
from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessResultV1
|
||||||
from ..services.deployment_broker import DeploymentBrokerClient
|
from ..services.deployment_broker import DeploymentBrokerClient
|
||||||
from ..services.deployment_control_plane import (
|
from ..services.deployment_control_plane import (
|
||||||
DeploymentControlError,
|
DeploymentControlError,
|
||||||
@@ -208,6 +209,29 @@ def get_forge_first_install(job_id: str, session: DeploymentSession):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/forge-bootstrap/first-install/{job_id}/readiness",
|
||||||
|
response_model=ForgeRuntimeReadinessResultV1,
|
||||||
|
)
|
||||||
|
def get_forge_first_install_readiness(
|
||||||
|
job_id: str,
|
||||||
|
request: Request,
|
||||||
|
session: DeploymentSession,
|
||||||
|
):
|
||||||
|
"""Return only a fresh, fixed controller observation for a known install."""
|
||||||
|
content_lengths = request.headers.getlist("content-length")
|
||||||
|
if (
|
||||||
|
request.query_params
|
||||||
|
or len(content_lengths) > 1
|
||||||
|
or (content_lengths and content_lengths[0] != "0")
|
||||||
|
or request.headers.getlist("transfer-encoding")
|
||||||
|
):
|
||||||
|
raise HTTPException(status_code=400, detail="invalid_readiness_request")
|
||||||
|
from ..services.forge_bootstrap_apply import ForgeBootstrapApplyService
|
||||||
|
|
||||||
|
return ForgeBootstrapApplyService().current_readiness(session, job_id)
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_deployments(session: DeploymentSession, limit: DeploymentLimit = 100):
|
def list_deployments(session: DeploymentSession, limit: DeploymentLimit = 100):
|
||||||
return {"items": DeploymentControlPlane().list_jobs(session, limit=limit)}
|
return {"items": DeploymentControlPlane().list_jobs(session, limit=limit)}
|
||||||
|
|||||||
@@ -0,0 +1,157 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr, model_validator
|
||||||
|
|
||||||
|
from .forge_secret_delivery import ForgeSecretDeliveryControllerArtifactV1
|
||||||
|
|
||||||
|
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||||
|
_REQUEST = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
_NONCE = re.compile(r"^[0-9a-f]{32}$")
|
||||||
|
_JOB = re.compile(r"^forge-install-[0-9a-f]{64}$")
|
||||||
|
_RUNTIME_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
|
||||||
|
|
||||||
|
|
||||||
|
class ForgeRuntimeReadinessProfileV1(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||||
|
|
||||||
|
revision: StrictStr = Field(pattern=_REVISION.pattern)
|
||||||
|
workspace_image_digest: StrictStr = Field(alias="workspaceImageDigest", pattern=_DIGEST.pattern)
|
||||||
|
frontdoor_image_digest: StrictStr = Field(alias="frontdoorImageDigest", pattern=_DIGEST.pattern)
|
||||||
|
|
||||||
|
|
||||||
|
class ForgeRuntimeServiceHealthV1(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||||
|
|
||||||
|
name: Literal["api", "dashboard", "itworx-forge", "postgres", "worker"]
|
||||||
|
status: Literal["healthy", "not_ready", "unavailable"]
|
||||||
|
|
||||||
|
|
||||||
|
class ForgeRuntimeHealthV1(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||||
|
|
||||||
|
status: Literal["healthy", "not_ready", "unavailable"]
|
||||||
|
services: list[ForgeRuntimeServiceHealthV1]
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def fixed_service_set(self) -> ForgeRuntimeHealthV1:
|
||||||
|
if tuple(item.name for item in self.services) != _RUNTIME_SERVICES:
|
||||||
|
raise ValueError("runtime service set is invalid")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ForgeRuntimeReadinessV1(BaseModel):
|
||||||
|
"""Strict controller response for one fresh, fixed Forge observation."""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True, strict=True)
|
||||||
|
|
||||||
|
api_version: Literal["itworx.forge-runtime-readiness/v1"] = Field(alias="apiVersion")
|
||||||
|
status: Literal["ready", "not_ready", "unavailable"]
|
||||||
|
repository: Literal["jens/itworx-forge"]
|
||||||
|
operation: Literal["forge_first_install"]
|
||||||
|
request: StrictStr = Field(pattern=_REQUEST.pattern)
|
||||||
|
profile: ForgeRuntimeReadinessProfileV1
|
||||||
|
observed_at: StrictInt = Field(alias="observedAt", ge=0)
|
||||||
|
nonce: StrictStr = Field(pattern=_NONCE.pattern)
|
||||||
|
health: ForgeRuntimeHealthV1
|
||||||
|
migration_completed: StrictBool = Field(alias="migrationCompleted")
|
||||||
|
controller_artifact: ForgeSecretDeliveryControllerArtifactV1 = Field(alias="controllerArtifact")
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def readiness_consistency(self) -> ForgeRuntimeReadinessV1:
|
||||||
|
service_statuses = {item.status for item in self.health.services}
|
||||||
|
expected_health = (
|
||||||
|
"unavailable"
|
||||||
|
if "unavailable" in service_statuses
|
||||||
|
else "not_ready"
|
||||||
|
if "not_ready" in service_statuses
|
||||||
|
else "healthy"
|
||||||
|
)
|
||||||
|
valid_overall = (
|
||||||
|
self.status == "unavailable"
|
||||||
|
if expected_health == "unavailable"
|
||||||
|
else self.status in {"not_ready", "unavailable"}
|
||||||
|
if expected_health == "not_ready"
|
||||||
|
else self.status == "ready"
|
||||||
|
if self.migration_completed
|
||||||
|
else self.status in {"not_ready", "unavailable"}
|
||||||
|
)
|
||||||
|
if self.health.status != expected_health or not valid_overall:
|
||||||
|
raise ValueError("readiness status does not match current health proof")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ForgeRuntimeReadinessResultV1(BaseModel):
|
||||||
|
"""Bounded operator projection; upstream diagnostics and nonce are omitted."""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True, strict=True)
|
||||||
|
|
||||||
|
status: Literal["ready", "not_ready", "unavailable"]
|
||||||
|
message: Literal[
|
||||||
|
"forge_runtime_ready",
|
||||||
|
"forge_runtime_not_ready",
|
||||||
|
"forge_runtime_readiness_unavailable",
|
||||||
|
]
|
||||||
|
target: Literal["itworx-forge"] = "itworx-forge"
|
||||||
|
job_id: StrictStr | None = Field(default=None, alias="jobId", pattern=_JOB.pattern)
|
||||||
|
revision: StrictStr | None = Field(default=None, pattern=_REVISION.pattern)
|
||||||
|
workspace_image_digest: StrictStr | None = Field(
|
||||||
|
default=None, alias="workspaceImageDigest", pattern=_DIGEST.pattern
|
||||||
|
)
|
||||||
|
frontdoor_image_digest: StrictStr | None = Field(
|
||||||
|
default=None, alias="frontdoorImageDigest", pattern=_DIGEST.pattern
|
||||||
|
)
|
||||||
|
observed_at: StrictInt | None = Field(default=None, alias="observedAt", ge=0)
|
||||||
|
health: ForgeRuntimeHealthV1
|
||||||
|
migration_completed: StrictBool = Field(default=False, alias="migrationCompleted")
|
||||||
|
controller_artifact: ForgeSecretDeliveryControllerArtifactV1 | None = Field(
|
||||||
|
default=None, alias="controllerArtifact"
|
||||||
|
)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def public_status_message(self) -> ForgeRuntimeReadinessResultV1:
|
||||||
|
expected = {
|
||||||
|
"ready": "forge_runtime_ready",
|
||||||
|
"not_ready": "forge_runtime_not_ready",
|
||||||
|
"unavailable": "forge_runtime_readiness_unavailable",
|
||||||
|
}[self.status]
|
||||||
|
if self.message != expected:
|
||||||
|
raise ValueError("readiness message does not match status")
|
||||||
|
service_statuses = {item.status for item in self.health.services}
|
||||||
|
expected_health = (
|
||||||
|
"unavailable"
|
||||||
|
if "unavailable" in service_statuses
|
||||||
|
else "not_ready"
|
||||||
|
if "not_ready" in service_statuses
|
||||||
|
else "healthy"
|
||||||
|
)
|
||||||
|
valid_overall = (
|
||||||
|
self.status == "unavailable"
|
||||||
|
if expected_health == "unavailable"
|
||||||
|
else self.status in {"not_ready", "unavailable"}
|
||||||
|
if expected_health == "not_ready"
|
||||||
|
else self.status == "ready"
|
||||||
|
if self.migration_completed
|
||||||
|
else self.status in {"not_ready", "unavailable"}
|
||||||
|
)
|
||||||
|
if self.health.status != expected_health or not valid_overall:
|
||||||
|
raise ValueError("public readiness is inconsistent")
|
||||||
|
identity_fields = (
|
||||||
|
self.job_id,
|
||||||
|
self.revision,
|
||||||
|
self.workspace_image_digest,
|
||||||
|
self.frontdoor_image_digest,
|
||||||
|
self.observed_at,
|
||||||
|
self.controller_artifact,
|
||||||
|
)
|
||||||
|
if self.job_id is None and (
|
||||||
|
self.status != "unavailable"
|
||||||
|
or any(value is not None for value in identity_fields[1:])
|
||||||
|
):
|
||||||
|
raise ValueError("public readiness identity is incomplete")
|
||||||
|
if self.job_id is not None and any(value is None for value in identity_fields):
|
||||||
|
raise ValueError("public readiness identity is incomplete")
|
||||||
|
return self
|
||||||
@@ -29,6 +29,7 @@ from ..core.redaction import redact_message
|
|||||||
from ..schemas.deployment import DeploymentPlanV1
|
from ..schemas.deployment import DeploymentPlanV1
|
||||||
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
|
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
|
||||||
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
||||||
|
from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessV1
|
||||||
|
|
||||||
_REQUEST_ID = re.compile(r"^[0-9a-f]{64}$")
|
_REQUEST_ID = re.compile(r"^[0-9a-f]{64}$")
|
||||||
_MAX_DELIVERY_STATUS_BYTES = 512 * 1024
|
_MAX_DELIVERY_STATUS_BYTES = 512 * 1024
|
||||||
@@ -40,6 +41,10 @@ _WORKMESH_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
|||||||
_FORGE_SECRET_DELIVERY_PATH = "/v1/appops/forge/bootstrap/secrets"
|
_FORGE_SECRET_DELIVERY_PATH = "/v1/appops/forge/bootstrap/secrets"
|
||||||
_MAX_FORGE_SECRET_DELIVERY_BODY_BYTES = 16 * 1024
|
_MAX_FORGE_SECRET_DELIVERY_BODY_BYTES = 16 * 1024
|
||||||
_MAX_FORGE_SECRET_DELIVERY_RESPONSE_BYTES = 64 * 1024
|
_MAX_FORGE_SECRET_DELIVERY_RESPONSE_BYTES = 64 * 1024
|
||||||
|
_FORGE_RUNTIME_READINESS_PATH = "/v1/appops/forge/bootstrap/readiness/"
|
||||||
|
_MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES = 32 * 1024
|
||||||
|
_FORGE_RUNTIME_READINESS_TOTAL_SECONDS = 35.0
|
||||||
|
_FORGE_RUNTIME_READINESS_READ_SECONDS = 20.0
|
||||||
|
|
||||||
|
|
||||||
def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||||
@@ -466,7 +471,7 @@ class DeploymentBrokerClient:
|
|||||||
bytes(response_body).decode("utf-8"),
|
bytes(response_body).decode("utf-8"),
|
||||||
object_pairs_hook=_unique_json_object,
|
object_pairs_hook=_unique_json_object,
|
||||||
)
|
)
|
||||||
except (UnicodeError, ValueError, TypeError):
|
except (UnicodeError, ValueError, TypeError, RecursionError):
|
||||||
response_body.clear()
|
response_body.clear()
|
||||||
return {
|
return {
|
||||||
"status": "outcome_unknown",
|
"status": "outcome_unknown",
|
||||||
@@ -508,6 +513,106 @@ class DeploymentBrokerClient:
|
|||||||
secret=secret,
|
secret=secret,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def forge_bootstrap_readiness(
|
||||||
|
self, request_id: str, *, secret: str
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Fetch one fresh fixed-profile runtime snapshot with no receipt fallback."""
|
||||||
|
if not _REQUEST_ID.fullmatch(request_id):
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
if (
|
||||||
|
not self.is_configured()
|
||||||
|
or not isinstance(secret, str)
|
||||||
|
or len(secret) < 32
|
||||||
|
or not secret.isascii()
|
||||||
|
or not secret.isprintable()
|
||||||
|
):
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
|
||||||
|
timestamp = str(int(time.time()))
|
||||||
|
nonce = uuid.uuid4().hex
|
||||||
|
path = f"{_FORGE_RUNTIME_READINESS_PATH}{request_id}"
|
||||||
|
headers = {
|
||||||
|
"Accept": "application/json",
|
||||||
|
"Accept-Encoding": "identity",
|
||||||
|
"Cache-Control": "no-store",
|
||||||
|
"Pragma": "no-cache",
|
||||||
|
"X-ITWorx-Deployment-Timestamp": timestamp,
|
||||||
|
"X-ITWorx-Deployment-Nonce": nonce,
|
||||||
|
"X-ITWorx-Deployment-Signature": self._signature(
|
||||||
|
secret, "GET", path, timestamp, nonce, b""
|
||||||
|
),
|
||||||
|
}
|
||||||
|
response_body = bytearray()
|
||||||
|
deadline = time.monotonic() + _FORGE_RUNTIME_READINESS_TOTAL_SECONDS
|
||||||
|
client_timeout = httpx.Timeout(
|
||||||
|
connect=min(self.timeout_seconds, 5.0),
|
||||||
|
read=_FORGE_RUNTIME_READINESS_READ_SECONDS,
|
||||||
|
write=min(self.timeout_seconds, 5.0),
|
||||||
|
pool=min(self.timeout_seconds, 5.0),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with httpx.Client(
|
||||||
|
timeout=client_timeout,
|
||||||
|
verify=self._verify_option(),
|
||||||
|
follow_redirects=False,
|
||||||
|
) as client, client.stream(
|
||||||
|
"GET",
|
||||||
|
f"{self.base_url}{path}",
|
||||||
|
content=b"",
|
||||||
|
headers=headers,
|
||||||
|
) as response:
|
||||||
|
if response.status_code != 200:
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
content_lengths = response.headers.get_list("content-length")
|
||||||
|
if len(content_lengths) > 1:
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
content_length = content_lengths[0] if content_lengths else None
|
||||||
|
if content_length is not None and (
|
||||||
|
not content_length.isascii()
|
||||||
|
or not content_length.isdecimal()
|
||||||
|
or len(content_length) > 10
|
||||||
|
or int(content_length) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES
|
||||||
|
):
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
content_encoding = response.headers.get("content-encoding", "identity").lower()
|
||||||
|
content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||||
|
if content_encoding != "identity" or content_type != "application/json":
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
for chunk in response.iter_raw():
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
response_body.clear()
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
response_body.extend(chunk)
|
||||||
|
if len(response_body) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES:
|
||||||
|
response_body.clear()
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
except Exception: # noqa: BLE001 - transport diagnostics may contain held credentials.
|
||||||
|
response_body.clear()
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
response_body.clear()
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
decoded = json.loads(
|
||||||
|
bytes(response_body).decode("utf-8", errors="strict"),
|
||||||
|
object_pairs_hook=_unique_json_object,
|
||||||
|
)
|
||||||
|
validated = ForgeRuntimeReadinessV1.model_validate(decoded)
|
||||||
|
except (UnicodeError, ValueError, TypeError, RecursionError):
|
||||||
|
response_body.clear()
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
response_body.clear()
|
||||||
|
now = int(time.time())
|
||||||
|
if (
|
||||||
|
validated.request != request_id
|
||||||
|
or validated.nonce != nonce
|
||||||
|
or validated.observed_at < now - 30
|
||||||
|
or validated.observed_at > now + 5
|
||||||
|
):
|
||||||
|
return {"status": "unavailable"}
|
||||||
|
return validated.model_dump(mode="json", by_alias=True)
|
||||||
|
|
||||||
def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]:
|
def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]:
|
||||||
"""Read the bootstrap-protected enrollment controller capability."""
|
"""Read the bootstrap-protected enrollment controller capability."""
|
||||||
return self._request(
|
return self._request(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import base64
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import UTC, datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import update
|
from sqlalchemy import update
|
||||||
@@ -17,6 +17,10 @@ from ..db.models import DeploymentJob, Policy
|
|||||||
from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT
|
from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT
|
||||||
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
|
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
|
||||||
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
||||||
|
from ..schemas.forge_runtime_readiness import (
|
||||||
|
ForgeRuntimeReadinessResultV1,
|
||||||
|
ForgeRuntimeReadinessV1,
|
||||||
|
)
|
||||||
from .audit_logger import log_event
|
from .audit_logger import log_event
|
||||||
from .deployment_broker import DeploymentBrokerClient
|
from .deployment_broker import DeploymentBrokerClient
|
||||||
from .enrollment_plane import EnrollmentPlane
|
from .enrollment_plane import EnrollmentPlane
|
||||||
@@ -83,6 +87,7 @@ _PROFILE_CHECKS = {
|
|||||||
_FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"}
|
_FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"}
|
||||||
_ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"}
|
_ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"}
|
||||||
_TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"}
|
_TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"}
|
||||||
|
_RUNTIME_READINESS_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
|
||||||
|
|
||||||
|
|
||||||
class ForgeBootstrapApplyError(ValueError):
|
class ForgeBootstrapApplyError(ValueError):
|
||||||
@@ -715,6 +720,113 @@ class ForgeBootstrapApplyService:
|
|||||||
return None
|
return None
|
||||||
return self._refresh_job(session, job)
|
return self._refresh_job(session, job)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _runtime_readiness_unavailable() -> dict[str, Any]:
|
||||||
|
return ForgeRuntimeReadinessResultV1.model_validate(
|
||||||
|
{
|
||||||
|
"status": "unavailable",
|
||||||
|
"message": "forge_runtime_readiness_unavailable",
|
||||||
|
"health": {
|
||||||
|
"status": "unavailable",
|
||||||
|
"services": [
|
||||||
|
{"name": name, "status": "unavailable"}
|
||||||
|
for name in _RUNTIME_READINESS_SERVICES
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"migrationCompleted": False,
|
||||||
|
}
|
||||||
|
).model_dump(mode="json", by_alias=True, exclude_none=True)
|
||||||
|
|
||||||
|
def current_readiness(self, session: Session, job_id: str) -> dict[str, Any]:
|
||||||
|
"""Read current controller state only for a verified successful Forge job."""
|
||||||
|
unavailable = self._runtime_readiness_unavailable()
|
||||||
|
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
||||||
|
return unavailable
|
||||||
|
try:
|
||||||
|
job = session.get(DeploymentJob, job_id)
|
||||||
|
except Exception: # noqa: BLE001 - do not expose protected store/DB failures.
|
||||||
|
return unavailable
|
||||||
|
if (
|
||||||
|
job is None
|
||||||
|
or job.application_id != _APPLICATION_ID
|
||||||
|
or job.repository != _REPOSITORY
|
||||||
|
or job.status != "deployed"
|
||||||
|
or job.stage not in {"controller_deployed", "resume_controller_deployed"}
|
||||||
|
or job.completed_at is None
|
||||||
|
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
|
||||||
|
):
|
||||||
|
return unavailable
|
||||||
|
try:
|
||||||
|
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return unavailable
|
||||||
|
if (
|
||||||
|
job.revision != profile.revision
|
||||||
|
or not isinstance(job.plan_json, dict)
|
||||||
|
or _sha256(job.plan_json) != job.plan_sha256
|
||||||
|
):
|
||||||
|
return unavailable
|
||||||
|
evidence = job.evidence_json
|
||||||
|
if (
|
||||||
|
not isinstance(evidence, dict)
|
||||||
|
or set(evidence) != {"profile", "controller"}
|
||||||
|
or evidence.get("profile") != "forge-first-install"
|
||||||
|
):
|
||||||
|
return unavailable
|
||||||
|
historical_status, _stage, _summary, _safe_evidence = _classify_controller_result(
|
||||||
|
evidence.get("controller"),
|
||||||
|
request_id=job.broker_request_id or "",
|
||||||
|
profile=profile,
|
||||||
|
)
|
||||||
|
if historical_status != "deployed":
|
||||||
|
return unavailable
|
||||||
|
|
||||||
|
# This protected per-target secret lookup is deliberately after every
|
||||||
|
# durable job/profile/receipt validation above.
|
||||||
|
secret = _controller_secret(session)
|
||||||
|
if secret is None:
|
||||||
|
return unavailable
|
||||||
|
try:
|
||||||
|
response = self.broker.forge_bootstrap_readiness(
|
||||||
|
job.broker_request_id or "", secret=secret
|
||||||
|
)
|
||||||
|
current = ForgeRuntimeReadinessV1.model_validate(response)
|
||||||
|
except Exception: # noqa: BLE001 - broker and schema errors are safe unavailable.
|
||||||
|
return unavailable
|
||||||
|
now = int(datetime.now(UTC).timestamp())
|
||||||
|
if (
|
||||||
|
current.request != job.broker_request_id
|
||||||
|
or current.repository != _REPOSITORY
|
||||||
|
or current.operation != _OPERATION
|
||||||
|
or current.profile.revision != profile.revision
|
||||||
|
or current.profile.workspace_image_digest != profile.workspace_image_digest
|
||||||
|
or current.profile.frontdoor_image_digest != profile.frontdoor_image_digest
|
||||||
|
or current.observed_at < now - 30
|
||||||
|
or current.observed_at > now + 5
|
||||||
|
):
|
||||||
|
return unavailable
|
||||||
|
message = {
|
||||||
|
"ready": "forge_runtime_ready",
|
||||||
|
"not_ready": "forge_runtime_not_ready",
|
||||||
|
"unavailable": "forge_runtime_readiness_unavailable",
|
||||||
|
}[current.status]
|
||||||
|
return ForgeRuntimeReadinessResultV1.model_validate(
|
||||||
|
{
|
||||||
|
"status": current.status,
|
||||||
|
"message": message,
|
||||||
|
"jobId": job.id,
|
||||||
|
"revision": current.profile.revision,
|
||||||
|
"workspaceImageDigest": current.profile.workspace_image_digest,
|
||||||
|
"frontdoorImageDigest": current.profile.frontdoor_image_digest,
|
||||||
|
"observedAt": current.observed_at,
|
||||||
|
"health": current.health.model_dump(mode="python"),
|
||||||
|
"migrationCompleted": current.migration_completed,
|
||||||
|
"controllerArtifact": current.controller_artifact.model_dump(
|
||||||
|
mode="python", by_alias=True
|
||||||
|
),
|
||||||
|
}
|
||||||
|
).model_dump(mode="json", by_alias=True, exclude_none=True)
|
||||||
|
|
||||||
def resume(self, session: Session, job_id: str) -> dict[str, Any] | None:
|
def resume(self, session: Session, job_id: str) -> dict[str, Any] | None:
|
||||||
"""Explicitly resume only after a signed read proves pre-migration pause."""
|
"""Explicitly resume only after a signed read proves pre-migration pause."""
|
||||||
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
|
||||||
|
|||||||
@@ -469,7 +469,7 @@ def test_appops_legacy_intent_is_root_private_and_compare_and_swap_bound(
|
|||||||
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
|
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
|
||||||
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
|
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
|
||||||
"candidateRevision": revision, "composeBlob": "3" * 40,
|
"candidateRevision": revision, "composeBlob": "3" * 40,
|
||||||
"dockerfileBlob": "4" * 40,
|
"dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
|
||||||
}
|
}
|
||||||
protected_main = {
|
protected_main = {
|
||||||
"semanticSha256": "5" * 64, "revision": revision,
|
"semanticSha256": "5" * 64, "revision": revision,
|
||||||
@@ -677,7 +677,7 @@ def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256
|
|||||||
"legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
|
"legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
|
||||||
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
|
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
|
||||||
"candidateRevision": revision, "composeBlob": "3" * 40,
|
"candidateRevision": revision, "composeBlob": "3" * 40,
|
||||||
"dockerfileBlob": "4" * 40,
|
"dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
|
||||||
}
|
}
|
||||||
protected_main = {
|
protected_main = {
|
||||||
"semanticSha256": "5" * 64, "revision": revision,
|
"semanticSha256": "5" * 64, "revision": revision,
|
||||||
@@ -730,6 +730,40 @@ def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256
|
|||||||
return repository, metadata, source_proof, protected_main, backup, intent
|
return repository, metadata, source_proof, protected_main, backup, intent
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_legacy_intent_accepts_only_the_reviewed_dockerfile_blob_pair(
|
||||||
|
deploy_module
|
||||||
|
):
|
||||||
|
_repository, _metadata, _source, _main, _backup, intent = (
|
||||||
|
_appops_rollback_intent_fixture(deploy_module, "c" * 40, "a" * 64)
|
||||||
|
)
|
||||||
|
deploy_module._appops_validate_intent_payload(intent)
|
||||||
|
|
||||||
|
reviewed_update = copy.deepcopy(intent)
|
||||||
|
reviewed_source = reviewed_update["migrationProof"]["source"]
|
||||||
|
reviewed_source["dockerfileBlob"] = "aae97717280b8495a93bc277e290eeb515b76862"
|
||||||
|
reviewed_evidence = reviewed_update["rollbackSnapshot"]["appops_legacy"]
|
||||||
|
reviewed_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(
|
||||||
|
reviewed_source
|
||||||
|
)
|
||||||
|
reviewed_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(
|
||||||
|
reviewed_update["migrationProof"]
|
||||||
|
)
|
||||||
|
deploy_module._appops_validate_intent_payload(reviewed_update)
|
||||||
|
|
||||||
|
unknown_update = copy.deepcopy(reviewed_update)
|
||||||
|
unknown_source = unknown_update["migrationProof"]["source"]
|
||||||
|
unknown_source["dockerfileBlob"] = "b" * 40
|
||||||
|
unknown_evidence = unknown_update["rollbackSnapshot"]["appops_legacy"]
|
||||||
|
unknown_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(
|
||||||
|
unknown_source
|
||||||
|
)
|
||||||
|
unknown_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(
|
||||||
|
unknown_update["migrationProof"]
|
||||||
|
)
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="source proof is invalid"):
|
||||||
|
deploy_module._appops_validate_intent_payload(unknown_update)
|
||||||
|
|
||||||
|
|
||||||
def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt(
|
def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt(
|
||||||
deploy_module, monkeypatch, tmp_path, capsys
|
deploy_module, monkeypatch, tmp_path, capsys
|
||||||
):
|
):
|
||||||
@@ -1333,7 +1367,7 @@ def test_appops_durable_rollback_requires_completed_intent_and_verified_backup(
|
|||||||
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
|
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
|
||||||
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
|
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
|
||||||
"candidateRevision": revision, "composeBlob": "3" * 40,
|
"candidateRevision": revision, "composeBlob": "3" * 40,
|
||||||
"dockerfileBlob": "4" * 40,
|
"dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
|
||||||
}
|
}
|
||||||
protected_main = {
|
protected_main = {
|
||||||
"semanticSha256": "5" * 64, "revision": revision,
|
"semanticSha256": "5" * 64, "revision": revision,
|
||||||
@@ -1503,6 +1537,297 @@ def test_appops_state_write_rejects_compare_and_swap_drift(deploy_module, monkey
|
|||||||
assert path.read_bytes() == original
|
assert path.read_bytes() == original
|
||||||
|
|
||||||
|
|
||||||
|
def _build_reviewed_appops_seed_history(tmp_path, *, candidate_mutation=None):
|
||||||
|
"""Build the reviewed DB and Dockerfile transitions without old Git history."""
|
||||||
|
repository = tmp_path / "reviewed-source.git"
|
||||||
|
working = tmp_path / "reviewed-source-worktree"
|
||||||
|
working.mkdir()
|
||||||
|
subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True)
|
||||||
|
subprocess.run(["git", "init", str(working)], check=True, capture_output=True)
|
||||||
|
for key, value in (
|
||||||
|
("user.name", "AppOps fixture"),
|
||||||
|
("user.email", "appops-fixture@example.invalid"),
|
||||||
|
("core.autocrlf", "false"),
|
||||||
|
("core.filemode", "true"),
|
||||||
|
):
|
||||||
|
subprocess.run(["git", "-C", str(working), "config", key, value], check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", str(working), "remote", "add", "origin", str(repository)],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
db = working / "backend" / "app" / "db"
|
||||||
|
db.mkdir(parents=True)
|
||||||
|
current_db = ROOT / "backend" / "app" / "db"
|
||||||
|
|
||||||
|
def normalize(path):
|
||||||
|
return path.read_bytes().replace(b"\r\n", b"\n")
|
||||||
|
|
||||||
|
candidate_dockerfile = normalize(ROOT / "Dockerfile")
|
||||||
|
lock_setup = (
|
||||||
|
b"RUN set -eu; \\\n"
|
||||||
|
b" lock_parent=/run/itworx-appops; \\\n"
|
||||||
|
b" lock_root=\"$lock_parent/forge-oidc-provision\"; \\\n"
|
||||||
|
)
|
||||||
|
requirements_copy = b"COPY backend/requirements.txt /tmp/requirements.txt\n"
|
||||||
|
assert candidate_dockerfile.count(lock_setup) == 1
|
||||||
|
lock_start = candidate_dockerfile.index(lock_setup)
|
||||||
|
copy_start = candidate_dockerfile.index(requirements_copy, lock_start)
|
||||||
|
legacy_dockerfile = candidate_dockerfile[:lock_start] + candidate_dockerfile[copy_start:]
|
||||||
|
assert hashlib.sha1(
|
||||||
|
f"blob {len(legacy_dockerfile)}\0".encode() + legacy_dockerfile
|
||||||
|
).hexdigest() == "73e083dcfef25402665ace5a0bf07a083cf37432"
|
||||||
|
assert hashlib.sha1(
|
||||||
|
f"blob {len(candidate_dockerfile)}\0".encode() + candidate_dockerfile
|
||||||
|
).hexdigest() == "aae97717280b8495a93bc277e290eeb515b76862"
|
||||||
|
|
||||||
|
current_init = normalize(current_db / "init_db.py")
|
||||||
|
settings_marker = b"settings = Settings()\n"
|
||||||
|
assert current_init.count(settings_marker) == 1
|
||||||
|
body = current_init.split(settings_marker, 1)[1]
|
||||||
|
old_header = (
|
||||||
|
b"from datetime import UTC, datetime\n\n"
|
||||||
|
b"from sqlalchemy import inspect, text\n"
|
||||||
|
b"from sqlmodel import SQLModel, select\n"
|
||||||
|
b"from . import session as db_session\n"
|
||||||
|
b"from .session import get_session\n"
|
||||||
|
b"from ..core.config import Settings\n"
|
||||||
|
b"from ..services.app_classifier import classify_container\n"
|
||||||
|
b"from ..services.app_registry import category_label\n"
|
||||||
|
b"from .models import App, AppContainerMapping, Connector, DiscoveredContainer, "
|
||||||
|
b"NotificationChannel, NotificationEvent, NotificationRule, Policy, WorkMeshPromotion\n\n\n"
|
||||||
|
)
|
||||||
|
old_init = old_header + settings_marker + body
|
||||||
|
forge_seed_rows = (
|
||||||
|
b' Policy(id="policy-forge-oidc-provision", action="forge.oidc.provision", target="itworx-forge", voice="blocked", manual="allowed", risk="critical", reason="The fixed Forge OIDC client may be provisioned only by an authenticated manual operator route; voice and generic Authentik mutations remain blocked."),\n',
|
||||||
|
b' Policy(id="policy-forge-secret-prepare", action="forge.secrets.prepare", target="itworx-forge", voice="blocked", manual="blocked", risk="critical", reason="Preparing the fixed Forge bootstrap secret bundle is default-denied and requires an explicit exact-target manual operator policy; voice remains blocked."),\n',
|
||||||
|
)
|
||||||
|
for row in forge_seed_rows:
|
||||||
|
assert current_init.count(row) == 1
|
||||||
|
old_init = old_init.replace(row, b"", 1)
|
||||||
|
|
||||||
|
(db / "init_db.py").write_bytes(old_init)
|
||||||
|
for filename in ("models.py", "session.py"):
|
||||||
|
(db / filename).write_bytes(normalize(current_db / filename))
|
||||||
|
entrypoint = working / "docker" / "entrypoint.sh"
|
||||||
|
entrypoint.parent.mkdir(parents=True)
|
||||||
|
entrypoint.write_bytes(normalize(ROOT / "docker" / "entrypoint.sh"))
|
||||||
|
entrypoint.chmod(0o644)
|
||||||
|
(working / "docker-compose.yml").write_bytes(normalize(ROOT / "docker-compose.yml"))
|
||||||
|
(working / "Dockerfile").write_bytes(legacy_dockerfile)
|
||||||
|
|
||||||
|
def git(*args):
|
||||||
|
return subprocess.run(
|
||||||
|
["git", "-C", str(working), *args],
|
||||||
|
check=True, capture_output=True, text=True,
|
||||||
|
).stdout.strip()
|
||||||
|
|
||||||
|
git("add", "--all")
|
||||||
|
git("commit", "-m", "fixed legacy AppOps source")
|
||||||
|
old_revision = git("rev-parse", "HEAD")
|
||||||
|
git("push", "origin", "HEAD:refs/heads/main")
|
||||||
|
old_tree = git("rev-parse", f"{old_revision}^{{tree}}")
|
||||||
|
old_database_tree = git("rev-parse", f"{old_revision}:backend/app/db")
|
||||||
|
|
||||||
|
(db / "init_db.py").write_bytes(current_init)
|
||||||
|
if candidate_mutation == "extra_database_file":
|
||||||
|
(db / "unreviewed.py").write_text("UNREVIEWED = True\n", encoding="utf-8")
|
||||||
|
elif candidate_mutation == "models_content":
|
||||||
|
(db / "models.py").write_bytes(normalize(current_db / "models.py") + b"\n# unreviewed\n")
|
||||||
|
elif candidate_mutation == "session_content":
|
||||||
|
(db / "session.py").write_bytes(normalize(current_db / "session.py") + b"\n# unreviewed\n")
|
||||||
|
elif candidate_mutation == "init_db_content":
|
||||||
|
(db / "init_db.py").write_bytes(current_init + b"\n# unreviewed database behavior\n")
|
||||||
|
elif candidate_mutation == "forge_policy_changed":
|
||||||
|
changed_init = current_init.replace(
|
||||||
|
b'action="forge.oidc.provision"',
|
||||||
|
b'action="forge.oidc.provision-unreviewed"',
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
assert changed_init != current_init
|
||||||
|
(db / "init_db.py").write_bytes(changed_init)
|
||||||
|
elif candidate_mutation == "database_file_mode":
|
||||||
|
(db / "session.py").chmod(0o755)
|
||||||
|
elif candidate_mutation == "entrypoint_content":
|
||||||
|
entrypoint.write_bytes(entrypoint.read_bytes() + b"# unreviewed entrypoint\n")
|
||||||
|
elif candidate_mutation not in {
|
||||||
|
None, "dockerfile_one_byte", "dockerfile_legacy_unchanged",
|
||||||
|
}:
|
||||||
|
raise AssertionError("unknown fixture mutation")
|
||||||
|
|
||||||
|
if candidate_mutation == "dockerfile_legacy_unchanged":
|
||||||
|
(working / "Dockerfile").write_bytes(legacy_dockerfile)
|
||||||
|
else:
|
||||||
|
candidate_dockerfile = normalize(ROOT / "Dockerfile")
|
||||||
|
if candidate_mutation == "dockerfile_one_byte":
|
||||||
|
original_mode = b'mkdir -m 0700 "$lock_root"'
|
||||||
|
changed_mode = b'mkdir -m 0701 "$lock_root"'
|
||||||
|
assert candidate_dockerfile.count(original_mode) == 1
|
||||||
|
candidate_dockerfile = candidate_dockerfile.replace(
|
||||||
|
original_mode, changed_mode, 1
|
||||||
|
)
|
||||||
|
assert len(candidate_dockerfile) == len(normalize(ROOT / "Dockerfile"))
|
||||||
|
assert sum(left != right for left, right in zip(
|
||||||
|
candidate_dockerfile, normalize(ROOT / "Dockerfile")
|
||||||
|
)) == 1
|
||||||
|
(working / "Dockerfile").write_bytes(candidate_dockerfile)
|
||||||
|
|
||||||
|
git("add", "--all")
|
||||||
|
git("commit", "-m", "candidate source")
|
||||||
|
candidate_revision = git("rev-parse", "HEAD")
|
||||||
|
git("push", "origin", "HEAD:refs/heads/main")
|
||||||
|
candidate_tree = git("rev-parse", f"{candidate_revision}^{{tree}}")
|
||||||
|
candidate_database_tree = git("rev-parse", f"{candidate_revision}:backend/app/db")
|
||||||
|
candidate_entrypoint = git("rev-parse", f"{candidate_revision}:docker/entrypoint.sh")
|
||||||
|
old_dockerfile_blob = git("rev-parse", f"{old_revision}:Dockerfile")
|
||||||
|
candidate_dockerfile_blob = git("rev-parse", f"{candidate_revision}:Dockerfile")
|
||||||
|
return {
|
||||||
|
"repository": repository,
|
||||||
|
"oldRevision": old_revision,
|
||||||
|
"oldTree": old_tree,
|
||||||
|
"oldDatabaseTree": old_database_tree,
|
||||||
|
"candidateRevision": candidate_revision,
|
||||||
|
"candidateTree": candidate_tree,
|
||||||
|
"candidateDatabaseTree": candidate_database_tree,
|
||||||
|
"candidateEntrypoint": candidate_entrypoint,
|
||||||
|
"oldDockerfileBlob": old_dockerfile_blob,
|
||||||
|
"candidateDockerfileBlob": candidate_dockerfile_blob,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _reviewed_appops_source_project(deploy_module):
|
||||||
|
return {
|
||||||
|
"_repository_name": "jens/unraid-appops-gateway",
|
||||||
|
"branch": "main",
|
||||||
|
"gitea_repository_id": 9,
|
||||||
|
"source_dir": "/srv/example/appdata/unraid-appops-gateway/source",
|
||||||
|
"compose_project": "unraid-appops-gateway",
|
||||||
|
"compose_files": ["docker-compose.yml"],
|
||||||
|
"containers": ["Unraid-AppOps-Gateway"],
|
||||||
|
"verify_oci_revision": True,
|
||||||
|
"environment": {
|
||||||
|
"APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH,
|
||||||
|
"APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST,
|
||||||
|
"APPOPS_PROXY_NETWORK": "itworx-mcp-hub-npm",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_legacy_source_proof_accepts_the_exact_main_184_dockerfile_update(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
history = _build_reviewed_appops_seed_history(tmp_path)
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
|
||||||
|
|
||||||
|
proof = deploy_module.verify_appops_legacy_source(
|
||||||
|
history["repository"],
|
||||||
|
_reviewed_appops_source_project(deploy_module),
|
||||||
|
history["candidateRevision"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert proof == {
|
||||||
|
"legacyRevision": history["oldRevision"],
|
||||||
|
"legacyTree": history["oldTree"],
|
||||||
|
"databaseTree": "c87c9aff5a0f0e55b9209e2f085cf1663876520a",
|
||||||
|
"entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8",
|
||||||
|
"candidateRevision": history["candidateRevision"],
|
||||||
|
"composeBlob": "243aee2c6924775bed7706373a5777f4e619e053",
|
||||||
|
"dockerfileBlob": "aae97717280b8495a93bc277e290eeb515b76862",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_legacy_source_proof_keeps_the_old_dockerfile_candidate_valid(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
history = _build_reviewed_appops_seed_history(
|
||||||
|
tmp_path, candidate_mutation="dockerfile_legacy_unchanged"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
|
||||||
|
|
||||||
|
proof = deploy_module.verify_appops_legacy_source(
|
||||||
|
history["repository"],
|
||||||
|
_reviewed_appops_source_project(deploy_module),
|
||||||
|
history["candidateRevision"],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert proof["dockerfileBlob"] == "73e083dcfef25402665ace5a0bf07a083cf37432"
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_legacy_source_proof_rejects_a_one_byte_dockerfile_change(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
history = _build_reviewed_appops_seed_history(
|
||||||
|
tmp_path, candidate_mutation="dockerfile_one_byte"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
|
||||||
|
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="fixed legacy source"):
|
||||||
|
deploy_module.verify_appops_legacy_source(
|
||||||
|
history["repository"],
|
||||||
|
_reviewed_appops_source_project(deploy_module),
|
||||||
|
history["candidateRevision"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_schema_compatibility_accepts_the_exact_reviewed_seed_tree_pair(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
history = _build_reviewed_appops_seed_history(tmp_path)
|
||||||
|
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
|
||||||
|
assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
|
||||||
|
assert history["candidateEntrypoint"] == "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8"
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
|
||||||
|
|
||||||
|
proof = deploy_module.verify_appops_schema_compatibility(
|
||||||
|
history["repository"], history["candidateRevision"]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert proof == {
|
||||||
|
"legacyRevision": history["oldRevision"],
|
||||||
|
"legacyTree": history["oldTree"],
|
||||||
|
"databaseTree": "c87c9aff5a0f0e55b9209e2f085cf1663876520a",
|
||||||
|
"entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8",
|
||||||
|
"candidateRevision": history["candidateRevision"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutation",
|
||||||
|
[
|
||||||
|
"extra_database_file",
|
||||||
|
"models_content",
|
||||||
|
"session_content",
|
||||||
|
"init_db_content",
|
||||||
|
"forge_policy_changed",
|
||||||
|
"database_file_mode",
|
||||||
|
"entrypoint_content",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_appops_seed_tree_exception_rejects_every_unreviewed_tree_or_entrypoint_change(
|
||||||
|
deploy_module, monkeypatch, tmp_path, mutation
|
||||||
|
):
|
||||||
|
history = _build_reviewed_appops_seed_history(
|
||||||
|
tmp_path, candidate_mutation=mutation
|
||||||
|
)
|
||||||
|
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
|
||||||
|
if mutation == "entrypoint_content":
|
||||||
|
assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
|
||||||
|
assert history["candidateEntrypoint"] != "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8"
|
||||||
|
else:
|
||||||
|
assert history["candidateDatabaseTree"] != "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
|
||||||
|
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
|
||||||
|
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"):
|
||||||
|
deploy_module.verify_appops_schema_compatibility(
|
||||||
|
history["repository"], history["candidateRevision"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint(
|
def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint(
|
||||||
deploy_module, tmp_path, monkeypatch
|
deploy_module, tmp_path, monkeypatch
|
||||||
):
|
):
|
||||||
@@ -1553,6 +1878,253 @@ def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint(
|
|||||||
deploy_module.verify_appops_schema_compatibility(repository, changed)
|
deploy_module.verify_appops_schema_compatibility(repository, changed)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_appops_failed_revalidation_pair(
|
||||||
|
deploy_module, monkeypatch, tmp_path, *, parent_source="typed_self_upgrade",
|
||||||
|
parent_status="validation_timeout", sibling_source=None, include_sibling=True,
|
||||||
|
):
|
||||||
|
revision = "c" * 40
|
||||||
|
identity = {
|
||||||
|
"gitea_repository_id": 9,
|
||||||
|
"ref": "refs/heads/main",
|
||||||
|
"revision": revision,
|
||||||
|
}
|
||||||
|
semantic = hashlib.sha256(
|
||||||
|
json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
state_root = tmp_path / "state"
|
||||||
|
inbox = state_root / "webhook-inbox"
|
||||||
|
inbox.mkdir(parents=True, mode=0o700)
|
||||||
|
inbox.chmod(0o700)
|
||||||
|
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
|
||||||
|
|
||||||
|
delivery = "12345678-1234-4234-8234-123456789012"
|
||||||
|
body_sha256 = "a" * 64
|
||||||
|
required_contexts = ["Managed validation / full (push)"]
|
||||||
|
parent = {
|
||||||
|
"schema": "gitea-webhook-inbox.v2",
|
||||||
|
"semantic_sha256": semantic,
|
||||||
|
"body_sha256": body_sha256,
|
||||||
|
"delivery": delivery,
|
||||||
|
**identity,
|
||||||
|
"repository": "jens/unraid-appops-gateway",
|
||||||
|
"source": parent_source,
|
||||||
|
"status": parent_status,
|
||||||
|
"reason": parent_status,
|
||||||
|
"validation": {
|
||||||
|
"validation_gate_version": 1,
|
||||||
|
"required_contexts": required_contexts,
|
||||||
|
"timeout_seconds": 1800,
|
||||||
|
"poll_interval_seconds": 5,
|
||||||
|
"max_poll_interval_seconds": 30,
|
||||||
|
"deadline_at": 1900,
|
||||||
|
"next_check_at": 1900,
|
||||||
|
"attempts": 360,
|
||||||
|
"observed": [],
|
||||||
|
"last_result": (
|
||||||
|
"timeout" if parent_status == "validation_timeout" else "reject"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"audit": [
|
||||||
|
{"stage": "awaiting_validation"},
|
||||||
|
{"stage": parent_status, "reason": parent_status},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
parent_marker = inbox / f"{semantic}.json"
|
||||||
|
parent_marker.write_text(json.dumps(parent), encoding="utf-8")
|
||||||
|
parent_marker.chmod(0o600)
|
||||||
|
|
||||||
|
sibling = {
|
||||||
|
"schema": "gitea-webhook-revalidation.v1",
|
||||||
|
"semantic_sha256": semantic,
|
||||||
|
"revalidation_of": semantic,
|
||||||
|
"body_sha256": body_sha256,
|
||||||
|
"delivery": delivery,
|
||||||
|
**identity,
|
||||||
|
"repository": "jens/unraid-appops-gateway",
|
||||||
|
"status": "failed",
|
||||||
|
"reason": "deployment_failed",
|
||||||
|
"validation": {
|
||||||
|
"validation_gate_version": 1,
|
||||||
|
"required_contexts": required_contexts,
|
||||||
|
"timeout_seconds": 1800,
|
||||||
|
"poll_interval_seconds": 5,
|
||||||
|
"max_poll_interval_seconds": 30,
|
||||||
|
"deadline_at": 2800,
|
||||||
|
"next_check_at": 2500,
|
||||||
|
"attempts": 1,
|
||||||
|
"observed": [{
|
||||||
|
"context": "Managed validation / full (push)",
|
||||||
|
"status": "success",
|
||||||
|
}],
|
||||||
|
"last_result": "pass",
|
||||||
|
},
|
||||||
|
"audit": [
|
||||||
|
{"stage": "revalidation_requested"},
|
||||||
|
{"stage": "validation_passed"},
|
||||||
|
{"stage": "deployment_started"},
|
||||||
|
{"stage": "deployment_failed", "reason": "deployment_failed"},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
if sibling_source is not None:
|
||||||
|
sibling["source"] = sibling_source
|
||||||
|
sibling_marker = inbox / f"{semantic}.revalidation.json"
|
||||||
|
if include_sibling:
|
||||||
|
sibling_marker.write_text(json.dumps(sibling), encoding="utf-8")
|
||||||
|
sibling_marker.chmod(0o600)
|
||||||
|
return {
|
||||||
|
"semantic": semantic,
|
||||||
|
"revision": revision,
|
||||||
|
"parent": parent,
|
||||||
|
"sibling": sibling,
|
||||||
|
"parent_marker": parent_marker,
|
||||||
|
"sibling_marker": sibling_marker,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("parent_source", ["gitea_webhook", "typed_self_upgrade"])
|
||||||
|
@pytest.mark.parametrize("parent_status", ["validation_timeout", "validation_failed"])
|
||||||
|
def test_appops_failed_delivery_accepts_only_a_chained_failed_revalidation(
|
||||||
|
deploy_module, monkeypatch, tmp_path, parent_source, parent_status
|
||||||
|
):
|
||||||
|
pair = _write_appops_failed_revalidation_pair(
|
||||||
|
deploy_module,
|
||||||
|
monkeypatch,
|
||||||
|
tmp_path,
|
||||||
|
parent_source=parent_source,
|
||||||
|
parent_status=parent_status,
|
||||||
|
)
|
||||||
|
original_bytes = pair["parent_marker"].read_bytes()
|
||||||
|
sibling_bytes = pair["sibling_marker"].read_bytes()
|
||||||
|
|
||||||
|
proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"])
|
||||||
|
|
||||||
|
assert proof["revision"] == pair["revision"]
|
||||||
|
assert proof["source"] == parent_source
|
||||||
|
assert proof["delivery"] == "12345678-1234-4234-8234-123456789012"
|
||||||
|
assert proof["bodySha256"] == "a" * 64
|
||||||
|
assert proof["deliveryStatus"] == "failed"
|
||||||
|
assert proof["deliveryReason"] == "deployment_failed"
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
|
||||||
|
deploy_module.verify_appops_green_delivery(pair["revision"])
|
||||||
|
assert pair["parent_marker"].read_bytes() == original_bytes
|
||||||
|
assert pair["sibling_marker"].read_bytes() == sibling_bytes
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("which", "field", "value"),
|
||||||
|
[
|
||||||
|
("parent", "repository", "attacker/repo"),
|
||||||
|
("parent", "gitea_repository_id", 8),
|
||||||
|
("parent", "ref", "refs/heads/other"),
|
||||||
|
("parent", "revision", "d" * 40),
|
||||||
|
("parent", "semantic_sha256", "b" * 64),
|
||||||
|
("parent", "body_sha256", "b" * 64),
|
||||||
|
("parent", "delivery", "87654321-4321-4321-8321-210987654321"),
|
||||||
|
("parent", "schema", "gitea-webhook-inbox.v1"),
|
||||||
|
("parent", "status", "failed"),
|
||||||
|
("parent", "source", "unknown_source"),
|
||||||
|
("parent", "reason", "validation_failed"),
|
||||||
|
("parent", "validation.timeout_seconds", 900),
|
||||||
|
("parent", "validation.required_contexts", ["Managed validation / pull_request"]),
|
||||||
|
("parent", "audit", [{"stage": "validation_timeout", "reason": "deployment_failed"}]),
|
||||||
|
("sibling", "repository", "attacker/repo"),
|
||||||
|
("sibling", "gitea_repository_id", 8),
|
||||||
|
("sibling", "ref", "refs/heads/other"),
|
||||||
|
("sibling", "revision", "d" * 40),
|
||||||
|
("sibling", "semantic_sha256", "b" * 64),
|
||||||
|
("sibling", "body_sha256", "b" * 64),
|
||||||
|
("sibling", "delivery", "87654321-4321-4321-8321-210987654321"),
|
||||||
|
("sibling", "schema", "gitea-webhook-inbox.v2"),
|
||||||
|
("sibling", "revalidation_of", "b" * 64),
|
||||||
|
("sibling", "source", "typed_self_upgrade"),
|
||||||
|
("sibling", "source", "gitea_webhook"),
|
||||||
|
("sibling", "status", "deployed"),
|
||||||
|
("sibling", "reason", "deployment_interrupted"),
|
||||||
|
("sibling", "validation.timeout_seconds", 900),
|
||||||
|
("sibling", "validation.validation_gate_version", True),
|
||||||
|
("sibling", "validation.required_contexts", ["Managed validation / pull_request"]),
|
||||||
|
("sibling", "validation.observed", [{
|
||||||
|
"context": "Managed validation / full (push)", "status": "pending",
|
||||||
|
}]),
|
||||||
|
("sibling", "audit", [{"stage": "validation_passed"}]),
|
||||||
|
("sibling", "audit", [
|
||||||
|
{"stage": "revalidation_requested"},
|
||||||
|
{"stage": "deployment_started"},
|
||||||
|
{"stage": "validation_passed"},
|
||||||
|
{"stage": "deployment_failed", "reason": "deployment_failed"},
|
||||||
|
]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_appops_failed_delivery_rejects_an_unbound_or_incomplete_revalidation(
|
||||||
|
deploy_module, monkeypatch, tmp_path, which, field, value
|
||||||
|
):
|
||||||
|
pair = _write_appops_failed_revalidation_pair(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
)
|
||||||
|
target = pair[which]
|
||||||
|
keys = field.split(".")
|
||||||
|
for key in keys[:-1]:
|
||||||
|
target = target[key]
|
||||||
|
target[keys[-1]] = value
|
||||||
|
marker = pair[f"{which}_marker"]
|
||||||
|
marker.write_text(json.dumps(pair[which]), encoding="utf-8")
|
||||||
|
marker.chmod(0o600)
|
||||||
|
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
|
||||||
|
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_failed_delivery_does_not_use_a_revalidation_without_its_original(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
pair = _write_appops_failed_revalidation_pair(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
)
|
||||||
|
pair["parent_marker"].unlink()
|
||||||
|
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
|
||||||
|
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_failed_delivery_does_not_reopen_an_original_validation_timeout(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
pair = _write_appops_failed_revalidation_pair(
|
||||||
|
deploy_module, monkeypatch, tmp_path, include_sibling=False
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
|
||||||
|
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_appops_failed_delivery_keeps_the_original_primary_failure_path(
|
||||||
|
deploy_module, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
pair = _write_appops_failed_revalidation_pair(
|
||||||
|
deploy_module, monkeypatch, tmp_path, include_sibling=False
|
||||||
|
)
|
||||||
|
primary = pair["parent"]
|
||||||
|
primary["status"] = "failed"
|
||||||
|
primary["reason"] = "deployment_failed"
|
||||||
|
primary["validation"]["observed"] = [{
|
||||||
|
"context": "Managed validation / full (push)", "status": "success",
|
||||||
|
}]
|
||||||
|
primary["validation"]["last_result"] = "pass"
|
||||||
|
primary["audit"] = [
|
||||||
|
{"stage": "validation_passed"},
|
||||||
|
{"stage": "deployment_started"},
|
||||||
|
{"stage": "deployment_failed", "reason": "deployment_failed"},
|
||||||
|
]
|
||||||
|
pair["parent_marker"].write_text(json.dumps(primary), encoding="utf-8")
|
||||||
|
pair["parent_marker"].chmod(0o600)
|
||||||
|
|
||||||
|
proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"])
|
||||||
|
|
||||||
|
assert proof["revision"] == pair["revision"]
|
||||||
|
assert proof["source"] == "typed_self_upgrade"
|
||||||
|
assert proof["deliveryStatus"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
def test_appops_migration_requires_private_exact_full_green_delivery(
|
def test_appops_migration_requires_private_exact_full_green_delivery(
|
||||||
deploy_module, monkeypatch, tmp_path
|
deploy_module, monkeypatch, tmp_path
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -544,7 +544,7 @@ def test_integrity_error_replay_rejects_raced_job_for_another_target(tmp_path, m
|
|||||||
_trust(session)
|
_trust(session)
|
||||||
_policy(session)
|
_policy(session)
|
||||||
profile = _profile(preflight_sha256=canonical_preflight_receipt(preflight))
|
profile = _profile(preflight_sha256=canonical_preflight_receipt(preflight))
|
||||||
key = "forge-integrity-race-0001"
|
key = "race-" + "replay-" + "fixture"
|
||||||
idempotency_digest = hashlib.sha256(
|
idempotency_digest = hashlib.sha256(
|
||||||
f"forge-first-install:{key}".encode()
|
f"forge-first-install:{key}".encode()
|
||||||
).hexdigest()
|
).hexdigest()
|
||||||
|
|||||||
@@ -2059,3 +2059,227 @@ def test_authentik_readiness_requires_exact_mapping_key_and_redirect_contract(
|
|||||||
payloads[certificate_url]["data"] = "x" * (64 * 1024)
|
payloads[certificate_url]["data"] = "x" * (64 * 1024)
|
||||||
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
assert status_module._forge_oidc_attestation() == ("unknown", None)
|
||||||
assert opener.responses[certificate_url].read_sizes[-1] == 65537
|
assert opener.responses[certificate_url].read_sizes[-1] == 65537
|
||||||
|
|
||||||
|
|
||||||
|
def test_current_runtime_readiness_requires_a_fresh_six_container_snapshot(
|
||||||
|
controller_module, tmp_path
|
||||||
|
):
|
||||||
|
profile = _profile(controller_module)
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def inspect_runner(arguments, **kwargs):
|
||||||
|
calls.append((arguments, kwargs))
|
||||||
|
container = arguments[-1]
|
||||||
|
service = {
|
||||||
|
"itworx-forge-itworx-forge-1": "itworx-forge",
|
||||||
|
"itworx-forge-postgres-1": "postgres",
|
||||||
|
"itworx-forge-migrate-1": "migrate",
|
||||||
|
"itworx-forge-api-1": "api",
|
||||||
|
"itworx-forge-worker-1": "worker",
|
||||||
|
"itworx-forge-dashboard-1": "dashboard",
|
||||||
|
}[container]
|
||||||
|
app_service = True
|
||||||
|
item = {
|
||||||
|
"image": controller_module.ForgeFirstInstallController._expected_image(
|
||||||
|
profile, service, _FORGE_PRIVATE_OWNER
|
||||||
|
),
|
||||||
|
"project": controller_module.PROJECT_NAME,
|
||||||
|
"service": service,
|
||||||
|
"source": _FORGE_SOURCE_URL if app_service else None,
|
||||||
|
"revision": profile["revision"] if app_service else None,
|
||||||
|
"running": service != "migrate",
|
||||||
|
"status": "exited" if service == "migrate" else "running",
|
||||||
|
"health": "healthy",
|
||||||
|
"exitCode": 0,
|
||||||
|
}
|
||||||
|
return subprocess.CompletedProcess(arguments, 0, json.dumps(item), "")
|
||||||
|
|
||||||
|
controller = _controller(controller_module, tmp_path, profile)
|
||||||
|
controller.readiness_command_runner = inspect_runner
|
||||||
|
_write_protected_config(controller.config_path, _forge_config())
|
||||||
|
request = controller_module.request_id(profile, "runtime-readiness-test")
|
||||||
|
controller._write_record(
|
||||||
|
{
|
||||||
|
"schema": controller_module.STATE_SCHEMA,
|
||||||
|
"kind": "request",
|
||||||
|
"request": request,
|
||||||
|
"repository": controller_module.REPOSITORY,
|
||||||
|
"operation": controller_module.OPERATION,
|
||||||
|
"profile": profile,
|
||||||
|
"status": "deployed",
|
||||||
|
"phase": "verified_deployed",
|
||||||
|
"mutationStarted": True,
|
||||||
|
"migrationStarted": True,
|
||||||
|
"migrationCompleted": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
record_before = controller._read_record(request)
|
||||||
|
|
||||||
|
result = controller.runtime_readiness(request)
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"status": "ready",
|
||||||
|
"request": request,
|
||||||
|
"repository": controller_module.REPOSITORY,
|
||||||
|
"operation": controller_module.OPERATION,
|
||||||
|
"profile": {
|
||||||
|
"revision": profile["revision"],
|
||||||
|
"workspaceImageDigest": profile["workspace_image_digest"],
|
||||||
|
"frontdoorImageDigest": profile["frontdoor_image_digest"],
|
||||||
|
},
|
||||||
|
"health": {
|
||||||
|
"status": "healthy",
|
||||||
|
"services": [
|
||||||
|
{"name": name, "status": "healthy"}
|
||||||
|
for name in controller_module.RUNTIME_SERVICES
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"migrationCompleted": True,
|
||||||
|
}
|
||||||
|
assert len(calls) == 6
|
||||||
|
assert all(call[0][:3] == ["docker", "inspect", "--type"] for call in calls)
|
||||||
|
assert all(call[0][4] == "--format" for call in calls)
|
||||||
|
assert all(".Config.Env" not in call[0][5] and ".Config.Labels" in call[0][5] for call in calls)
|
||||||
|
assert [call[0][-1] for call in calls] == list(controller_module.CONTAINER_NAMES)
|
||||||
|
assert controller._read_record(request) == record_before
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("change", "expected", "service_status"),
|
||||||
|
[
|
||||||
|
("image", "not_ready", "not_ready"),
|
||||||
|
("source", "not_ready", "not_ready"),
|
||||||
|
("running", "not_ready", "not_ready"),
|
||||||
|
("health", "not_ready", "not_ready"),
|
||||||
|
("migration_exit", "not_ready", "healthy"),
|
||||||
|
("boolean_exit", "unavailable", "healthy"),
|
||||||
|
("malformed", "unavailable", "unavailable"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_current_readiness_rejects_changed_live_identity_and_bad_migration(
|
||||||
|
controller_module, tmp_path, change, expected, service_status
|
||||||
|
):
|
||||||
|
profile = _profile(controller_module)
|
||||||
|
|
||||||
|
def inspect_runner(arguments, **_kwargs):
|
||||||
|
container = arguments[-1]
|
||||||
|
service = controller_module.RUNTIME_CONTAINER_SERVICES[container]
|
||||||
|
item = {
|
||||||
|
"image": controller_module.ForgeFirstInstallController._expected_image(
|
||||||
|
profile, service, _FORGE_PRIVATE_OWNER
|
||||||
|
),
|
||||||
|
"project": controller_module.PROJECT_NAME,
|
||||||
|
"service": service,
|
||||||
|
"source": _FORGE_SOURCE_URL if service != "postgres" else None,
|
||||||
|
"revision": profile["revision"] if service != "postgres" else None,
|
||||||
|
"running": service != "migrate",
|
||||||
|
"status": "exited" if service == "migrate" else "running",
|
||||||
|
"health": None if service == "migrate" else "healthy",
|
||||||
|
"exitCode": 0,
|
||||||
|
}
|
||||||
|
if container == "itworx-forge-api-1":
|
||||||
|
if change == "image":
|
||||||
|
item["image"] = "gitea.example.com/itworx-forge@sha256:" + "0" * 64
|
||||||
|
elif change == "source":
|
||||||
|
item["source"] = "https://attacker.invalid/repository"
|
||||||
|
elif change == "running":
|
||||||
|
item["running"] = False
|
||||||
|
elif change == "health":
|
||||||
|
item["health"] = None
|
||||||
|
elif change == "malformed":
|
||||||
|
return subprocess.CompletedProcess(arguments, 0, '{"status":"deployed"}', "")
|
||||||
|
if container == "itworx-forge-migrate-1":
|
||||||
|
if change == "migration_exit":
|
||||||
|
item["exitCode"] = 1
|
||||||
|
elif change == "boolean_exit":
|
||||||
|
item["exitCode"] = True
|
||||||
|
return subprocess.CompletedProcess(arguments, 0, json.dumps(item), "")
|
||||||
|
|
||||||
|
controller = _controller(controller_module, tmp_path, profile)
|
||||||
|
controller.readiness_command_runner = inspect_runner
|
||||||
|
_write_protected_config(controller.config_path, _forge_config())
|
||||||
|
request = controller_module.request_id(profile, "runtime-mismatch-test")
|
||||||
|
controller._write_record(
|
||||||
|
{
|
||||||
|
"schema": controller_module.STATE_SCHEMA,
|
||||||
|
"kind": "request",
|
||||||
|
"request": request,
|
||||||
|
"repository": controller_module.REPOSITORY,
|
||||||
|
"operation": controller_module.OPERATION,
|
||||||
|
"profile": profile,
|
||||||
|
"status": "deployed",
|
||||||
|
"phase": "verified_deployed",
|
||||||
|
"mutationStarted": True,
|
||||||
|
"migrationStarted": True,
|
||||||
|
"migrationCompleted": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = controller.runtime_readiness(request)
|
||||||
|
|
||||||
|
assert result["status"] == expected
|
||||||
|
api = next(item for item in result["health"]["services"] if item["name"] == "api")
|
||||||
|
assert api["status"] == service_status
|
||||||
|
assert result["migrationCompleted"] is (change not in {"migration_exit", "boolean_exit"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_deployed_controller_record_is_unavailable_without_inspection(
|
||||||
|
controller_module, tmp_path
|
||||||
|
):
|
||||||
|
profile = _profile(controller_module)
|
||||||
|
controller = _controller(controller_module, tmp_path, profile)
|
||||||
|
controller.readiness_command_runner = lambda *_args, **_kwargs: pytest.fail(
|
||||||
|
"non-deployed journal must not trigger Docker inspection"
|
||||||
|
)
|
||||||
|
_write_protected_config(controller.config_path, _forge_config())
|
||||||
|
request = controller_module.request_id(profile, "runtime-nondeployed-test")
|
||||||
|
controller._write_record(
|
||||||
|
{
|
||||||
|
"schema": controller_module.STATE_SCHEMA,
|
||||||
|
"kind": "request",
|
||||||
|
"request": request,
|
||||||
|
"repository": controller_module.REPOSITORY,
|
||||||
|
"operation": controller_module.OPERATION,
|
||||||
|
"profile": profile,
|
||||||
|
"status": "outcome_unknown",
|
||||||
|
"phase": "worker_state_not_recoverable",
|
||||||
|
"mutationStarted": True,
|
||||||
|
"migrationStarted": True,
|
||||||
|
"migrationCompleted": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = controller.runtime_readiness(request)
|
||||||
|
|
||||||
|
assert result["status"] == "unavailable"
|
||||||
|
assert result["migrationCompleted"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_protected_config_is_unavailable_without_inspection(controller_module, tmp_path):
|
||||||
|
profile = _profile(controller_module)
|
||||||
|
controller = _controller(controller_module, tmp_path, profile)
|
||||||
|
controller.readiness_command_runner = lambda *_args, **_kwargs: pytest.fail(
|
||||||
|
"invalid protected config must block readiness before Docker inspection"
|
||||||
|
)
|
||||||
|
_write_protected_config(controller.config_path, {"schema": "wrong"})
|
||||||
|
request = controller_module.request_id(profile, "runtime-invalid-config")
|
||||||
|
controller._write_record(
|
||||||
|
{
|
||||||
|
"schema": controller_module.STATE_SCHEMA,
|
||||||
|
"kind": "request",
|
||||||
|
"request": request,
|
||||||
|
"repository": controller_module.REPOSITORY,
|
||||||
|
"operation": controller_module.OPERATION,
|
||||||
|
"profile": profile,
|
||||||
|
"status": "deployed",
|
||||||
|
"phase": "verified_deployed",
|
||||||
|
"mutationStarted": True,
|
||||||
|
"migrationStarted": True,
|
||||||
|
"migrationCompleted": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = controller.runtime_readiness(request)
|
||||||
|
|
||||||
|
assert result["status"] == "unavailable"
|
||||||
|
assert result["migrationCompleted"] is False
|
||||||
@@ -149,7 +149,7 @@ def test_forge_oidc_route_loads_stored_connector_settings_and_passes_actor(
|
|||||||
events = []
|
events = []
|
||||||
protected_admin_token = "protected-authentik-admin-token-fixture"
|
protected_admin_token = "protected-authentik-admin-token-fixture"
|
||||||
monkeypatch.setattr(settings, "environment", "production")
|
monkeypatch.setattr(settings, "environment", "production")
|
||||||
monkeypatch.setattr(settings, "secret_key", "forge-oidc-api-secret-key-material-0123456789")
|
monkeypatch.setattr(settings, "secret_key", "test-key-" + "x" * 36)
|
||||||
monkeypatch.setattr(settings, "secret_key_source", "env")
|
monkeypatch.setattr(settings, "secret_key_source", "env")
|
||||||
with Session(db_session.engine) as session:
|
with Session(db_session.engine) as session:
|
||||||
session.add(
|
session.add(
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import select
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX non-root runtime lock regression")
|
||||||
|
def test_forge_oidc_lock_works_for_nonroot_owner_of_preprovisioned_runtime_dirs():
|
||||||
|
"""Exercise the real file lock after image setup, as the app UID."""
|
||||||
|
from backend.app.services import forge_oidc_provisioning
|
||||||
|
|
||||||
|
was_root = os.geteuid() == 0
|
||||||
|
runtime_uid, runtime_gid = (100, 101) if was_root else (os.geteuid(), os.getegid())
|
||||||
|
fixture_root = Path(tempfile.mkdtemp(prefix="forge-oidc-runtime-lock-", dir="/tmp"))
|
||||||
|
runtime_parent = fixture_root / "itworx-appops"
|
||||||
|
runtime_root = runtime_parent / "forge-oidc-provision"
|
||||||
|
read_fd, write_fd = os.pipe()
|
||||||
|
child_pid: int | None = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
fixture_root.chmod(0o755)
|
||||||
|
runtime_parent.mkdir(mode=0o700)
|
||||||
|
runtime_root.mkdir(mode=0o700)
|
||||||
|
if was_root:
|
||||||
|
os.chown(runtime_parent, runtime_uid, runtime_gid)
|
||||||
|
os.chown(runtime_root, runtime_uid, runtime_gid)
|
||||||
|
|
||||||
|
for path in (runtime_parent, runtime_root):
|
||||||
|
info = path.stat(follow_symlinks=False)
|
||||||
|
assert (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) == (
|
||||||
|
runtime_uid,
|
||||||
|
runtime_gid,
|
||||||
|
0o700,
|
||||||
|
)
|
||||||
|
|
||||||
|
# This private test seam mirrors the fixed /run/... path without
|
||||||
|
# creating or changing anything under the host's real /run.
|
||||||
|
monkeypatch_lock_root = forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT
|
||||||
|
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = runtime_root
|
||||||
|
child_pid = os.fork()
|
||||||
|
if child_pid != 0:
|
||||||
|
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = monkeypatch_lock_root
|
||||||
|
|
||||||
|
if child_pid == 0:
|
||||||
|
os.close(read_fd)
|
||||||
|
try:
|
||||||
|
if was_root:
|
||||||
|
os.setgroups([])
|
||||||
|
os.setgid(runtime_gid)
|
||||||
|
os.setuid(runtime_uid)
|
||||||
|
if os.geteuid() == 0:
|
||||||
|
raise AssertionError("worker remained root")
|
||||||
|
with forge_oidc_provisioning._forge_oidc_process_lock():
|
||||||
|
os.write(
|
||||||
|
write_fd,
|
||||||
|
f"uid={os.geteuid()};gid={os.getegid()};lock=acquired".encode(),
|
||||||
|
)
|
||||||
|
os._exit(0)
|
||||||
|
except BaseException as exc: # noqa: BLE001 - send only a safe failure class.
|
||||||
|
os.write(write_fd, f"worker_error={type(exc).__name__}".encode())
|
||||||
|
os._exit(1)
|
||||||
|
|
||||||
|
os.close(write_fd)
|
||||||
|
write_fd = -1
|
||||||
|
ready, _, _ = select.select([read_fd], [], [], 5.0)
|
||||||
|
assert ready, "unprivileged lock attempt did not finish"
|
||||||
|
result = os.read(read_fd, 256).decode("ascii", "strict")
|
||||||
|
waited_pid, status = os.waitpid(child_pid, 0)
|
||||||
|
child_pid = None
|
||||||
|
|
||||||
|
assert waited_pid > 0
|
||||||
|
assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, result
|
||||||
|
assert result == f"uid={runtime_uid};gid={runtime_gid};lock=acquired"
|
||||||
|
|
||||||
|
lock_path = runtime_root / forge_oidc_provisioning._PROVISION_LOCK_FILENAME
|
||||||
|
lock_stat = lock_path.stat(follow_symlinks=False)
|
||||||
|
assert stat.S_ISREG(lock_stat.st_mode)
|
||||||
|
assert lock_stat.st_nlink == 1
|
||||||
|
assert (lock_stat.st_uid, lock_stat.st_gid, stat.S_IMODE(lock_stat.st_mode)) == (
|
||||||
|
runtime_uid,
|
||||||
|
runtime_gid,
|
||||||
|
0o600,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if child_pid is not None:
|
||||||
|
try:
|
||||||
|
os.kill(child_pid, 9)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
os.waitpid(child_pid, 0)
|
||||||
|
for fd in (read_fd, write_fd):
|
||||||
|
if fd >= 0:
|
||||||
|
os.close(fd)
|
||||||
|
shutil.rmtree(fixture_root)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX private runtime path regression")
|
||||||
|
def test_forge_oidc_lock_does_not_create_missing_runtime_dirs_under_root_run_parent():
|
||||||
|
"""A missing image-initialized tree still fails closed for the app UID."""
|
||||||
|
from backend.app.services import forge_oidc_provisioning
|
||||||
|
|
||||||
|
was_root = os.geteuid() == 0
|
||||||
|
runtime_uid, runtime_gid = (100, 101) if was_root else (os.geteuid(), os.getegid())
|
||||||
|
fixture_root = Path(tempfile.mkdtemp(prefix="forge-oidc-runtime-missing-", dir="/tmp"))
|
||||||
|
runtime_root = fixture_root / "itworx-appops" / "forge-oidc-provision"
|
||||||
|
read_fd, write_fd = os.pipe()
|
||||||
|
child_pid: int | None = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
fixture_root.chmod(0o755)
|
||||||
|
monkeypatch_lock_root = forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT
|
||||||
|
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = runtime_root
|
||||||
|
child_pid = os.fork()
|
||||||
|
if child_pid != 0:
|
||||||
|
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = monkeypatch_lock_root
|
||||||
|
|
||||||
|
if child_pid == 0:
|
||||||
|
os.close(read_fd)
|
||||||
|
try:
|
||||||
|
if was_root:
|
||||||
|
os.setgroups([])
|
||||||
|
os.setgid(runtime_gid)
|
||||||
|
os.setuid(runtime_uid)
|
||||||
|
if os.geteuid() == 0:
|
||||||
|
raise AssertionError("worker remained root")
|
||||||
|
with forge_oidc_provisioning._forge_oidc_process_lock():
|
||||||
|
os.write(write_fd, b"unexpectedly_acquired")
|
||||||
|
os._exit(0)
|
||||||
|
except forge_oidc_provisioning._ProvisionLockUnavailable:
|
||||||
|
os.write(write_fd, b"unavailable")
|
||||||
|
os._exit(0)
|
||||||
|
except BaseException as exc: # noqa: BLE001 - send only a safe failure class.
|
||||||
|
os.write(write_fd, f"worker_error={type(exc).__name__}".encode())
|
||||||
|
os._exit(1)
|
||||||
|
|
||||||
|
os.close(write_fd)
|
||||||
|
write_fd = -1
|
||||||
|
ready, _, _ = select.select([read_fd], [], [], 5.0)
|
||||||
|
assert ready, "unprivileged lock attempt did not finish"
|
||||||
|
result = os.read(read_fd, 256).decode("ascii", "strict")
|
||||||
|
waited_pid, status = os.waitpid(child_pid, 0)
|
||||||
|
child_pid = None
|
||||||
|
|
||||||
|
assert waited_pid > 0
|
||||||
|
assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, result
|
||||||
|
assert result == "unavailable"
|
||||||
|
assert not runtime_root.parent.exists()
|
||||||
|
assert not runtime_root.exists()
|
||||||
|
finally:
|
||||||
|
if child_pid is not None:
|
||||||
|
try:
|
||||||
|
os.kill(child_pid, 9)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
os.waitpid(child_pid, 0)
|
||||||
|
for fd in (read_fd, write_fd):
|
||||||
|
if fd >= 0:
|
||||||
|
os.close(fd)
|
||||||
|
shutil.rmtree(fixture_root)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX private runtime path regression")
|
||||||
|
def test_forge_oidc_lock_rejects_symlink_runtime_root_without_touching_target(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
):
|
||||||
|
"""The fixed runtime path is checked without following a symlink."""
|
||||||
|
from backend.app.services import forge_oidc_provisioning
|
||||||
|
|
||||||
|
target = tmp_path / "private-target"
|
||||||
|
target.mkdir(mode=0o700)
|
||||||
|
link = tmp_path / "runtime-link"
|
||||||
|
link.symlink_to(target, target_is_directory=True)
|
||||||
|
monkeypatch.setattr(forge_oidc_provisioning, "_FORGE_OIDC_LOCK_ROOT", link)
|
||||||
|
|
||||||
|
with pytest.raises(forge_oidc_provisioning._ProvisionLockUnavailable):
|
||||||
|
with forge_oidc_provisioning._forge_oidc_process_lock():
|
||||||
|
pytest.fail("symlinked runtime root must not be used")
|
||||||
|
|
||||||
|
assert list(target.iterdir()) == []
|
||||||
@@ -0,0 +1,721 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import http.client
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import types
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from sqlmodel import Session, SQLModel, create_engine
|
||||||
|
|
||||||
|
from backend.app.core.time import utc_now
|
||||||
|
from backend.app.db.models import DeploymentJob
|
||||||
|
from backend.app.schemas.forge_runtime_readiness import (
|
||||||
|
ForgeRuntimeReadinessResultV1,
|
||||||
|
ForgeRuntimeReadinessV1,
|
||||||
|
)
|
||||||
|
from backend.app.services import user_access
|
||||||
|
|
||||||
|
_REAL_CURRENT_USER = user_access.current_user
|
||||||
|
_JOB_ID = "forge-install-" + "a" * 64
|
||||||
|
_REQUEST = "b" * 64
|
||||||
|
_REVISION = "c" * 40
|
||||||
|
_WORKSPACE_DIGEST = "sha256:" + "d" * 64
|
||||||
|
_FRONTDOOR_DIGEST = "sha256:" + "e" * 64
|
||||||
|
_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
|
||||||
|
_ARTIFACT = {
|
||||||
|
"controllerVersion": "1.2.3",
|
||||||
|
"sourceRepository": "jens/unraid-appops-gateway",
|
||||||
|
"sourceRevision": "f" * 40,
|
||||||
|
"sourceTree": "1" * 40,
|
||||||
|
"artifactSha256": "2" * 64,
|
||||||
|
}
|
||||||
|
_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def _profile() -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"api_version": "itworx.forge-first-install/v1",
|
||||||
|
"revision": _REVISION,
|
||||||
|
"workspace_image_digest": _WORKSPACE_DIGEST,
|
||||||
|
"frontdoor_image_digest": _FRONTDOOR_DIGEST,
|
||||||
|
"preflight_job_id": "forge-preflight-" + "3" * 16,
|
||||||
|
"preflight_sha256": "4" * 64,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_payload(*, nonce: str = "5" * 32, observed_at: int | None = None) -> dict:
|
||||||
|
return {
|
||||||
|
"apiVersion": "itworx.forge-runtime-readiness/v1",
|
||||||
|
"status": "ready",
|
||||||
|
"repository": "jens/itworx-forge",
|
||||||
|
"operation": "forge_first_install",
|
||||||
|
"request": _REQUEST,
|
||||||
|
"profile": {
|
||||||
|
"revision": _REVISION,
|
||||||
|
"workspaceImageDigest": _WORKSPACE_DIGEST,
|
||||||
|
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
|
||||||
|
},
|
||||||
|
"observedAt": int(datetime.now(timezone.utc).timestamp()) if observed_at is None else observed_at,
|
||||||
|
"nonce": nonce,
|
||||||
|
"health": {
|
||||||
|
"status": "healthy",
|
||||||
|
"services": [{"name": name, "status": "healthy"} for name in _SERVICES],
|
||||||
|
},
|
||||||
|
"migrationCompleted": True,
|
||||||
|
"controllerArtifact": dict(_ARTIFACT),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_schema_requires_exact_service_set_and_aggregate_consistency():
|
||||||
|
parsed = ForgeRuntimeReadinessV1.model_validate(_runtime_payload())
|
||||||
|
assert parsed.status == "ready"
|
||||||
|
|
||||||
|
unavailable_child = _runtime_payload()
|
||||||
|
unavailable_child["health"]["services"][0]["status"] = "unavailable"
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ForgeRuntimeReadinessV1.model_validate(unavailable_child)
|
||||||
|
|
||||||
|
duplicate_service = _runtime_payload()
|
||||||
|
duplicate_service["health"]["services"][0]["name"] = "worker"
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ForgeRuntimeReadinessV1.model_validate(duplicate_service)
|
||||||
|
|
||||||
|
boolean_timestamp = _runtime_payload()
|
||||||
|
boolean_timestamp["observedAt"] = True
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ForgeRuntimeReadinessV1.model_validate(boolean_timestamp)
|
||||||
|
|
||||||
|
unexpected_field = _runtime_payload()
|
||||||
|
unexpected_field["diagnostic"] = "must never be projected"
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
ForgeRuntimeReadinessV1.model_validate(unexpected_field)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("status", ["ready", "not_ready"])
|
||||||
|
def test_public_readiness_requires_bound_identity_for_observed_results(status):
|
||||||
|
result = {
|
||||||
|
"status": status,
|
||||||
|
"message": f"forge_runtime_{status}",
|
||||||
|
"health": {
|
||||||
|
"status": "healthy",
|
||||||
|
"services": [{"name": name, "status": "healthy"} for name in _SERVICES],
|
||||||
|
},
|
||||||
|
"migrationCompleted": status == "ready",
|
||||||
|
}
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="identity is incomplete"):
|
||||||
|
ForgeRuntimeReadinessResultV1.model_validate(result)
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_readiness_allows_anonymous_fixed_unavailable_projection():
|
||||||
|
result = ForgeRuntimeReadinessResultV1.model_validate(
|
||||||
|
{
|
||||||
|
"status": "unavailable",
|
||||||
|
"message": "forge_runtime_readiness_unavailable",
|
||||||
|
"health": {
|
||||||
|
"status": "unavailable",
|
||||||
|
"services": [
|
||||||
|
{"name": name, "status": "unavailable"} for name in _SERVICES
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"migrationCompleted": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "unavailable"
|
||||||
|
assert result.job_id is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_broker_sends_one_fixed_fresh_hmac_request(monkeypatch):
|
||||||
|
from backend.app.services import deployment_broker
|
||||||
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
||||||
|
|
||||||
|
observed = {}
|
||||||
|
secret = "s" * 48
|
||||||
|
nonce = "6" * 32
|
||||||
|
payload = _runtime_payload(nonce=nonce, observed_at=2000)
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
monotonic_now = [0.0]
|
||||||
|
monkeypatch.setattr(deployment_broker.time, "time", lambda: 2000)
|
||||||
|
monkeypatch.setattr(deployment_broker.time, "monotonic", lambda: monotonic_now[0])
|
||||||
|
monkeypatch.setattr(
|
||||||
|
deployment_broker.uuid,
|
||||||
|
"uuid4",
|
||||||
|
lambda: types.SimpleNamespace(hex=nonce),
|
||||||
|
)
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
status_code = 200
|
||||||
|
headers = httpx.Headers(
|
||||||
|
{"content-length": str(len(body)), "content-type": "application/json"}
|
||||||
|
)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def iter_raw(self):
|
||||||
|
yield body
|
||||||
|
|
||||||
|
class Client:
|
||||||
|
def __init__(self, *, timeout, verify, follow_redirects):
|
||||||
|
observed["client"] = (timeout, verify, follow_redirects)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def stream(self, method, url, *, content, headers):
|
||||||
|
observed.update(method=method, url=url, content=content, headers=headers)
|
||||||
|
# The fixed controller may spend fifteen seconds completing its
|
||||||
|
# six-container snapshot before it can send response headers.
|
||||||
|
monotonic_now[0] = 16.0
|
||||||
|
return Response()
|
||||||
|
|
||||||
|
monkeypatch.setattr(deployment_broker.httpx, "Client", Client)
|
||||||
|
broker = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True)
|
||||||
|
|
||||||
|
result = broker.forge_bootstrap_readiness(_REQUEST, secret=secret)
|
||||||
|
|
||||||
|
path = f"/v1/appops/forge/bootstrap/readiness/{_REQUEST}"
|
||||||
|
assert result == payload
|
||||||
|
assert isinstance(observed["client"][0], httpx.Timeout)
|
||||||
|
assert observed["client"][0].read == 20.0
|
||||||
|
assert monotonic_now[0] == 16.0
|
||||||
|
assert observed["client"][1:] == (True, False)
|
||||||
|
assert observed["method"] == "GET"
|
||||||
|
assert observed["url"] == f"https://controller.example{path}"
|
||||||
|
assert observed["content"] == b""
|
||||||
|
assert observed["headers"]["Accept-Encoding"] == "identity"
|
||||||
|
assert observed["headers"]["Cache-Control"] == "no-store"
|
||||||
|
assert observed["headers"]["Pragma"] == "no-cache"
|
||||||
|
assert observed["headers"]["X-ITWorx-Deployment-Signature"] == broker._signature(
|
||||||
|
secret,
|
||||||
|
"GET",
|
||||||
|
path,
|
||||||
|
"2000",
|
||||||
|
nonce,
|
||||||
|
b"",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"failure",
|
||||||
|
[
|
||||||
|
"redirect", "stale", "wrong_nonce", "duplicate", "boolean_timestamp",
|
||||||
|
"future", "oversized", "long_length", "stream_oversized", "slow_drip", "invalid_utf8",
|
||||||
|
"deeply_nested",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_readiness_broker_fails_closed_on_unfresh_or_ambiguous_response(
|
||||||
|
monkeypatch, failure
|
||||||
|
):
|
||||||
|
from backend.app.services import deployment_broker
|
||||||
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
||||||
|
|
||||||
|
nonce = "7" * 32
|
||||||
|
current_time = 5000
|
||||||
|
payload = _runtime_payload(nonce=nonce, observed_at=current_time)
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
status_code = 302 if failure == "redirect" else 200
|
||||||
|
if failure == "stale":
|
||||||
|
payload["observedAt"] = current_time - 31
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
elif failure == "wrong_nonce":
|
||||||
|
payload["nonce"] = "8" * 32
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
elif failure == "duplicate":
|
||||||
|
body = body[:-1] + b',"status":"ready"}'
|
||||||
|
elif failure == "boolean_timestamp":
|
||||||
|
payload["observedAt"] = True
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
elif failure == "future":
|
||||||
|
payload["observedAt"] = current_time + 6
|
||||||
|
body = json.dumps(payload, separators=(",", ":")).encode()
|
||||||
|
content_length = str(len(body))
|
||||||
|
if failure == "oversized":
|
||||||
|
content_length = str(32 * 1024 + 1)
|
||||||
|
elif failure == "long_length":
|
||||||
|
content_length = "9" * 11
|
||||||
|
elif failure == "stream_oversized":
|
||||||
|
content_length = None
|
||||||
|
body = b"x" * (32 * 1024 + 1)
|
||||||
|
elif failure == "invalid_utf8":
|
||||||
|
body = b"\xff"
|
||||||
|
elif failure == "deeply_nested":
|
||||||
|
body = b"[" * 1200 + b"]" * 1200
|
||||||
|
monkeypatch.setattr(deployment_broker.time, "time", lambda: current_time)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
deployment_broker.uuid,
|
||||||
|
"uuid4",
|
||||||
|
lambda: types.SimpleNamespace(hex=nonce),
|
||||||
|
)
|
||||||
|
if failure == "slow_drip":
|
||||||
|
clock_values = iter((0.0, 0.0, 36.0))
|
||||||
|
monkeypatch.setattr(deployment_broker.time, "monotonic", lambda: next(clock_values))
|
||||||
|
|
||||||
|
response_headers = {"content-type": "application/json"}
|
||||||
|
if content_length is not None:
|
||||||
|
response_headers["content-length"] = content_length
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
headers = httpx.Headers(response_headers)
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.status_code = status_code
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def iter_raw(self):
|
||||||
|
if failure == "slow_drip":
|
||||||
|
yield body[:8]
|
||||||
|
yield body[8:]
|
||||||
|
else:
|
||||||
|
yield body
|
||||||
|
|
||||||
|
class Client:
|
||||||
|
def __init__(self, **_kwargs):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def stream(self, *_args, **_kwargs):
|
||||||
|
return Response()
|
||||||
|
|
||||||
|
monkeypatch.setattr(deployment_broker.httpx, "Client", Client)
|
||||||
|
result = DeploymentBrokerClient(
|
||||||
|
base_url="https://controller.example", verify_tls=True
|
||||||
|
).forge_bootstrap_readiness(_REQUEST, secret="s" * 48)
|
||||||
|
assert result == {"status": "unavailable"}
|
||||||
|
|
||||||
|
|
||||||
|
class _ReadinessBroker:
|
||||||
|
def __init__(self, response):
|
||||||
|
self.response = response
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def forge_bootstrap_readiness(self, request_id, *, secret):
|
||||||
|
self.calls.append((request_id, secret))
|
||||||
|
return self.response
|
||||||
|
|
||||||
|
|
||||||
|
def _deployed_job(session, *, overrides=None):
|
||||||
|
plan = _profile()
|
||||||
|
if overrides:
|
||||||
|
plan.update(overrides)
|
||||||
|
canonical_plan = json.dumps(plan, sort_keys=True, separators=(",", ":"))
|
||||||
|
historical = {
|
||||||
|
"apiVersion": "itworx.forge-first-install/v1",
|
||||||
|
"status": "deployed",
|
||||||
|
"request": _REQUEST,
|
||||||
|
"repository": "jens/itworx-forge",
|
||||||
|
"revision": _REVISION,
|
||||||
|
"operation": "forge_first_install",
|
||||||
|
"workspaceImageDigest": _WORKSPACE_DIGEST,
|
||||||
|
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
|
||||||
|
"receipt": {
|
||||||
|
"revision": _REVISION,
|
||||||
|
"workspaceImageDigest": _WORKSPACE_DIGEST,
|
||||||
|
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
|
||||||
|
"migration": {"service": "migrate", "status": "completed", "result": "succeeded"},
|
||||||
|
"health": {"status": "healthy", "services": list(_SERVICES)},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
row = DeploymentJob(
|
||||||
|
id=_JOB_ID,
|
||||||
|
application_id="itworx-forge",
|
||||||
|
repository="jens/itworx-forge",
|
||||||
|
revision=_REVISION,
|
||||||
|
plan_sha256=hashlib.sha256(canonical_plan.encode()).hexdigest(),
|
||||||
|
idempotency_sha256="9" * 64,
|
||||||
|
status="deployed",
|
||||||
|
stage="controller_deployed",
|
||||||
|
plan_json=plan,
|
||||||
|
preflight_json={},
|
||||||
|
evidence_json={"profile": "forge-first-install", "controller": historical},
|
||||||
|
broker_request_id=_REQUEST,
|
||||||
|
result_summary="completed",
|
||||||
|
completed_at=utc_now(),
|
||||||
|
)
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
session.refresh(row)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def test_historical_deployed_receipt_alone_never_means_current_runtime_ready(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
from backend.app.services import forge_bootstrap_apply
|
||||||
|
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
|
||||||
|
|
||||||
|
engine = create_engine(f"sqlite:///{tmp_path / 'readiness-historical.db'}")
|
||||||
|
SQLModel.metadata.create_all(engine)
|
||||||
|
with Session(engine) as session:
|
||||||
|
_deployed_job(session)
|
||||||
|
secret_reads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
forge_bootstrap_apply,
|
||||||
|
"_controller_secret",
|
||||||
|
lambda _session: secret_reads.append(True) or "s" * 48,
|
||||||
|
)
|
||||||
|
broker = _ReadinessBroker(
|
||||||
|
{
|
||||||
|
"apiVersion": "itworx.forge-first-install/v1",
|
||||||
|
"status": "deployed",
|
||||||
|
"receipt": {"health": {"status": "healthy"}},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
result = ForgeBootstrapApplyService(broker=broker).current_readiness(
|
||||||
|
session, _JOB_ID
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "unavailable"
|
||||||
|
assert broker.calls == [(_REQUEST, "s" * 48)]
|
||||||
|
assert secret_reads == [True]
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_service_validates_current_identity_and_projects_no_nonce(tmp_path, monkeypatch):
|
||||||
|
from backend.app.services import forge_bootstrap_apply
|
||||||
|
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
|
||||||
|
|
||||||
|
engine = create_engine(f"sqlite:///{tmp_path / 'readiness-current.db'}")
|
||||||
|
SQLModel.metadata.create_all(engine)
|
||||||
|
with Session(engine) as session:
|
||||||
|
_deployed_job(session)
|
||||||
|
current_job = session.get(DeploymentJob, _JOB_ID)
|
||||||
|
from backend.app.schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
|
||||||
|
from backend.app.services.forge_bootstrap_apply import _classify_controller_result
|
||||||
|
|
||||||
|
parsed_profile = ForgeBootstrapApplyV1.model_validate(current_job.plan_json)
|
||||||
|
assert _classify_controller_result(
|
||||||
|
current_job.evidence_json["controller"],
|
||||||
|
request_id=_REQUEST,
|
||||||
|
profile=parsed_profile,
|
||||||
|
)[0] == "deployed"
|
||||||
|
assert ForgeRuntimeReadinessV1.model_validate(_runtime_payload()).status == "ready"
|
||||||
|
secret_reads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
forge_bootstrap_apply,
|
||||||
|
"_controller_secret",
|
||||||
|
lambda _session: secret_reads.append(True) or "s" * 48,
|
||||||
|
)
|
||||||
|
broker = _ReadinessBroker(_runtime_payload())
|
||||||
|
|
||||||
|
result = ForgeBootstrapApplyService(broker=broker).current_readiness(
|
||||||
|
session, _JOB_ID
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "ready"
|
||||||
|
assert result["jobId"] == _JOB_ID
|
||||||
|
assert result["controllerArtifact"] == _ARTIFACT
|
||||||
|
assert "nonce" not in result
|
||||||
|
assert broker.calls == [(_REQUEST, "s" * 48)]
|
||||||
|
assert secret_reads == [True]
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutation",
|
||||||
|
["request", "revision", "workspace_digest", "artifact", "aggregate", "extra"],
|
||||||
|
)
|
||||||
|
def test_readiness_service_rejects_unbound_or_malformed_current_response(
|
||||||
|
tmp_path, monkeypatch, mutation
|
||||||
|
):
|
||||||
|
from backend.app.services import forge_bootstrap_apply
|
||||||
|
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
|
||||||
|
|
||||||
|
engine = create_engine(f"sqlite:///{tmp_path / f'readiness-{mutation}.db'}")
|
||||||
|
SQLModel.metadata.create_all(engine)
|
||||||
|
response = _runtime_payload()
|
||||||
|
if mutation == "request":
|
||||||
|
response["request"] = "0" * 64
|
||||||
|
elif mutation == "revision":
|
||||||
|
response["profile"]["revision"] = "0" * 40
|
||||||
|
elif mutation == "workspace_digest":
|
||||||
|
response["profile"]["workspaceImageDigest"] = "sha256:" + "0" * 64
|
||||||
|
elif mutation == "artifact":
|
||||||
|
response["controllerArtifact"]["sourceRepository"] = "another/repository"
|
||||||
|
elif mutation == "aggregate":
|
||||||
|
response["health"]["services"][0]["status"] = "unavailable"
|
||||||
|
elif mutation == "extra":
|
||||||
|
response["upstreamReason"] = "must not be projected"
|
||||||
|
with Session(engine) as session:
|
||||||
|
_deployed_job(session)
|
||||||
|
monkeypatch.setattr(forge_bootstrap_apply, "_controller_secret", lambda _session: "s" * 48)
|
||||||
|
result = ForgeBootstrapApplyService(
|
||||||
|
broker=_ReadinessBroker(response)
|
||||||
|
).current_readiness(session, _JOB_ID)
|
||||||
|
assert result["status"] == "unavailable"
|
||||||
|
assert "upstreamReason" not in result
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_or_not_deployed_job_never_reads_controller_secret_or_calls_broker(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
from backend.app.services import forge_bootstrap_apply
|
||||||
|
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
|
||||||
|
|
||||||
|
engine = create_engine(f"sqlite:///{tmp_path / 'readiness-invalid.db'}")
|
||||||
|
SQLModel.metadata.create_all(engine)
|
||||||
|
secret_reads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
forge_bootstrap_apply,
|
||||||
|
"_controller_secret",
|
||||||
|
lambda _session: secret_reads.append(True) or "s" * 48,
|
||||||
|
)
|
||||||
|
broker = _ReadinessBroker(_runtime_payload())
|
||||||
|
with Session(engine) as session:
|
||||||
|
_deployed_job(session)
|
||||||
|
row = session.get(DeploymentJob, _JOB_ID)
|
||||||
|
row.status = "applying"
|
||||||
|
session.add(row)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
service = ForgeBootstrapApplyService(broker=broker)
|
||||||
|
invalid_format = service.current_readiness(session, "bad-job-id")
|
||||||
|
not_deployed = service.current_readiness(session, _JOB_ID)
|
||||||
|
absent = service.current_readiness(session, "forge-install-" + "0" * 64)
|
||||||
|
|
||||||
|
assert invalid_format["status"] == not_deployed["status"] == absent["status"] == "unavailable"
|
||||||
|
assert secret_reads == []
|
||||||
|
assert broker.calls == []
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def readiness_operator_client(tmp_path, monkeypatch):
|
||||||
|
from backend.app.core import config
|
||||||
|
from backend.app.db import session as db_session
|
||||||
|
from backend.app.db.init_db import init_db
|
||||||
|
from backend.app.main import app
|
||||||
|
|
||||||
|
engine = create_engine(
|
||||||
|
f"sqlite:///{tmp_path / 'readiness-api.db'}",
|
||||||
|
connect_args={"check_same_thread": False},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(db_session, "engine", engine)
|
||||||
|
monkeypatch.setenv("APPOPS_REQUIRE_OPERATOR_AUTH", "false")
|
||||||
|
monkeypatch.setattr(config.settings, "operator_proxy_auth_enabled", False)
|
||||||
|
monkeypatch.setattr(config.settings, "auth_admin_subjects", ["authentik:owner"])
|
||||||
|
monkeypatch.setattr(config.settings, "auth_admin_emails", [])
|
||||||
|
monkeypatch.setattr(config.settings, "data_dir", str(tmp_path))
|
||||||
|
monkeypatch.setattr(user_access, "current_user", _REAL_CURRENT_USER)
|
||||||
|
SQLModel.metadata.create_all(engine)
|
||||||
|
init_db()
|
||||||
|
client = TestClient(app, client=("198.18.10.20", 42000))
|
||||||
|
yield client
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def _sign_in_operator(client):
|
||||||
|
from backend.app.db import session as db_session
|
||||||
|
from backend.app.services.user_access import issue_session, provision_user
|
||||||
|
|
||||||
|
with Session(db_session.engine) as session:
|
||||||
|
user = provision_user(
|
||||||
|
session,
|
||||||
|
"authentik",
|
||||||
|
"https://auth.example/application/o/appops/",
|
||||||
|
{"sub": "owner", "email": "owner@example.invalid"},
|
||||||
|
)
|
||||||
|
cookie = issue_session(session, user)
|
||||||
|
client.cookies.set("appops_user_session", cookie)
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_route_requires_operator_and_rejects_query_without_runtime_call(
|
||||||
|
readiness_operator_client, monkeypatch
|
||||||
|
):
|
||||||
|
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
|
||||||
|
|
||||||
|
client = readiness_operator_client
|
||||||
|
path = f"/api/deployments/forge-bootstrap/first-install/{_JOB_ID}/readiness"
|
||||||
|
assert client.get(path).status_code == 401
|
||||||
|
_sign_in_operator(client)
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ForgeBootstrapApplyService,
|
||||||
|
"current_readiness",
|
||||||
|
lambda self, _session, _job_id: calls.append(True) or {"status": "unavailable"},
|
||||||
|
)
|
||||||
|
|
||||||
|
query = client.get(path + "?profile=caller-selected")
|
||||||
|
body = client.request(
|
||||||
|
"GET", path, content=b"{}", headers={"Content-Type": "application/json"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert query.status_code == 400
|
||||||
|
assert body.status_code == 400
|
||||||
|
assert calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_route_returns_fixed_unavailable_for_unknown_job_without_broker_call(
|
||||||
|
readiness_operator_client, monkeypatch
|
||||||
|
):
|
||||||
|
from backend.app.services import forge_bootstrap_apply
|
||||||
|
from backend.app.services.deployment_broker import DeploymentBrokerClient
|
||||||
|
|
||||||
|
client = readiness_operator_client
|
||||||
|
_sign_in_operator(client)
|
||||||
|
secret_reads = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
forge_bootstrap_apply,
|
||||||
|
"_controller_secret",
|
||||||
|
lambda _session: secret_reads.append(True) or "s" * 48,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
DeploymentBrokerClient,
|
||||||
|
"forge_bootstrap_readiness",
|
||||||
|
lambda *_args, **_kwargs: pytest.fail("unknown jobs must not contact the controller"),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = client.get(
|
||||||
|
"/api/deployments/forge-bootstrap/first-install/forge-install-"
|
||||||
|
+ "0" * 64
|
||||||
|
+ "/readiness"
|
||||||
|
)
|
||||||
|
invalid_path = client.get(
|
||||||
|
"/api/deployments/forge-bootstrap/first-install/not-a-job/readiness"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["status"] == "unavailable"
|
||||||
|
assert response.json()["message"] == "forge_runtime_readiness_unavailable"
|
||||||
|
assert invalid_path.status_code == 200
|
||||||
|
assert invalid_path.json()["status"] == "unavailable"
|
||||||
|
assert secret_reads == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def forge_status_module():
|
||||||
|
path = _ROOT / "ops" / "gitea-deploy-broker" / "status_server.py"
|
||||||
|
spec = importlib.util.spec_from_file_location("forge_runtime_readiness_status_test", path)
|
||||||
|
assert spec and spec.loader
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_controller_readiness_route_is_signed_replay_safe_and_uses_live_probe_only(
|
||||||
|
forge_status_module, tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
module = forge_status_module
|
||||||
|
secret = b"controller-readiness-secret-" + b"x" * 32
|
||||||
|
secret_root = tmp_path / "deployment-secrets"
|
||||||
|
secret_root.mkdir(mode=0o700)
|
||||||
|
secret_path = secret_root / module.FORGE_BOOTSTRAP_DEPLOYMENT_SECRET_NAME
|
||||||
|
secret_path.write_bytes(secret)
|
||||||
|
secret_path.chmod(0o600)
|
||||||
|
nonce_root = tmp_path / "nonces"
|
||||||
|
monkeypatch.setattr(module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root)
|
||||||
|
monkeypatch.setattr(module, "APPOPS_NONCE_ROOT", nonce_root)
|
||||||
|
request_id = "9" * 64
|
||||||
|
runtime_result = {
|
||||||
|
"status": "ready",
|
||||||
|
"request": request_id,
|
||||||
|
"repository": "jens/itworx-forge",
|
||||||
|
"operation": "forge_first_install",
|
||||||
|
"profile": {
|
||||||
|
"revision": _REVISION,
|
||||||
|
"workspaceImageDigest": _WORKSPACE_DIGEST,
|
||||||
|
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
|
||||||
|
},
|
||||||
|
"health": {
|
||||||
|
"status": "healthy",
|
||||||
|
"services": [{"name": name, "status": "healthy"} for name in _SERVICES],
|
||||||
|
},
|
||||||
|
"migrationCompleted": True,
|
||||||
|
}
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class Controller:
|
||||||
|
def runtime_readiness(self, supplied_request):
|
||||||
|
calls.append(supplied_request)
|
||||||
|
return runtime_result
|
||||||
|
|
||||||
|
def status(self, _supplied_request):
|
||||||
|
pytest.fail("readiness must not invoke the mutating status accessor")
|
||||||
|
|
||||||
|
release = {
|
||||||
|
**_ARTIFACT,
|
||||||
|
"apiVersion": "itworx.controller-release/v1",
|
||||||
|
"builtAt": "2026-10-04T00:00:00Z",
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(module, "_forge_first_install_controller", Controller())
|
||||||
|
monkeypatch.setattr(module, "_forge_controller_artifact_identity", lambda: release)
|
||||||
|
server = module.BoundedThreadingHTTPServer(("127.0.0.1", 0), module.Handler)
|
||||||
|
serving = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
serving.start()
|
||||||
|
|
||||||
|
def get_readiness(nonce: str):
|
||||||
|
path = f"{module.FORGE_BOOTSTRAP_READINESS_PREFIX}{request_id}"
|
||||||
|
timestamp = str(int(module.time.time()))
|
||||||
|
signature = module._request_signature(
|
||||||
|
secret,
|
||||||
|
method="GET",
|
||||||
|
path=path,
|
||||||
|
timestamp=timestamp,
|
||||||
|
nonce=nonce,
|
||||||
|
raw_body=b"",
|
||||||
|
)
|
||||||
|
connection = http.client.HTTPConnection(*server.server_address, timeout=3)
|
||||||
|
connection.request(
|
||||||
|
"GET",
|
||||||
|
path,
|
||||||
|
headers={
|
||||||
|
"X-ITWorx-Deployment-Timestamp": timestamp,
|
||||||
|
"X-ITWorx-Deployment-Nonce": nonce,
|
||||||
|
"X-ITWorx-Deployment-Signature": signature,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response = connection.getresponse()
|
||||||
|
status = response.status
|
||||||
|
payload = json.loads(response.read())
|
||||||
|
connection.close()
|
||||||
|
return status, payload
|
||||||
|
|
||||||
|
try:
|
||||||
|
status, payload = get_readiness("a" * 32)
|
||||||
|
assert status == 200
|
||||||
|
assert payload["apiVersion"] == "itworx.forge-runtime-readiness/v1"
|
||||||
|
assert payload["nonce"] == "a" * 32
|
||||||
|
assert payload["controllerArtifact"] == _ARTIFACT
|
||||||
|
assert set(payload) == {
|
||||||
|
"apiVersion", "status", "request", "repository", "operation", "profile",
|
||||||
|
"health", "migrationCompleted", "observedAt", "nonce", "controllerArtifact",
|
||||||
|
}
|
||||||
|
assert secret.decode() not in json.dumps(payload)
|
||||||
|
assert calls == [request_id]
|
||||||
|
|
||||||
|
replay_status, replay_payload = get_readiness("a" * 32)
|
||||||
|
assert replay_status == 403
|
||||||
|
assert replay_payload == {"status": "authentication_rejected"}
|
||||||
|
assert calls == [request_id]
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
server.server_close()
|
||||||
|
serving.join(timeout=2)
|
||||||
@@ -336,7 +336,7 @@ def _delivery_setup(tmp_path, monkeypatch, *, manual="allowed"):
|
|||||||
SQLModel.metadata.create_all(engine)
|
SQLModel.metadata.create_all(engine)
|
||||||
session = Session(engine)
|
session = Session(engine)
|
||||||
monkeypatch.setattr(settings, "environment", "production")
|
monkeypatch.setattr(settings, "environment", "production")
|
||||||
monkeypatch.setattr(settings, "secret_key", "delivery-test-secret-key-material-0123456789")
|
monkeypatch.setattr(settings, "secret_key", "test-key-" + "x" * 35)
|
||||||
monkeypatch.setattr(settings, "secret_key_source", "env")
|
monkeypatch.setattr(settings, "secret_key_source", "env")
|
||||||
for key, value in {
|
for key, value in {
|
||||||
"AUTHENTIK_OIDC_AUTHORIZATION_FLOW": "forge-authz-flow",
|
"AUTHENTIK_OIDC_AUTHORIZATION_FLOW": "forge-authz-flow",
|
||||||
|
|||||||
Reference in new issue
Block a user