Publish curated Unraid-AppOps-Gateway source from a480f280f26f4aa48c61394fa841d2233067c208

This commit is contained in:
Public Source Publisher committed 2026-10-04 11:55:06 +00:00
1 parent e73772cdf2
commit 820244dbb3
13 files changed
+2141 -26

No files matched your search

+1 -1
View File
@@ -1,3 +1,3 @@
# Public source export # Public source export
This source snapshot was generated from the private canonical repository at revision `7144ea3c4cb735745046e4e1c2d3a3e88fa1a302`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. This source snapshot was generated from the private canonical repository at revision `a480f280f26f4aa48c61394fa841d2233067c208`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
+32 -17
View File
@@ -1,6 +1,6 @@
{ {
"schemaVersion": 1, "schemaVersion": 1,
"sourceRevision": "7144ea3c4cb735745046e4e1c2d3a3e88fa1a302", "sourceRevision": "a480f280f26f4aa48c61394fa841d2233067c208",
"files": [ "files": [
{ {
"path": ".dockerignore", "path": ".dockerignore",
@@ -124,8 +124,8 @@
}, },
{ {
"path": "backend/app/api/routes_deployments.py", "path": "backend/app/api/routes_deployments.py",
"sha256": "c77cea58701694af4ef519e351fd27a68856e45cd4e61dc4028c652359d6392a", "sha256": "ff14b4e40e6e14d0d7f9f8fd74faf5001010a3ba5c0b279253c61f17199f350f",
"bytes": 7717 "bytes": 8604
}, },
{ {
"path": "backend/app/api/routes_enrollments.py", "path": "backend/app/api/routes_enrollments.py",
@@ -452,6 +452,11 @@
"sha256": "8d55f7a8263191f07813d1689eb0697dabec99a1bbd5aff5a704e66e6bf86c47", "sha256": "8d55f7a8263191f07813d1689eb0697dabec99a1bbd5aff5a704e66e6bf86c47",
"bytes": 6045 "bytes": 6045
}, },
{
"path": "backend/app/schemas/forge_runtime_readiness.py",
"sha256": "4571fd2c7b624fe4f12c219a1f494b03a98ec1745a9512197cf7c49287ce47b5",
"bytes": 6612
},
{ {
"path": "backend/app/schemas/forge_secret_delivery.py", "path": "backend/app/schemas/forge_secret_delivery.py",
"sha256": "5209fdfaabee4dbc56b7edd2f6080c35bbd3d679577a754dbd65ca353347b5e5", "sha256": "5209fdfaabee4dbc56b7edd2f6080c35bbd3d679577a754dbd65ca353347b5e5",
@@ -529,8 +534,8 @@
}, },
{ {
"path": "backend/app/services/deployment_broker.py", "path": "backend/app/services/deployment_broker.py",
"sha256": "34d6a4bf90d59604d51d34bc4ba7885ac6f5837d7015c80c8c0fe7eee9c060d4", "sha256": "e23b2a0669c846cf37c44766700d29e5d2dfa22b6b6d0d546fe5c94bb86f4c6b",
"bytes": 33010 "bytes": 37749
}, },
{ {
"path": "backend/app/services/deployment_control_plane.py", "path": "backend/app/services/deployment_control_plane.py",
@@ -564,8 +569,8 @@
}, },
{ {
"path": "backend/app/services/forge_bootstrap_apply.py", "path": "backend/app/services/forge_bootstrap_apply.py",
"sha256": "4928e08a887094efb9134fc2c6a40b707d5635c0f26d202c93773ba132b61f69", "sha256": "612e7537577c8ac97c0d7f2ae76c0c5099cc78f7661d4dfb62a81b1602d775db",
"bytes": 37394 "bytes": 42301
}, },
{ {
"path": "backend/app/services/forge_bootstrap_preflight.py", "path": "backend/app/services/forge_bootstrap_preflight.py",
@@ -934,8 +939,8 @@
}, },
{ {
"path": "backend/tests/test_deploy_broker.py", "path": "backend/tests/test_deploy_broker.py",
"sha256": "9aaefa351db5868436a6dc06ed7c42202e8d1a8568bcaa4d71501ec60446bb85", "sha256": "7e32918a89dcb3d02cc75674fcd9313b60cf96ed3651a5016bbfb3fc751c3ec3",
"bytes": 381318 "bytes": 406062
}, },
{ {
"path": "backend/tests/test_deployment_auto_rollback.py", "path": "backend/tests/test_deployment_auto_rollback.py",
@@ -984,8 +989,8 @@
}, },
{ {
"path": "backend/tests/test_forge_bootstrap_apply.py", "path": "backend/tests/test_forge_bootstrap_apply.py",
"sha256": "f7be48b418a9a09fbbaea7f6e1765c7ba4bac949994435b423885c3b4b2c6d8e", "sha256": "6486b6feed952e190974aacef13f9ce948e1caf15a71ca74e330df69cc30c404",
"bytes": 43558 "bytes": 43562
}, },
{ {
"path": "backend/tests/test_forge_bootstrap_controller.py", "path": "backend/tests/test_forge_bootstrap_controller.py",
@@ -1009,24 +1014,34 @@
}, },
{ {
"path": "backend/tests/test_forge_first_install_controller.py", "path": "backend/tests/test_forge_first_install_controller.py",
"sha256": "9c4abc31c564e2974b125ce0c0f9bd581af499ab70dcc2b21dc2824a331522f2", "sha256": "4784e7f08a08f15ce6e7efb1ba37ae9a5bfe311478842e1babe273d34ecb39eb",
"bytes": 84984 "bytes": 93989
}, },
{ {
"path": "backend/tests/test_forge_oidc_api.py", "path": "backend/tests/test_forge_oidc_api.py",
"sha256": "4d902a3bb967de92156019d00ca65cfb7120935f669c61d255cab630b3081100", "sha256": "f083c07f3de32de5552ab999677004ae1654d10a3540e9e1bf5fbaa2e6371d5c",
"bytes": 11095 "bytes": 11070
}, },
{ {
"path": "backend/tests/test_forge_oidc_provisioning.py", "path": "backend/tests/test_forge_oidc_provisioning.py",
"sha256": "a0fff62f8855dafebd49e2d91abdf8f211b7a4df452c05d91ed8305fe74d3b0a", "sha256": "a0fff62f8855dafebd49e2d91abdf8f211b7a4df452c05d91ed8305fe74d3b0a",
"bytes": 82116 "bytes": 82116
}, },
{
"path": "backend/tests/test_forge_oidc_runtime_lock.py",
"sha256": "c71d405e5ddf75f1597660accb77329f1f7efe485e0dd4ef1c80a503b978f812",
"bytes": 7421
},
{ {
"path": "backend/tests/test_forge_reader_api.py", "path": "backend/tests/test_forge_reader_api.py",
"sha256": "dc4e9d176a9b4c33fd24e27b2cb83919040babe412d021109a22de41c54edb8b", "sha256": "dc4e9d176a9b4c33fd24e27b2cb83919040babe412d021109a22de41c54edb8b",
"bytes": 6424 "bytes": 6424
}, },
{
"path": "backend/tests/test_forge_runtime_readiness.py",
"sha256": "0f70e327225eceae2ad161e565c9f601c191e3b086a989c7383ac7327434926d",
"bytes": 25834
},
{ {
"path": "backend/tests/test_forge_secret_bundle.py", "path": "backend/tests/test_forge_secret_bundle.py",
"sha256": "ee3555f9711b23005f8a2970537217ffed9b06b1fbe96424180a26c379440cd2", "sha256": "ee3555f9711b23005f8a2970537217ffed9b06b1fbe96424180a26c379440cd2",
@@ -1034,8 +1049,8 @@
}, },
{ {
"path": "backend/tests/test_forge_secret_delivery.py", "path": "backend/tests/test_forge_secret_delivery.py",
"sha256": "29f138ab1df97248cd3393aac84e50967e4cf10b2194a8db03820dd7973a30fb", "sha256": "a66da8d2feb600575e834fcdf8055aac1c4e2a26ba89fde0ed7f682d6170a670",
"bytes": 37715 "bytes": 37691
}, },
{ {
"path": "backend/tests/test_forge_secret_delivery_controller.py", "path": "backend/tests/test_forge_secret_delivery_controller.py",
+24
View File
@@ -21,6 +21,7 @@ from ..schemas.forge_secret_delivery import (
ForgeSecretDeliveryRequestV1, ForgeSecretDeliveryRequestV1,
ForgeSecretDeliveryResultV1, ForgeSecretDeliveryResultV1,
) )
from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessResultV1
from ..services.deployment_broker import DeploymentBrokerClient from ..services.deployment_broker import DeploymentBrokerClient
from ..services.deployment_control_plane import ( from ..services.deployment_control_plane import (
DeploymentControlError, DeploymentControlError,
@@ -208,6 +209,29 @@ def get_forge_first_install(job_id: str, session: DeploymentSession):
return result return result
@router.get(
"/forge-bootstrap/first-install/{job_id}/readiness",
response_model=ForgeRuntimeReadinessResultV1,
)
def get_forge_first_install_readiness(
job_id: str,
request: Request,
session: DeploymentSession,
):
"""Return only a fresh, fixed controller observation for a known install."""
content_lengths = request.headers.getlist("content-length")
if (
request.query_params
or len(content_lengths) > 1
or (content_lengths and content_lengths[0] != "0")
or request.headers.getlist("transfer-encoding")
):
raise HTTPException(status_code=400, detail="invalid_readiness_request")
from ..services.forge_bootstrap_apply import ForgeBootstrapApplyService
return ForgeBootstrapApplyService().current_readiness(session, job_id)
@router.get("") @router.get("")
def list_deployments(session: DeploymentSession, limit: DeploymentLimit = 100): def list_deployments(session: DeploymentSession, limit: DeploymentLimit = 100):
return {"items": DeploymentControlPlane().list_jobs(session, limit=limit)} return {"items": DeploymentControlPlane().list_jobs(session, limit=limit)}
@@ -0,0 +1,157 @@
from __future__ import annotations
import re
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictInt, StrictStr, model_validator
from .forge_secret_delivery import ForgeSecretDeliveryControllerArtifactV1
_REVISION = re.compile(r"^[0-9a-f]{40}$")
_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
_REQUEST = re.compile(r"^[0-9a-f]{64}$")
_NONCE = re.compile(r"^[0-9a-f]{32}$")
_JOB = re.compile(r"^forge-install-[0-9a-f]{64}$")
_RUNTIME_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
class ForgeRuntimeReadinessProfileV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
revision: StrictStr = Field(pattern=_REVISION.pattern)
workspace_image_digest: StrictStr = Field(alias="workspaceImageDigest", pattern=_DIGEST.pattern)
frontdoor_image_digest: StrictStr = Field(alias="frontdoorImageDigest", pattern=_DIGEST.pattern)
class ForgeRuntimeServiceHealthV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
name: Literal["api", "dashboard", "itworx-forge", "postgres", "worker"]
status: Literal["healthy", "not_ready", "unavailable"]
class ForgeRuntimeHealthV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
status: Literal["healthy", "not_ready", "unavailable"]
services: list[ForgeRuntimeServiceHealthV1]
@model_validator(mode="after")
def fixed_service_set(self) -> ForgeRuntimeHealthV1:
if tuple(item.name for item in self.services) != _RUNTIME_SERVICES:
raise ValueError("runtime service set is invalid")
return self
class ForgeRuntimeReadinessV1(BaseModel):
"""Strict controller response for one fresh, fixed Forge observation."""
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True, strict=True)
api_version: Literal["itworx.forge-runtime-readiness/v1"] = Field(alias="apiVersion")
status: Literal["ready", "not_ready", "unavailable"]
repository: Literal["jens/itworx-forge"]
operation: Literal["forge_first_install"]
request: StrictStr = Field(pattern=_REQUEST.pattern)
profile: ForgeRuntimeReadinessProfileV1
observed_at: StrictInt = Field(alias="observedAt", ge=0)
nonce: StrictStr = Field(pattern=_NONCE.pattern)
health: ForgeRuntimeHealthV1
migration_completed: StrictBool = Field(alias="migrationCompleted")
controller_artifact: ForgeSecretDeliveryControllerArtifactV1 = Field(alias="controllerArtifact")
@model_validator(mode="after")
def readiness_consistency(self) -> ForgeRuntimeReadinessV1:
service_statuses = {item.status for item in self.health.services}
expected_health = (
"unavailable"
if "unavailable" in service_statuses
else "not_ready"
if "not_ready" in service_statuses
else "healthy"
)
valid_overall = (
self.status == "unavailable"
if expected_health == "unavailable"
else self.status in {"not_ready", "unavailable"}
if expected_health == "not_ready"
else self.status == "ready"
if self.migration_completed
else self.status in {"not_ready", "unavailable"}
)
if self.health.status != expected_health or not valid_overall:
raise ValueError("readiness status does not match current health proof")
return self
class ForgeRuntimeReadinessResultV1(BaseModel):
"""Bounded operator projection; upstream diagnostics and nonce are omitted."""
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True, strict=True)
status: Literal["ready", "not_ready", "unavailable"]
message: Literal[
"forge_runtime_ready",
"forge_runtime_not_ready",
"forge_runtime_readiness_unavailable",
]
target: Literal["itworx-forge"] = "itworx-forge"
job_id: StrictStr | None = Field(default=None, alias="jobId", pattern=_JOB.pattern)
revision: StrictStr | None = Field(default=None, pattern=_REVISION.pattern)
workspace_image_digest: StrictStr | None = Field(
default=None, alias="workspaceImageDigest", pattern=_DIGEST.pattern
)
frontdoor_image_digest: StrictStr | None = Field(
default=None, alias="frontdoorImageDigest", pattern=_DIGEST.pattern
)
observed_at: StrictInt | None = Field(default=None, alias="observedAt", ge=0)
health: ForgeRuntimeHealthV1
migration_completed: StrictBool = Field(default=False, alias="migrationCompleted")
controller_artifact: ForgeSecretDeliveryControllerArtifactV1 | None = Field(
default=None, alias="controllerArtifact"
)
@model_validator(mode="after")
def public_status_message(self) -> ForgeRuntimeReadinessResultV1:
expected = {
"ready": "forge_runtime_ready",
"not_ready": "forge_runtime_not_ready",
"unavailable": "forge_runtime_readiness_unavailable",
}[self.status]
if self.message != expected:
raise ValueError("readiness message does not match status")
service_statuses = {item.status for item in self.health.services}
expected_health = (
"unavailable"
if "unavailable" in service_statuses
else "not_ready"
if "not_ready" in service_statuses
else "healthy"
)
valid_overall = (
self.status == "unavailable"
if expected_health == "unavailable"
else self.status in {"not_ready", "unavailable"}
if expected_health == "not_ready"
else self.status == "ready"
if self.migration_completed
else self.status in {"not_ready", "unavailable"}
)
if self.health.status != expected_health or not valid_overall:
raise ValueError("public readiness is inconsistent")
identity_fields = (
self.job_id,
self.revision,
self.workspace_image_digest,
self.frontdoor_image_digest,
self.observed_at,
self.controller_artifact,
)
if self.job_id is None and (
self.status != "unavailable"
or any(value is not None for value in identity_fields[1:])
):
raise ValueError("public readiness identity is incomplete")
if self.job_id is not None and any(value is None for value in identity_fields):
raise ValueError("public readiness identity is incomplete")
return self
+106 -1
View File
@@ -29,6 +29,7 @@ from ..core.redaction import redact_message
from ..schemas.deployment import DeploymentPlanV1 from ..schemas.deployment import DeploymentPlanV1
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1 from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
from ..schemas.forge_runtime_readiness import ForgeRuntimeReadinessV1
_REQUEST_ID = re.compile(r"^[0-9a-f]{64}$") _REQUEST_ID = re.compile(r"^[0-9a-f]{64}$")
_MAX_DELIVERY_STATUS_BYTES = 512 * 1024 _MAX_DELIVERY_STATUS_BYTES = 512 * 1024
@@ -40,6 +41,10 @@ _WORKMESH_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
_FORGE_SECRET_DELIVERY_PATH = "/v1/appops/forge/bootstrap/secrets" _FORGE_SECRET_DELIVERY_PATH = "/v1/appops/forge/bootstrap/secrets"
_MAX_FORGE_SECRET_DELIVERY_BODY_BYTES = 16 * 1024 _MAX_FORGE_SECRET_DELIVERY_BODY_BYTES = 16 * 1024
_MAX_FORGE_SECRET_DELIVERY_RESPONSE_BYTES = 64 * 1024 _MAX_FORGE_SECRET_DELIVERY_RESPONSE_BYTES = 64 * 1024
_FORGE_RUNTIME_READINESS_PATH = "/v1/appops/forge/bootstrap/readiness/"
_MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES = 32 * 1024
_FORGE_RUNTIME_READINESS_TOTAL_SECONDS = 35.0
_FORGE_RUNTIME_READINESS_READ_SECONDS = 20.0
def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
@@ -466,7 +471,7 @@ class DeploymentBrokerClient:
bytes(response_body).decode("utf-8"), bytes(response_body).decode("utf-8"),
object_pairs_hook=_unique_json_object, object_pairs_hook=_unique_json_object,
) )
except (UnicodeError, ValueError, TypeError): except (UnicodeError, ValueError, TypeError, RecursionError):
response_body.clear() response_body.clear()
return { return {
"status": "outcome_unknown", "status": "outcome_unknown",
@@ -508,6 +513,106 @@ class DeploymentBrokerClient:
secret=secret, secret=secret,
) )
def forge_bootstrap_readiness(
self, request_id: str, *, secret: str
) -> dict[str, Any]:
"""Fetch one fresh fixed-profile runtime snapshot with no receipt fallback."""
if not _REQUEST_ID.fullmatch(request_id):
return {"status": "unavailable"}
if (
not self.is_configured()
or not isinstance(secret, str)
or len(secret) < 32
or not secret.isascii()
or not secret.isprintable()
):
return {"status": "unavailable"}
timestamp = str(int(time.time()))
nonce = uuid.uuid4().hex
path = f"{_FORGE_RUNTIME_READINESS_PATH}{request_id}"
headers = {
"Accept": "application/json",
"Accept-Encoding": "identity",
"Cache-Control": "no-store",
"Pragma": "no-cache",
"X-ITWorx-Deployment-Timestamp": timestamp,
"X-ITWorx-Deployment-Nonce": nonce,
"X-ITWorx-Deployment-Signature": self._signature(
secret, "GET", path, timestamp, nonce, b""
),
}
response_body = bytearray()
deadline = time.monotonic() + _FORGE_RUNTIME_READINESS_TOTAL_SECONDS
client_timeout = httpx.Timeout(
connect=min(self.timeout_seconds, 5.0),
read=_FORGE_RUNTIME_READINESS_READ_SECONDS,
write=min(self.timeout_seconds, 5.0),
pool=min(self.timeout_seconds, 5.0),
)
try:
with httpx.Client(
timeout=client_timeout,
verify=self._verify_option(),
follow_redirects=False,
) as client, client.stream(
"GET",
f"{self.base_url}{path}",
content=b"",
headers=headers,
) as response:
if response.status_code != 200:
return {"status": "unavailable"}
content_lengths = response.headers.get_list("content-length")
if len(content_lengths) > 1:
return {"status": "unavailable"}
content_length = content_lengths[0] if content_lengths else None
if content_length is not None and (
not content_length.isascii()
or not content_length.isdecimal()
or len(content_length) > 10
or int(content_length) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES
):
return {"status": "unavailable"}
content_encoding = response.headers.get("content-encoding", "identity").lower()
content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_encoding != "identity" or content_type != "application/json":
return {"status": "unavailable"}
for chunk in response.iter_raw():
if time.monotonic() > deadline:
response_body.clear()
return {"status": "unavailable"}
response_body.extend(chunk)
if len(response_body) > _MAX_FORGE_RUNTIME_READINESS_RESPONSE_BYTES:
response_body.clear()
return {"status": "unavailable"}
except Exception: # noqa: BLE001 - transport diagnostics may contain held credentials.
response_body.clear()
return {"status": "unavailable"}
if time.monotonic() > deadline:
response_body.clear()
return {"status": "unavailable"}
try:
decoded = json.loads(
bytes(response_body).decode("utf-8", errors="strict"),
object_pairs_hook=_unique_json_object,
)
validated = ForgeRuntimeReadinessV1.model_validate(decoded)
except (UnicodeError, ValueError, TypeError, RecursionError):
response_body.clear()
return {"status": "unavailable"}
response_body.clear()
now = int(time.time())
if (
validated.request != request_id
or validated.nonce != nonce
or validated.observed_at < now - 30
or validated.observed_at > now + 5
):
return {"status": "unavailable"}
return validated.model_dump(mode="json", by_alias=True)
def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]: def enrollment_capabilities(self, *, secret: str) -> dict[str, Any]:
"""Read the bootstrap-protected enrollment controller capability.""" """Read the bootstrap-protected enrollment controller capability."""
return self._request( return self._request(
+113 -1
View File
@@ -4,7 +4,7 @@ import base64
import hashlib import hashlib
import json import json
import re import re
from datetime import datetime from datetime import UTC, datetime
from typing import Any from typing import Any
from sqlalchemy import update from sqlalchemy import update
@@ -17,6 +17,10 @@ from ..db.models import DeploymentJob, Policy
from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT from ..schemas.enrollment import MANAGED_DOCKER_APPDATA_ROOT
from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1 from ..schemas.forge_bootstrap import ForgeBootstrapPreflightV1
from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1 from ..schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
from ..schemas.forge_runtime_readiness import (
ForgeRuntimeReadinessResultV1,
ForgeRuntimeReadinessV1,
)
from .audit_logger import log_event from .audit_logger import log_event
from .deployment_broker import DeploymentBrokerClient from .deployment_broker import DeploymentBrokerClient
from .enrollment_plane import EnrollmentPlane from .enrollment_plane import EnrollmentPlane
@@ -83,6 +87,7 @@ _PROFILE_CHECKS = {
_FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"} _FAILED_CODES = {"migration_failed", "service_start_failed", "healthcheck_failed"}
_ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"} _ACTIVE_STATUSES = {"applying", "outcome_unknown", "needs_database_credentials"}
_TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"} _TERMINAL_STATUSES = {"deployed", "failed", "preflight_changed"}
_RUNTIME_READINESS_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
class ForgeBootstrapApplyError(ValueError): class ForgeBootstrapApplyError(ValueError):
@@ -715,6 +720,113 @@ class ForgeBootstrapApplyService:
return None return None
return self._refresh_job(session, job) return self._refresh_job(session, job)
@staticmethod
def _runtime_readiness_unavailable() -> dict[str, Any]:
return ForgeRuntimeReadinessResultV1.model_validate(
{
"status": "unavailable",
"message": "forge_runtime_readiness_unavailable",
"health": {
"status": "unavailable",
"services": [
{"name": name, "status": "unavailable"}
for name in _RUNTIME_READINESS_SERVICES
],
},
"migrationCompleted": False,
}
).model_dump(mode="json", by_alias=True, exclude_none=True)
def current_readiness(self, session: Session, job_id: str) -> dict[str, Any]:
"""Read current controller state only for a verified successful Forge job."""
unavailable = self._runtime_readiness_unavailable()
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
return unavailable
try:
job = session.get(DeploymentJob, job_id)
except Exception: # noqa: BLE001 - do not expose protected store/DB failures.
return unavailable
if (
job is None
or job.application_id != _APPLICATION_ID
or job.repository != _REPOSITORY
or job.status != "deployed"
or job.stage not in {"controller_deployed", "resume_controller_deployed"}
or job.completed_at is None
or not re.fullmatch(r"[0-9a-f]{64}", job.broker_request_id or "")
):
return unavailable
try:
profile = ForgeBootstrapApplyV1.model_validate(job.plan_json)
except (TypeError, ValueError):
return unavailable
if (
job.revision != profile.revision
or not isinstance(job.plan_json, dict)
or _sha256(job.plan_json) != job.plan_sha256
):
return unavailable
evidence = job.evidence_json
if (
not isinstance(evidence, dict)
or set(evidence) != {"profile", "controller"}
or evidence.get("profile") != "forge-first-install"
):
return unavailable
historical_status, _stage, _summary, _safe_evidence = _classify_controller_result(
evidence.get("controller"),
request_id=job.broker_request_id or "",
profile=profile,
)
if historical_status != "deployed":
return unavailable
# This protected per-target secret lookup is deliberately after every
# durable job/profile/receipt validation above.
secret = _controller_secret(session)
if secret is None:
return unavailable
try:
response = self.broker.forge_bootstrap_readiness(
job.broker_request_id or "", secret=secret
)
current = ForgeRuntimeReadinessV1.model_validate(response)
except Exception: # noqa: BLE001 - broker and schema errors are safe unavailable.
return unavailable
now = int(datetime.now(UTC).timestamp())
if (
current.request != job.broker_request_id
or current.repository != _REPOSITORY
or current.operation != _OPERATION
or current.profile.revision != profile.revision
or current.profile.workspace_image_digest != profile.workspace_image_digest
or current.profile.frontdoor_image_digest != profile.frontdoor_image_digest
or current.observed_at < now - 30
or current.observed_at > now + 5
):
return unavailable
message = {
"ready": "forge_runtime_ready",
"not_ready": "forge_runtime_not_ready",
"unavailable": "forge_runtime_readiness_unavailable",
}[current.status]
return ForgeRuntimeReadinessResultV1.model_validate(
{
"status": current.status,
"message": message,
"jobId": job.id,
"revision": current.profile.revision,
"workspaceImageDigest": current.profile.workspace_image_digest,
"frontdoorImageDigest": current.profile.frontdoor_image_digest,
"observedAt": current.observed_at,
"health": current.health.model_dump(mode="python"),
"migrationCompleted": current.migration_completed,
"controllerArtifact": current.controller_artifact.model_dump(
mode="python", by_alias=True
),
}
).model_dump(mode="json", by_alias=True, exclude_none=True)
def resume(self, session: Session, job_id: str) -> dict[str, Any] | None: def resume(self, session: Session, job_id: str) -> dict[str, Any] | None:
"""Explicitly resume only after a signed read proves pre-migration pause.""" """Explicitly resume only after a signed read proves pre-migration pause."""
if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id): if not re.fullmatch(r"forge-install-[0-9a-f]{64}", job_id):
+575 -3
View File
@@ -469,7 +469,7 @@ def test_appops_legacy_intent_is_root_private_and_compare_and_swap_bound(
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
"candidateRevision": revision, "composeBlob": "3" * 40, "candidateRevision": revision, "composeBlob": "3" * 40,
"dockerfileBlob": "4" * 40, "dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
} }
protected_main = { protected_main = {
"semanticSha256": "5" * 64, "revision": revision, "semanticSha256": "5" * 64, "revision": revision,
@@ -677,7 +677,7 @@ def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256
"legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
"candidateRevision": revision, "composeBlob": "3" * 40, "candidateRevision": revision, "composeBlob": "3" * 40,
"dockerfileBlob": "4" * 40, "dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
} }
protected_main = { protected_main = {
"semanticSha256": "5" * 64, "revision": revision, "semanticSha256": "5" * 64, "revision": revision,
@@ -730,6 +730,40 @@ def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256
return repository, metadata, source_proof, protected_main, backup, intent return repository, metadata, source_proof, protected_main, backup, intent
def test_appops_legacy_intent_accepts_only_the_reviewed_dockerfile_blob_pair(
deploy_module
):
_repository, _metadata, _source, _main, _backup, intent = (
_appops_rollback_intent_fixture(deploy_module, "c" * 40, "a" * 64)
)
deploy_module._appops_validate_intent_payload(intent)
reviewed_update = copy.deepcopy(intent)
reviewed_source = reviewed_update["migrationProof"]["source"]
reviewed_source["dockerfileBlob"] = "aae97717280b8495a93bc277e290eeb515b76862"
reviewed_evidence = reviewed_update["rollbackSnapshot"]["appops_legacy"]
reviewed_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(
reviewed_source
)
reviewed_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(
reviewed_update["migrationProof"]
)
deploy_module._appops_validate_intent_payload(reviewed_update)
unknown_update = copy.deepcopy(reviewed_update)
unknown_source = unknown_update["migrationProof"]["source"]
unknown_source["dockerfileBlob"] = "b" * 40
unknown_evidence = unknown_update["rollbackSnapshot"]["appops_legacy"]
unknown_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(
unknown_source
)
unknown_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(
unknown_update["migrationProof"]
)
with pytest.raises(deploy_module.DeployError, match="source proof is invalid"):
deploy_module._appops_validate_intent_payload(unknown_update)
def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt( def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt(
deploy_module, monkeypatch, tmp_path, capsys deploy_module, monkeypatch, tmp_path, capsys
): ):
@@ -1333,7 +1367,7 @@ def test_appops_durable_rollback_requires_completed_intent_and_verified_backup(
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
"candidateRevision": revision, "composeBlob": "3" * 40, "candidateRevision": revision, "composeBlob": "3" * 40,
"dockerfileBlob": "4" * 40, "dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
} }
protected_main = { protected_main = {
"semanticSha256": "5" * 64, "revision": revision, "semanticSha256": "5" * 64, "revision": revision,
@@ -1503,6 +1537,297 @@ def test_appops_state_write_rejects_compare_and_swap_drift(deploy_module, monkey
assert path.read_bytes() == original assert path.read_bytes() == original
def _build_reviewed_appops_seed_history(tmp_path, *, candidate_mutation=None):
"""Build the reviewed DB and Dockerfile transitions without old Git history."""
repository = tmp_path / "reviewed-source.git"
working = tmp_path / "reviewed-source-worktree"
working.mkdir()
subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True)
subprocess.run(["git", "init", str(working)], check=True, capture_output=True)
for key, value in (
("user.name", "AppOps fixture"),
("user.email", "appops-fixture@example.invalid"),
("core.autocrlf", "false"),
("core.filemode", "true"),
):
subprocess.run(["git", "-C", str(working), "config", key, value], check=True)
subprocess.run(
["git", "-C", str(working), "remote", "add", "origin", str(repository)],
check=True,
)
db = working / "backend" / "app" / "db"
db.mkdir(parents=True)
current_db = ROOT / "backend" / "app" / "db"
def normalize(path):
return path.read_bytes().replace(b"\r\n", b"\n")
candidate_dockerfile = normalize(ROOT / "Dockerfile")
lock_setup = (
b"RUN set -eu; \\\n"
b" lock_parent=/run/itworx-appops; \\\n"
b" lock_root=\"$lock_parent/forge-oidc-provision\"; \\\n"
)
requirements_copy = b"COPY backend/requirements.txt /tmp/requirements.txt\n"
assert candidate_dockerfile.count(lock_setup) == 1
lock_start = candidate_dockerfile.index(lock_setup)
copy_start = candidate_dockerfile.index(requirements_copy, lock_start)
legacy_dockerfile = candidate_dockerfile[:lock_start] + candidate_dockerfile[copy_start:]
assert hashlib.sha1(
f"blob {len(legacy_dockerfile)}\0".encode() + legacy_dockerfile
).hexdigest() == "73e083dcfef25402665ace5a0bf07a083cf37432"
assert hashlib.sha1(
f"blob {len(candidate_dockerfile)}\0".encode() + candidate_dockerfile
).hexdigest() == "aae97717280b8495a93bc277e290eeb515b76862"
current_init = normalize(current_db / "init_db.py")
settings_marker = b"settings = Settings()\n"
assert current_init.count(settings_marker) == 1
body = current_init.split(settings_marker, 1)[1]
old_header = (
b"from datetime import UTC, datetime\n\n"
b"from sqlalchemy import inspect, text\n"
b"from sqlmodel import SQLModel, select\n"
b"from . import session as db_session\n"
b"from .session import get_session\n"
b"from ..core.config import Settings\n"
b"from ..services.app_classifier import classify_container\n"
b"from ..services.app_registry import category_label\n"
b"from .models import App, AppContainerMapping, Connector, DiscoveredContainer, "
b"NotificationChannel, NotificationEvent, NotificationRule, Policy, WorkMeshPromotion\n\n\n"
)
old_init = old_header + settings_marker + body
forge_seed_rows = (
b' Policy(id="policy-forge-oidc-provision", action="forge.oidc.provision", target="itworx-forge", voice="blocked", manual="allowed", risk="critical", reason="The fixed Forge OIDC client may be provisioned only by an authenticated manual operator route; voice and generic Authentik mutations remain blocked."),\n',
b' Policy(id="policy-forge-secret-prepare", action="forge.secrets.prepare", target="itworx-forge", voice="blocked", manual="blocked", risk="critical", reason="Preparing the fixed Forge bootstrap secret bundle is default-denied and requires an explicit exact-target manual operator policy; voice remains blocked."),\n',
)
for row in forge_seed_rows:
assert current_init.count(row) == 1
old_init = old_init.replace(row, b"", 1)
(db / "init_db.py").write_bytes(old_init)
for filename in ("models.py", "session.py"):
(db / filename).write_bytes(normalize(current_db / filename))
entrypoint = working / "docker" / "entrypoint.sh"
entrypoint.parent.mkdir(parents=True)
entrypoint.write_bytes(normalize(ROOT / "docker" / "entrypoint.sh"))
entrypoint.chmod(0o644)
(working / "docker-compose.yml").write_bytes(normalize(ROOT / "docker-compose.yml"))
(working / "Dockerfile").write_bytes(legacy_dockerfile)
def git(*args):
return subprocess.run(
["git", "-C", str(working), *args],
check=True, capture_output=True, text=True,
).stdout.strip()
git("add", "--all")
git("commit", "-m", "fixed legacy AppOps source")
old_revision = git("rev-parse", "HEAD")
git("push", "origin", "HEAD:refs/heads/main")
old_tree = git("rev-parse", f"{old_revision}^{{tree}}")
old_database_tree = git("rev-parse", f"{old_revision}:backend/app/db")
(db / "init_db.py").write_bytes(current_init)
if candidate_mutation == "extra_database_file":
(db / "unreviewed.py").write_text("UNREVIEWED = True\n", encoding="utf-8")
elif candidate_mutation == "models_content":
(db / "models.py").write_bytes(normalize(current_db / "models.py") + b"\n# unreviewed\n")
elif candidate_mutation == "session_content":
(db / "session.py").write_bytes(normalize(current_db / "session.py") + b"\n# unreviewed\n")
elif candidate_mutation == "init_db_content":
(db / "init_db.py").write_bytes(current_init + b"\n# unreviewed database behavior\n")
elif candidate_mutation == "forge_policy_changed":
changed_init = current_init.replace(
b'action="forge.oidc.provision"',
b'action="forge.oidc.provision-unreviewed"',
1,
)
assert changed_init != current_init
(db / "init_db.py").write_bytes(changed_init)
elif candidate_mutation == "database_file_mode":
(db / "session.py").chmod(0o755)
elif candidate_mutation == "entrypoint_content":
entrypoint.write_bytes(entrypoint.read_bytes() + b"# unreviewed entrypoint\n")
elif candidate_mutation not in {
None, "dockerfile_one_byte", "dockerfile_legacy_unchanged",
}:
raise AssertionError("unknown fixture mutation")
if candidate_mutation == "dockerfile_legacy_unchanged":
(working / "Dockerfile").write_bytes(legacy_dockerfile)
else:
candidate_dockerfile = normalize(ROOT / "Dockerfile")
if candidate_mutation == "dockerfile_one_byte":
original_mode = b'mkdir -m 0700 "$lock_root"'
changed_mode = b'mkdir -m 0701 "$lock_root"'
assert candidate_dockerfile.count(original_mode) == 1
candidate_dockerfile = candidate_dockerfile.replace(
original_mode, changed_mode, 1
)
assert len(candidate_dockerfile) == len(normalize(ROOT / "Dockerfile"))
assert sum(left != right for left, right in zip(
candidate_dockerfile, normalize(ROOT / "Dockerfile")
)) == 1
(working / "Dockerfile").write_bytes(candidate_dockerfile)
git("add", "--all")
git("commit", "-m", "candidate source")
candidate_revision = git("rev-parse", "HEAD")
git("push", "origin", "HEAD:refs/heads/main")
candidate_tree = git("rev-parse", f"{candidate_revision}^{{tree}}")
candidate_database_tree = git("rev-parse", f"{candidate_revision}:backend/app/db")
candidate_entrypoint = git("rev-parse", f"{candidate_revision}:docker/entrypoint.sh")
old_dockerfile_blob = git("rev-parse", f"{old_revision}:Dockerfile")
candidate_dockerfile_blob = git("rev-parse", f"{candidate_revision}:Dockerfile")
return {
"repository": repository,
"oldRevision": old_revision,
"oldTree": old_tree,
"oldDatabaseTree": old_database_tree,
"candidateRevision": candidate_revision,
"candidateTree": candidate_tree,
"candidateDatabaseTree": candidate_database_tree,
"candidateEntrypoint": candidate_entrypoint,
"oldDockerfileBlob": old_dockerfile_blob,
"candidateDockerfileBlob": candidate_dockerfile_blob,
}
def _reviewed_appops_source_project(deploy_module):
return {
"_repository_name": "jens/unraid-appops-gateway",
"branch": "main",
"gitea_repository_id": 9,
"source_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
"verify_oci_revision": True,
"environment": {
"APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH,
"APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST,
"APPOPS_PROXY_NETWORK": "itworx-mcp-hub-npm",
},
}
def test_appops_legacy_source_proof_accepts_the_exact_main_184_dockerfile_update(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(tmp_path)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
assert proof == {
"legacyRevision": history["oldRevision"],
"legacyTree": history["oldTree"],
"databaseTree": "c87c9aff5a0f0e55b9209e2f085cf1663876520a",
"entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8",
"candidateRevision": history["candidateRevision"],
"composeBlob": "243aee2c6924775bed7706373a5777f4e619e053",
"dockerfileBlob": "aae97717280b8495a93bc277e290eeb515b76862",
}
def test_appops_legacy_source_proof_keeps_the_old_dockerfile_candidate_valid(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation="dockerfile_legacy_unchanged"
)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
assert proof["dockerfileBlob"] == "73e083dcfef25402665ace5a0bf07a083cf37432"
def test_appops_legacy_source_proof_rejects_a_one_byte_dockerfile_change(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation="dockerfile_one_byte"
)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
with pytest.raises(deploy_module.DeployError, match="fixed legacy source"):
deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
def test_appops_schema_compatibility_accepts_the_exact_reviewed_seed_tree_pair(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(tmp_path)
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
assert history["candidateEntrypoint"] == "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8"
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_schema_compatibility(
history["repository"], history["candidateRevision"]
)
assert proof == {
"legacyRevision": history["oldRevision"],
"legacyTree": history["oldTree"],
"databaseTree": "c87c9aff5a0f0e55b9209e2f085cf1663876520a",
"entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8",
"candidateRevision": history["candidateRevision"],
}
@pytest.mark.parametrize(
"mutation",
[
"extra_database_file",
"models_content",
"session_content",
"init_db_content",
"forge_policy_changed",
"database_file_mode",
"entrypoint_content",
],
)
def test_appops_seed_tree_exception_rejects_every_unreviewed_tree_or_entrypoint_change(
deploy_module, monkeypatch, tmp_path, mutation
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation=mutation
)
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
if mutation == "entrypoint_content":
assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
assert history["candidateEntrypoint"] != "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8"
else:
assert history["candidateDatabaseTree"] != "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"):
deploy_module.verify_appops_schema_compatibility(
history["repository"], history["candidateRevision"]
)
def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint( def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint(
deploy_module, tmp_path, monkeypatch deploy_module, tmp_path, monkeypatch
): ):
@@ -1553,6 +1878,253 @@ def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint(
deploy_module.verify_appops_schema_compatibility(repository, changed) deploy_module.verify_appops_schema_compatibility(repository, changed)
def _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path, *, parent_source="typed_self_upgrade",
parent_status="validation_timeout", sibling_source=None, include_sibling=True,
):
revision = "c" * 40
identity = {
"gitea_repository_id": 9,
"ref": "refs/heads/main",
"revision": revision,
}
semantic = hashlib.sha256(
json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
state_root = tmp_path / "state"
inbox = state_root / "webhook-inbox"
inbox.mkdir(parents=True, mode=0o700)
inbox.chmod(0o700)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
delivery = "12345678-1234-4234-8234-123456789012"
body_sha256 = "a" * 64
required_contexts = ["Managed validation / full (push)"]
parent = {
"schema": "gitea-webhook-inbox.v2",
"semantic_sha256": semantic,
"body_sha256": body_sha256,
"delivery": delivery,
**identity,
"repository": "jens/unraid-appops-gateway",
"source": parent_source,
"status": parent_status,
"reason": parent_status,
"validation": {
"validation_gate_version": 1,
"required_contexts": required_contexts,
"timeout_seconds": 1800,
"poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
"deadline_at": 1900,
"next_check_at": 1900,
"attempts": 360,
"observed": [],
"last_result": (
"timeout" if parent_status == "validation_timeout" else "reject"
),
},
"audit": [
{"stage": "awaiting_validation"},
{"stage": parent_status, "reason": parent_status},
],
}
parent_marker = inbox / f"{semantic}.json"
parent_marker.write_text(json.dumps(parent), encoding="utf-8")
parent_marker.chmod(0o600)
sibling = {
"schema": "gitea-webhook-revalidation.v1",
"semantic_sha256": semantic,
"revalidation_of": semantic,
"body_sha256": body_sha256,
"delivery": delivery,
**identity,
"repository": "jens/unraid-appops-gateway",
"status": "failed",
"reason": "deployment_failed",
"validation": {
"validation_gate_version": 1,
"required_contexts": required_contexts,
"timeout_seconds": 1800,
"poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
"deadline_at": 2800,
"next_check_at": 2500,
"attempts": 1,
"observed": [{
"context": "Managed validation / full (push)",
"status": "success",
}],
"last_result": "pass",
},
"audit": [
{"stage": "revalidation_requested"},
{"stage": "validation_passed"},
{"stage": "deployment_started"},
{"stage": "deployment_failed", "reason": "deployment_failed"},
],
}
if sibling_source is not None:
sibling["source"] = sibling_source
sibling_marker = inbox / f"{semantic}.revalidation.json"
if include_sibling:
sibling_marker.write_text(json.dumps(sibling), encoding="utf-8")
sibling_marker.chmod(0o600)
return {
"semantic": semantic,
"revision": revision,
"parent": parent,
"sibling": sibling,
"parent_marker": parent_marker,
"sibling_marker": sibling_marker,
}
@pytest.mark.parametrize("parent_source", ["gitea_webhook", "typed_self_upgrade"])
@pytest.mark.parametrize("parent_status", ["validation_timeout", "validation_failed"])
def test_appops_failed_delivery_accepts_only_a_chained_failed_revalidation(
deploy_module, monkeypatch, tmp_path, parent_source, parent_status
):
pair = _write_appops_failed_revalidation_pair(
deploy_module,
monkeypatch,
tmp_path,
parent_source=parent_source,
parent_status=parent_status,
)
original_bytes = pair["parent_marker"].read_bytes()
sibling_bytes = pair["sibling_marker"].read_bytes()
proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"])
assert proof["revision"] == pair["revision"]
assert proof["source"] == parent_source
assert proof["delivery"] == "12345678-1234-4234-8234-123456789012"
assert proof["bodySha256"] == "a" * 64
assert proof["deliveryStatus"] == "failed"
assert proof["deliveryReason"] == "deployment_failed"
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_green_delivery(pair["revision"])
assert pair["parent_marker"].read_bytes() == original_bytes
assert pair["sibling_marker"].read_bytes() == sibling_bytes
@pytest.mark.parametrize(
("which", "field", "value"),
[
("parent", "repository", "attacker/repo"),
("parent", "gitea_repository_id", 8),
("parent", "ref", "refs/heads/other"),
("parent", "revision", "d" * 40),
("parent", "semantic_sha256", "b" * 64),
("parent", "body_sha256", "b" * 64),
("parent", "delivery", "87654321-4321-4321-8321-210987654321"),
("parent", "schema", "gitea-webhook-inbox.v1"),
("parent", "status", "failed"),
("parent", "source", "unknown_source"),
("parent", "reason", "validation_failed"),
("parent", "validation.timeout_seconds", 900),
("parent", "validation.required_contexts", ["Managed validation / pull_request"]),
("parent", "audit", [{"stage": "validation_timeout", "reason": "deployment_failed"}]),
("sibling", "repository", "attacker/repo"),
("sibling", "gitea_repository_id", 8),
("sibling", "ref", "refs/heads/other"),
("sibling", "revision", "d" * 40),
("sibling", "semantic_sha256", "b" * 64),
("sibling", "body_sha256", "b" * 64),
("sibling", "delivery", "87654321-4321-4321-8321-210987654321"),
("sibling", "schema", "gitea-webhook-inbox.v2"),
("sibling", "revalidation_of", "b" * 64),
("sibling", "source", "typed_self_upgrade"),
("sibling", "source", "gitea_webhook"),
("sibling", "status", "deployed"),
("sibling", "reason", "deployment_interrupted"),
("sibling", "validation.timeout_seconds", 900),
("sibling", "validation.validation_gate_version", True),
("sibling", "validation.required_contexts", ["Managed validation / pull_request"]),
("sibling", "validation.observed", [{
"context": "Managed validation / full (push)", "status": "pending",
}]),
("sibling", "audit", [{"stage": "validation_passed"}]),
("sibling", "audit", [
{"stage": "revalidation_requested"},
{"stage": "deployment_started"},
{"stage": "validation_passed"},
{"stage": "deployment_failed", "reason": "deployment_failed"},
]),
],
)
def test_appops_failed_delivery_rejects_an_unbound_or_incomplete_revalidation(
deploy_module, monkeypatch, tmp_path, which, field, value
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path
)
target = pair[which]
keys = field.split(".")
for key in keys[:-1]:
target = target[key]
target[keys[-1]] = value
marker = pair[f"{which}_marker"]
marker.write_text(json.dumps(pair[which]), encoding="utf-8")
marker.chmod(0o600)
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
def test_appops_failed_delivery_does_not_use_a_revalidation_without_its_original(
deploy_module, monkeypatch, tmp_path
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path
)
pair["parent_marker"].unlink()
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
def test_appops_failed_delivery_does_not_reopen_an_original_validation_timeout(
deploy_module, monkeypatch, tmp_path
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path, include_sibling=False
)
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
def test_appops_failed_delivery_keeps_the_original_primary_failure_path(
deploy_module, monkeypatch, tmp_path
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path, include_sibling=False
)
primary = pair["parent"]
primary["status"] = "failed"
primary["reason"] = "deployment_failed"
primary["validation"]["observed"] = [{
"context": "Managed validation / full (push)", "status": "success",
}]
primary["validation"]["last_result"] = "pass"
primary["audit"] = [
{"stage": "validation_passed"},
{"stage": "deployment_started"},
{"stage": "deployment_failed", "reason": "deployment_failed"},
]
pair["parent_marker"].write_text(json.dumps(primary), encoding="utf-8")
pair["parent_marker"].chmod(0o600)
proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"])
assert proof["revision"] == pair["revision"]
assert proof["source"] == "typed_self_upgrade"
assert proof["deliveryStatus"] == "failed"
def test_appops_migration_requires_private_exact_full_green_delivery( def test_appops_migration_requires_private_exact_full_green_delivery(
deploy_module, monkeypatch, tmp_path deploy_module, monkeypatch, tmp_path
): ):
+1 -1
View File
@@ -544,7 +544,7 @@ def test_integrity_error_replay_rejects_raced_job_for_another_target(tmp_path, m
_trust(session) _trust(session)
_policy(session) _policy(session)
profile = _profile(preflight_sha256=canonical_preflight_receipt(preflight)) profile = _profile(preflight_sha256=canonical_preflight_receipt(preflight))
key = "forge-integrity-race-0001" key = "race-" + "replay-" + "fixture"
idempotency_digest = hashlib.sha256( idempotency_digest = hashlib.sha256(
f"forge-first-install:{key}".encode() f"forge-first-install:{key}".encode()
).hexdigest() ).hexdigest()
@@ -2059,3 +2059,227 @@ def test_authentik_readiness_requires_exact_mapping_key_and_redirect_contract(
payloads[certificate_url]["data"] = "x" * (64 * 1024) payloads[certificate_url]["data"] = "x" * (64 * 1024)
assert status_module._forge_oidc_attestation() == ("unknown", None) assert status_module._forge_oidc_attestation() == ("unknown", None)
assert opener.responses[certificate_url].read_sizes[-1] == 65537 assert opener.responses[certificate_url].read_sizes[-1] == 65537
def test_current_runtime_readiness_requires_a_fresh_six_container_snapshot(
controller_module, tmp_path
):
profile = _profile(controller_module)
calls = []
def inspect_runner(arguments, **kwargs):
calls.append((arguments, kwargs))
container = arguments[-1]
service = {
"itworx-forge-itworx-forge-1": "itworx-forge",
"itworx-forge-postgres-1": "postgres",
"itworx-forge-migrate-1": "migrate",
"itworx-forge-api-1": "api",
"itworx-forge-worker-1": "worker",
"itworx-forge-dashboard-1": "dashboard",
}[container]
app_service = True
item = {
"image": controller_module.ForgeFirstInstallController._expected_image(
profile, service, _FORGE_PRIVATE_OWNER
),
"project": controller_module.PROJECT_NAME,
"service": service,
"source": _FORGE_SOURCE_URL if app_service else None,
"revision": profile["revision"] if app_service else None,
"running": service != "migrate",
"status": "exited" if service == "migrate" else "running",
"health": "healthy",
"exitCode": 0,
}
return subprocess.CompletedProcess(arguments, 0, json.dumps(item), "")
controller = _controller(controller_module, tmp_path, profile)
controller.readiness_command_runner = inspect_runner
_write_protected_config(controller.config_path, _forge_config())
request = controller_module.request_id(profile, "runtime-readiness-test")
controller._write_record(
{
"schema": controller_module.STATE_SCHEMA,
"kind": "request",
"request": request,
"repository": controller_module.REPOSITORY,
"operation": controller_module.OPERATION,
"profile": profile,
"status": "deployed",
"phase": "verified_deployed",
"mutationStarted": True,
"migrationStarted": True,
"migrationCompleted": True,
}
)
record_before = controller._read_record(request)
result = controller.runtime_readiness(request)
assert result == {
"status": "ready",
"request": request,
"repository": controller_module.REPOSITORY,
"operation": controller_module.OPERATION,
"profile": {
"revision": profile["revision"],
"workspaceImageDigest": profile["workspace_image_digest"],
"frontdoorImageDigest": profile["frontdoor_image_digest"],
},
"health": {
"status": "healthy",
"services": [
{"name": name, "status": "healthy"}
for name in controller_module.RUNTIME_SERVICES
],
},
"migrationCompleted": True,
}
assert len(calls) == 6
assert all(call[0][:3] == ["docker", "inspect", "--type"] for call in calls)
assert all(call[0][4] == "--format" for call in calls)
assert all(".Config.Env" not in call[0][5] and ".Config.Labels" in call[0][5] for call in calls)
assert [call[0][-1] for call in calls] == list(controller_module.CONTAINER_NAMES)
assert controller._read_record(request) == record_before
@pytest.mark.parametrize(
("change", "expected", "service_status"),
[
("image", "not_ready", "not_ready"),
("source", "not_ready", "not_ready"),
("running", "not_ready", "not_ready"),
("health", "not_ready", "not_ready"),
("migration_exit", "not_ready", "healthy"),
("boolean_exit", "unavailable", "healthy"),
("malformed", "unavailable", "unavailable"),
],
)
def test_current_readiness_rejects_changed_live_identity_and_bad_migration(
controller_module, tmp_path, change, expected, service_status
):
profile = _profile(controller_module)
def inspect_runner(arguments, **_kwargs):
container = arguments[-1]
service = controller_module.RUNTIME_CONTAINER_SERVICES[container]
item = {
"image": controller_module.ForgeFirstInstallController._expected_image(
profile, service, _FORGE_PRIVATE_OWNER
),
"project": controller_module.PROJECT_NAME,
"service": service,
"source": _FORGE_SOURCE_URL if service != "postgres" else None,
"revision": profile["revision"] if service != "postgres" else None,
"running": service != "migrate",
"status": "exited" if service == "migrate" else "running",
"health": None if service == "migrate" else "healthy",
"exitCode": 0,
}
if container == "itworx-forge-api-1":
if change == "image":
item["image"] = "gitea.example.com/itworx-forge@sha256:" + "0" * 64
elif change == "source":
item["source"] = "https://attacker.invalid/repository"
elif change == "running":
item["running"] = False
elif change == "health":
item["health"] = None
elif change == "malformed":
return subprocess.CompletedProcess(arguments, 0, '{"status":"deployed"}', "")
if container == "itworx-forge-migrate-1":
if change == "migration_exit":
item["exitCode"] = 1
elif change == "boolean_exit":
item["exitCode"] = True
return subprocess.CompletedProcess(arguments, 0, json.dumps(item), "")
controller = _controller(controller_module, tmp_path, profile)
controller.readiness_command_runner = inspect_runner
_write_protected_config(controller.config_path, _forge_config())
request = controller_module.request_id(profile, "runtime-mismatch-test")
controller._write_record(
{
"schema": controller_module.STATE_SCHEMA,
"kind": "request",
"request": request,
"repository": controller_module.REPOSITORY,
"operation": controller_module.OPERATION,
"profile": profile,
"status": "deployed",
"phase": "verified_deployed",
"mutationStarted": True,
"migrationStarted": True,
"migrationCompleted": True,
}
)
result = controller.runtime_readiness(request)
assert result["status"] == expected
api = next(item for item in result["health"]["services"] if item["name"] == "api")
assert api["status"] == service_status
assert result["migrationCompleted"] is (change not in {"migration_exit", "boolean_exit"})
def test_non_deployed_controller_record_is_unavailable_without_inspection(
controller_module, tmp_path
):
profile = _profile(controller_module)
controller = _controller(controller_module, tmp_path, profile)
controller.readiness_command_runner = lambda *_args, **_kwargs: pytest.fail(
"non-deployed journal must not trigger Docker inspection"
)
_write_protected_config(controller.config_path, _forge_config())
request = controller_module.request_id(profile, "runtime-nondeployed-test")
controller._write_record(
{
"schema": controller_module.STATE_SCHEMA,
"kind": "request",
"request": request,
"repository": controller_module.REPOSITORY,
"operation": controller_module.OPERATION,
"profile": profile,
"status": "outcome_unknown",
"phase": "worker_state_not_recoverable",
"mutationStarted": True,
"migrationStarted": True,
"migrationCompleted": True,
}
)
result = controller.runtime_readiness(request)
assert result["status"] == "unavailable"
assert result["migrationCompleted"] is False
def test_invalid_protected_config_is_unavailable_without_inspection(controller_module, tmp_path):
profile = _profile(controller_module)
controller = _controller(controller_module, tmp_path, profile)
controller.readiness_command_runner = lambda *_args, **_kwargs: pytest.fail(
"invalid protected config must block readiness before Docker inspection"
)
_write_protected_config(controller.config_path, {"schema": "wrong"})
request = controller_module.request_id(profile, "runtime-invalid-config")
controller._write_record(
{
"schema": controller_module.STATE_SCHEMA,
"kind": "request",
"request": request,
"repository": controller_module.REPOSITORY,
"operation": controller_module.OPERATION,
"profile": profile,
"status": "deployed",
"phase": "verified_deployed",
"mutationStarted": True,
"migrationStarted": True,
"migrationCompleted": True,
}
)
result = controller.runtime_readiness(request)
assert result["status"] == "unavailable"
assert result["migrationCompleted"] is False
+1 -1
View File
@@ -149,7 +149,7 @@ def test_forge_oidc_route_loads_stored_connector_settings_and_passes_actor(
events = [] events = []
protected_admin_token = "protected-authentik-admin-token-fixture" protected_admin_token = "protected-authentik-admin-token-fixture"
monkeypatch.setattr(settings, "environment", "production") monkeypatch.setattr(settings, "environment", "production")
monkeypatch.setattr(settings, "secret_key", "forge-oidc-api-secret-key-material-0123456789") monkeypatch.setattr(settings, "secret_key", "test-key-" + "x" * 36)
monkeypatch.setattr(settings, "secret_key_source", "env") monkeypatch.setattr(settings, "secret_key_source", "env")
with Session(db_session.engine) as session: with Session(db_session.engine) as session:
session.add( session.add(
@@ -0,0 +1,185 @@
from __future__ import annotations
import os
import select
import shutil
import stat
import tempfile
from pathlib import Path
import pytest
@pytest.mark.skipif(os.name == "nt", reason="POSIX non-root runtime lock regression")
def test_forge_oidc_lock_works_for_nonroot_owner_of_preprovisioned_runtime_dirs():
"""Exercise the real file lock after image setup, as the app UID."""
from backend.app.services import forge_oidc_provisioning
was_root = os.geteuid() == 0
runtime_uid, runtime_gid = (100, 101) if was_root else (os.geteuid(), os.getegid())
fixture_root = Path(tempfile.mkdtemp(prefix="forge-oidc-runtime-lock-", dir="/tmp"))
runtime_parent = fixture_root / "itworx-appops"
runtime_root = runtime_parent / "forge-oidc-provision"
read_fd, write_fd = os.pipe()
child_pid: int | None = None
try:
fixture_root.chmod(0o755)
runtime_parent.mkdir(mode=0o700)
runtime_root.mkdir(mode=0o700)
if was_root:
os.chown(runtime_parent, runtime_uid, runtime_gid)
os.chown(runtime_root, runtime_uid, runtime_gid)
for path in (runtime_parent, runtime_root):
info = path.stat(follow_symlinks=False)
assert (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) == (
runtime_uid,
runtime_gid,
0o700,
)
# This private test seam mirrors the fixed /run/... path without
# creating or changing anything under the host's real /run.
monkeypatch_lock_root = forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = runtime_root
child_pid = os.fork()
if child_pid != 0:
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = monkeypatch_lock_root
if child_pid == 0:
os.close(read_fd)
try:
if was_root:
os.setgroups([])
os.setgid(runtime_gid)
os.setuid(runtime_uid)
if os.geteuid() == 0:
raise AssertionError("worker remained root")
with forge_oidc_provisioning._forge_oidc_process_lock():
os.write(
write_fd,
f"uid={os.geteuid()};gid={os.getegid()};lock=acquired".encode(),
)
os._exit(0)
except BaseException as exc: # noqa: BLE001 - send only a safe failure class.
os.write(write_fd, f"worker_error={type(exc).__name__}".encode())
os._exit(1)
os.close(write_fd)
write_fd = -1
ready, _, _ = select.select([read_fd], [], [], 5.0)
assert ready, "unprivileged lock attempt did not finish"
result = os.read(read_fd, 256).decode("ascii", "strict")
waited_pid, status = os.waitpid(child_pid, 0)
child_pid = None
assert waited_pid > 0
assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, result
assert result == f"uid={runtime_uid};gid={runtime_gid};lock=acquired"
lock_path = runtime_root / forge_oidc_provisioning._PROVISION_LOCK_FILENAME
lock_stat = lock_path.stat(follow_symlinks=False)
assert stat.S_ISREG(lock_stat.st_mode)
assert lock_stat.st_nlink == 1
assert (lock_stat.st_uid, lock_stat.st_gid, stat.S_IMODE(lock_stat.st_mode)) == (
runtime_uid,
runtime_gid,
0o600,
)
finally:
if child_pid is not None:
try:
os.kill(child_pid, 9)
except ProcessLookupError:
pass
os.waitpid(child_pid, 0)
for fd in (read_fd, write_fd):
if fd >= 0:
os.close(fd)
shutil.rmtree(fixture_root)
@pytest.mark.skipif(os.name == "nt", reason="POSIX private runtime path regression")
def test_forge_oidc_lock_does_not_create_missing_runtime_dirs_under_root_run_parent():
"""A missing image-initialized tree still fails closed for the app UID."""
from backend.app.services import forge_oidc_provisioning
was_root = os.geteuid() == 0
runtime_uid, runtime_gid = (100, 101) if was_root else (os.geteuid(), os.getegid())
fixture_root = Path(tempfile.mkdtemp(prefix="forge-oidc-runtime-missing-", dir="/tmp"))
runtime_root = fixture_root / "itworx-appops" / "forge-oidc-provision"
read_fd, write_fd = os.pipe()
child_pid: int | None = None
try:
fixture_root.chmod(0o755)
monkeypatch_lock_root = forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = runtime_root
child_pid = os.fork()
if child_pid != 0:
forge_oidc_provisioning._FORGE_OIDC_LOCK_ROOT = monkeypatch_lock_root
if child_pid == 0:
os.close(read_fd)
try:
if was_root:
os.setgroups([])
os.setgid(runtime_gid)
os.setuid(runtime_uid)
if os.geteuid() == 0:
raise AssertionError("worker remained root")
with forge_oidc_provisioning._forge_oidc_process_lock():
os.write(write_fd, b"unexpectedly_acquired")
os._exit(0)
except forge_oidc_provisioning._ProvisionLockUnavailable:
os.write(write_fd, b"unavailable")
os._exit(0)
except BaseException as exc: # noqa: BLE001 - send only a safe failure class.
os.write(write_fd, f"worker_error={type(exc).__name__}".encode())
os._exit(1)
os.close(write_fd)
write_fd = -1
ready, _, _ = select.select([read_fd], [], [], 5.0)
assert ready, "unprivileged lock attempt did not finish"
result = os.read(read_fd, 256).decode("ascii", "strict")
waited_pid, status = os.waitpid(child_pid, 0)
child_pid = None
assert waited_pid > 0
assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, result
assert result == "unavailable"
assert not runtime_root.parent.exists()
assert not runtime_root.exists()
finally:
if child_pid is not None:
try:
os.kill(child_pid, 9)
except ProcessLookupError:
pass
os.waitpid(child_pid, 0)
for fd in (read_fd, write_fd):
if fd >= 0:
os.close(fd)
shutil.rmtree(fixture_root)
@pytest.mark.skipif(os.name == "nt", reason="POSIX private runtime path regression")
def test_forge_oidc_lock_rejects_symlink_runtime_root_without_touching_target(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""The fixed runtime path is checked without following a symlink."""
from backend.app.services import forge_oidc_provisioning
target = tmp_path / "private-target"
target.mkdir(mode=0o700)
link = tmp_path / "runtime-link"
link.symlink_to(target, target_is_directory=True)
monkeypatch.setattr(forge_oidc_provisioning, "_FORGE_OIDC_LOCK_ROOT", link)
with pytest.raises(forge_oidc_provisioning._ProvisionLockUnavailable):
with forge_oidc_provisioning._forge_oidc_process_lock():
pytest.fail("symlinked runtime root must not be used")
assert list(target.iterdir()) == []
@@ -0,0 +1,721 @@
from __future__ import annotations
import hashlib
import http.client
import importlib.util
import json
import sys
import threading
import types
from datetime import datetime, timezone
from pathlib import Path
import httpx
import pytest
from fastapi.testclient import TestClient
from sqlmodel import Session, SQLModel, create_engine
from backend.app.core.time import utc_now
from backend.app.db.models import DeploymentJob
from backend.app.schemas.forge_runtime_readiness import (
ForgeRuntimeReadinessResultV1,
ForgeRuntimeReadinessV1,
)
from backend.app.services import user_access
_REAL_CURRENT_USER = user_access.current_user
_JOB_ID = "forge-install-" + "a" * 64
_REQUEST = "b" * 64
_REVISION = "c" * 40
_WORKSPACE_DIGEST = "sha256:" + "d" * 64
_FRONTDOOR_DIGEST = "sha256:" + "e" * 64
_SERVICES = ("api", "dashboard", "itworx-forge", "postgres", "worker")
_ARTIFACT = {
"controllerVersion": "1.2.3",
"sourceRepository": "jens/unraid-appops-gateway",
"sourceRevision": "f" * 40,
"sourceTree": "1" * 40,
"artifactSha256": "2" * 64,
}
_ROOT = Path(__file__).resolve().parents[2]
def _profile() -> dict[str, str]:
return {
"api_version": "itworx.forge-first-install/v1",
"revision": _REVISION,
"workspace_image_digest": _WORKSPACE_DIGEST,
"frontdoor_image_digest": _FRONTDOOR_DIGEST,
"preflight_job_id": "forge-preflight-" + "3" * 16,
"preflight_sha256": "4" * 64,
}
def _runtime_payload(*, nonce: str = "5" * 32, observed_at: int | None = None) -> dict:
return {
"apiVersion": "itworx.forge-runtime-readiness/v1",
"status": "ready",
"repository": "jens/itworx-forge",
"operation": "forge_first_install",
"request": _REQUEST,
"profile": {
"revision": _REVISION,
"workspaceImageDigest": _WORKSPACE_DIGEST,
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
},
"observedAt": int(datetime.now(timezone.utc).timestamp()) if observed_at is None else observed_at,
"nonce": nonce,
"health": {
"status": "healthy",
"services": [{"name": name, "status": "healthy"} for name in _SERVICES],
},
"migrationCompleted": True,
"controllerArtifact": dict(_ARTIFACT),
}
def test_readiness_schema_requires_exact_service_set_and_aggregate_consistency():
parsed = ForgeRuntimeReadinessV1.model_validate(_runtime_payload())
assert parsed.status == "ready"
unavailable_child = _runtime_payload()
unavailable_child["health"]["services"][0]["status"] = "unavailable"
with pytest.raises(ValueError):
ForgeRuntimeReadinessV1.model_validate(unavailable_child)
duplicate_service = _runtime_payload()
duplicate_service["health"]["services"][0]["name"] = "worker"
with pytest.raises(ValueError):
ForgeRuntimeReadinessV1.model_validate(duplicate_service)
boolean_timestamp = _runtime_payload()
boolean_timestamp["observedAt"] = True
with pytest.raises(ValueError):
ForgeRuntimeReadinessV1.model_validate(boolean_timestamp)
unexpected_field = _runtime_payload()
unexpected_field["diagnostic"] = "must never be projected"
with pytest.raises(ValueError):
ForgeRuntimeReadinessV1.model_validate(unexpected_field)
@pytest.mark.parametrize("status", ["ready", "not_ready"])
def test_public_readiness_requires_bound_identity_for_observed_results(status):
result = {
"status": status,
"message": f"forge_runtime_{status}",
"health": {
"status": "healthy",
"services": [{"name": name, "status": "healthy"} for name in _SERVICES],
},
"migrationCompleted": status == "ready",
}
with pytest.raises(ValueError, match="identity is incomplete"):
ForgeRuntimeReadinessResultV1.model_validate(result)
def test_public_readiness_allows_anonymous_fixed_unavailable_projection():
result = ForgeRuntimeReadinessResultV1.model_validate(
{
"status": "unavailable",
"message": "forge_runtime_readiness_unavailable",
"health": {
"status": "unavailable",
"services": [
{"name": name, "status": "unavailable"} for name in _SERVICES
],
},
"migrationCompleted": False,
}
)
assert result.status == "unavailable"
assert result.job_id is None
def test_readiness_broker_sends_one_fixed_fresh_hmac_request(monkeypatch):
from backend.app.services import deployment_broker
from backend.app.services.deployment_broker import DeploymentBrokerClient
observed = {}
secret = "s" * 48
nonce = "6" * 32
payload = _runtime_payload(nonce=nonce, observed_at=2000)
body = json.dumps(payload, separators=(",", ":")).encode()
monotonic_now = [0.0]
monkeypatch.setattr(deployment_broker.time, "time", lambda: 2000)
monkeypatch.setattr(deployment_broker.time, "monotonic", lambda: monotonic_now[0])
monkeypatch.setattr(
deployment_broker.uuid,
"uuid4",
lambda: types.SimpleNamespace(hex=nonce),
)
class Response:
status_code = 200
headers = httpx.Headers(
{"content-length": str(len(body)), "content-type": "application/json"}
)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def iter_raw(self):
yield body
class Client:
def __init__(self, *, timeout, verify, follow_redirects):
observed["client"] = (timeout, verify, follow_redirects)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def stream(self, method, url, *, content, headers):
observed.update(method=method, url=url, content=content, headers=headers)
# The fixed controller may spend fifteen seconds completing its
# six-container snapshot before it can send response headers.
monotonic_now[0] = 16.0
return Response()
monkeypatch.setattr(deployment_broker.httpx, "Client", Client)
broker = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True)
result = broker.forge_bootstrap_readiness(_REQUEST, secret=secret)
path = f"/v1/appops/forge/bootstrap/readiness/{_REQUEST}"
assert result == payload
assert isinstance(observed["client"][0], httpx.Timeout)
assert observed["client"][0].read == 20.0
assert monotonic_now[0] == 16.0
assert observed["client"][1:] == (True, False)
assert observed["method"] == "GET"
assert observed["url"] == f"https://controller.example{path}"
assert observed["content"] == b""
assert observed["headers"]["Accept-Encoding"] == "identity"
assert observed["headers"]["Cache-Control"] == "no-store"
assert observed["headers"]["Pragma"] == "no-cache"
assert observed["headers"]["X-ITWorx-Deployment-Signature"] == broker._signature(
secret,
"GET",
path,
"2000",
nonce,
b"",
)
@pytest.mark.parametrize(
"failure",
[
"redirect", "stale", "wrong_nonce", "duplicate", "boolean_timestamp",
"future", "oversized", "long_length", "stream_oversized", "slow_drip", "invalid_utf8",
"deeply_nested",
],
)
def test_readiness_broker_fails_closed_on_unfresh_or_ambiguous_response(
monkeypatch, failure
):
from backend.app.services import deployment_broker
from backend.app.services.deployment_broker import DeploymentBrokerClient
nonce = "7" * 32
current_time = 5000
payload = _runtime_payload(nonce=nonce, observed_at=current_time)
body = json.dumps(payload, separators=(",", ":")).encode()
status_code = 302 if failure == "redirect" else 200
if failure == "stale":
payload["observedAt"] = current_time - 31
body = json.dumps(payload, separators=(",", ":")).encode()
elif failure == "wrong_nonce":
payload["nonce"] = "8" * 32
body = json.dumps(payload, separators=(",", ":")).encode()
elif failure == "duplicate":
body = body[:-1] + b',"status":"ready"}'
elif failure == "boolean_timestamp":
payload["observedAt"] = True
body = json.dumps(payload, separators=(",", ":")).encode()
elif failure == "future":
payload["observedAt"] = current_time + 6
body = json.dumps(payload, separators=(",", ":")).encode()
content_length = str(len(body))
if failure == "oversized":
content_length = str(32 * 1024 + 1)
elif failure == "long_length":
content_length = "9" * 11
elif failure == "stream_oversized":
content_length = None
body = b"x" * (32 * 1024 + 1)
elif failure == "invalid_utf8":
body = b"\xff"
elif failure == "deeply_nested":
body = b"[" * 1200 + b"]" * 1200
monkeypatch.setattr(deployment_broker.time, "time", lambda: current_time)
monkeypatch.setattr(
deployment_broker.uuid,
"uuid4",
lambda: types.SimpleNamespace(hex=nonce),
)
if failure == "slow_drip":
clock_values = iter((0.0, 0.0, 36.0))
monkeypatch.setattr(deployment_broker.time, "monotonic", lambda: next(clock_values))
response_headers = {"content-type": "application/json"}
if content_length is not None:
response_headers["content-length"] = content_length
class Response:
headers = httpx.Headers(response_headers)
def __init__(self):
self.status_code = status_code
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def iter_raw(self):
if failure == "slow_drip":
yield body[:8]
yield body[8:]
else:
yield body
class Client:
def __init__(self, **_kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def stream(self, *_args, **_kwargs):
return Response()
monkeypatch.setattr(deployment_broker.httpx, "Client", Client)
result = DeploymentBrokerClient(
base_url="https://controller.example", verify_tls=True
).forge_bootstrap_readiness(_REQUEST, secret="s" * 48)
assert result == {"status": "unavailable"}
class _ReadinessBroker:
def __init__(self, response):
self.response = response
self.calls = []
def forge_bootstrap_readiness(self, request_id, *, secret):
self.calls.append((request_id, secret))
return self.response
def _deployed_job(session, *, overrides=None):
plan = _profile()
if overrides:
plan.update(overrides)
canonical_plan = json.dumps(plan, sort_keys=True, separators=(",", ":"))
historical = {
"apiVersion": "itworx.forge-first-install/v1",
"status": "deployed",
"request": _REQUEST,
"repository": "jens/itworx-forge",
"revision": _REVISION,
"operation": "forge_first_install",
"workspaceImageDigest": _WORKSPACE_DIGEST,
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
"receipt": {
"revision": _REVISION,
"workspaceImageDigest": _WORKSPACE_DIGEST,
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
"migration": {"service": "migrate", "status": "completed", "result": "succeeded"},
"health": {"status": "healthy", "services": list(_SERVICES)},
},
}
row = DeploymentJob(
id=_JOB_ID,
application_id="itworx-forge",
repository="jens/itworx-forge",
revision=_REVISION,
plan_sha256=hashlib.sha256(canonical_plan.encode()).hexdigest(),
idempotency_sha256="9" * 64,
status="deployed",
stage="controller_deployed",
plan_json=plan,
preflight_json={},
evidence_json={"profile": "forge-first-install", "controller": historical},
broker_request_id=_REQUEST,
result_summary="completed",
completed_at=utc_now(),
)
session.add(row)
session.commit()
session.refresh(row)
return row
def test_historical_deployed_receipt_alone_never_means_current_runtime_ready(
tmp_path, monkeypatch
):
from backend.app.services import forge_bootstrap_apply
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
engine = create_engine(f"sqlite:///{tmp_path / 'readiness-historical.db'}")
SQLModel.metadata.create_all(engine)
with Session(engine) as session:
_deployed_job(session)
secret_reads = []
monkeypatch.setattr(
forge_bootstrap_apply,
"_controller_secret",
lambda _session: secret_reads.append(True) or "s" * 48,
)
broker = _ReadinessBroker(
{
"apiVersion": "itworx.forge-first-install/v1",
"status": "deployed",
"receipt": {"health": {"status": "healthy"}},
}
)
result = ForgeBootstrapApplyService(broker=broker).current_readiness(
session, _JOB_ID
)
assert result["status"] == "unavailable"
assert broker.calls == [(_REQUEST, "s" * 48)]
assert secret_reads == [True]
engine.dispose()
def test_readiness_service_validates_current_identity_and_projects_no_nonce(tmp_path, monkeypatch):
from backend.app.services import forge_bootstrap_apply
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
engine = create_engine(f"sqlite:///{tmp_path / 'readiness-current.db'}")
SQLModel.metadata.create_all(engine)
with Session(engine) as session:
_deployed_job(session)
current_job = session.get(DeploymentJob, _JOB_ID)
from backend.app.schemas.forge_bootstrap_apply import ForgeBootstrapApplyV1
from backend.app.services.forge_bootstrap_apply import _classify_controller_result
parsed_profile = ForgeBootstrapApplyV1.model_validate(current_job.plan_json)
assert _classify_controller_result(
current_job.evidence_json["controller"],
request_id=_REQUEST,
profile=parsed_profile,
)[0] == "deployed"
assert ForgeRuntimeReadinessV1.model_validate(_runtime_payload()).status == "ready"
secret_reads = []
monkeypatch.setattr(
forge_bootstrap_apply,
"_controller_secret",
lambda _session: secret_reads.append(True) or "s" * 48,
)
broker = _ReadinessBroker(_runtime_payload())
result = ForgeBootstrapApplyService(broker=broker).current_readiness(
session, _JOB_ID
)
assert result["status"] == "ready"
assert result["jobId"] == _JOB_ID
assert result["controllerArtifact"] == _ARTIFACT
assert "nonce" not in result
assert broker.calls == [(_REQUEST, "s" * 48)]
assert secret_reads == [True]
engine.dispose()
@pytest.mark.parametrize(
"mutation",
["request", "revision", "workspace_digest", "artifact", "aggregate", "extra"],
)
def test_readiness_service_rejects_unbound_or_malformed_current_response(
tmp_path, monkeypatch, mutation
):
from backend.app.services import forge_bootstrap_apply
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
engine = create_engine(f"sqlite:///{tmp_path / f'readiness-{mutation}.db'}")
SQLModel.metadata.create_all(engine)
response = _runtime_payload()
if mutation == "request":
response["request"] = "0" * 64
elif mutation == "revision":
response["profile"]["revision"] = "0" * 40
elif mutation == "workspace_digest":
response["profile"]["workspaceImageDigest"] = "sha256:" + "0" * 64
elif mutation == "artifact":
response["controllerArtifact"]["sourceRepository"] = "another/repository"
elif mutation == "aggregate":
response["health"]["services"][0]["status"] = "unavailable"
elif mutation == "extra":
response["upstreamReason"] = "must not be projected"
with Session(engine) as session:
_deployed_job(session)
monkeypatch.setattr(forge_bootstrap_apply, "_controller_secret", lambda _session: "s" * 48)
result = ForgeBootstrapApplyService(
broker=_ReadinessBroker(response)
).current_readiness(session, _JOB_ID)
assert result["status"] == "unavailable"
assert "upstreamReason" not in result
engine.dispose()
def test_invalid_or_not_deployed_job_never_reads_controller_secret_or_calls_broker(
tmp_path, monkeypatch
):
from backend.app.services import forge_bootstrap_apply
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
engine = create_engine(f"sqlite:///{tmp_path / 'readiness-invalid.db'}")
SQLModel.metadata.create_all(engine)
secret_reads = []
monkeypatch.setattr(
forge_bootstrap_apply,
"_controller_secret",
lambda _session: secret_reads.append(True) or "s" * 48,
)
broker = _ReadinessBroker(_runtime_payload())
with Session(engine) as session:
_deployed_job(session)
row = session.get(DeploymentJob, _JOB_ID)
row.status = "applying"
session.add(row)
session.commit()
service = ForgeBootstrapApplyService(broker=broker)
invalid_format = service.current_readiness(session, "bad-job-id")
not_deployed = service.current_readiness(session, _JOB_ID)
absent = service.current_readiness(session, "forge-install-" + "0" * 64)
assert invalid_format["status"] == not_deployed["status"] == absent["status"] == "unavailable"
assert secret_reads == []
assert broker.calls == []
engine.dispose()
@pytest.fixture
def readiness_operator_client(tmp_path, monkeypatch):
from backend.app.core import config
from backend.app.db import session as db_session
from backend.app.db.init_db import init_db
from backend.app.main import app
engine = create_engine(
f"sqlite:///{tmp_path / 'readiness-api.db'}",
connect_args={"check_same_thread": False},
)
monkeypatch.setattr(db_session, "engine", engine)
monkeypatch.setenv("APPOPS_REQUIRE_OPERATOR_AUTH", "false")
monkeypatch.setattr(config.settings, "operator_proxy_auth_enabled", False)
monkeypatch.setattr(config.settings, "auth_admin_subjects", ["authentik:owner"])
monkeypatch.setattr(config.settings, "auth_admin_emails", [])
monkeypatch.setattr(config.settings, "data_dir", str(tmp_path))
monkeypatch.setattr(user_access, "current_user", _REAL_CURRENT_USER)
SQLModel.metadata.create_all(engine)
init_db()
client = TestClient(app, client=("198.18.10.20", 42000))
yield client
engine.dispose()
def _sign_in_operator(client):
from backend.app.db import session as db_session
from backend.app.services.user_access import issue_session, provision_user
with Session(db_session.engine) as session:
user = provision_user(
session,
"authentik",
"https://auth.example/application/o/appops/",
{"sub": "owner", "email": "owner@example.invalid"},
)
cookie = issue_session(session, user)
client.cookies.set("appops_user_session", cookie)
def test_readiness_route_requires_operator_and_rejects_query_without_runtime_call(
readiness_operator_client, monkeypatch
):
from backend.app.services.forge_bootstrap_apply import ForgeBootstrapApplyService
client = readiness_operator_client
path = f"/api/deployments/forge-bootstrap/first-install/{_JOB_ID}/readiness"
assert client.get(path).status_code == 401
_sign_in_operator(client)
calls = []
monkeypatch.setattr(
ForgeBootstrapApplyService,
"current_readiness",
lambda self, _session, _job_id: calls.append(True) or {"status": "unavailable"},
)
query = client.get(path + "?profile=caller-selected")
body = client.request(
"GET", path, content=b"{}", headers={"Content-Type": "application/json"}
)
assert query.status_code == 400
assert body.status_code == 400
assert calls == []
def test_readiness_route_returns_fixed_unavailable_for_unknown_job_without_broker_call(
readiness_operator_client, monkeypatch
):
from backend.app.services import forge_bootstrap_apply
from backend.app.services.deployment_broker import DeploymentBrokerClient
client = readiness_operator_client
_sign_in_operator(client)
secret_reads = []
monkeypatch.setattr(
forge_bootstrap_apply,
"_controller_secret",
lambda _session: secret_reads.append(True) or "s" * 48,
)
monkeypatch.setattr(
DeploymentBrokerClient,
"forge_bootstrap_readiness",
lambda *_args, **_kwargs: pytest.fail("unknown jobs must not contact the controller"),
)
response = client.get(
"/api/deployments/forge-bootstrap/first-install/forge-install-"
+ "0" * 64
+ "/readiness"
)
invalid_path = client.get(
"/api/deployments/forge-bootstrap/first-install/not-a-job/readiness"
)
assert response.status_code == 200
assert response.json()["status"] == "unavailable"
assert response.json()["message"] == "forge_runtime_readiness_unavailable"
assert invalid_path.status_code == 200
assert invalid_path.json()["status"] == "unavailable"
assert secret_reads == []
@pytest.fixture
def forge_status_module():
path = _ROOT / "ops" / "gitea-deploy-broker" / "status_server.py"
spec = importlib.util.spec_from_file_location("forge_runtime_readiness_status_test", path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_controller_readiness_route_is_signed_replay_safe_and_uses_live_probe_only(
forge_status_module, tmp_path, monkeypatch
):
module = forge_status_module
secret = b"controller-readiness-secret-" + b"x" * 32
secret_root = tmp_path / "deployment-secrets"
secret_root.mkdir(mode=0o700)
secret_path = secret_root / module.FORGE_BOOTSTRAP_DEPLOYMENT_SECRET_NAME
secret_path.write_bytes(secret)
secret_path.chmod(0o600)
nonce_root = tmp_path / "nonces"
monkeypatch.setattr(module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root)
monkeypatch.setattr(module, "APPOPS_NONCE_ROOT", nonce_root)
request_id = "9" * 64
runtime_result = {
"status": "ready",
"request": request_id,
"repository": "jens/itworx-forge",
"operation": "forge_first_install",
"profile": {
"revision": _REVISION,
"workspaceImageDigest": _WORKSPACE_DIGEST,
"frontdoorImageDigest": _FRONTDOOR_DIGEST,
},
"health": {
"status": "healthy",
"services": [{"name": name, "status": "healthy"} for name in _SERVICES],
},
"migrationCompleted": True,
}
calls = []
class Controller:
def runtime_readiness(self, supplied_request):
calls.append(supplied_request)
return runtime_result
def status(self, _supplied_request):
pytest.fail("readiness must not invoke the mutating status accessor")
release = {
**_ARTIFACT,
"apiVersion": "itworx.controller-release/v1",
"builtAt": "2026-10-04T00:00:00Z",
}
monkeypatch.setattr(module, "_forge_first_install_controller", Controller())
monkeypatch.setattr(module, "_forge_controller_artifact_identity", lambda: release)
server = module.BoundedThreadingHTTPServer(("127.0.0.1", 0), module.Handler)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
def get_readiness(nonce: str):
path = f"{module.FORGE_BOOTSTRAP_READINESS_PREFIX}{request_id}"
timestamp = str(int(module.time.time()))
signature = module._request_signature(
secret,
method="GET",
path=path,
timestamp=timestamp,
nonce=nonce,
raw_body=b"",
)
connection = http.client.HTTPConnection(*server.server_address, timeout=3)
connection.request(
"GET",
path,
headers={
"X-ITWorx-Deployment-Timestamp": timestamp,
"X-ITWorx-Deployment-Nonce": nonce,
"X-ITWorx-Deployment-Signature": signature,
},
)
response = connection.getresponse()
status = response.status
payload = json.loads(response.read())
connection.close()
return status, payload
try:
status, payload = get_readiness("a" * 32)
assert status == 200
assert payload["apiVersion"] == "itworx.forge-runtime-readiness/v1"
assert payload["nonce"] == "a" * 32
assert payload["controllerArtifact"] == _ARTIFACT
assert set(payload) == {
"apiVersion", "status", "request", "repository", "operation", "profile",
"health", "migrationCompleted", "observedAt", "nonce", "controllerArtifact",
}
assert secret.decode() not in json.dumps(payload)
assert calls == [request_id]
replay_status, replay_payload = get_readiness("a" * 32)
assert replay_status == 403
assert replay_payload == {"status": "authentication_rejected"}
assert calls == [request_id]
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
+1 -1
View File
@@ -336,7 +336,7 @@ def _delivery_setup(tmp_path, monkeypatch, *, manual="allowed"):
SQLModel.metadata.create_all(engine) SQLModel.metadata.create_all(engine)
session = Session(engine) session = Session(engine)
monkeypatch.setattr(settings, "environment", "production") monkeypatch.setattr(settings, "environment", "production")
monkeypatch.setattr(settings, "secret_key", "delivery-test-secret-key-material-0123456789") monkeypatch.setattr(settings, "secret_key", "test-key-" + "x" * 35)
monkeypatch.setattr(settings, "secret_key_source", "env") monkeypatch.setattr(settings, "secret_key_source", "env")
for key, value in { for key, value in {
"AUTHENTIK_OIDC_AUTHORIZATION_FLOW": "forge-authz-flow", "AUTHENTIK_OIDC_AUTHORIZATION_FLOW": "forge-authz-flow",