Publish curated Unraid-AppOps-Gateway source from c61e0e1e292914893b40354f904f23dd9fd4a958
This commit is contained in:
1 parent
b47ce43436
commit
9b94d8d0f5
18 files changed
+1651
-75
No files matched your search
@@ -343,7 +343,8 @@ def _mcp_action_registry(session: Session) -> dict[str, Any]:
|
||||
target = str(item.get("target") or "")
|
||||
capability = str(item.get("capability") or "operator")
|
||||
mcp_policy = evaluate_policy(
|
||||
session, action=action, target=target, source="automation"
|
||||
session, action=action, target=target,
|
||||
source=mcp_operations.mcp_policy_source(action, target)
|
||||
)
|
||||
original_decision = str(mcp_policy.get("decision") or "blocked")
|
||||
|
||||
@@ -417,7 +418,8 @@ def preview_action(action: str, target: str) -> dict[str, Any]:
|
||||
safe_action = action[:160]
|
||||
with Session(engine) as session:
|
||||
decision = evaluate_policy(
|
||||
session, action=safe_action, target=target[:240], source="automation"
|
||||
session, action=safe_action, target=target[:240],
|
||||
source=mcp_operations.mcp_policy_source(safe_action, target[:240])
|
||||
)
|
||||
mcp_mode, mcp_supported = _mcp_preview_mode(
|
||||
safe_action, str(decision["decision"])
|
||||
|
||||
@@ -55,6 +55,8 @@ APPROVAL_EXECUTOR_ACTIONS = {
|
||||
"runner.fleet.upgrade_v2",
|
||||
"runner.fleet.specialize",
|
||||
"runner.statistics.configure",
|
||||
"runner.acceptance.preflight",
|
||||
"runner.acceptance.quiesce",
|
||||
"hub.migration.prepare_permissions",
|
||||
"deployment.targets.sync",
|
||||
"deployment.runnerless.provision",
|
||||
@@ -125,7 +127,12 @@ _DIRECT_STEPS = {
|
||||
def action_execution_profile(
|
||||
action: str, policy_decision: str, capability: str
|
||||
) -> dict[str, Any]:
|
||||
if policy_decision == "blocked":
|
||||
if action in {"runner.acceptance.preflight", "runner.acceptance.quiesce"} and policy_decision != "blocked":
|
||||
# The fixed MCP typed authority is separate from Command Center's
|
||||
# manual route; do not advertise an unsupported manual executor.
|
||||
mode = "governance_only"
|
||||
surface = None
|
||||
elif policy_decision == "blocked":
|
||||
mode = "blocked"
|
||||
surface = None
|
||||
elif action in GENERIC_EXECUTOR_ACTIONS:
|
||||
@@ -1669,6 +1676,31 @@ def _execute_controller_artifact_promote(
|
||||
return operation_result
|
||||
|
||||
|
||||
def _execute_runner_acceptance(session: Session, *, action: str, target: str, source: str,
|
||||
risk: str, reason: str, original_policy_decision: str) -> dict[str, Any]:
|
||||
from ..db.models import GatewayUser
|
||||
from .user_access import actor_identity
|
||||
|
||||
actor = actor_identity.get()
|
||||
admin = False
|
||||
if actor:
|
||||
provider, _, subject = actor.partition(":")
|
||||
user = session.exec(select(GatewayUser).where(GatewayUser.provider == provider,
|
||||
GatewayUser.subject == subject)).first()
|
||||
admin = bool(user and user.enabled and user.role == "admin")
|
||||
authorized = (source == "chatgpt-mcp" and actor is None) or admin
|
||||
if (target != "tower/ci-acceptance-residue" or not authorized
|
||||
or source not in {"manual", "chatgpt-mcp", "approved_operator"}
|
||||
or original_policy_decision not in {"allowed", "approval_required"}):
|
||||
return {"success": False, "status": "blocked", "result": "blocked",
|
||||
"message": "Fixed CI maintenance requires the authenticated administrator boundary.",
|
||||
"policyDecision": "blocked"}
|
||||
return _execute_runner_fleet_upgrade_v2(
|
||||
session, source=source, risk=risk, reason=reason,
|
||||
original_policy_decision=original_policy_decision, acceptance=action.rsplit(".", 1)[-1],
|
||||
)
|
||||
|
||||
|
||||
def _execute_runner_fleet_upgrade_v2(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -1678,12 +1710,15 @@ def _execute_runner_fleet_upgrade_v2(
|
||||
original_policy_decision: str,
|
||||
specialized: bool = False,
|
||||
statistics_configuration: bool = False,
|
||||
acceptance: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from .deployment_broker import DeploymentBrokerClient
|
||||
|
||||
action = ("runner.statistics.configure" if statistics_configuration else
|
||||
action = ("runner.acceptance." + acceptance if acceptance else
|
||||
"runner.statistics.configure" if statistics_configuration else
|
||||
"runner.fleet.specialize" if specialized else "runner.fleet.upgrade_v2")
|
||||
target = ("tower/projectbrain-runner-statistics" if statistics_configuration else
|
||||
target = ("tower/ci-acceptance-residue" if acceptance else
|
||||
"tower/projectbrain-runner-statistics" if statistics_configuration else
|
||||
"tower/runner-fleet-specialized" if specialized else "tower/runner-fleet-v2")
|
||||
steps = [
|
||||
"Validate exact fixed runner-fleet target",
|
||||
@@ -1697,6 +1732,9 @@ def _execute_runner_fleet_upgrade_v2(
|
||||
steps = ["Verify fixed read-only statistics target", "Prove protected controller/runtime source",
|
||||
"Lock production configuration", "Provision private independent credential with durable CAS",
|
||||
"Configure ProjectBrain for its governed delivery"]
|
||||
if acceptance:
|
||||
steps = ["Prove protected controller/runtime source", "Attest finite CI origin and complete private identities",
|
||||
"Verify dispatch barrier and production exclusion", "Preserve all resources and data"]
|
||||
job = create_job(
|
||||
session,
|
||||
action_id=action,
|
||||
@@ -1710,26 +1748,57 @@ def _execute_runner_fleet_upgrade_v2(
|
||||
set_job_progress(session, job, 25, f"Requesting {action} through the fixed controller endpoint.")
|
||||
try:
|
||||
broker = DeploymentBrokerClient()
|
||||
result = (broker.runner_statistics_configure(reason=reason) if statistics_configuration else
|
||||
result = (broker.runner_acceptance(reason=reason, preflight=acceptance == "preflight") if acceptance else
|
||||
broker.runner_statistics_configure(reason=reason) if statistics_configuration else
|
||||
broker.runner_fleet_specialize(reason=reason) if specialized else broker.runner_fleet_upgrade_v2(reason=reason))
|
||||
if acceptance:
|
||||
allowed_blockers = {
|
||||
"terminal_origin_claim_missing", "terminal_origin_task_binding_unavailable",
|
||||
"terminal_origin_unavailable", "active_product_receipt_exclusion_unavailable",
|
||||
"dispatch_barrier_guard_unverified", "preflight_evidence_rejected",
|
||||
"acceptance_evidence_rejected", "term_timeout", "controller_response_untrusted",
|
||||
"maintenance_outcome_unknown", "maintenance_receipt_untrusted", "maintenance_execution_deadline",
|
||||
}
|
||||
blockers = result.get("blockers", [])
|
||||
clean_blockers = (blockers if isinstance(blockers, list) and len(blockers) <= 20
|
||||
and all(isinstance(code, str) and code in allowed_blockers for code in blockers)
|
||||
else ["controller_response_untrusted"])
|
||||
result = {
|
||||
"status": result.get("status") if result.get("status") in {
|
||||
"ready", "blocked", "partial", "quiesced", "already_quiesced", "unknown", "in_progress"
|
||||
} else "blocked",
|
||||
"revision": result.get("revision") if isinstance(result.get("revision"), str)
|
||||
and len(result["revision"]) == 40 and all(char in "0123456789abcdef" for char in result["revision"]) else None,
|
||||
"blockers": clean_blockers,
|
||||
"preservedContainers": 18 if result.get("preservedContainers") == 18 else None,
|
||||
"effectsPerformed": result.get("effectsPerformed") if type(result.get("effectsPerformed")) is bool else None,
|
||||
"noMutation": result.get("noMutation") if type(result.get("noMutation")) is bool else None,
|
||||
"maintenancePhase": result.get("maintenance", {}).get("phase") if isinstance(result.get("maintenance"), dict)
|
||||
and result["maintenance"].get("phase") in {"guarding", "quiescing", "restoring", "complete"} else result.get("phase")
|
||||
if result.get("phase") in {"guarding", "quiescing", "restoring", "complete"} else None,
|
||||
}
|
||||
status = str(result.get("status") or "error")
|
||||
success = status in ({"configured", "already_configured"} if statistics_configuration else
|
||||
success = status in ({"ready"} if acceptance == "preflight" else {"quiesced", "already_quiesced"} if acceptance else
|
||||
{"configured", "already_configured"} if statistics_configuration else
|
||||
{"specialized", "already_applied"} if specialized else {"upgraded", "already_applied"})
|
||||
message = str(
|
||||
result.get("reason")
|
||||
or (
|
||||
("ProjectBrain read-only statistics configuration verified." if statistics_configuration else
|
||||
("Finite CI acceptance maintenance verified." if acceptance else
|
||||
"ProjectBrain read-only statistics configuration verified." if statistics_configuration else
|
||||
"Tower runner specialization verified." if specialized
|
||||
else "Tower runner-fleet v2 migration verified.")
|
||||
if success else
|
||||
("ProjectBrain statistics configuration was rejected or failed." if statistics_configuration else
|
||||
("Maintenance outcome is unknown or still active; use read-only preflight status before any explicit resume." if acceptance and status in {"unknown", "in_progress", "partial"} else
|
||||
"Finite CI acceptance maintenance is blocked by required evidence." if acceptance else
|
||||
"ProjectBrain statistics configuration was rejected or failed." if statistics_configuration else
|
||||
"Tower runner specialization was rejected or failed." if specialized
|
||||
else "Tower runner-fleet v2 migration was rejected or failed.")
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
result = {"status": "error"}
|
||||
status = "error"
|
||||
result = {"status": "unknown" if acceptance == "quiesce" else "error", "effectsPerformed": None}
|
||||
status = result["status"]
|
||||
success = False
|
||||
message = f"{type(exc).__name__}: fixed controller operation failed"
|
||||
|
||||
@@ -1749,7 +1818,7 @@ def _execute_runner_fleet_upgrade_v2(
|
||||
risk=risk,
|
||||
policy_decision="allowed",
|
||||
reason=reason,
|
||||
result="success" if success else "failed",
|
||||
result="success" if success else "unknown" if acceptance and status in {"unknown", "in_progress", "partial"} else "failed",
|
||||
execution_plan={"steps": steps, "jobId": completed.id},
|
||||
after_state={
|
||||
"status": status,
|
||||
@@ -1768,7 +1837,7 @@ def _execute_runner_fleet_upgrade_v2(
|
||||
return {
|
||||
"success": success,
|
||||
"status": "success" if success else status,
|
||||
"result": "success" if success else "failed",
|
||||
"result": "success" if success else "unknown" if acceptance and status in {"unknown", "in_progress", "partial"} else "failed",
|
||||
"message": message,
|
||||
"jobId": completed.id,
|
||||
"auditId": audit.id,
|
||||
@@ -1778,6 +1847,11 @@ def _execute_runner_fleet_upgrade_v2(
|
||||
"revision": result.get("revision"),
|
||||
"verifiedRunners": result.get("verifiedRunners"),
|
||||
"projectbrainDeliveryRequired": result.get("projectbrainDeliveryRequired"),
|
||||
"blockers": result.get("blockers") if acceptance else None,
|
||||
"preservedContainers": result.get("preservedContainers") if acceptance else None,
|
||||
"effectsPerformed": result.get("effectsPerformed") if acceptance else None,
|
||||
"noMutation": result.get("noMutation") if acceptance else None,
|
||||
"maintenancePhase": result.get("maintenancePhase") if acceptance else None,
|
||||
}
|
||||
|
||||
|
||||
@@ -2065,6 +2139,10 @@ def execute_governed_action(
|
||||
"originalPolicyDecision": original_policy_decision,
|
||||
}
|
||||
|
||||
if action in {"runner.acceptance.preflight", "runner.acceptance.quiesce"}:
|
||||
return _execute_runner_acceptance(session, action=action, target=target, source=source,
|
||||
risk=risk, reason=reason, original_policy_decision=original_policy_decision)
|
||||
|
||||
if action == "controller.artifact.promote":
|
||||
if target.strip().lower() != "tower/deployment-controller":
|
||||
return {
|
||||
@@ -2603,6 +2681,13 @@ def execute_approved_action(session: Session, approval: Approval) -> dict[str, A
|
||||
original_policy_decision="approval_required",
|
||||
)
|
||||
|
||||
if approval.action in {"runner.acceptance.preflight", "runner.acceptance.quiesce"}:
|
||||
return _execute_runner_acceptance(
|
||||
session, action=approval.action, target=approval.target, source="approved_operator",
|
||||
risk=approval.risk, reason=approval.reason or f"Approved through {approval.id}.",
|
||||
original_policy_decision="approval_required",
|
||||
)
|
||||
|
||||
if approval.action == "runner.fleet.upgrade_v2":
|
||||
if approval.target.strip().lower() != "tower/runner-fleet-v2":
|
||||
return {
|
||||
|
||||
@@ -9,6 +9,8 @@ ACTIONS = [
|
||||
'runner.fleet.upgrade_v2',
|
||||
'runner.fleet.specialize',
|
||||
'runner.statistics.configure',
|
||||
'runner.acceptance.preflight',
|
||||
'runner.acceptance.quiesce',
|
||||
'controller.artifact.promote',
|
||||
'controller.managed-storage.initialize',
|
||||
'hub.migration.prepare_permissions',
|
||||
@@ -222,6 +224,8 @@ ACTION_RISK = {
|
||||
'runner.fleet.upgrade_v2': 'high',
|
||||
'runner.fleet.specialize': 'high',
|
||||
'runner.statistics.configure': 'high',
|
||||
'runner.acceptance.preflight': 'low',
|
||||
'runner.acceptance.quiesce': 'high',
|
||||
'controller.artifact.promote': 'high',
|
||||
'controller.managed-storage.initialize': 'high',
|
||||
'hub.migration.prepare_permissions': 'high',
|
||||
@@ -553,6 +557,8 @@ def action_default_target(action: str) -> str:
|
||||
return 'tower/runner-fleet-v2'
|
||||
if action == 'runner.fleet.specialize':
|
||||
return 'tower/runner-fleet-specialized'
|
||||
if action in {'runner.acceptance.preflight', 'runner.acceptance.quiesce'}:
|
||||
return 'tower/ci-acceptance-residue'
|
||||
if action == 'runner.statistics.configure':
|
||||
return 'tower/projectbrain-runner-statistics'
|
||||
if action == 'deployment.targets.sync':
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import Any
|
||||
|
||||
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
|
||||
HELPER_NAME = "itworx-controller-artifact-promoter-modern"
|
||||
CONTROLLER_VERSION = "2.1.20"
|
||||
CONTROLLER_VERSION = "2.1.21"
|
||||
HELPER_LABEL = "itworx.controller.promoter"
|
||||
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
@@ -174,6 +174,7 @@ class DeploymentBrokerClient:
|
||||
secret: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
error_response_fields: frozenset[str] = frozenset(),
|
||||
request_timeout: httpx.Timeout | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if not self.is_configured():
|
||||
return {"status": "missing_config", "reason": self.configuration_error()}
|
||||
@@ -194,7 +195,7 @@ class DeploymentBrokerClient:
|
||||
headers["Content-Type"] = "application/json"
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.timeout_seconds,
|
||||
timeout=request_timeout or self.timeout_seconds,
|
||||
verify=self._verify_option(),
|
||||
) as client:
|
||||
response = client.request(method, f"{self.base_url}{path}", content=body, headers=headers)
|
||||
@@ -990,6 +991,17 @@ class DeploymentBrokerClient:
|
||||
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
|
||||
)
|
||||
|
||||
def runner_acceptance(self, *, reason: str, preflight: bool) -> dict[str, Any]:
|
||||
result = self._request(
|
||||
"POST", "/v1/appops/runner-acceptance/preflight" if preflight else "/v1/appops/runner-acceptance/quiesce",
|
||||
secret=settings.mcp_token or "", payload={"reason": reason[:1000]},
|
||||
request_timeout=httpx.Timeout(connect=5, write=5, pool=5, read=350),
|
||||
)
|
||||
if not preflight and result.get("status") in {"unreachable", "error", "rejected"}:
|
||||
return {"status": "unknown", "noMutation": False, "effectsPerformed": None,
|
||||
"blockers": ["maintenance_outcome_unknown"]}
|
||||
return result
|
||||
|
||||
def hub_migration_prepare_permissions(self, *, reason: str) -> dict[str, Any]:
|
||||
"""Prepare only the fixed Hub backup and controller-state permissions."""
|
||||
return self._request(
|
||||
|
||||
@@ -59,6 +59,14 @@ def _reconciliation_required_response(
|
||||
}
|
||||
|
||||
|
||||
def mcp_policy_source(action: str, target: str) -> str:
|
||||
"""Normalize only the fixed acceptance authority at the authenticated MCP boundary."""
|
||||
if (action in {"runner.acceptance.preflight", "runner.acceptance.quiesce"}
|
||||
and target == "tower/ci-acceptance-residue"):
|
||||
return "chatgpt-mcp"
|
||||
return "automation"
|
||||
|
||||
|
||||
def request_operation(
|
||||
session: Session,
|
||||
*,
|
||||
@@ -97,7 +105,7 @@ def request_operation(
|
||||
session,
|
||||
action=safe_action,
|
||||
target=safe_target,
|
||||
source="automation",
|
||||
source=mcp_policy_source(safe_action, safe_target),
|
||||
)
|
||||
risk = str(policy.get("risk") or ACTION_RISK.get(safe_action, "critical"))
|
||||
if capability == "read_only":
|
||||
|
||||
@@ -210,6 +210,16 @@ def evaluate_policy(
|
||||
"scope_decision": "allowed" if allowed else None,
|
||||
}
|
||||
|
||||
if action in {'runner.acceptance.preflight', 'runner.acceptance.quiesce'}:
|
||||
# Dedicated finite maintenance authority; wildcard and database rows
|
||||
# cannot redirect this source-qualified action to other containers.
|
||||
eligible = target == 'tower/ci-acceptance-residue' and source in {'manual', 'chatgpt-mcp'}
|
||||
return {
|
||||
'decision': ('allowed' if action.endswith('.preflight') else 'approval_required') if eligible else 'blocked',
|
||||
'reason': 'Fixed CI acceptance maintenance requires current native origin, dispatch and product-exclusion proofs.',
|
||||
'risk': ACTION_RISK[action], 'policy_id': None, 'scope_decision': None,
|
||||
}
|
||||
|
||||
if action in _DEFAULT_DECISIONS:
|
||||
reason = _DEFAULT_REASONS.get(action, 'Action uses default governance policy.')
|
||||
return {
|
||||
|
||||
@@ -191,6 +191,12 @@ def test_action_registry_reports_real_execution_modes(api_client):
|
||||
assert items['stack.redeploy']['executionMode'] == 'governance_only'
|
||||
assert items['jellyfin.scan_library']['executionMode'] == 'blocked'
|
||||
assert items['deployment.target.promote_workmesh']['executionMode'] == 'approval_then_execute'
|
||||
for action in ('runner.acceptance.preflight', 'runner.acceptance.quiesce'):
|
||||
assert items[action]['target'] == 'tower/ci-acceptance-residue'
|
||||
assert items[action]['executionMode'] == 'governance_only'
|
||||
assert items[action]['executionSupported'] is False
|
||||
assert items[action]['commandExecutionSupported'] is False
|
||||
assert items[action]['executionSurface'] is None
|
||||
assert payload['summary']['commandExecutable'] >= 1
|
||||
assert payload['summary']['approvalExecutable'] == 5
|
||||
|
||||
|
||||
@@ -589,7 +589,7 @@ def _health(
|
||||
*,
|
||||
revision: str = REVISION,
|
||||
tree: str = TREE,
|
||||
version: str = "2.1.20",
|
||||
version: str = "2.1.21",
|
||||
artifact_sha256: str = "1" * 64,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
@@ -1347,7 +1347,7 @@ def test_current_214_baseline_is_verified_and_bound_to_new_helper():
|
||||
"itworx.controller.promoter": "true",
|
||||
"itworx.source.repository": "jens/unraid-appops-gateway",
|
||||
"itworx.source.revision": REVISION,
|
||||
"itworx.controller.version": "2.1.20",
|
||||
"itworx.controller.version": "2.1.21",
|
||||
"itworx.controller.bootstrap.target.id": MODERN_CONTROLLER_CONTAINER_ID,
|
||||
"itworx.controller.bootstrap.target.image": CONTROLLER_IMAGE_ID,
|
||||
"itworx.controller.bootstrap.target.revision": MODERN_CONTROLLER_REVISION,
|
||||
@@ -1826,7 +1826,7 @@ def test_successful_child_requires_exact_source_tree_version_and_health():
|
||||
|
||||
assert result["status"] == "promoted"
|
||||
assert result["revision"] == REVISION
|
||||
assert result["controllerVersion"] == "2.1.20"
|
||||
assert result["controllerVersion"] == "2.1.21"
|
||||
assert result["artifactSha256"] == "1" * 64
|
||||
assert docker.started == [MODERN_HELPER_ID]
|
||||
assert docker.created[0]["Image"] == CONTROLLER_IMAGE_ID
|
||||
@@ -2086,32 +2086,33 @@ def test_modern_upgrade_path_still_requires_private_archive_and_image_id(
|
||||
def test_current_helper_version_is_distinct_from_fixed_controller_baselines():
|
||||
source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes()
|
||||
namespace = bootstrap._safe_modern_spec_namespace(source)
|
||||
assert namespace["CONTROLLER_VERSION"] == "2.1.20"
|
||||
assert bootstrap._verified_historical_helper_version(source) == "2.1.20"
|
||||
assert namespace["CONTROLLER_VERSION"] == "2.1.21"
|
||||
assert bootstrap._verified_historical_helper_version(source) == "2.1.21"
|
||||
assert [baseline.version for baseline in bootstrap.CONTROLLER_BASELINES] == ["2.1.12", "2.1.14"]
|
||||
with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
|
||||
bootstrap._safe_modern_spec_namespace(RETIRED_MODERN_HELPER_SOURCE.encode("utf-8"))
|
||||
assert bootstrap._verified_historical_helper_version(RETIRED_MODERN_HELPER_SOURCE.encode("utf-8")) == "2.1.14"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", ["2.1.16", "2.1.21", "9.9.9"])
|
||||
@pytest.mark.parametrize("version", ["2.1.16", "2.1.22", "9.9.9"])
|
||||
def test_current_and_historical_helper_reject_unreviewed_target_versions(version):
|
||||
source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes()
|
||||
foreign = source.replace(b'CONTROLLER_VERSION = "2.1.20"', f'CONTROLLER_VERSION = "{version}"'.encode())
|
||||
foreign = source.replace(b'CONTROLLER_VERSION = "2.1.21"', f'CONTROLLER_VERSION = "{version}"'.encode())
|
||||
with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
|
||||
bootstrap._safe_modern_spec_namespace(foreign)
|
||||
with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
|
||||
bootstrap._verified_historical_helper_version(foreign)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", ["2.1.14", "2.1.16", "2.1.21"])
|
||||
@pytest.mark.parametrize("version", ["2.1.14", "2.1.16", "2.1.22"])
|
||||
def test_promoted_health_cannot_substitute_a_baseline_or_unknown_version(version):
|
||||
assert not bootstrap._health_matches(_health(version=version), REVISION, TREE)
|
||||
|
||||
|
||||
def test_previous_reviewed_helper_is_historical_only():
|
||||
@pytest.mark.parametrize("version", ["2.1.19", "2.1.20"])
|
||||
def test_previous_reviewed_helper_is_historical_only(version):
|
||||
source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes()
|
||||
prior = source.replace(b'CONTROLLER_VERSION = "2.1.20"', b'CONTROLLER_VERSION = "2.1.19"')
|
||||
prior = source.replace(b'CONTROLLER_VERSION = "2.1.21"', f'CONTROLLER_VERSION = "{version}"'.encode())
|
||||
with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"):
|
||||
bootstrap._safe_modern_spec_namespace(prior)
|
||||
assert bootstrap._verified_historical_helper_version(prior) == "2.1.19"
|
||||
assert bootstrap._verified_historical_helper_version(prior) == version
|
||||
@@ -14,20 +14,33 @@ def test_promoter_archive_covers_every_required_installer_source():
|
||||
|
||||
spec = helper_container_spec(revision="a" * 40, image_id="sha256:" + "b" * 64)
|
||||
command = spec["Cmd"][2].replace("\\\n", " ")
|
||||
archive = re.search(r'archive --format=tar "\$revision"(.*?)> "\$work/controller.tar"', command, re.S)
|
||||
archive = re.search(
|
||||
r'archive --format=tar "\$revision"(.*?)> "\$work/controller.tar"',
|
||||
command,
|
||||
re.S,
|
||||
)
|
||||
assert archive is not None
|
||||
packaged = shlex.split(archive.group(1))
|
||||
installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8")
|
||||
installer = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
required_block = re.search(r"(?m)^source_paths=\(\n(.*?)\n\)", installer, re.S)
|
||||
assert required_block is not None
|
||||
required = shlex.split(required_block.group(1))
|
||||
assert required
|
||||
assert all(any(path == prefix or path.startswith(prefix + "/") for prefix in packaged) for path in required)
|
||||
assert all(
|
||||
any(path == prefix or path.startswith(prefix + "/") for prefix in packaged)
|
||||
for path in required
|
||||
)
|
||||
|
||||
|
||||
def test_controller_promoter_action_is_fixed_and_registered():
|
||||
registry = (ROOT / "backend/app/services/action_registry.py").read_text(encoding="utf-8")
|
||||
execution = (ROOT / "backend/app/services/action_execution.py").read_text(encoding="utf-8")
|
||||
registry = (ROOT / "backend/app/services/action_registry.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
execution = (ROOT / "backend/app/services/action_execution.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert "'controller.artifact.promote'" in registry
|
||||
assert "return 'tower/deployment-controller'" in registry
|
||||
assert 'action == "controller.artifact.promote"' in execution
|
||||
@@ -46,7 +59,7 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded():
|
||||
spec = helper_container_spec(revision=revision, image_id=image_id)
|
||||
|
||||
assert HELPER_NAME == "itworx-controller-artifact-promoter-modern"
|
||||
assert CONTROLLER_VERSION == "2.1.20"
|
||||
assert CONTROLLER_VERSION == "2.1.21"
|
||||
assert re.fullmatch(r"2\.1\.\d+", CONTROLLER_VERSION)
|
||||
assert spec["Image"] == image_id
|
||||
assert spec["HostConfig"]["Privileged"] is False
|
||||
@@ -68,23 +81,29 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded():
|
||||
command = spec["Cmd"][2]
|
||||
assert "mktemp -d -p /tmp itworx-controller-artifact-promoter-" in command
|
||||
assert f"itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX" in command
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-" not in command
|
||||
assert (
|
||||
"/mnt/cache/appdata/gitea-deploy-runners/.controller-promoter-" not in command
|
||||
)
|
||||
assert "work='/mnt/cache" not in command
|
||||
assert "rm -rf -- \"$work\"" in command
|
||||
assert "scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"" in command
|
||||
assert "chmod 0600 \"$work/controller.tar\"" in command
|
||||
assert 'rm -rf -- "$work"' in command
|
||||
assert (
|
||||
'scripts/upgrade_deployment_controller.sh > "$work/controller.tar"' in command
|
||||
)
|
||||
assert 'chmod 0600 "$work/controller.tar"' in command
|
||||
assert "scripts/prepare_controller_hub_release_volume.py" in command
|
||||
assert "scripts/run_repopulse_adoption.py" in command
|
||||
assert "UPGRADE_DEPLOYMENT_CONTROLLER_V2" in command
|
||||
assert revision in command
|
||||
assert "--image-id" in command
|
||||
assert CONTROLLER_VERSION in command
|
||||
assert "version='2.1.20'" in command
|
||||
assert "version='2.1.21'" in command
|
||||
assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command
|
||||
archive_start = command.index('git --git-dir="$repo" archive')
|
||||
archive_end = command.index('> "$work/controller.tar"', archive_start)
|
||||
assert "scripts/vendor" in command[archive_start:archive_end]
|
||||
assert "scripts/upgrade_deployment_controller.sh > \"$work/controller.tar\"" in command
|
||||
assert (
|
||||
'scripts/upgrade_deployment_controller.sh > "$work/controller.tar"' in command
|
||||
)
|
||||
assert '--image-id "$image_id"' in command
|
||||
|
||||
|
||||
@@ -110,7 +129,7 @@ def test_controller_promoter_bootstrap_child_spec_binds_legacy_target_identity()
|
||||
"itworx.controller.promoter": "true",
|
||||
"itworx.source.repository": "jens/unraid-appops-gateway",
|
||||
"itworx.source.revision": revision,
|
||||
"itworx.controller.version": "2.1.20",
|
||||
"itworx.controller.version": "2.1.21",
|
||||
"itworx.controller.bootstrap.target.id": target_id,
|
||||
"itworx.controller.bootstrap.target.image": image_id,
|
||||
"itworx.controller.bootstrap.target.revision": target_revision,
|
||||
@@ -142,6 +161,7 @@ def test_controller_promoter_rejects_partial_or_mutable_bootstrap_binding():
|
||||
current_controller_version="2.1.12",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"revision,image_id",
|
||||
[
|
||||
@@ -151,7 +171,9 @@ def test_controller_promoter_rejects_partial_or_mutable_bootstrap_binding():
|
||||
("a" * 40, "sha256:" + "z" * 64),
|
||||
],
|
||||
)
|
||||
def test_controller_promoter_rejects_mutable_or_invalid_identity(revision: str, image_id: str):
|
||||
def test_controller_promoter_rejects_mutable_or_invalid_identity(
|
||||
revision: str, image_id: str
|
||||
):
|
||||
from backend.app.services.controller_artifact_promoter import helper_container_spec
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
@@ -159,7 +181,9 @@ def test_controller_promoter_rejects_mutable_or_invalid_identity(revision: str,
|
||||
|
||||
|
||||
def test_controller_promoter_uses_only_protected_main_and_push_gate():
|
||||
source = (ROOT / "backend/app/services/controller_artifact_promoter.py").read_text(encoding="utf-8")
|
||||
source = (ROOT / "backend/app/services/controller_artifact_promoter.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert 'branch = "main"' in source
|
||||
assert 'REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"' in source
|
||||
assert "controller_promoter_main_not_validated" in source
|
||||
@@ -167,7 +191,29 @@ def test_controller_promoter_uses_only_protected_main_and_push_gate():
|
||||
|
||||
|
||||
def test_deployment_broker_exposes_bounded_controller_health():
|
||||
source = (ROOT / "backend/app/services/deployment_broker.py").read_text(encoding="utf-8")
|
||||
source = (ROOT / "backend/app/services/deployment_broker.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert "def controller_health(self)" in source
|
||||
assert 'f"{self.base_url}/health"' in source
|
||||
assert "_MAX_CONTROLLER_HEALTH_BYTES" in source
|
||||
|
||||
|
||||
def test_modern_promoter_21_binds_actual_installed_19_without_widening_bootstrap_baselines():
|
||||
from backend.app.services.controller_artifact_promoter import helper_container_spec
|
||||
|
||||
actual_revision = "0f89e1506065c5365c48968b1d7845d666a18be8"
|
||||
spec = helper_container_spec(
|
||||
revision="a" * 40,
|
||||
image_id="sha256:" + "b" * 64,
|
||||
current_controller_id="c" * 64,
|
||||
current_controller_revision=actual_revision,
|
||||
current_controller_tree="d" * 40,
|
||||
current_controller_version="2.1.19",
|
||||
)
|
||||
labels = spec["Labels"]
|
||||
assert labels["itworx.controller.version"] == "2.1.21"
|
||||
assert labels["itworx.controller.bootstrap.target.revision"] == actual_revision
|
||||
assert labels["itworx.controller.bootstrap.target.version"] == "2.1.19"
|
||||
assert labels["itworx.controller.bootstrap.target.id"] == "c" * 64
|
||||
assert labels["itworx.controller.bootstrap.target.tree"] == "d" * 40
|
||||
@@ -2441,6 +2441,34 @@ def test_storage_command_diagnostics_never_retain_private_paths(stderr, conditio
|
||||
assert 'fixture-secret' not in diagnostic
|
||||
|
||||
|
||||
@pytest.mark.parametrize('condition', [
|
||||
'image_metadata_failed', 'inspect_failed', 'identity_invalid', 'reinspect_failed',
|
||||
'identity_changed', 'remove_failed', 'remove_storage_full', 'remove_read_only',
|
||||
'remove_permission_denied', 'remove_daemon_unavailable', 'remove_busy',
|
||||
'remove_container_missing', 'remove_api_version_unsupported', 'absence_failed', 'unverified',
|
||||
])
|
||||
def test_probe_child_diagnostics_survive_promotion_without_raw_output(monkeypatch, condition):
|
||||
stderr = ('resource_cgroup_probe_cleanup_' + condition + '\n').encode()
|
||||
monkeypatch.setattr(module.subprocess, 'run', lambda *a, **kw: types.SimpleNamespace(
|
||||
returncode=1, stderr=stderr, stdout=b'Bearer fixture-secret'))
|
||||
with pytest.raises(module.VolumeTransitionError) as error:
|
||||
module._run(['python3', 'qualified-validator'], diagnostic_action='probe_cleanup', max_output=4096)
|
||||
assert module._safe_command_diagnostic(error.value.command_failure) == (
|
||||
'probe_cleanup_exit_1_probe_' + condition)
|
||||
assert 'fixture-secret' not in str(error.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('stderr', [
|
||||
b'resource_cgroup_probe_cleanup_remove_failed Bearer fixture-secret',
|
||||
b'resource_cgroup_probe_cleanup_remove_failed\nBearer fixture-secret',
|
||||
b'resource_cgroup_probe_cleanup_fixture-secret', b'permission denied fixture-secret',
|
||||
b'resource_cgroup_probe_cleanup_\xff',
|
||||
b'resource_cgroup_probe_cleanup_remove_failed' + b'x' * 4096,
|
||||
])
|
||||
def test_probe_child_rejects_untrusted_or_multiline_diagnostics(stderr):
|
||||
assert module._command_stderr_condition(stderr, 'probe_cleanup', max_bytes=4096) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('read_only,free_bytes', [(True, 1024**3), (False, 0)])
|
||||
def test_stop_rejects_failed_storage_before_recording_intent_or_stopping(monkeypatch, read_only, free_bytes):
|
||||
receipt = {'phase': 'prepared'}
|
||||
@@ -2628,6 +2656,7 @@ def test_installer_release_order_matches_renderer_contract():
|
||||
assert module.LEGACY_RELEASE_FILES_WITH_CONFIG_ROOT == HISTORICAL_CONFIG_ROOT_RELEASE_FILES
|
||||
assert module._KNOWN_RELEASE_FILE_SETS == (
|
||||
module.RELEASE_FILES,
|
||||
module.LEGACY_RELEASE_FILES_PRE_QUIESCENCE,
|
||||
module.LEGACY_RELEASE_FILES_PRE_COMPONENT,
|
||||
module.LEGACY_RELEASE_FILES_WITH_ROLE_HELPER,
|
||||
module.LEGACY_RELEASE_FILES_WITHOUT_ROLE_HELPER,
|
||||
@@ -2656,7 +2685,7 @@ def test_installer_release_order_matches_renderer_contract():
|
||||
)
|
||||
assert module.LEGACY_RELEASE_FILES_WITH_ROLE_HELPER == tuple(
|
||||
name for name in module.RELEASE_FILES
|
||||
if name not in {"forge_backup_paths.py", "forge_secret_bundle.py", "immutable_service_promotion.py", "registered_legacy_adoption.py", "registered_legacy_transport.py", "registered_build_context.py"}
|
||||
if name not in {"forge_backup_paths.py", "forge_secret_bundle.py", "immutable_service_promotion.py", "registered_legacy_adoption.py", "registered_legacy_transport.py", "registered_build_context.py", "runner_acceptance_quiescence.py"}
|
||||
)
|
||||
assert module.LEGACY_RELEASE_FILES_WITHOUT_ROLE_HELPER == tuple(
|
||||
name for name in module.LEGACY_RELEASE_FILES_WITH_ROLE_HELPER
|
||||
@@ -3605,7 +3634,7 @@ def test_promotion_cleanup_uses_incoming_code_but_only_attested_previous_source(
|
||||
assert args[0] == "python3"
|
||||
assert Path(args[1]).name == "runner_resource_envelope.py"
|
||||
assert args[2:] == ["--cleanup-stopped-host-probe", IMAGE_ID, old_source]
|
||||
assert kw == {"max_output": 4096}
|
||||
assert kw == {"max_output": 4096, "diagnostic_action": "probe_cleanup"}
|
||||
events.append("cleanup")
|
||||
return b""
|
||||
monkeypatch.setattr(module, "_run", run)
|
||||
|
||||
@@ -1482,3 +1482,42 @@ def test_mcp_sdk_redaction_preserves_schema_valid_calls_and_domain_denials():
|
||||
assert "policy_denied" in denied.content[0].text
|
||||
asyncio.run(exercise())
|
||||
assert calls == [True, False]
|
||||
|
||||
|
||||
def test_public_mcp_acceptance_list_preview_request_and_replay_are_consistent(integration_client, monkeypatch):
|
||||
from backend.app import mcp_server
|
||||
from backend.app.services import deployment_broker
|
||||
from backend.app.db.models import Approval, AuditEvent
|
||||
_, engine = integration_client
|
||||
calls = []
|
||||
def broker(*args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return {"status": "ready" if kwargs["preflight"] else "quiesced", "revision": "d"*40,
|
||||
"preservedContainers": 18, "private": "Bearer SECRET-CANARY"}
|
||||
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_acceptance", broker)
|
||||
async def exercise():
|
||||
async with Client(mcp_server.appops_mcp) as client:
|
||||
listed = (await client.call_tool("get_action_registry", {})).structured_content
|
||||
by_id = {item["id"]: item for item in listed["items"]}
|
||||
for action in ("runner.acceptance.preflight", "runner.acceptance.quiesce"):
|
||||
assert by_id[action]["mcpExecutionMode"] == "direct_typed"
|
||||
assert by_id[action]["manualExecutionMode"] == "governance_only"
|
||||
preview = (await client.call_tool("preview_action", {"action": action, "target": "tower/ci-acceptance-residue"})).structured_content
|
||||
assert preview["mcpExecutionSupported"] is True and preview["executed"] is False
|
||||
args = {"action": action, "target": "tower/ci-acceptance-residue",
|
||||
"reason": "Preserve finite residue. Authorization: Bearer SECRET-CANARY",
|
||||
"idempotencyKey": "public-acceptance-"+action}
|
||||
result = (await client.call_tool("server.operation.request", args)).structured_content
|
||||
assert result["executed"] is True and result["executionSupported"] is True
|
||||
assert result["controllerStatus"] == ("ready" if action.endswith("preflight") else "quiesced")
|
||||
assert result["originalPolicyDecision"] == ("allowed" if action.endswith("preflight") else "approval_required")
|
||||
replay = (await client.call_tool("server.operation.request", args)).structured_content
|
||||
assert replay["idempotentReplay"] is True and replay["jobId"] == result["jobId"]
|
||||
assert "SECRET-CANARY" not in json.dumps(result)
|
||||
denied = (await client.call_tool("preview_action", {"action": "container.stop", "target": "tower/ci-acceptance-residue"})).structured_content
|
||||
assert denied["policyDecision"] == "blocked"
|
||||
asyncio.run(exercise())
|
||||
assert len(calls) == 2
|
||||
with Session(engine) as session:
|
||||
assert session.exec(select(Approval)).all() == []
|
||||
assert "SECRET-CANARY" not in json.dumps([row.model_dump(mode="json") for row in session.exec(select(AuditEvent)).all()])
|
||||
@@ -0,0 +1,931 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
BROKER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def module(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(BROKER))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"acceptance_quiescence_test", BROKER / "runner_acceptance_quiescence.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module.fleet, "_migration_lock", lambda **kwargs: nullcontext())
|
||||
return module
|
||||
|
||||
|
||||
class Driver:
|
||||
def __init__(self, module):
|
||||
self.module = module
|
||||
self.state = {
|
||||
"daemon": {
|
||||
"legacyAttested": True,
|
||||
"engineId": "fixed-private-engine",
|
||||
"outerId": "a" * 64,
|
||||
"dockerRootDir": module.ROOTLESS_STORAGE,
|
||||
"rootless": True,
|
||||
},
|
||||
"productExclusionVerified": True,
|
||||
"dispatchBarrierVerified": True,
|
||||
"resources": {
|
||||
"attachmentsVerified": True,
|
||||
"ownershipVerified": True,
|
||||
"networks": ["fixed-network"],
|
||||
"volumes": ["preserved-data"],
|
||||
},
|
||||
"rows": copy.deepcopy(module._fixed_rows()),
|
||||
"origins": {},
|
||||
}
|
||||
for row in self.state["rows"]:
|
||||
row["ownerLabelDigest"] = "b" * 64 if row["ownerLabelPresent"] else None
|
||||
row["labelDigest"] = module._digest(row["composeLabels"])
|
||||
for project, claim in module.ORIGINS.items():
|
||||
if claim is None:
|
||||
self.state["origins"][project] = None
|
||||
continue
|
||||
attempt = claim["attempt"] if claim["attempt"] is not None else 0
|
||||
self.state["origins"][project] = {
|
||||
"repository": "jens/workmesh",
|
||||
"run": claim["run"],
|
||||
"head": claim["head"],
|
||||
"status": "completed",
|
||||
"conclusion": "failure",
|
||||
"attempt": attempt,
|
||||
"task": claim["task"],
|
||||
"taskHead": claim["head"],
|
||||
"taskAttempt": attempt,
|
||||
"jobsTerminal": True,
|
||||
}
|
||||
self.runners = [
|
||||
{
|
||||
"id": 9,
|
||||
"name": "legacy-linux-1",
|
||||
"labels": ["linux", "docker"],
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"name": "legacy-linux-2",
|
||||
"labels": ["linux", "docker"],
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
},
|
||||
{
|
||||
"id": 700,
|
||||
"name": "windows-preserved",
|
||||
"labels": ["windows"],
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
},
|
||||
]
|
||||
self.effects = []
|
||||
self.pending = []
|
||||
self.timeout = False
|
||||
self.after_term = None
|
||||
|
||||
def maintenance(self):
|
||||
return nullcontext()
|
||||
|
||||
def capture_guards(self):
|
||||
return {"runner": {"synthetic": True}, "host": {"synthetic": True}}
|
||||
|
||||
def install_guards(self, receipt, persist):
|
||||
for name in ("runner", "host"):
|
||||
if receipt["guardProgress"].get(name) in {
|
||||
"installed",
|
||||
"restore_intent",
|
||||
"restored",
|
||||
}:
|
||||
continue
|
||||
receipt["guardProgress"][name] = "install_intent"
|
||||
persist()
|
||||
receipt["guardProgress"][name] = "installed"
|
||||
persist()
|
||||
|
||||
def restore_guards(self, receipt, persist):
|
||||
for name in ("runner", "host"):
|
||||
if receipt["guardProgress"].get(name) == "restored":
|
||||
continue
|
||||
receipt["guardProgress"][name] = "restore_intent"
|
||||
persist()
|
||||
receipt["guardProgress"][name] = "restored"
|
||||
persist()
|
||||
|
||||
def snapshot(self):
|
||||
return copy.deepcopy(self.state)
|
||||
|
||||
def registrations(self):
|
||||
return copy.deepcopy(self.runners)
|
||||
|
||||
def attest_registrations(self, rows):
|
||||
assert {row["id"] for row in rows if "linux" in row["labels"]} == {9, 22}
|
||||
|
||||
def pending_jobs(self):
|
||||
return copy.deepcopy(self.pending)
|
||||
|
||||
def pause(self):
|
||||
pass
|
||||
|
||||
def set_registration_disabled(self, original, value):
|
||||
self.effects.append(("registration", original["id"], value))
|
||||
next(row for row in self.runners if row["id"] == original["id"])["disabled"] = (
|
||||
value
|
||||
)
|
||||
|
||||
def disable_restart(self, cid):
|
||||
self.effects.append(("restart", cid))
|
||||
next(row for row in self.state["rows"] if row["id"] == cid)["restartPolicy"] = {
|
||||
"Name": "no",
|
||||
"MaximumRetryCount": 0,
|
||||
}
|
||||
|
||||
def term(self, cid):
|
||||
row = next(row for row in self.state["rows"] if row["id"] == cid)
|
||||
if row["state"]["Running"]:
|
||||
self.effects.append(("TERM", cid))
|
||||
if not self.timeout:
|
||||
row["state"].update(
|
||||
{
|
||||
"Running": False,
|
||||
"Status": "exited",
|
||||
"Pid": 0,
|
||||
"FinishedAt": "2026-10-09T20:00:00Z",
|
||||
}
|
||||
)
|
||||
if self.after_term:
|
||||
self.after_term(self)
|
||||
|
||||
def wait_exited(self, cid):
|
||||
return not self.timeout
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def proven(module, monkeypatch):
|
||||
receipt = {"value": None}
|
||||
monkeypatch.setattr(
|
||||
module, "_read_receipt", lambda: copy.deepcopy(receipt["value"])
|
||||
)
|
||||
|
||||
def save(value, expected):
|
||||
current = receipt["value"]
|
||||
actual = module._digest(current) if current is not None else None
|
||||
module._require(actual == expected, "maintenance_receipt_cas_failed")
|
||||
receipt["value"] = copy.deepcopy(value)
|
||||
return module._digest(value)
|
||||
|
||||
monkeypatch.setattr(module, "_save_receipt", save)
|
||||
return Driver(module), receipt
|
||||
|
||||
|
||||
def test_missing_backup_origin_never_becomes_authority(module, monkeypatch):
|
||||
monkeypatch.setitem(module.ORIGINS, "workmesh-backup-exercise-39a1a242e4b9", None)
|
||||
monkeypatch.setattr(module, "_read_receipt", lambda: None)
|
||||
driver = Driver(module)
|
||||
result = module.preflight(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "blocked"
|
||||
assert result["effectsPerformed"] is False
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
def test_positive_preserves_resources_and_windows_then_idempotent(module, proven):
|
||||
driver, receipt = proven
|
||||
data_before = copy.deepcopy(driver.state["resources"])
|
||||
ids_before = {row["id"] for row in driver.state["rows"]}
|
||||
assert (
|
||||
module.preflight(revision="d" * 40, projects={}, driver=driver)["status"]
|
||||
== "ready"
|
||||
)
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "quiesced"
|
||||
assert result["preservedData"] is True
|
||||
assert driver.state["resources"] == data_before
|
||||
assert {row["id"] for row in driver.state["rows"]} == ids_before
|
||||
assert all(not row["state"]["Running"] for row in driver.state["rows"])
|
||||
assert sum(effect[0] == "TERM" for effect in driver.effects) == 12
|
||||
assert driver.runners[2]["disabled"] is False
|
||||
assert all(
|
||||
effect[0] in {"registration", "restart", "TERM"} for effect in driver.effects
|
||||
)
|
||||
assert receipt["value"]["phase"] == "complete"
|
||||
count = len(driver.effects)
|
||||
assert (
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"]
|
||||
== "already_quiesced"
|
||||
)
|
||||
assert len(driver.effects) == count
|
||||
with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"):
|
||||
module.quiesce(revision="e" * 40, projects={}, driver=driver)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"])
|
||||
def test_initial_new_work_refuses_all_effects(module, proven, kind):
|
||||
driver, _ = proven
|
||||
driver.pending = [{"id": 99, "status": kind}]
|
||||
with pytest.raises(module.QuiescenceError, match="global_ci_work_present"):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["queued", "in_progress", "pending"])
|
||||
def test_new_work_during_execution_blocks_next_container(module, proven, kind):
|
||||
driver, receipt = proven
|
||||
driver.after_term = lambda self: setattr(
|
||||
self, "pending", [{"id": 99, "status": kind}]
|
||||
)
|
||||
with pytest.raises(module.QuiescenceError, match="global_ci_work_present"):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert sum(e[0] == "restart" for e in driver.effects) == 1
|
||||
assert receipt["value"]["phase"] != "complete"
|
||||
assert all(row["disabled"] for row in driver.runners[:2])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
lambda s: s["daemon"].update({"rootless": False}),
|
||||
lambda s: s["daemon"].update({"dockerRootDir": "/srv/example/appdata/production"}),
|
||||
lambda s: s["rows"][0].update({"id": "f" * 64}),
|
||||
lambda s: s["rows"][0].update({"imageId": "sha256:" + "f" * 64}),
|
||||
lambda s: s["rows"][0]["composeLabels"].update(
|
||||
{"com.docker.compose.config-hash": "f" * 64}
|
||||
),
|
||||
lambda s: s["rows"][0]["composeLabels"].update(
|
||||
{"com.docker.compose.project.working_dir": "/production"}
|
||||
),
|
||||
lambda s: s["rows"][0]["mounts"].append(
|
||||
{"Type": "bind", "Source": "/production", "RW": True}
|
||||
),
|
||||
lambda s: s["rows"][0]["networks"].update({"foreign": "f" * 64}),
|
||||
lambda s: s["resources"].update({"attachmentsVerified": False}),
|
||||
lambda s: s["resources"].update({"ownershipVerified": False}),
|
||||
lambda s: s.update({"productExclusionVerified": False}),
|
||||
lambda s: s.update({"dispatchBarrierVerified": False}),
|
||||
lambda s: s["rows"].append(copy.deepcopy(s["rows"][0])),
|
||||
lambda s: s["rows"][0].update({"ownerLabelDigest": "e" * 64}),
|
||||
],
|
||||
)
|
||||
def test_identity_or_scope_drift_has_no_effect(module, proven, change):
|
||||
driver, _ = proven
|
||||
change(driver.state)
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("attempt", None),
|
||||
("attempt", 2),
|
||||
("taskAttempt", 3),
|
||||
("task", 999),
|
||||
("head", "e" * 40),
|
||||
("jobsTerminal", False),
|
||||
],
|
||||
)
|
||||
def test_origin_drift_refuses_effects(module, proven, field, value):
|
||||
driver, _ = proven
|
||||
driver.state["origins"]["workmesh-product-21300-7bace5f2edf0d259"][field] = value
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert driver.effects == []
|
||||
|
||||
|
||||
def test_legacy_attempt_zero_is_actual_sentinel_and_missing_is_not_assumed(
|
||||
module, proven
|
||||
):
|
||||
driver, _ = proven
|
||||
module.ORIGINS["workmesh-runtime-19421"]["attempt"] = 0
|
||||
driver.state["origins"]["workmesh-runtime-19421"].update(attempt=0, taskAttempt=0)
|
||||
module.attest_snapshot(driver.snapshot())
|
||||
driver.state["origins"]["workmesh-runtime-19421"]["attempt"] = None
|
||||
with pytest.raises(module.QuiescenceError, match="attempt_unavailable"):
|
||||
module.attest_snapshot(driver.snapshot())
|
||||
|
||||
|
||||
def test_timeout_retains_owned_guard_and_resumes_only_own_transition(module, proven):
|
||||
driver, receipt = proven
|
||||
driver.timeout = True
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "partial" and result["guardHeld"] is True
|
||||
assert receipt["value"]["progress"][driver.state["rows"][0]["id"]] == "term_intent"
|
||||
driver.timeout = False
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] == "quiesced"
|
||||
|
||||
|
||||
def test_registration_drift_after_partial_never_restores_foreign_identity(
|
||||
module, proven
|
||||
):
|
||||
driver, _ = proven
|
||||
driver.timeout = True
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
before = len(driver.effects)
|
||||
driver.runners[0]["labels"].append("foreign-project")
|
||||
with pytest.raises(module.QuiescenceError, match="legacy_registration_changed"):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert len(driver.effects) == before
|
||||
|
||||
|
||||
def test_no_arbitrary_or_destructive_docker_effect(module, monkeypatch):
|
||||
calls = []
|
||||
|
||||
class Result:
|
||||
returncode = 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_run", lambda args, **kwargs: calls.append(args) or Result()
|
||||
)
|
||||
driver = module.ControllerDriver({})
|
||||
monkeypatch.setattr(driver, "assert_barrier", lambda: None)
|
||||
monkeypatch.setattr(driver, "_guard_binding", lambda: {})
|
||||
monkeypatch.setattr(driver, "_effect_admission", lambda cid, args: None)
|
||||
cid = module._fixed_rows()[0]["id"]
|
||||
driver.disable_restart(cid)
|
||||
assert calls == [
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
"-e",
|
||||
"DOCKER_HOST=tcp://127.0.0.1:2375",
|
||||
"gitea-runner-dind",
|
||||
"docker",
|
||||
"update",
|
||||
"--restart=no",
|
||||
cid,
|
||||
]
|
||||
]
|
||||
with pytest.raises(module.QuiescenceError, match="container_target_invalid"):
|
||||
driver.disable_restart("user-selected")
|
||||
|
||||
|
||||
def test_public_preflight_never_serializes_private_canary(module):
|
||||
class BadDriver:
|
||||
def maintenance(self):
|
||||
return nullcontext()
|
||||
|
||||
def pending_jobs(self):
|
||||
raise ValueError("Bearer SECRET-CANARY")
|
||||
|
||||
def proof_gaps(self):
|
||||
return ["terminal_origin_unavailable"]
|
||||
|
||||
result = module.preflight(revision="d" * 40, projects={}, driver=BadDriver())
|
||||
assert "SECRET-CANARY" not in str(result)
|
||||
assert result["effectsPerformed"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("boundary", range(1, 93))
|
||||
def test_every_durable_write_boundary_resumes_without_adoption(
|
||||
module, proven, monkeypatch, boundary
|
||||
):
|
||||
driver, receipt = proven
|
||||
save = module._save_receipt
|
||||
count = 0
|
||||
|
||||
def crash(value, expected):
|
||||
nonlocal count
|
||||
count += 1
|
||||
result = save(value, expected)
|
||||
if count == boundary:
|
||||
raise OSError("simulated power loss after durable commit")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(module, "_save_receipt", crash)
|
||||
try:
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
except OSError:
|
||||
pass
|
||||
monkeypatch.setattr(module, "_save_receipt", save)
|
||||
result = module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert result["status"] in {"quiesced", "already_quiesced"}
|
||||
assert receipt["value"]["phase"] == "complete"
|
||||
assert all(not row["state"]["Running"] for row in driver.state["rows"])
|
||||
assert sum(e[0] == "TERM" for e in driver.effects) == 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"effect", ["set_registration_disabled", "disable_restart", "term"]
|
||||
)
|
||||
def test_effect_commit_before_receipt_completion_is_owned_resume(
|
||||
module, proven, monkeypatch, effect
|
||||
):
|
||||
driver, _ = proven
|
||||
original = getattr(driver, effect)
|
||||
crashed = False
|
||||
|
||||
def wrapper(*args):
|
||||
nonlocal crashed
|
||||
original(*args)
|
||||
if not crashed:
|
||||
crashed = True
|
||||
raise OSError("simulated loss after external effect")
|
||||
|
||||
monkeypatch.setattr(driver, effect, wrapper)
|
||||
with pytest.raises(OSError):
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
monkeypatch.setattr(driver, effect, original)
|
||||
assert (
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)["status"]
|
||||
== "quiesced"
|
||||
)
|
||||
|
||||
|
||||
def test_current_fixed_origins_include_reviewed_backup_and_attempts(module):
|
||||
assert module.ControllerDriver({}).proof_gaps() == []
|
||||
assert module.ORIGINS["workmesh-backup-exercise-39a1a242e4b9"]["run"] == 20184
|
||||
assert all(row["attempt"] == 1 for row in module.ORIGINS.values())
|
||||
|
||||
|
||||
def test_malformed_complete_receipt_cannot_reenable_maintenance(module, proven):
|
||||
driver, receipt = proven
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert module.maintenance_fenced() is False
|
||||
receipt["value"]["progress"].pop(next(iter(receipt["value"]["progress"])))
|
||||
assert module.maintenance_fenced() is True
|
||||
|
||||
|
||||
def test_failure_projection_never_misreports_partial_effects(module, proven):
|
||||
driver, _ = proven
|
||||
assert module.failure_projection("d" * 40)["effectsPerformed"] is False
|
||||
driver.timeout = True
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
result = module.failure_projection("d" * 40)
|
||||
assert result["status"] == "partial"
|
||||
assert result["effectsPerformed"] is True and result["guardHeld"] is True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_origins(module, monkeypatch):
|
||||
dates = {
|
||||
19421: ("2026-09-29T00:53:46Z", "2026-09-29T00:57:44Z", 768),
|
||||
19617: ("2026-09-29T07:36:50Z", "2026-09-29T07:41:39Z", 834),
|
||||
20184: ("2026-09-29T22:34:11Z", "2026-09-29T22:35:35Z", 941),
|
||||
21300: ("2026-10-01T20:53:43Z", "2026-10-01T20:54:47Z", 1128),
|
||||
}
|
||||
tasks, runs, jobs = [], {}, {}
|
||||
for claim in module.ORIGINS.values():
|
||||
start, end, number = dates[claim["run"]]
|
||||
conclusion = "cancelled" if claim["run"] == 20184 else "failure"
|
||||
tasks.append(
|
||||
{
|
||||
"id": claim["task"],
|
||||
"run_number": number,
|
||||
"head_sha": claim["head"],
|
||||
"workflow_id": "runtime-acceptance.yml",
|
||||
"status": conclusion,
|
||||
"created_at": start,
|
||||
"run_started_at": start,
|
||||
"updated_at": end,
|
||||
"url": f"http://127.0.0.1:3000/Jens/WorkMesh/actions/runs/{claim['run']}",
|
||||
}
|
||||
)
|
||||
runs[claim["run"]] = {
|
||||
"id": claim["run"],
|
||||
"run_number": number,
|
||||
"head_sha": claim["head"],
|
||||
"status": "completed",
|
||||
"conclusion": conclusion,
|
||||
"run_attempt": 1,
|
||||
"run_started_at": None,
|
||||
"updated_at": end,
|
||||
}
|
||||
jobs[claim["run"]] = [
|
||||
{
|
||||
"id": claim["job"],
|
||||
"run_id": claim["run"],
|
||||
"head_sha": claim["head"],
|
||||
"run_attempt": 1,
|
||||
"runner_id": claim["runner"],
|
||||
"started_at": start,
|
||||
"status": "completed",
|
||||
"conclusion": conclusion,
|
||||
}
|
||||
]
|
||||
# Pagination includes unrelated metadata but never returns it publicly.
|
||||
tasks.extend({"id": i} for i in range(50000, 50055))
|
||||
calls = []
|
||||
|
||||
def api(path, **kwargs):
|
||||
calls.append(path)
|
||||
if "/actions/tasks?" in path:
|
||||
page = int(path.rsplit("=", 1)[-1])
|
||||
return {
|
||||
"total_count": len(tasks),
|
||||
"workflow_runs": copy.deepcopy(tasks[(page - 1) * 50 : page * 50]),
|
||||
}
|
||||
run = int(path.split("/runs/")[1].split("/")[0])
|
||||
return (
|
||||
{"total_count": len(jobs[run]), "jobs": copy.deepcopy(jobs[run])}
|
||||
if "/jobs?" in path
|
||||
else copy.deepcopy(runs[run])
|
||||
)
|
||||
|
||||
monkeypatch.setattr(module.fleet, "_api_json", api)
|
||||
return tasks, runs, jobs, calls
|
||||
|
||||
|
||||
def test_native_documented_task_binding_current_attempt_and_null_run_start(
|
||||
module, native_origins
|
||||
):
|
||||
result = module.ControllerDriver({})._origins()
|
||||
module._attest_origins(result)
|
||||
assert all(row["attempt"] == 1 and row["jobsTerminal"] for row in result.values())
|
||||
assert len([path for path in native_origins[3] if "/tasks?" in path]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
"task-url",
|
||||
"task-head",
|
||||
"task-workflow",
|
||||
"task-time",
|
||||
"foreign-job",
|
||||
"duplicate-job-start",
|
||||
"later-attempt",
|
||||
"active-job",
|
||||
"missing-job",
|
||||
"missing-attempt",
|
||||
"terminal-task-changed",
|
||||
"duplicate-task",
|
||||
],
|
||||
)
|
||||
def test_native_origin_drift_refuses_authority(module, native_origins, change):
|
||||
tasks, runs, jobs, _ = native_origins
|
||||
claim = module.ORIGINS[next(iter(module.ORIGINS))]
|
||||
task = next(row for row in tasks if row["id"] == claim["task"])
|
||||
job = jobs[claim["run"]][0]
|
||||
if change == "task-url":
|
||||
task["url"] = task["url"].replace("127.0.0.1", "foreign.invalid")
|
||||
elif change == "task-head":
|
||||
task["head_sha"] = "f" * 40
|
||||
elif change == "task-workflow":
|
||||
task["workflow_id"] = "other.yml"
|
||||
elif change == "task-time":
|
||||
task["created_at"] = "2026-10-02T00:00:00Z"
|
||||
elif change == "foreign-job":
|
||||
job["runner_id"] = 800
|
||||
elif change == "duplicate-job-start":
|
||||
jobs[claim["run"]].append(copy.deepcopy(job))
|
||||
elif change == "later-attempt":
|
||||
runs[claim["run"]]["run_attempt"] = 2
|
||||
elif change == "active-job":
|
||||
job.update(status="in_progress", conclusion=None)
|
||||
elif change == "missing-job":
|
||||
jobs[claim["run"]] = []
|
||||
elif change == "missing-attempt":
|
||||
runs[claim["run"]].pop("run_attempt")
|
||||
elif change == "terminal-task-changed":
|
||||
task["status"] = "success"
|
||||
elif change == "duplicate-task":
|
||||
tasks[-1] = copy.deepcopy(tasks[0])
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module._attest_origins(module.ControllerDriver({})._origins())
|
||||
|
||||
|
||||
def test_exact_inspect_inventory_cannot_omit_foreign_volume_attachment(
|
||||
module, monkeypatch
|
||||
):
|
||||
driver = module.ControllerDriver({})
|
||||
ids = ["a" * 64, "b" * 64]
|
||||
monkeypatch.setattr(driver, "_docker", lambda *args: [{"Id": ids[0]}])
|
||||
with pytest.raises(module.QuiescenceError, match="legacy_inventory_unavailable"):
|
||||
driver._snapshot_from_ids({}, ids)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def product(module, monkeypatch):
|
||||
source = "/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh"
|
||||
names = [
|
||||
"workmesh-postgres-1",
|
||||
"workmesh-api-1",
|
||||
"workmesh-n8n-1",
|
||||
"workmesh-workmesh-1",
|
||||
]
|
||||
target = {
|
||||
"slug": "workmesh",
|
||||
"branch": "main",
|
||||
"exact_source": True,
|
||||
"source_dir": source,
|
||||
"env_source": "/srv/example/appdata/workmesh/.env",
|
||||
"compose_files": ["compose.yaml"],
|
||||
"compose_project": "workmesh",
|
||||
"containers": names,
|
||||
}
|
||||
rows = {
|
||||
name: {
|
||||
"Id": str(i + 1) * 64,
|
||||
"Name": "/" + name,
|
||||
"Image": "sha256:" + str(i + 1) * 64,
|
||||
"Created": "2026-09-01T00:00:00Z",
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Config": {
|
||||
"Labels": {
|
||||
"com.docker.compose.project": "workmesh",
|
||||
"com.docker.compose.service": name.removeprefix(
|
||||
"workmesh-"
|
||||
).removesuffix("-1"),
|
||||
"com.docker.compose.project.working_dir": source,
|
||||
"com.docker.compose.project.config_files": source + "/compose.yaml",
|
||||
}
|
||||
},
|
||||
"Mounts": [
|
||||
{
|
||||
"Type": "volume",
|
||||
"Name": "production_data",
|
||||
"Source": "/var/lib/docker/volumes/production_data/_data",
|
||||
"Destination": "/data",
|
||||
"RW": True,
|
||||
}
|
||||
],
|
||||
"NetworkSettings": {
|
||||
"Networks": {"workmesh_default": {"NetworkID": "e" * 64}}
|
||||
},
|
||||
}
|
||||
for i, name in enumerate(names)
|
||||
}
|
||||
info = {"ID": "host-engine", "DockerRootDir": "/var/lib/docker"}
|
||||
inventory = [row["Id"] for row in rows.values()]
|
||||
|
||||
def run(argv, **kw):
|
||||
import json
|
||||
|
||||
if argv[1] == "info":
|
||||
return json.dumps(info)
|
||||
if argv[1] == "inspect":
|
||||
return json.dumps([rows[argv[2]]])
|
||||
assert argv[1:3] == ["ps", "-a"]
|
||||
return "\n".join(inventory)
|
||||
|
||||
monkeypatch.setattr(module.fleet, "_run", run)
|
||||
return target, rows, info, inventory
|
||||
|
||||
|
||||
def test_manual_product_observation_protects_exact_healthy_target_without_fake_receipt(
|
||||
module, product
|
||||
):
|
||||
target, _, _, _ = product
|
||||
proof = module.ControllerDriver({"jens/workmesh": target})._product_binding(
|
||||
{"engineId": "private"}, []
|
||||
)
|
||||
assert (
|
||||
len(proof["rows"]) == 4
|
||||
and proof["receiptStatus"] == "legacy-physical-observation"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
["source", "engine", "root", "cid", "health", "mount", "network", "extra-product"],
|
||||
)
|
||||
def test_product_disjointness_fails_on_changed_identity_or_overlap(
|
||||
module, product, change
|
||||
):
|
||||
target, rows, info, inventory = product
|
||||
row = next(iter(rows.values()))
|
||||
private = [
|
||||
{
|
||||
"id": "a" * 64,
|
||||
"mounts": [{"Source": "/private-data"}],
|
||||
"networks": {"private": "b" * 64},
|
||||
}
|
||||
]
|
||||
if change == "source":
|
||||
row["Config"]["Labels"]["com.docker.compose.project.config_files"] = (
|
||||
"/foreign/compose.yaml"
|
||||
)
|
||||
elif change == "engine":
|
||||
info["ID"] = "private"
|
||||
elif change == "root":
|
||||
info["DockerRootDir"] = module.ROOTLESS_STORAGE
|
||||
elif change == "cid":
|
||||
row["Id"] = "a" * 64
|
||||
elif change == "health":
|
||||
row["State"]["Health"]["Status"] = "unhealthy"
|
||||
elif change == "mount":
|
||||
row["Mounts"][0]["Source"] = "/private-data"
|
||||
elif change == "network":
|
||||
row["NetworkSettings"]["Networks"]["workmesh_default"]["NetworkID"] = "b" * 64
|
||||
elif change == "extra-product":
|
||||
inventory.append("f" * 64)
|
||||
with pytest.raises(module.QuiescenceError):
|
||||
module.ControllerDriver({"jens/workmesh": target})._product_binding(
|
||||
{"engineId": "private"}, private
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def real_guards(module, proven, monkeypatch, tmp_path):
|
||||
import hashlib
|
||||
|
||||
driver = module.ControllerDriver({})
|
||||
monkeypatch.setattr(driver, "assert_barrier", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
driver, "registrations", lambda: copy.deepcopy(proven[0].runners)
|
||||
)
|
||||
monkeypatch.setattr(driver, "attest_registrations", lambda rows: None)
|
||||
monkeypatch.setattr(driver, "pending_jobs", lambda: [])
|
||||
sources = {}
|
||||
for name in ("runner", "host"):
|
||||
stanza = (
|
||||
"if ! flock -n 8; then\nexit 0\nfi"
|
||||
if name == "runner"
|
||||
else "flock -n 8 || exit 0"
|
||||
)
|
||||
raw = (
|
||||
'#!/bin/bash\nreadonly MAINTENANCE_LOCK="/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock"\nexec 8>"$MAINTENANCE_LOCK"\n'
|
||||
+ stanza
|
||||
+ "\n# private SECRET-CANARY\n"
|
||||
).encode()
|
||||
path = tmp_path / name
|
||||
path.write_bytes(raw)
|
||||
sources[name] = (path, hashlib.sha256(raw).hexdigest())
|
||||
monkeypatch.setattr(module, "GUARDS", sources)
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_read_source_file", lambda path, **kw: path.read_bytes()
|
||||
)
|
||||
|
||||
def write(path, raw, *, mode, expected_sha256):
|
||||
module._require(
|
||||
hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256,
|
||||
"guard_cas_failed",
|
||||
)
|
||||
path.write_bytes(raw)
|
||||
|
||||
monkeypatch.setattr(module.fleet, "_atomic_write_owned", write)
|
||||
# Only the filesystem-independent binding is exercised on Windows here.
|
||||
monkeypatch.setattr(
|
||||
module.os, "fstat", lambda fd: type("Info", (), {"st_dev": 1, "st_ino": 2})()
|
||||
)
|
||||
return driver, sources, proven[1]
|
||||
|
||||
|
||||
def test_actual_guard_wrapper_owned_cas_and_exact_restore(module, real_guards):
|
||||
driver, sources, store = real_guards
|
||||
captured = driver.capture_guards()
|
||||
value = {
|
||||
"guards": captured,
|
||||
"guardProgress": {},
|
||||
"registrations": driver.registrations(),
|
||||
"registrationProgress": {},
|
||||
"progress": {row["id"]: "quiesced" for row in module._fixed_rows()},
|
||||
}
|
||||
store["value"] = copy.deepcopy(value)
|
||||
|
||||
def persist():
|
||||
store["value"] = copy.deepcopy(value)
|
||||
|
||||
driver.install_guards(value, persist)
|
||||
assert all(
|
||||
module.GUARD_MARKER.encode() in path.read_bytes()
|
||||
for path, _ in sources.values()
|
||||
)
|
||||
value["registrationProgress"] = {"9": "restored", "22": "restored"}
|
||||
persist()
|
||||
driver.restore_guards(value, persist)
|
||||
assert value["guardProgress"] == {"runner": "restored", "host": "restored"}
|
||||
assert all(
|
||||
module.hashlib.sha256(path.read_bytes()).hexdigest() == digest
|
||||
for path, digest in sources.values()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stage", [None, "install_intent", "installed", "restore_intent", "restored"]
|
||||
)
|
||||
def test_foreign_guard_changes_are_never_adopted_or_overwritten(
|
||||
module, real_guards, stage
|
||||
):
|
||||
driver, sources, store = real_guards
|
||||
captured = driver.capture_guards()
|
||||
store["value"] = {
|
||||
"guards": captured,
|
||||
"guardProgress": {"runner": stage} if stage else {},
|
||||
}
|
||||
path = sources["runner"][0]
|
||||
path.write_bytes(path.read_bytes() + b"foreign-change")
|
||||
before = path.read_bytes()
|
||||
with pytest.raises(module.QuiescenceError, match="dispatch_guard_changed"):
|
||||
driver._guard_binding()
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"failure", ["short-write", "temp-fsync", "before-link", "after-link"]
|
||||
)
|
||||
def test_initial_receipt_atomic_publication_faults(
|
||||
module, monkeypatch, tmp_path, failure
|
||||
):
|
||||
import json
|
||||
|
||||
monkeypatch.setattr(module, "STATE_PATH", tmp_path / "receipt.json")
|
||||
monkeypatch.setattr(module.fleet, "_trusted_directory", lambda path: None)
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_trusted_private_file", lambda path, **kw: path.read_bytes()
|
||||
)
|
||||
receipt = {
|
||||
"schema": module.SCHEMA,
|
||||
"revision": "d" * 40,
|
||||
"manifestDigest": module._digest(module.OBSERVATION),
|
||||
"phase": "guarding",
|
||||
"baseline": {},
|
||||
"registrations": [],
|
||||
"registrationProgress": {},
|
||||
"progress": {},
|
||||
"guards": {"runner": {}, "host": {}},
|
||||
"guardProgress": {},
|
||||
}
|
||||
real_open, real_link, real_fsync = module.os.fdopen, module.os.link, module.os.fsync
|
||||
if failure == "short-write":
|
||||
|
||||
class Short:
|
||||
def __init__(self, fd, mode):
|
||||
self.stream = real_open(fd, mode)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stream.close()
|
||||
|
||||
def write(self, raw):
|
||||
return self.stream.write(raw[:1])
|
||||
|
||||
monkeypatch.setattr(module.os, "fdopen", Short)
|
||||
elif failure == "temp-fsync":
|
||||
monkeypatch.setattr(
|
||||
module.os, "fsync", lambda fd: (_ for _ in ()).throw(OSError("fault"))
|
||||
)
|
||||
else:
|
||||
|
||||
def link(src, dst, **kw):
|
||||
if failure == "after-link":
|
||||
real_link(src, dst, **kw)
|
||||
raise OSError("fault")
|
||||
|
||||
monkeypatch.setattr(module.os, "link", link)
|
||||
with pytest.raises((OSError, module.QuiescenceError)):
|
||||
module._save_receipt(receipt, None)
|
||||
if failure == "after-link":
|
||||
assert json.loads(module.STATE_PATH.read_bytes()) == receipt
|
||||
assert module._read_receipt() == receipt
|
||||
else:
|
||||
assert not module.STATE_PATH.exists()
|
||||
assert list(tmp_path.glob(".acceptance-receipt-*")) == []
|
||||
|
||||
|
||||
def test_deadline_expires_before_native_read_or_effect(module, monkeypatch):
|
||||
driver = module.ControllerDriver({})
|
||||
driver._deadline = module.time.monotonic() - 1
|
||||
monkeypatch.setattr(
|
||||
module.fleet, "_run", lambda *a, **kw: pytest.fail("no effect after deadline")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module.fleet,
|
||||
"_api_json",
|
||||
lambda *a, **kw: pytest.fail("no read after deadline"),
|
||||
)
|
||||
with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"):
|
||||
driver._run(["docker", "info"])
|
||||
with pytest.raises(module.QuiescenceError, match="maintenance_execution_deadline"):
|
||||
driver._api_json("/admin/actions/runners")
|
||||
|
||||
|
||||
def test_readonly_poll_reports_real_partial_and_complete_without_resuming(
|
||||
module, proven
|
||||
):
|
||||
driver, receipt = proven
|
||||
driver.timeout = True
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert module.operation_status()["status"] == "partial"
|
||||
assert module.operation_status(active=True)["status"] == "in_progress"
|
||||
before = len(driver.effects)
|
||||
assert module.operation_status()["phase"] == "quiescing"
|
||||
assert len(driver.effects) == before
|
||||
driver.timeout = False
|
||||
module.quiesce(revision="d" * 40, projects={}, driver=driver)
|
||||
assert module.operation_status()["status"] == "quiesced"
|
||||
assert module.operation_status()["completedContainers"] == 18
|
||||
|
||||
|
||||
def test_external_migration_lock_is_nonblocking_and_readonly_busy(module, proven, monkeypatch):
|
||||
def busy(**kwargs):
|
||||
assert kwargs == {"blocking": False}
|
||||
raise module.fleet.RunnerFleetSpecializeError("runner_fleet_specialization_recovery_busy")
|
||||
monkeypatch.setattr(module.fleet, "_migration_lock", busy)
|
||||
driver, _ = proven
|
||||
result = module.preflight(revision="d"*40, projects={}, driver=driver)
|
||||
assert result["status"] == "in_progress" and result["effectsPerformed"] is False
|
||||
assert driver.effects == []
|
||||
@@ -0,0 +1,260 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import AuditEvent, GatewayUser, JobRun, Policy
|
||||
from backend.app.services import (
|
||||
action_execution,
|
||||
deployment_broker,
|
||||
policy_engine,
|
||||
user_access,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'acceptance.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as value:
|
||||
yield value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"action", ["runner.acceptance.preflight", "runner.acceptance.quiesce"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"target,source,expected",
|
||||
[
|
||||
("tower/ci-acceptance-residue", "chatgpt-mcp", True),
|
||||
("tower/ci-acceptance-residue", "manual", True),
|
||||
("tower/ci-acceptance-residue", "voice", False),
|
||||
("other-target", "chatgpt-mcp", False),
|
||||
("*", "manual", False),
|
||||
],
|
||||
)
|
||||
def test_finite_policy_never_inherits_wildcard_or_voice(
|
||||
session, action, target, source, expected
|
||||
):
|
||||
session.add(
|
||||
Policy(
|
||||
id="wildcard",
|
||||
action=action,
|
||||
target="*",
|
||||
manual="allowed",
|
||||
voice="allowed",
|
||||
risk="low",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
result = policy_engine.evaluate_policy(
|
||||
session, action=action, target=target, source=source
|
||||
)
|
||||
assert (result["decision"] != "blocked") is expected
|
||||
assert result["policy_id"] is None
|
||||
if expected and action.endswith(".quiesce"):
|
||||
assert result["decision"] == "approval_required"
|
||||
assert (
|
||||
policy_engine.evaluate_policy(
|
||||
session, action="container.stop", target=target, source=source
|
||||
)["decision"]
|
||||
== "blocked"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["operator", "read_only", "custom"])
|
||||
@pytest.mark.parametrize(
|
||||
"action", ["runner.acceptance.preflight", "runner.acceptance.quiesce"]
|
||||
)
|
||||
def test_human_nonadmin_cannot_call_new_executor(session, monkeypatch, role, action):
|
||||
user = GatewayUser(
|
||||
id="human",
|
||||
provider="authentik",
|
||||
issuer="https://issuer",
|
||||
subject="jens",
|
||||
role=role,
|
||||
)
|
||||
session.add(user)
|
||||
session.commit()
|
||||
actor = user_access.actor_identity.set("authentik:jens")
|
||||
try:
|
||||
monkeypatch.setattr(
|
||||
deployment_broker.DeploymentBrokerClient,
|
||||
"runner_acceptance",
|
||||
lambda *a, **k: pytest.fail("must not call broker"),
|
||||
)
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action=action,
|
||||
target="tower/ci-acceptance-residue",
|
||||
source="manual",
|
||||
risk="high",
|
||||
reason="inspect fixture",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
assert result["policyDecision"] == "blocked"
|
||||
assert session.exec(select(JobRun)).all() == []
|
||||
finally:
|
||||
user_access.actor_identity.reset(actor)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"action,status,success",
|
||||
[
|
||||
("runner.acceptance.preflight", "ready", True),
|
||||
("runner.acceptance.preflight", "blocked", False),
|
||||
("runner.acceptance.quiesce", "quiesced", True),
|
||||
("runner.acceptance.quiesce", "already_quiesced", True),
|
||||
("runner.acceptance.quiesce", "partial", False),
|
||||
("runner.acceptance.quiesce", "blocked", False),
|
||||
("runner.acceptance.quiesce", "ready", False),
|
||||
],
|
||||
)
|
||||
def test_typed_mcp_records_actual_controller_result(
|
||||
session, monkeypatch, action, status, success
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
deployment_broker.DeploymentBrokerClient,
|
||||
"runner_acceptance",
|
||||
lambda *_, **kw: {
|
||||
"status": status,
|
||||
"revision": "a" * 40,
|
||||
"blockers": ["terminal_origin_claim_missing"],
|
||||
},
|
||||
)
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action=action,
|
||||
target="tower/ci-acceptance-residue",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="inspect fixture",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
assert result["success"] is success
|
||||
assert session.get(JobRun, result["jobId"]).status == (
|
||||
"succeeded" if success else "failed"
|
||||
)
|
||||
assert (
|
||||
session.get(AuditEvent, result["auditId"]).target
|
||||
== "tower/ci-acceptance-residue"
|
||||
)
|
||||
assert result["blockers"] == ["terminal_origin_claim_missing"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,source,decision",
|
||||
[
|
||||
("other", "chatgpt-mcp", "allowed"),
|
||||
("tower/ci-acceptance-residue", "voice", "allowed"),
|
||||
("tower/ci-acceptance-residue", "chatgpt-mcp", "blocked"),
|
||||
("tower/ci-acceptance-residue", "manual", "allowed"),
|
||||
],
|
||||
)
|
||||
def test_unsupported_target_or_auth_refuses_before_job(
|
||||
session, monkeypatch, target, source, decision
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
deployment_broker.DeploymentBrokerClient,
|
||||
"runner_acceptance",
|
||||
lambda *a, **k: pytest.fail("must not call broker"),
|
||||
)
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action="runner.acceptance.quiesce",
|
||||
target=target,
|
||||
source=source,
|
||||
risk="high",
|
||||
reason="inspect fixture",
|
||||
original_policy_decision=decision,
|
||||
)
|
||||
assert result["policyDecision"] == "blocked"
|
||||
assert session.exec(select(JobRun)).all() == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"preflight,path",
|
||||
[
|
||||
(True, "/v1/appops/runner-acceptance/preflight"),
|
||||
(False, "/v1/appops/runner-acceptance/quiesce"),
|
||||
],
|
||||
)
|
||||
def test_broker_has_fixed_signed_path_and_bounded_body(monkeypatch, preflight, path):
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
deployment_broker.DeploymentBrokerClient,
|
||||
"_request",
|
||||
lambda _, *a, **kw: calls.append((a, kw)) or {},
|
||||
)
|
||||
deployment_broker.DeploymentBrokerClient().runner_acceptance(
|
||||
reason="x" * 1100, preflight=preflight
|
||||
)
|
||||
assert calls[0][0] == ("POST", path)
|
||||
assert calls[0][1]["payload"] == {"reason": "x" * 1000}
|
||||
assert "secret" in calls[0][1]
|
||||
|
||||
|
||||
def test_controller_canary_is_discarded_before_job_and_audit(session, monkeypatch):
|
||||
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_acceptance", lambda *a, **k: {
|
||||
"status": "Bearer SECRET-CANARY", "reason": "Bearer SECRET-CANARY", "revision": "SECRET-CANARY",
|
||||
"blockers": ["SECRET-CANARY"], "verifiedRunners": ["SECRET-CANARY"], "projectbrainDeliveryRequired": "SECRET-CANARY",
|
||||
})
|
||||
result = action_execution.execute_governed_action(session, action="runner.acceptance.quiesce", target="tower/ci-acceptance-residue",
|
||||
source="chatgpt-mcp", risk="high", reason="inspect fixture",
|
||||
original_policy_decision="allowed")
|
||||
assert result["success"] is False
|
||||
assert result["blockers"] == ["controller_response_untrusted"]
|
||||
assert "SECRET-CANARY" not in str(result)
|
||||
assert "SECRET-CANARY" not in str(session.get(JobRun, result["jobId"]))
|
||||
assert "SECRET-CANARY" not in str(session.get(AuditEvent, result["auditId"]))
|
||||
|
||||
|
||||
def test_acceptance_transport_loss_is_unknown_never_effect_free(monkeypatch):
|
||||
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "_request", lambda *a, **kw: {"status": "unreachable"})
|
||||
result = deployment_broker.DeploymentBrokerClient().runner_acceptance(reason="preserve residue", preflight=False)
|
||||
assert result["status"] == "unknown" and result["noMutation"] is False
|
||||
assert result["effectsPerformed"] is None
|
||||
|
||||
|
||||
def test_acceptance_unknown_transport_is_preserved_in_job_result_and_audit(session, monkeypatch):
|
||||
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_acceptance", lambda *a, **kw:
|
||||
{"status": "unknown", "noMutation": False, "effectsPerformed": None,
|
||||
"blockers": ["maintenance_outcome_unknown"]})
|
||||
result = action_execution.execute_governed_action(session, action="runner.acceptance.quiesce",
|
||||
target="tower/ci-acceptance-residue", source="chatgpt-mcp", risk="high", reason="preserve residue",
|
||||
original_policy_decision="allowed")
|
||||
assert result["controllerStatus"] == "unknown" and result["result"] == "unknown"
|
||||
assert result["noMutation"] is False and result["effectsPerformed"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["runner.acceptance.preflight", "runner.acceptance.quiesce"])
|
||||
def test_actual_mcp_request_preserves_actor_guard_and_never_grants_automation(session, monkeypatch, action):
|
||||
from backend.app.services import mcp_operations
|
||||
session.add(GatewayUser(id="operator", provider="authentik", issuer="test", subject="operator", role="operator"))
|
||||
session.commit()
|
||||
token = user_access.actor_identity.set("authentik:operator")
|
||||
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_acceptance",
|
||||
lambda *a, **k: pytest.fail("nonadmin must not reach broker"))
|
||||
try:
|
||||
result = mcp_operations.request_operation(session, action=action, target="tower/ci-acceptance-residue",
|
||||
reason="preserve finite residue", idempotency_key="nonadmin-"+action)
|
||||
finally:
|
||||
user_access.actor_identity.reset(token)
|
||||
assert result["policyDecision"] == "blocked"
|
||||
assert session.exec(select(JobRun)).all() == []
|
||||
assert policy_engine.evaluate_policy(session, action=action, target="tower/ci-acceptance-residue", source="automation")["decision"] == "blocked"
|
||||
profile = action_execution.action_execution_profile(action, "allowed", "operator")
|
||||
assert profile["executionMode"] == "governance_only" and profile["commandExecutionSupported"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action,target", [("runner.acceptance.quiesce", "other-target"),
|
||||
("runner.acceptance.preflight", "*"),
|
||||
("container.stop", "tower/ci-acceptance-residue")])
|
||||
def test_actual_mcp_request_keeps_wrong_target_and_generic_stop_denied(session, monkeypatch, action, target):
|
||||
from backend.app.services import mcp_operations
|
||||
monkeypatch.setattr(deployment_broker.DeploymentBrokerClient, "runner_acceptance", lambda *a, **k: pytest.fail("must remain denied"))
|
||||
result = mcp_operations.request_operation(session, action=action, target=target,
|
||||
reason="preserve finite residue", idempotency_key="deny-"+action)
|
||||
assert result["policyDecision"] == "blocked" and result["executed"] is False
|
||||
assert session.exec(select(JobRun)).all() == []
|
||||
assert mcp_operations.mcp_policy_source(action, target) == "automation"
|
||||
@@ -4,6 +4,7 @@ import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -649,7 +650,11 @@ def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(mon
|
||||
inspected = iter([probe, copy.deepcopy(probe), None])
|
||||
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
||||
calls = []
|
||||
monkeypatch.setattr(envelope, "_run", lambda args, **kw: calls.append(args))
|
||||
def remove(args, **kw):
|
||||
calls.append(args)
|
||||
assert kw == {"check": False, "capture_output": True, "timeout": 10}
|
||||
return SimpleNamespace(returncode=0, stderr=b"")
|
||||
monkeypatch.setattr(envelope.subprocess, "run", remove)
|
||||
assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
||||
assert calls == [["docker", "rm", probe["Id"]]]
|
||||
|
||||
@@ -671,10 +676,85 @@ def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch,
|
||||
inspected = iter([probe, fresh])
|
||||
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
||||
monkeypatch.setattr(envelope, "_run", lambda *a, **kw: pytest.fail("No removal or probe execution"))
|
||||
monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda *a: pytest.fail("No removal"))
|
||||
with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_"):
|
||||
envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stderr,condition", [
|
||||
(b"private-canary: no space left on device", "remove_storage_full"),
|
||||
(b"private-canary: read-only file system", "remove_read_only"),
|
||||
(b"private-canary: permission denied", "remove_permission_denied"),
|
||||
(b"Cannot connect to the Docker daemon at private-canary", "remove_daemon_unavailable"),
|
||||
(b"private-canary: device or resource busy", "remove_busy"),
|
||||
(b"Removal of container private-canary is already in progress", "remove_busy"),
|
||||
(b"No such container: private-canary", "remove_container_missing"),
|
||||
(b"private-canary: client version is too old", "remove_api_version_unsupported"),
|
||||
(b"removal of container private-canary failed unexpectedly", "remove_failed"),
|
||||
(b"private-canary", "remove_failed"),
|
||||
(b"no space left on device" + b"x" * 4096, "remove_failed"),
|
||||
])
|
||||
def test_probe_removal_has_finite_diagnostics_and_never_forces(monkeypatch, stderr, condition):
|
||||
envelope = _module()
|
||||
identifier = "d" * 64
|
||||
def run(args, **kw):
|
||||
assert args == ["docker", "rm", identifier]
|
||||
assert kw == {"check": False, "capture_output": True, "timeout": 10}
|
||||
return SimpleNamespace(returncode=1, stderr=stderr, stdout=b"private-canary")
|
||||
monkeypatch.setattr(envelope.subprocess, "run", run)
|
||||
with pytest.raises(envelope.HostProbeCleanupError) as error:
|
||||
envelope._remove_cleanup_probe(identifier)
|
||||
assert str(error.value) == "resource_cgroup_probe_cleanup_" + condition
|
||||
assert "private-canary" not in str(error.value)
|
||||
assert error.value.__cause__ is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identifier", ["probe", "d" * 12, "d" * 64 + " --force", "../private-canary"])
|
||||
def test_probe_removal_rejects_selectors_before_any_command(monkeypatch, identifier):
|
||||
envelope = _module()
|
||||
monkeypatch.setattr(envelope.subprocess, "run", lambda *a, **kw: pytest.fail("No command"))
|
||||
with pytest.raises(envelope.HostProbeCleanupError, match="cleanup_identity_invalid"):
|
||||
envelope._remove_cleanup_probe(identifier)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["image_metadata_failed", "inspect_failed", "reinspect_failed", "absence_failed"])
|
||||
def test_probe_read_failures_drop_untrusted_exception_text(stage):
|
||||
envelope = _module()
|
||||
def fail():
|
||||
raise RuntimeError("Bearer private-canary")
|
||||
with pytest.raises(envelope.HostProbeCleanupError) as error:
|
||||
envelope._cleanup_probe_read(stage, fail)
|
||||
assert str(error.value) == "resource_cgroup_probe_cleanup_" + stage
|
||||
assert error.value.__cause__ is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name == "nt", reason="Docker CLI shim uses POSIX executable semantics")
|
||||
@pytest.mark.parametrize("failure,condition", [("read-only file system private-canary", "remove_read_only"),
|
||||
("Bearer private-canary", "remove_failed")])
|
||||
def test_real_probe_cleanup_cli_exposes_only_finite_child_failure(tmp_path, failure, condition):
|
||||
envelope = _module()
|
||||
probe, image_id, code_source = _host_probe_fixture(envelope)
|
||||
fixture = tmp_path / "fixture.json"
|
||||
fixture.write_text(json.dumps({"probe": probe, "image": image_id, "failure": failure}))
|
||||
shim = tmp_path / "docker"
|
||||
shim.write_text("#!" + sys.executable + "\n" +
|
||||
"import json,sys,pathlib\n" +
|
||||
"v=json.loads(pathlib.Path(" + repr(str(fixture)) + ").read_text())\n" +
|
||||
"a=sys.argv[1:]\n" +
|
||||
"if a[:2]==['image','inspect']: print(json.dumps([{'Id':v['image'],'Config':{'Labels':{}}}]))\n" +
|
||||
"elif a[:2]==['ps','-a']: print(v['probe']['Id'])\n" +
|
||||
"elif a[0]=='inspect': print(json.dumps([v['probe']]))\n" +
|
||||
"elif a==['rm',v['probe']['Id']]: print(v['failure'],file=sys.stderr); sys.exit(1)\n" +
|
||||
"else: sys.exit(99)\n")
|
||||
shim.chmod(0o700)
|
||||
result = subprocess.run([sys.executable, str(ENVELOPE_PATH), "--cleanup-stopped-host-probe",
|
||||
image_id, code_source], capture_output=True, timeout=15,
|
||||
env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", "")})
|
||||
assert result.returncode == 1
|
||||
assert result.stdout == b""
|
||||
assert result.stderr == ("resource_cgroup_probe_cleanup_" + condition + "\n").encode()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.title": "runner"}])
|
||||
def test_probe_label_metadata_comes_from_exact_image(monkeypatch, labels):
|
||||
envelope = _module()
|
||||
|
||||
@@ -156,3 +156,54 @@ def test_statistics_configuration_caller_cannot_select_credential_or_endpoint(co
|
||||
monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration'))
|
||||
instance.do_POST()
|
||||
assert responses == [(400, {'status': 'invalid_request'})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["preflight", "quiesce"])
|
||||
def test_acceptance_uses_fixed_signed_route_and_protected_source(controller, monkeypatch, stage):
|
||||
import runner_acceptance_quiescence
|
||||
instance, responses, signatures = handler(controller, monkeypatch)
|
||||
instance.path = getattr(controller, "RUNNER_ACCEPTANCE_" + stage.upper() + "_PATH")
|
||||
calls = []
|
||||
monkeypatch.setattr(runner_acceptance_quiescence, stage,
|
||||
lambda **kw: calls.append(kw) or {"status": "blocked", "blockers": ["terminal_origin_claim_missing"]})
|
||||
instance.do_POST()
|
||||
assert signatures == [{"method": "POST", "path": instance.path}]
|
||||
assert calls == [{"revision": "a" * 40, "projects": {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}}}]
|
||||
assert responses == [(200, {"status": "blocked", "blockers": ["terminal_origin_claim_missing"]})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["preflight", "quiesce"])
|
||||
@pytest.mark.parametrize("gate", ["untrusted", "invalid_signature", "source_mismatch", "push_pending", "wrong_head"])
|
||||
def test_acceptance_security_gate_refuses_before_executor(controller, monkeypatch, stage, gate):
|
||||
import runner_acceptance_quiescence
|
||||
instance, responses, _ = handler(controller, monkeypatch)
|
||||
instance.path = getattr(controller, "RUNNER_ACCEPTANCE_" + stage.upper() + "_PATH")
|
||||
monkeypatch.setattr(runner_acceptance_quiescence, stage, lambda **_: pytest.fail("no executor before source/auth gate"))
|
||||
if gate == "untrusted":
|
||||
monkeypatch.setattr(instance, "_trusted_client", lambda: False)
|
||||
elif gate == "invalid_signature":
|
||||
monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_, **__: False)
|
||||
elif gate == "source_mismatch":
|
||||
monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "b" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY})
|
||||
elif gate == "push_pending":
|
||||
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _: {"status": "preflight_rejected"})
|
||||
else:
|
||||
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: False)
|
||||
instance.do_POST()
|
||||
assert responses[0][0] in {403, 409}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["containerId", "project", "revision", "daemon", "command", "credential", "path"])
|
||||
def test_acceptance_caller_cannot_choose_any_effect_identity(controller, monkeypatch, field):
|
||||
instance, responses, _ = handler(controller, monkeypatch, {"reason": "inspect fixture", field: "SECRET-CANARY"})
|
||||
instance.path = controller.RUNNER_ACCEPTANCE_QUIESCE_PATH
|
||||
instance.do_POST()
|
||||
assert responses == [(400, {"status": "invalid_request"})]
|
||||
assert "SECRET-CANARY" not in str(responses)
|
||||
|
||||
|
||||
def test_pending_acceptance_receipt_fences_legacy_maintenance(controller, monkeypatch):
|
||||
import runner_acceptance_quiescence
|
||||
monkeypatch.setattr(runner_acceptance_quiescence, "maintenance_fenced", lambda: True)
|
||||
monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("no re-enable during partial receipt"))
|
||||
assert controller._runner_maintenance_cycle() == "acceptance_maintenance_owned"
|
||||
Reference in new issue
Block a user