Jens/Unraid-AppOps-Gateway-Public
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
8dd4b40252bf235711837e43e88140382aecd88b
Unraid-AppOps-Gateway-Public/backend/tests/test_deploy_broker.py
T

10086 lines
406 KiB
Python
Raw Blame History

from __future__ import annotations
import copy
import hashlib
import hmac
import http.client
import importlib.util
import io
import ipaddress
import json
import os
import socket
import ssl
import stat
import subprocess
import sys
import tarfile
import threading
import types
from contextlib import contextmanager
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
BROKER_ROOT = ROOT / "ops" / "gitea-deploy-broker"
def _load_module(name: str, path: Path):
if "fcntl" not in sys.modules:
fcntl = types.ModuleType("fcntl")
fcntl.LOCK_EX = 2
fcntl.LOCK_UN = 8
fcntl.flock = lambda *_args: None
sys.modules["fcntl"] = fcntl
spec = importlib.util.spec_from_file_location(name, path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.fixture
def deploy_module():
return _load_module("appops_deploy_broker_test", BROKER_ROOT / "deploy.py")
@pytest.fixture
def status_module():
return _load_module("appops_deploy_status_test", BROKER_ROOT / "status_server.py")
def test_controller_state_uses_the_physical_cache_path(deploy_module, status_module):
expected_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
assert deploy_module.STATE_ROOT == expected_root
assert deploy_module.GLOBAL_DEPLOY_LOCK == expected_root.parent / "deploy.lock"
assert status_module.STATE_ROOT == expected_root
assert status_module.WEBHOOK_INBOX_ROOT == expected_root / "webhook-inbox"
def test_normal_deploy_still_takes_its_own_lock(deploy_module, monkeypatch, tmp_path):
project_lock = tmp_path / "repopulse.lock"
other_lock = tmp_path / "other.lock"
monkeypatch.setitem(
sys.modules,
"repopulse_adoption_store",
types.SimpleNamespace(transaction_lock_held=lambda path: path == project_lock),
)
with deploy_module.exclusive_lock(project_lock, "project repopulse"):
assert project_lock.exists()
with deploy_module.exclusive_lock(other_lock, "other project"):
assert other_lock.exists()
def test_adoption_deploy_requires_both_held_locks_before_entering_core(
deploy_module, monkeypatch
):
monkeypatch.setitem(
sys.modules,
"repopulse_adoption_store",
types.SimpleNamespace(transaction_lock_held=lambda _path: False),
)
monkeypatch.setattr(
deploy_module,
"main",
lambda _argv: pytest.fail("deploy core must not run without the transaction"),
)
with pytest.raises(deploy_module.DeployError, match="requires both fixed transaction locks"):
deploy_module.deploy_repopulse_under_adoption_transaction("a" * 40)
def test_adoption_deploy_rechecks_exact_receipt_after_existing_core(
deploy_module, monkeypatch, tmp_path
):
revision = "a" * 40
fixed_locks = {
deploy_module.STATE_ROOT / "locks" / "repopulse.lock",
deploy_module.GLOBAL_DEPLOY_LOCK,
}
monkeypatch.setitem(
sys.modules,
"repopulse_adoption_store",
types.SimpleNamespace(transaction_lock_held=lambda path: path in fixed_locks),
)
observed = []
project = {
"managed_profile": "repopulse-authenticated",
"branch": "main",
"source_dir": str(tmp_path),
"slug": "repopulse",
"exact_source": True,
}
monkeypatch.setattr(
deploy_module,
"load_projects",
lambda: {"jens/repopulse": project},
)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda *_args: revision)
monkeypatch.setattr(
deploy_module,
"deployment_metadata",
lambda *_args: {"tree": "b" * 40},
)
for name in (
"validate_project_config",
"validate_forward_revision",
"sync_source",
"copy_environment",
"revalidate_deployment_candidate",
"validate_managed_v1_runtime",
"ensure_docker_capacity",
"deploy_project",
"write_state",
):
monkeypatch.setattr(
deploy_module,
name,
lambda *_args, _name=name, **_kwargs: observed.append(_name),
)
receipt = {"repository": "jens/repopulse", "revision": revision}
monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt)
monkeypatch.setattr(
deploy_module,
"verified_current_receipt",
lambda _repo, _project, _revision, _tree: receipt,
)
assert deploy_module.deploy_repopulse_under_adoption_transaction(revision) == receipt
assert observed == [
"validate_project_config",
"validate_forward_revision",
"sync_source",
"copy_environment",
"revalidate_deployment_candidate",
"validate_managed_v1_runtime",
"ensure_docker_capacity",
"deploy_project",
"write_state",
]
def test_global_lock_uses_only_the_physical_cache_lock(deploy_module, monkeypatch):
events: list[tuple[str, Path] | tuple[str]] = []
@contextmanager
def observed_lock(path: Path, _description: str):
events.append(("enter", path))
try:
yield
finally:
events.append(("exit", path))
monkeypatch.setattr(deploy_module, "exclusive_lock", observed_lock)
with deploy_module.exclusive_global_deploy_lock():
events.append(("body",))
physical = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
assert events == [
("enter", physical),
("body",),
("exit", physical),
]
def test_static_rollback_environment_accepts_application_database_url(deploy_module, tmp_path):
path = tmp_path / ".env"
path.write_text("DATABASE_URL='postgresql://fixture:fixture@localhost:5432/projectbrain'\n")
assert deploy_module.static_projectbrain_environment(path) == {
"DATABASE_URL": "postgresql://fixture:fixture@localhost:5432/projectbrain"
}
@pytest.mark.parametrize("changed", [False, True])
def test_project_config_is_rechecked_after_lock_before_work(deploy_module, monkeypatch, tmp_path, changed):
repository = "jens/unraid-appops-gateway"
config = {"slug": "unraid-appops-gateway", "branch": "main",
"source_dir": str(tmp_path / "source"), "compose_project": "appops",
"containers": ["Unraid-AppOps-Gateway"]}
locked = False
reads = 0
@contextmanager
def lock(*_args):
nonlocal locked
locked = True
try:
yield
finally:
locked = False
def load():
nonlocal reads
reads += 1
if reads == 2:
assert locked, "configuration must be rechecked inside the project lock"
value = copy.deepcopy(config)
if reads == 2 and changed:
value["source_dir"] = str(tmp_path / "migrated-source")
return {repository: value}
monkeypatch.setattr(sys, "argv", ["deploy.py", "check", repository, "a" * 40])
monkeypatch.setattr(deploy_module, "load_projects", load)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda *_: "a" * 40)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_: {})
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", lock)
if changed:
with pytest.raises(deploy_module.DeployError, match="configuration changed"):
deploy_module.main()
else:
assert deploy_module.main() == 0
assert reads == 2
def _project(tmp_path: Path) -> dict[str, object]:
return {
"slug": "unraid-appops-gateway",
"source_dir": str(tmp_path / "source"),
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
"verify_oci_revision": True,
"_repository_name": "jens/unraid-appops-gateway",
}
def _appops_known_legacy_state() -> dict[str, object]:
old_revision = "fe0772fdf5eb7ad772312c9263560d70ebddb536"
failed_revision = "034c9f844d2455850ba37b31379b7edaa786583e"
image_id = "sha256:2b15f07e5be8e0e51075e282354ca8f9feeda1a4146c1f55313a7dd9e9b74fbe"
stale_container_id = "6632964b26056cb05832bd52cc19a32cb654cdc1e68ec9d569553271e88b1aea"
stale_container = {
"container_id": stale_container_id,
"image_id": image_id,
"image_ref": "unraid-appops-gateway:production",
"container_revision": old_revision,
"image_revision": old_revision,
"image_source": "jens/unraid-appops-gateway",
"image_tree": "89fae3afc35222c2f176cf50f1fcb19e535e2539",
"image_version": "1.8.0-platform-fe0772fdf5eb",
"image_created": "2026-09-29T00:00:00Z",
}
stale_artifacts = {"containers": {"Unraid-AppOps-Gateway": stale_container}}
stale_receipt = {
"schema": "gitea-deploy-receipt.v1",
"repository": "jens/unraid-appops-gateway",
"revision": old_revision,
"tree": stale_container["image_tree"],
"treeDigest": "657e133a772e839b1d917f081b5135f2b985f774aeb825cc7a00f29091b242a9",
"build": {"revision": old_revision, "tree": stale_container["image_tree"]},
"containers": {"Unraid-AppOps-Gateway": stale_container},
"live": None,
}
failure = "Existing production containers require a valid healthy live deployment state"
attempt_metadata = {
"builtAt": "2026-10-01T18:38:29Z",
"id": failed_revision[:12],
"repository": "jens/unraid-appops-gateway",
"revision": failed_revision,
"tree": "a" * 40,
"treeDigest": hashlib.sha256(f"git-tree:{'a' * 40}".encode()).hexdigest(),
}
return {
"schema_version": 3,
"repository": "jens/unraid-appops-gateway",
"slug": "unraid-appops-gateway",
"compose_project": "unraid-appops-gateway",
"revision": old_revision,
"live_revision": old_revision,
"live_status": "unknown",
"live_legacy_provenance": False,
"live_artifacts": copy.deepcopy(stale_artifacts),
"live_receipt": copy.deepcopy(stale_receipt),
"rollback_snapshot": {
"revision": old_revision,
"legacy_provenance": False,
"receipt": copy.deepcopy(stale_receipt),
},
"status": "failed",
"message": failure,
"updated_at_epoch": 1,
"artifacts": copy.deepcopy(stale_artifacts),
"last_attempt": {
"revision": failed_revision,
"status": "failed",
"message": failure,
"updated_at_epoch": 1,
"metadata": attempt_metadata,
},
"previous_revision": "",
"rollback_restored_revision": "",
}
def test_appops_legacy_state_anchor_accepts_only_known_preimage_or_current_green_attempt(
deploy_module, monkeypatch
):
state = _appops_known_legacy_state()
raw = (json.dumps(state, indent=2) + "\n").encode()
monkeypatch.setattr(
deploy_module,
"APPOPS_LEGACY_ATTEMPT_METADATA_SHA256",
hashlib.sha256(json.dumps(
state["last_attempt"]["metadata"], sort_keys=True, separators=(",", ":")
).encode()).hexdigest(),
)
monkeypatch.setattr(
deploy_module,
"APPOPS_LEGACY_STATE_SHA256",
hashlib.sha256(raw).hexdigest(),
)
assert deploy_module.appops_legacy_state_anchor(
state, raw, "034c9f844d2455850ba37b31379b7edaa786583e",
protected_main_green=False,
)
current = "c" * 40
rewritten = copy.deepcopy(state)
rewritten["message"] = deploy_module.APPOPS_LEGACY_FAILURE_MESSAGE
rewritten["updated_at_epoch"] = 2
candidate_tree = "c" * 40
rewritten["last_attempt"].update({
"revision": current, "updated_at_epoch": 2,
"metadata": {
"builtAt": "2026-10-02T10:00:00Z", "id": current[:12],
"repository": "jens/unraid-appops-gateway", "revision": current,
"tree": candidate_tree,
"treeDigest": hashlib.sha256(f"git-tree:{candidate_tree}".encode()).hexdigest(),
},
})
expected = copy.deepcopy(rewritten)
expected.pop("updated_at_epoch")
expected.pop("last_attempt")
projection = hashlib.sha256(
json.dumps(expected, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_PROJECTION_SHA256", projection)
rewritten_raw = (json.dumps(rewritten, indent=2) + "\n").encode()
assert deploy_module.appops_legacy_state_anchor(
rewritten, rewritten_raw, current,
candidate_metadata=rewritten["last_attempt"]["metadata"],
protected_main_green=True,
)
assert not deploy_module.appops_legacy_state_anchor(
rewritten, rewritten_raw, current, protected_main_green=False,
)
tampered = copy.deepcopy(rewritten)
tampered["rollback_snapshot"]["revision"] = "d" * 40
tampered_raw = (json.dumps(tampered, indent=2) + "\n").encode()
assert not deploy_module.appops_legacy_state_anchor(
tampered, tampered_raw, current,
candidate_metadata=rewritten["last_attempt"]["metadata"],
protected_main_green=True,
)
def test_appops_legacy_runtime_requires_pinned_source_image_and_resolved_model(
deploy_module, tmp_path
):
project = _project(tmp_path)
project.update({
"branch": "main", "gitea_repository_id": 9,
"compose_project": "unraid-appops-gateway",
"containers": ["Unraid-AppOps-Gateway"],
"compose_files": ["docker-compose.yml"],
"environment": {
"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216",
"APPOPS_PUBLISH_HOST": "198.18.10.150",
},
})
source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway"
old_revision = "5553c04ce497c08a35ed301052dc5e0b02e5d9b4"
image_id = "sha256:9b5eddf32c8eb5361106fc35cc0e056c49d287a20d1fb26490662cc8447f09d4"
container = {
"Id": "72daa9f638defc4f8897d8aad23c4181054b2a1bc04c993078a5ac9597eeb0ec",
"Image": image_id,
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Config": {
"Image": "unraid-appops-gateway:platform-5553c04ce497",
"Hostname": "unraid-appops-gateway",
"Labels": {
"com.docker.compose.project": "unraid-appops-gateway",
"com.docker.compose.service": "appops-gateway",
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
"org.opencontainers.image.source": source,
"org.opencontainers.image.revision": old_revision,
"org.opencontainers.image.source-tree": "1624e1748053f998d8427c8c8a4203277d22f13c",
"org.opencontainers.image.version": "1.8.0-platform-5553c04ce497",
"org.opencontainers.image.created": "2026-10-01T16:32:01Z",
},
},
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
"Mounts": [
{"Type": "bind", "Source": "/srv/example/appdata/unraid-appops-gateway-1216",
"Destination": "/data", "RW": True},
{"Type": "bind", "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models",
"Destination": "/mnt/privatemuse-comfyui-models", "RW": True},
],
"NetworkSettings": {
"Ports": {"1216/tcp": [{"HostIp": "198.18.10.150", "HostPort": "1216"}]},
"Networks": {
"itworx-mcp-hub-npm": {},
"unraid-appops-gateway_default": {},
},
},
}
image = {
"Id": image_id,
"Config": {"Labels": container["Config"]["Labels"]},
}
observed = deploy_module.verify_appops_legacy_runtime(project, container, image)
assert observed["imageId"] == image_id
assert observed["imageRef"] == "unraid-appops-gateway:platform-5553c04ce497"
assert observed["sourceRepository"] == source
wrong = copy.deepcopy(container)
wrong["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "127.0.0.1"
with pytest.raises(deploy_module.DeployError, match="fixed AppOps legacy runtime"):
deploy_module.verify_appops_legacy_runtime(project, wrong, image)
wrong_image = copy.deepcopy(image)
wrong_image["Id"] = "sha256:" + "0" * 64
with pytest.raises(deploy_module.DeployError, match="fixed AppOps legacy runtime"):
deploy_module.verify_appops_legacy_runtime(project, container, wrong_image)
def test_appops_legacy_intent_is_root_private_and_compare_and_swap_bound(
deploy_module, monkeypatch, tmp_path
):
root = tmp_path / "controller-config" / "appops-legacy-upgrade"
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", root)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_PATH", root / "unraid-appops-gateway.json")
revision = "c" * 40
tree = "d" * 40
metadata = {
"id": revision[:12], "repository": "jens/unraid-appops-gateway",
"revision": revision, "tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-10-02T10:00:00.000Z",
}
source_proof = {
"legacyRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
"candidateRevision": revision, "composeBlob": "3" * 40,
"dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
}
protected_main = {
"semanticSha256": "5" * 64, "revision": revision,
"source": "gitea_webhook",
"validationContext": "Managed validation / full (push)",
"validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000",
"bodySha256": "6" * 64, "validationSha256": "7" * 64,
}
runtime_proof = {
"resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64,
"runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64,
"healthSha256": "c" * 64,
}
backup_receipt = {
"schema": "itworx.appops-data-backup/v1",
"source": deploy_module.APPOPS_LEGACY_DATA_PATH,
"archive": "/srv/example/appdata/unraid-appops-gateway-backups/one/appdata.tar.gz",
"sha256": "d" * 64, "bytes": 12, "requestedRevision": revision,
"previousRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"createdAt": "2026-10-02T10:00:00.000Z",
}
backup = {
**backup_receipt,
"receiptSha256": deploy_module._canonical_json_sha256(backup_receipt),
}
snapshot = {
"revision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"legacy_provenance": True, "receipt": {},
"compose_containers": ["Unraid-AppOps-Gateway"],
"containers": [{
"name": "Unraid-AppOps-Gateway",
"container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID,
"image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID,
"image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF,
"restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF,
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
"was_running": True,
}],
"captured_at_epoch": 1790884709,
}
payload = deploy_module._appops_legacy_intent_payload(
revision=revision, metadata=metadata, state_sha256="e" * 64,
observed_state_sha256="e" * 64, source_proof=source_proof,
protected_main=protected_main, runtime_proof=runtime_proof,
runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID},
backup=backup, snapshot=snapshot,
)
payload["status"] = "prepared"
digest = deploy_module.write_appops_legacy_intent(payload)
loaded, observed_digest = deploy_module.read_appops_legacy_intent()
assert loaded == payload
assert observed_digest == digest
assert root.stat().st_mode & 0o777 == 0o700
assert deploy_module.APPOPS_LEGACY_INTENT_PATH.stat().st_mode & 0o777 == 0o600
captured_snapshot = copy.deepcopy(payload["rollbackSnapshot"])
updated = deploy_module._appops_legacy_rebind_observed_state(payload, "f" * 64)
assert updated["stateObservedSha256"] == "f" * 64
assert updated["rollbackSnapshot"] == captured_snapshot
updated["status"] = "snapshot_ready"
next_digest = deploy_module.write_appops_legacy_intent(updated, expected_sha256=digest)
assert next_digest != digest
with pytest.raises(deploy_module.DeployError, match="compare-and-swap"):
deploy_module.write_appops_legacy_intent(payload, expected_sha256=digest)
loaded, _ = deploy_module.read_appops_legacy_intent()
assert loaded["stateObservedSha256"] == "f" * 64
assert loaded["rollbackSnapshot"] == captured_snapshot
tampered = copy.deepcopy(loaded)
tampered["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] = "f" * 64
with pytest.raises(deploy_module.DeployError, match="intent and rollback snapshot differ"):
deploy_module.write_appops_legacy_intent(tampered, expected_sha256=next_digest)
hardlink = root / "hardlink.json"
os.link(deploy_module.APPOPS_LEGACY_INTENT_PATH, hardlink)
with pytest.raises(deploy_module.DeployError, match="unsafe"):
deploy_module.read_appops_legacy_intent()
def test_appops_legacy_main_dispatch_uses_prepared_snapshot_backup_and_state_cas(
deploy_module, monkeypatch, tmp_path, capsys
):
repository = "jens/unraid-appops-gateway"
revision = "c" * 40
project_config = {
"slug": "unraid-appops-gateway", "branch": "main",
"source_dir": str(tmp_path / "source"),
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
}
projects = {repository: project_config}
metadata = {
"id": revision[:12], "repository": repository, "revision": revision,
"tree": "d" * 40,
"treeDigest": hashlib.sha256(f"git-tree:{'d' * 40}".encode()).hexdigest(),
"builtAt": "2026-10-02T10:00:00.000Z",
}
legacy_state = _appops_known_legacy_state()
raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode()
state_sha256 = hashlib.sha256(raw_state).hexdigest()
post_state = {"repository": repository, "live_revision": revision, "live_status": "healthy"}
events = []
snapshot = {
"revision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"legacy_provenance": True,
"receipt": {},
"containers": [{"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497"}],
}
backup = {"sha256": "e" * 64, "receiptSha256": "f" * 64}
intent = {"status": "deploying"}
migration = {
"intent": intent, "intent_sha256": "1" * 64,
"state_sha256": state_sha256, "state": legacy_state,
"raw_state": raw_state, "snapshot": snapshot, "backup": backup,
}
@contextmanager
def project_lock(path, _description):
events.append(("project-lock", path))
yield
@contextmanager
def global_lock():
events.append(("global-lock",))
yield
monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision])
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: metadata)
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock)
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None)
monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state)
monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None)
monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None)
monkeypatch.setattr(deploy_module, "validate_managed_v1_runtime", lambda *_args: None)
monkeypatch.setattr(deploy_module, "ensure_docker_capacity", lambda: None)
monkeypatch.setattr(deploy_module, "capture_compose_rollback", lambda *_args, **_kwargs: pytest.fail("migration snapshot must be reused"))
monkeypatch.setattr(deploy_module, "capture_appops_data_backup", lambda *_args: pytest.fail("migration backup must be reused"))
def read_preimage(_project):
events.append(("state-preimage",))
return (legacy_state, raw_state, state_sha256) if events.count(("state-preimage",)) == 1 else (
post_state, b"post-state", "2" * 64
)
def prepare(project, _repository, _repo, _revision, _metadata, state, raw, digest):
assert events[-1] == ("state-preimage",)
assert state == legacy_state and raw == raw_state and digest == state_sha256
assert events.count(("global-lock",)) == 1
project["_appops_legacy_migration"] = migration
events.append(("prepare",))
return migration
monkeypatch.setattr(deploy_module, "read_appops_state_preimage", read_preimage)
monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_appops_legacy_reconcile_pending_rollback", lambda *_args, **_kwargs: False)
monkeypatch.setattr(deploy_module, "_appops_legacy_recover_interrupted_candidate", lambda *_args: False)
monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare)
monkeypatch.setattr(deploy_module, "write_appops_progress", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({}, "a" * 64, "b" * 64))
monkeypatch.setattr(deploy_module, "deploy_project", lambda *_args: events.append(("deploy",)))
receipt = {"repository": repository, "revision": revision, "tree": metadata["tree"]}
monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt)
monkeypatch.setattr(deploy_module, "finish_compose_migration", lambda *_args: None)
def write_state(*_args, **kwargs):
events.append(("write-state", kwargs))
def write_intent(payload, *, expected_sha256=None):
events.append(("write-intent", payload["status"], expected_sha256))
return "3" * 64
monkeypatch.setattr(deploy_module, "write_state", write_state)
monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", write_intent)
monkeypatch.setattr(deploy_module, "cleanup_rollback_image_tags", lambda *_args, **_kwargs: None)
assert deploy_module.main() == 0
assert ("prepare",) in events and ("deploy",) in events
state_event = next(event for event in events if event[0] == "write-state")
assert state_event[1]["expected_state_sha256"] == state_sha256
assert state_event[1]["rollback_snapshot"] == snapshot
assert events.index(("global-lock",)) < events.index(("prepare",)) < events.index(("deploy",))
assert ("write-intent", "deployed", "1" * 64) in events
assert json.loads(capsys.readouterr().out)["status"] == "deployed"
def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256: str):
repository = "jens/unraid-appops-gateway"
tree = "d" * 40
old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION
metadata = {
"id": revision[:12], "repository": repository, "revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-10-02T10:00:00.000Z",
}
source_proof = {
"legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
"candidateRevision": revision, "composeBlob": "3" * 40,
"dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
}
protected_main = {
"semanticSha256": "5" * 64, "revision": revision,
"source": "gitea_webhook",
"validationContext": "Managed validation / full (push)",
"validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000",
"bodySha256": "6" * 64, "validationSha256": "7" * 64,
}
runtime_proof = {
"resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64,
"runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64,
"healthSha256": "c" * 64,
}
backup_receipt = {
"schema": "itworx.appops-data-backup/v1",
"source": deploy_module.APPOPS_LEGACY_DATA_PATH,
"archive": "/tmp/appops-rollback-fixture/appdata.tar.gz",
"sha256": "d" * 64, "bytes": 12, "requestedRevision": revision,
"previousRevision": old_revision, "createdAt": "2026-10-02T10:00:00.000Z",
}
backup = {
**backup_receipt,
"receiptSha256": deploy_module._canonical_json_sha256(backup_receipt),
}
snapshot = {
"revision": old_revision,
"legacy_provenance": True,
"receipt": {},
"compose_containers": ["Unraid-AppOps-Gateway"],
"containers": [{
"name": "Unraid-AppOps-Gateway",
"container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID,
"image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID,
"image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF,
"restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF,
"rollback_ref": (
"gitea-deploy-rollback/unraid-appops-gateway-0:" + old_revision[:12]
),
"was_running": True,
}],
"captured_at_epoch": 1790884709,
}
intent = deploy_module._appops_legacy_intent_payload(
revision=revision, metadata=metadata, state_sha256=capture_sha256,
observed_state_sha256=capture_sha256, source_proof=source_proof,
protected_main=protected_main, runtime_proof=runtime_proof,
runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID},
backup=backup, snapshot=snapshot, status="deploying",
)
return repository, metadata, source_proof, protected_main, backup, intent
def test_appops_legacy_intent_accepts_only_the_reviewed_dockerfile_blob_pair(
deploy_module
):
_repository, _metadata, _source, _main, _backup, intent = (
_appops_rollback_intent_fixture(deploy_module, "c" * 40, "a" * 64)
)
deploy_module._appops_validate_intent_payload(intent)
reviewed_update = copy.deepcopy(intent)
reviewed_source = reviewed_update["migrationProof"]["source"]
reviewed_source["dockerfileBlob"] = "aae97717280b8495a93bc277e290eeb515b76862"
reviewed_evidence = reviewed_update["rollbackSnapshot"]["appops_legacy"]
reviewed_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(
reviewed_source
)
reviewed_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(
reviewed_update["migrationProof"]
)
deploy_module._appops_validate_intent_payload(reviewed_update)
unknown_update = copy.deepcopy(reviewed_update)
unknown_source = unknown_update["migrationProof"]["source"]
unknown_source["dockerfileBlob"] = "b" * 40
unknown_evidence = unknown_update["rollbackSnapshot"]["appops_legacy"]
unknown_evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(
unknown_source
)
unknown_evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(
unknown_update["migrationProof"]
)
with pytest.raises(deploy_module.DeployError, match="source proof is invalid"):
deploy_module._appops_validate_intent_payload(unknown_update)
def test_new_database_pair_keeps_protected_full_and_runtime_intent_proofs_strict(
deploy_module,
):
_repository, _metadata, _source, _main, _backup, intent = (
_appops_rollback_intent_fixture(deploy_module, "c" * 40, "a" * 64)
)
def rebind_migration_proof(value):
proof = value["migrationProof"]
evidence = value["rollbackSnapshot"]["appops_legacy"]
evidence["sourceProofSha256"] = deploy_module._canonical_json_sha256(proof["source"])
evidence["protectedMainSha256"] = deploy_module._canonical_json_sha256(proof["protectedMain"])
evidence["migrationProofSha256"] = deploy_module._canonical_json_sha256(proof)
intent["migrationProof"]["source"]["databaseTree"] = (
"21d29c1799118fdc43946c2b6fd5585ccc2c4db1"
)
rebind_migration_proof(intent)
deploy_module._appops_validate_intent_payload(intent)
weak_protected = copy.deepcopy(intent)
weak_protected["migrationProof"]["protectedMain"]["validationContext"] = (
"Managed validation / partial (push)"
)
rebind_migration_proof(weak_protected)
with pytest.raises(deploy_module.DeployError, match="protected-main proof is invalid"):
deploy_module._appops_validate_intent_payload(weak_protected)
incomplete_runtime = copy.deepcopy(intent)
incomplete_runtime["migrationProof"]["runtime"].pop("healthSha256")
rebind_migration_proof(incomplete_runtime)
with pytest.raises(deploy_module.DeployError, match="legacy runtime proof is invalid"):
deploy_module._appops_validate_intent_payload(incomplete_runtime)
def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt(
deploy_module, monkeypatch, tmp_path, capsys
):
repository = "jens/unraid-appops-gateway"
revision = "c" * 40
old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION
state_root = tmp_path / "state"
state_root.mkdir()
intent_root = tmp_path / "appops-legacy-intent"
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root)
monkeypatch.setattr(
deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json"
)
capture_sha256 = hashlib.sha256(b"immutable legacy capture preimage\n").hexdigest()
repository, metadata, source_proof, protected_main, backup, intent = (
_appops_rollback_intent_fixture(deploy_module, revision, capture_sha256)
)
backup_root = tmp_path / "backups"
backup_dir = backup_root / "one"
backup_dir.mkdir(parents=True, mode=0o700)
backup_dir.chmod(0o700)
backup_root.chmod(0o700)
archive_path = backup_dir / "appdata.tar.gz"
archive_bytes = b"verified AppOps rollback backup fixture"
archive_path.write_bytes(archive_bytes)
archive_path.chmod(0o600)
monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", backup_root)
backup["archive"] = str(archive_path)
backup["sha256"] = hashlib.sha256(archive_bytes).hexdigest()
backup["bytes"] = len(archive_bytes)
backup_receipt = {key: value for key, value in backup.items() if key != "receiptSha256"}
backup["receiptSha256"] = deploy_module._canonical_json_sha256(backup_receipt)
receipt_path = backup_dir / "receipt.json"
receipt_path.write_text(json.dumps(backup_receipt, sort_keys=True), encoding="utf-8")
receipt_path.chmod(0o600)
intent["dataBackup"] = backup
intent["rollbackSnapshot"]["appops_legacy"]["dataBackupSha256"] = backup["sha256"]
snapshot = intent["rollbackSnapshot"]
project_config = {
"slug": "unraid-appops-gateway", "branch": "main",
"source_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
"gitea_repository_id": 9, "verify_oci_revision": True,
"environment": {
"APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH,
"APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST,
},
}
projects = {repository: project_config}
receipt = {
"repository": repository, "revision": revision, "tree": metadata["tree"],
"build": metadata,
}
candidate_artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "sha256:" + "e" * 64}}}
candidate_state = {
"schema_version": 3, "repository": repository, "slug": "unraid-appops-gateway",
"compose_project": "unraid-appops-gateway", "revision": revision,
"live_revision": revision, "live_status": "healthy",
"live_legacy_provenance": False, "live_artifacts": candidate_artifacts,
"live_receipt": receipt, "rollback_snapshot": snapshot,
"status": "success", "message": "verified candidate",
"updated_at_epoch": 100, "artifacts": candidate_artifacts,
"last_attempt": {
"revision": revision, "status": "success", "message": "verified candidate",
"updated_at_epoch": 100, "metadata": metadata,
},
"previous_revision": old_revision, "rollback_restored_revision": "",
}
state_path = state_root / "unraid-appops-gateway.json"
state_path.write_text(json.dumps(candidate_state, indent=2) + "\n", encoding="utf-8")
state_path.chmod(0o644)
original_snapshot = copy.deepcopy(snapshot)
deploy_module.write_appops_legacy_intent(intent)
@contextmanager
def no_lock(*_args):
yield
monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", repository, revision])
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata)
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock)
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: receipt)
monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None)
monkeypatch.setattr(
deploy_module,
"validate_recorded_receipt",
lambda _project, observed, observed_repository, observed_revision: (
metadata
if observed == receipt and observed_repository == repository
and observed_revision == revision
else pytest.fail("rollback receipt did not match the strict candidate fixture")
),
)
monkeypatch.setattr(
deploy_module, "inspect_image",
lambda _ref: {"Id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID},
)
monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: {
"containers": {"Unraid-AppOps-Gateway": {
"container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID,
"image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID,
"image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF,
"container_revision": old_revision,
"image_revision": old_revision,
"image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0],
"image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION,
"image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED,
}},
})
restored = {"runtime": False}
def restore(project, _snapshot, _repository, _failed_revision):
assert _snapshot == original_snapshot
restored["runtime"] = True
project["_appops_legacy_restored"] = True
def inspect_restored_runtime(_project, *, require_original_container):
assert require_original_container is False
assert restored["runtime"] is True
return ({}, {}, {}, {})
runtime_kinds = []
def classify_runtime(_project, observed_intent):
assert observed_intent["rollbackSnapshot"] == original_snapshot
runtime_kinds.append("candidate")
return "candidate"
monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", classify_runtime)
monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored_runtime)
monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main)
monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof)
original_verify_backup = deploy_module.verify_appops_data_backup
def verify_backup(observed, requested, previous):
assert observed == backup
assert requested == revision and previous == old_revision
original_verify_backup(observed, requested, previous)
monkeypatch.setattr(deploy_module, "verify_appops_data_backup", verify_backup)
monkeypatch.setattr(
deploy_module, "_appops_legacy_tag_snapshot",
lambda observed: observed == original_snapshot or pytest.fail("rollback snapshot changed"),
)
monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore)
original_write_intent = deploy_module.write_appops_legacy_intent
fail_after_state_write = {"enabled": True}
def interrupt_completion(payload, **kwargs):
if payload.get("status") == "rolled_back" and fail_after_state_write["enabled"]:
fail_after_state_write["enabled"] = False
raise OSError("simulated crash after the state CAS")
return original_write_intent(payload, **kwargs)
monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", interrupt_completion)
# Reconcile the earlier state-write/intent-write gap against the real
# private intent file. Its rollback snapshot remains the original capture.
monkeypatch.setattr(deploy_module, "read_appops_state_preimage", deploy_module.read_appops_state_preimage)
deploy_module.reconcile_appops_deployed_intent(
_project(tmp_path) | {"_repository_name": repository, "branch": "main"},
repository, tmp_path, revision, metadata, receipt,
)
deployed_intent, _ = deploy_module.read_appops_legacy_intent()
candidate_preimage = deploy_module.read_appops_state_preimage(_project(tmp_path))[2]
assert deployed_intent["status"] == "deployed"
assert deployed_intent["stateObservedSha256"] == candidate_preimage
assert deployed_intent["rollbackSnapshot"] == original_snapshot
assert deployed_intent["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] == capture_sha256
candidate_raw = state_path.read_bytes()
drifted_state = json.loads(candidate_raw.decode("utf-8"))
drifted_state["message"] = "unexpected external state edit"
state_path.write_text(json.dumps(drifted_state, indent=2) + "\n", encoding="utf-8")
state_path.chmod(0o644)
with pytest.raises(deploy_module.DeployError, match="live state changed"):
deploy_module.main()
still_deployed, _ = deploy_module.read_appops_legacy_intent()
assert still_deployed["status"] == "deployed"
assert runtime_kinds == []
state_path.write_bytes(candidate_raw)
state_path.chmod(0o644)
# Verify backup assertions on rollback calls; deployment checkpoint
# reconciliation above is intentionally outside the rollback fixture.
monkeypatch.setattr(
deploy_module, "verify_appops_data_backup",
lambda observed, requested, previous: (
observed == backup and requested == revision and previous == old_revision
) or pytest.fail("rollback data backup did not match its immutable intent"),
)
with pytest.raises(OSError, match="simulated crash after the state CAS"):
deploy_module.main()
pending_intent, _ = deploy_module.read_appops_legacy_intent()
restored_state, restored_raw, restored_sha256 = deploy_module.read_appops_state_preimage(
_project(tmp_path)
)
assert pending_intent["status"] == "rollback_pending"
assert restored_state["live_status"] == "healthy"
assert restored_state["live_revision"] == old_revision
assert restored_state["live_legacy_provenance"] is True
assert restored_state["live_receipt"] == {}
assert restored_state["rollback_snapshot"] == {}
assert pending_intent["stateObservedSha256"] == candidate_preimage
assert pending_intent["rollbackSnapshot"] == original_snapshot
assert hashlib.sha256(restored_raw).hexdigest() == restored_sha256
# The actual next CLI retry recognizes the exact controller-written
# legacy state and only closes the durable intent checkpoint.
assert deploy_module.main() == 0
completed_intent, _ = deploy_module.read_appops_legacy_intent()
final_state, final_raw, final_sha256 = deploy_module.read_appops_state_preimage(_project(tmp_path))
assert completed_intent["status"] == "rolled_back"
assert completed_intent["stateObservedSha256"] == final_sha256
assert completed_intent["rollbackSnapshot"] == original_snapshot
assert completed_intent["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] == capture_sha256
assert final_state["live_receipt"] == {}
assert final_state["rollback_snapshot"] == {}
assert hashlib.sha256(final_raw).hexdigest() == final_sha256
assert runtime_kinds == ["candidate"]
assert json.loads(capsys.readouterr().out)["status"] == "rolled-back"
@pytest.mark.parametrize(
"runtime_kind", ["candidate", "absent", "legacy_partial", "unknown"]
)
def test_appops_candidate_crash_recovers_before_prepare_and_fails_forward(
deploy_module, monkeypatch, tmp_path, runtime_kind
):
repository = "jens/unraid-appops-gateway"
revision = "c" * 40
old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION
state_root = tmp_path / "state"
state_root.mkdir()
intent_root = tmp_path / "appops-legacy-intent"
intent_root.mkdir(mode=0o700)
intent_root.chmod(0o700)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root)
monkeypatch.setattr(
deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json"
)
legacy_state = _appops_known_legacy_state()
raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode()
state_sha256 = hashlib.sha256(raw_state).hexdigest()
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256)
monkeypatch.setattr(
deploy_module,
"APPOPS_LEGACY_ATTEMPT_METADATA_SHA256",
deploy_module._canonical_json_sha256(legacy_state["last_attempt"]["metadata"]),
)
repository, metadata, source_proof, protected_main, backup, intent = (
_appops_rollback_intent_fixture(deploy_module, revision, state_sha256)
)
intent["status"] = "deploying"
state_path = state_root / "unraid-appops-gateway.json"
state_path.write_bytes(raw_state)
state_path.chmod(0o644)
deploy_module.write_appops_legacy_intent(intent)
project_config = {
"slug": "unraid-appops-gateway", "branch": "main",
"source_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
"gitea_repository_id": 9, "verify_oci_revision": True,
"environment": {
"APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH,
"APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST,
},
}
projects = {repository: project_config}
original_snapshot = copy.deepcopy(intent["rollbackSnapshot"])
restored = {"runtime": False, "snapshot": None}
classification_calls = []
@contextmanager
def no_lock(*_args):
yield
def classify(_project, observed_intent):
assert observed_intent["targetRevision"] == revision
assert observed_intent["rollbackSnapshot"] == original_snapshot
classification_calls.append(runtime_kind)
return runtime_kind
def restore(project, observed_snapshot, _repository, _failed_revision):
assert observed_snapshot == original_snapshot
restored["runtime"] = True
restored["snapshot"] = observed_snapshot
project["_appops_legacy_restored"] = True
def inspect_restored(_project, *, require_original_container):
assert require_original_container is False
assert restored["runtime"] is True
return ({}, {}, {}, {})
old_artifacts = {"containers": {"Unraid-AppOps-Gateway": {
"container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID,
"image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID,
"image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF,
"container_revision": old_revision,
"image_revision": old_revision,
"image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0],
"image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION,
"image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED,
}}}
monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision])
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata)
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock)
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None)
monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state)
monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None)
monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None)
monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None)
monkeypatch.setattr(
deploy_module, "prepare_appops_legacy_migration",
lambda *_args: pytest.fail("partial AppOps runtime must recover before migration preparation"),
)
monkeypatch.setattr(deploy_module, "write_appops_progress", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: old_artifacts)
monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", classify)
monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored)
monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main)
monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof)
monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda observed: (
observed == original_snapshot or pytest.fail("recovery snapshot was not intent-owned")
))
monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore)
if runtime_kind == "unknown":
with pytest.raises(deploy_module.AppOpsLegacyNoMutation):
deploy_module.main()
untouched_intent, _ = deploy_module.read_appops_legacy_intent()
untouched_state, untouched_raw, untouched_sha256 = deploy_module.read_appops_state_preimage(
_project(tmp_path)
)
assert untouched_intent["status"] == "deploying"
assert untouched_state == legacy_state
assert untouched_raw == raw_state
assert untouched_sha256 == state_sha256
assert restored["runtime"] is False
assert classification_calls == [runtime_kind]
return
with pytest.raises(deploy_module.AppOpsLegacyRecoveryComplete):
deploy_module.main()
rolled_state, rolled_raw, rolled_sha256 = deploy_module.read_appops_state_preimage(
_project(tmp_path)
)
recovered_intent, _ = deploy_module.read_appops_legacy_intent()
assert classification_calls == [runtime_kind, runtime_kind]
assert restored["snapshot"] == original_snapshot
assert recovered_intent["status"] == "rolled_back"
assert recovered_intent["stateObservedSha256"] == rolled_sha256
assert recovered_intent["rollbackSnapshot"] == original_snapshot
assert rolled_state["live_revision"] == old_revision
assert rolled_state["live_status"] == "healthy"
assert rolled_state["live_legacy_provenance"] is True
assert rolled_state["live_receipt"] == {}
assert rolled_state["rollback_snapshot"] == {}
assert hashlib.sha256(rolled_raw).hexdigest() == rolled_sha256
retry_prepared = []
runtime_proof = dict(recovered_intent["migrationProof"]["runtime"])
def prepare_same_candidate(*_args):
retry_prepared.append(True)
raise deploy_module.AppOpsLegacyNoMutation("same candidate reached safe preparation")
monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({}, "a" * 64, "b" * 64))
monkeypatch.setattr(deploy_module, "appops_legacy_runtime_hashes", lambda *_args: runtime_proof)
monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare_same_candidate)
monkeypatch.setattr(deploy_module, "restore_live_source", lambda *_args: None)
with pytest.raises(deploy_module.AppOpsLegacyNoMutation, match="same candidate reached safe preparation"):
deploy_module.main()
assert retry_prepared == [True]
def test_appops_forward_failure_rollback_is_pending_before_state_commit_and_resumes(
deploy_module, monkeypatch, tmp_path
):
repository = "jens/unraid-appops-gateway"
revision = "c" * 40
old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION
missing_docker_root = tmp_path / "missing-docker-root"
assert not missing_docker_root.exists()
monkeypatch.setattr(deploy_module, "DOCKER_ROOT", missing_docker_root)
state_root = tmp_path / "state"
state_root.mkdir()
intent_root = tmp_path / "appops-legacy-intent"
intent_root.mkdir(mode=0o700)
intent_root.chmod(0o700)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root)
monkeypatch.setattr(
deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json"
)
legacy_state = _appops_known_legacy_state()
raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode()
state_sha256 = hashlib.sha256(raw_state).hexdigest()
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256)
monkeypatch.setattr(
deploy_module,
"APPOPS_LEGACY_ATTEMPT_METADATA_SHA256",
deploy_module._canonical_json_sha256(legacy_state["last_attempt"]["metadata"]),
)
repository, metadata, source_proof, protected_main, backup, intent = (
_appops_rollback_intent_fixture(deploy_module, revision, state_sha256)
)
intent["status"] = "deploying"
state_path = state_root / "unraid-appops-gateway.json"
state_path.write_bytes(raw_state)
state_path.chmod(0o644)
intent_digest = deploy_module.write_appops_legacy_intent(intent)
project_config = {
"slug": "unraid-appops-gateway", "branch": "main",
"source_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
"gitea_repository_id": 9, "verify_oci_revision": True,
"environment": {
"APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH,
"APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST,
},
}
projects = {repository: project_config}
snapshot = copy.deepcopy(intent["rollbackSnapshot"])
old_artifacts = {"containers": {"Unraid-AppOps-Gateway": {
"container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID,
"image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID,
"image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF,
"container_revision": old_revision,
"image_revision": old_revision,
"image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0],
"image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION,
"image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED,
}}}
runtime_kinds = iter(["legacy", "candidate"])
restored = {"runtime": False}
progress = []
@contextmanager
def no_lock(*_args):
yield
def prepare(project, _repository, _repo, _revision, _metadata, state, raw, digest):
assert digest == state_sha256
assert raw == raw_state and state == legacy_state
migration = {
"intent": intent,
"intent_sha256": intent_digest,
"state_sha256": digest,
"state": state,
"raw_state": raw,
"snapshot": snapshot,
"backup": backup,
}
project["_appops_legacy_migration"] = migration
project["_compose_rollback"] = snapshot
project["_appops_data_backup"] = backup
return migration
def restore(project, observed_snapshot, _repository, _failed_revision):
assert observed_snapshot == snapshot
pending_intent, _ = deploy_module.read_appops_legacy_intent()
assert pending_intent["status"] == "rollback_pending"
restored["runtime"] = True
project["_rollback_restored_revision"] = old_revision
project["_appops_legacy_restored"] = True
def inspect_restored(_project, *, require_original_container):
assert require_original_container is False
assert restored["runtime"] is True
return ({}, {}, {}, {})
monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision])
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata)
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock)
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None)
monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state)
monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None)
monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None)
monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None)
monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare)
monkeypatch.setattr(deploy_module, "write_appops_progress", lambda _revision, phase, **_kwargs: progress.append(phase))
monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: old_artifacts)
monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", lambda *_args: next(runtime_kinds))
monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored)
monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main)
monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof)
monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda observed: (
observed == snapshot or pytest.fail("forward failure did not use the intent-owned snapshot")
))
monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore)
capacity_checks = []
def skip_docker_capacity_for_rollback_test():
assert not deploy_module.DOCKER_ROOT.exists()
capacity_checks.append(True)
monkeypatch.setattr(deploy_module, "ensure_docker_capacity", skip_docker_capacity_for_rollback_test)
deploy_calls = []
def fail_deploy(*_args):
deploy_calls.append(revision)
raise RuntimeError("simulated candidate failure")
monkeypatch.setattr(deploy_module, "deploy_project", fail_deploy)
original_write_intent = deploy_module.write_appops_legacy_intent
def crash_before_terminal_intent(payload, **kwargs):
if payload.get("status") == "rolled_back":
observed_state, _raw, _digest = deploy_module.read_appops_state_preimage(
_project(tmp_path)
)
assert observed_state["status"] == "success"
assert observed_state["live_revision"] == old_revision
raise OSError("simulated crash after rollback state CAS")
return original_write_intent(payload, **kwargs)
monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", crash_before_terminal_intent)
with pytest.raises(deploy_module.AppOpsLegacyRecoveryPending):
deploy_module.main()
assert capacity_checks == [True]
assert deploy_calls == [revision]
pending_intent, _ = deploy_module.read_appops_legacy_intent()
pending_state, pending_raw, pending_sha256 = deploy_module.read_appops_state_preimage(
_project(tmp_path)
)
assert pending_intent["status"] == "rollback_pending"
assert pending_intent["stateObservedSha256"] == state_sha256
assert pending_intent["rollbackSnapshot"] == snapshot
assert pending_state["status"] == "success"
assert pending_state["live_revision"] == old_revision
assert pending_state["live_status"] == "healthy"
assert pending_state["live_receipt"] == {}
assert hashlib.sha256(pending_raw).hexdigest() == pending_sha256
assert progress[-2:] == ["rolling_back", "outcome_unknown"]
monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", original_write_intent)
with pytest.raises(deploy_module.AppOpsLegacyRecoveryComplete):
deploy_module.main()
completed_intent, _ = deploy_module.read_appops_legacy_intent()
assert completed_intent["status"] == "rolled_back"
assert completed_intent["stateObservedSha256"] == pending_sha256
def test_appops_durable_rollback_requires_completed_intent_and_verified_backup(
deploy_module, monkeypatch, tmp_path
):
revision = "c" * 40
tree = "d" * 40
repository = "jens/unraid-appops-gateway"
metadata = {
"id": revision[:12], "repository": repository, "revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-10-02T10:00:00.000Z",
}
source_proof = {
"legacyRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE,
"databaseTree": "1" * 40, "entrypointBlob": "2" * 40,
"candidateRevision": revision, "composeBlob": "3" * 40,
"dockerfileBlob": "73e083dcfef25402665ace5a0bf07a083cf37432",
}
protected_main = {
"semanticSha256": "5" * 64, "revision": revision,
"source": "gitea_webhook",
"validationContext": "Managed validation / full (push)",
"validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000",
"bodySha256": "6" * 64, "validationSha256": "7" * 64,
}
runtime_proof = {
"resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64,
"runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64,
"healthSha256": "c" * 64,
}
backup_receipt = {
"schema": "itworx.appops-data-backup/v1",
"source": deploy_module.APPOPS_LEGACY_DATA_PATH,
"archive": "/srv/example/appdata/unraid-appops-gateway-backups/one/appdata.tar.gz",
"sha256": "d" * 64, "bytes": 12, "requestedRevision": revision,
"previousRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"createdAt": "2026-10-02T10:00:00.000Z",
}
backup = {
**backup_receipt,
"receiptSha256": deploy_module._canonical_json_sha256(backup_receipt),
}
snapshot = {
"revision": deploy_module.APPOPS_LEGACY_OLD_REVISION,
"legacy_provenance": True, "receipt": {},
"compose_containers": ["Unraid-AppOps-Gateway"],
"containers": [{
"name": "Unraid-AppOps-Gateway",
"container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID,
"image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID,
"image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF,
"restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF,
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
"was_running": True,
}],
"captured_at_epoch": 1790884709,
}
intent = deploy_module._appops_legacy_intent_payload(
revision=revision, metadata=metadata, state_sha256="e" * 64,
observed_state_sha256="e" * 64, source_proof=source_proof,
protected_main=protected_main, runtime_proof=runtime_proof,
runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID},
backup=backup, snapshot=snapshot, status="deployed",
)
project = _project(tmp_path)
project.update({"branch": "main", "_repository_name": repository})
current_receipt = {
"repository": repository, "revision": revision, "tree": tree,
"build": metadata,
}
state = {
"schema_version": 3, "repository": repository,
"revision": revision, "live_revision": revision,
"live_status": "healthy", "live_legacy_provenance": False,
"live_receipt": current_receipt, "rollback_snapshot": intent["rollbackSnapshot"],
}
monkeypatch.setattr(deploy_module, "read_state", lambda _project: state)
monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "1" * 64))
monkeypatch.setattr(deploy_module, "validate_recorded_receipt", lambda *_args: metadata)
backup_checks = []
monkeypatch.setattr(
deploy_module,
"verify_appops_data_backup",
lambda observed, requested, previous: backup_checks.append((observed, requested, previous)),
)
monkeypatch.setattr(
deploy_module,
"inspect_image",
lambda _ref: {"Id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID},
)
assert deploy_module.durable_rollback_snapshot(project, repository, revision) == intent["rollbackSnapshot"]
assert backup_checks == [(backup, revision, deploy_module.APPOPS_LEGACY_OLD_REVISION)]
intent["status"] = "rolled_back"
with pytest.raises(deploy_module.DeployError, match="completed migration intent"):
deploy_module.durable_rollback_snapshot(project, repository, revision)
def test_appops_candidate_crash_reconcile_writes_only_from_exact_anchored_state(
deploy_module, monkeypatch, tmp_path
):
repository = "jens/unraid-appops-gateway"
revision = "c" * 40
tree = "d" * 40
metadata = {
"id": revision[:12], "repository": repository, "revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-10-02T10:00:00.000Z",
}
state = _appops_known_legacy_state()
raw = (json.dumps(state, indent=2) + "\n").encode()
state_sha256 = hashlib.sha256(raw).hexdigest()
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", hashlib.sha256(
json.dumps(state["last_attempt"]["metadata"], sort_keys=True, separators=(",", ":")).encode()
).hexdigest())
protected_main = {"revision": revision, "delivery": "same-green-main"}
source_proof = {"candidateRevision": revision, "databaseTree": "1" * 40}
intent = {
"status": "deploying", "targetRevision": revision,
"targetMetadata": metadata, "stateObservedSha256": state_sha256,
"migrationProof": {"protectedMain": protected_main, "source": source_proof},
"rollbackSnapshot": {}, "dataBackup": {},
}
project = _project(tmp_path)
project["_repository_path"] = str(tmp_path)
receipt = {
"repository": repository, "revision": revision, "tree": tree,
"build": metadata,
}
writes = []
intent_writes = []
monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "a" * 64))
monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main)
monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof)
monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None)
monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({"name": project["compose_project"]}, "b" * 64, "c" * 64))
monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_validate_appops_legacy_snapshot", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda *_args: None)
monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt)
monkeypatch.setattr(deploy_module, "write_state", lambda *args, **kwargs: writes.append((args, kwargs)))
monkeypatch.setattr(deploy_module, "read_appops_state_preimage", lambda _project: (state, raw, state_sha256))
monkeypatch.setattr(
deploy_module,
"write_appops_legacy_intent",
lambda payload, **kwargs: intent_writes.append((payload, kwargs)) or "d" * 64,
)
observed = deploy_module.reconcile_appops_legacy_candidate(
project, repository, tmp_path, revision, metadata, state, raw, state_sha256,
)
assert observed == receipt
assert len(writes) == 1 and writes[0][1]["expected_state_sha256"] == state_sha256
assert writes[0][1]["rollback_snapshot"] == {}
assert intent_writes[0][0]["status"] == "deployed"
writes.clear()
changed_raw = raw + b" "
assert deploy_module.reconcile_appops_legacy_candidate(
project, repository, tmp_path, revision, metadata, state, changed_raw,
hashlib.sha256(changed_raw).hexdigest(),
) is None
assert writes == []
def test_appops_state_write_rejects_compare_and_swap_drift(deploy_module, monkeypatch, tmp_path):
state_root = tmp_path / "state"
state_root.mkdir()
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
project = _project(tmp_path)
path = state_root / "unraid-appops-gateway.json"
original = json.dumps({"schema_version": 3, "repository": "jens/unraid-appops-gateway"}).encode()
path.write_bytes(original)
path.chmod(0o644)
with pytest.raises(deploy_module.DeployError, match="compare-and-swap"):
deploy_module.write_state(
"jens/unraid-appops-gateway", project, "c" * 40, "failed", "blocked",
expected_state_sha256="0" * 64,
)
assert path.read_bytes() == original
def test_registered_legacy_state_cas_drift_preserves_external_state_and_records_recovery(
deploy_module, monkeypatch, tmp_path
):
repository = "jens/quiz-monitor"
request_id = "6f9619ff-8b86-4011-b42d-00c04fc964ff"
revision = "a" * 40
state_root = tmp_path / "state"
state_root.mkdir()
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "_registered_legacy_receipt_key", lambda: b"k" * 32)
project = {
"slug": "quiz-monitor", "branch": "main", "compose_project": "quiz-monitor",
"compose_files": ["docker-compose.yml"],
"containers": ["quiz-monitor-quiz-server-1"],
"source_dir": "/srv/example/appdata/quiz-monitor/source",
"_registered_legacy_request_id": request_id,
"_registered_legacy_authorization": {
"requestId": request_id, "revision": revision,
"expectedStateSha256": "0" * 64,
},
}
path = state_root / "quiz-monitor.json"
external_state = b'{"schema_version":3,"live_revision":"independent-update"}\n'
path.write_bytes(external_state)
with pytest.raises(deploy_module.DeployError, match="compare-and-swap"):
deploy_module.write_state(
repository, project, revision, "failed", "deployment failed",
expected_state_sha256="0" * 64,
)
assert path.read_bytes() == external_state
helper = deploy_module._load_registered_legacy_adoption()
outcome = helper.status(
{"repository": repository, "requestId": request_id},
registry={repository: project}, receipt_key=b"k" * 32, state_root=state_root,
)
assert outcome["status"] == "recovery-required"
assert outcome["reasonCode"] == "controller_state_cas_failed"
def _build_reviewed_appops_seed_history(tmp_path, *, candidate_mutation=None):
"""Build the reviewed DB and Dockerfile transitions without old Git history."""
repository = tmp_path / "reviewed-source.git"
working = tmp_path / "reviewed-source-worktree"
working.mkdir()
subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True)
subprocess.run(["git", "init", str(working)], check=True, capture_output=True)
for key, value in (
("user.name", "AppOps fixture"),
("user.email", "appops-fixture@example.invalid"),
("core.autocrlf", "false"),
("core.filemode", "true"),
):
subprocess.run(["git", "-C", str(working), "config", key, value], check=True)
subprocess.run(
["git", "-C", str(working), "remote", "add", "origin", str(repository)],
check=True,
)
db = working / "backend" / "app" / "db"
db.mkdir(parents=True)
current_db = ROOT / "backend" / "app" / "db"
def normalize(path):
return path.read_bytes().replace(b"\r\n", b"\n")
candidate_dockerfile = normalize(ROOT / "Dockerfile")
lock_setup = (
b"RUN set -eu; \\\n"
b" lock_parent=/run/itworx-appops; \\\n"
b" lock_root=\"$lock_parent/forge-oidc-provision\"; \\\n"
)
requirements_copy = b"COPY backend/requirements.txt /tmp/requirements.txt\n"
assert candidate_dockerfile.count(lock_setup) == 1
lock_start = candidate_dockerfile.index(lock_setup)
copy_start = candidate_dockerfile.index(requirements_copy, lock_start)
legacy_dockerfile = candidate_dockerfile[:lock_start] + candidate_dockerfile[copy_start:]
assert hashlib.sha1(
f"blob {len(legacy_dockerfile)}\0".encode() + legacy_dockerfile
).hexdigest() == "73e083dcfef25402665ace5a0bf07a083cf37432"
assert hashlib.sha1(
f"blob {len(candidate_dockerfile)}\0".encode() + candidate_dockerfile
).hexdigest() == "aae97717280b8495a93bc277e290eeb515b76862"
current_init = normalize(current_db / "init_db.py")
settings_marker = b"settings = Settings()\n"
assert current_init.count(settings_marker) == 1
body = current_init.split(settings_marker, 1)[1]
old_header = (
b"from datetime import UTC, datetime\n\n"
b"from sqlalchemy import inspect, text\n"
b"from sqlmodel import SQLModel, select\n"
b"from . import session as db_session\n"
b"from .session import get_session\n"
b"from ..core.config import Settings\n"
b"from ..services.app_classifier import classify_container\n"
b"from ..services.app_registry import category_label\n"
b"from .models import App, AppContainerMapping, Connector, DiscoveredContainer, "
b"NotificationChannel, NotificationEvent, NotificationRule, Policy, WorkMeshPromotion\n\n\n"
)
old_init = old_header + settings_marker + body
forge_seed_rows = (
b' Policy(id="policy-forge-oidc-provision", action="forge.oidc.provision", target="itworx-forge", voice="blocked", manual="allowed", risk="critical", reason="The fixed Forge OIDC client may be provisioned only by an authenticated manual operator route; voice and generic Authentik mutations remain blocked."),\n',
b' Policy(id="policy-forge-secret-prepare", action="forge.secrets.prepare", target="itworx-forge", voice="blocked", manual="blocked", risk="critical", reason="Preparing the fixed Forge bootstrap secret bundle is default-denied and requires an explicit exact-target manual operator policy; voice remains blocked."),\n',
b' Policy(id="policy-forge-route-stage", action="forge.route.stage", target="itworx-forge", voice="blocked", manual="blocked", risk="critical", reason="Staging the fixed disabled Forge proxy host is default-denied; voice remains blocked."),\n',
)
for row in forge_seed_rows:
assert current_init.count(row) == 1
old_init = old_init.replace(row, b"", 1)
(db / "init_db.py").write_bytes(old_init)
for filename in ("models.py", "session.py"):
(db / filename).write_bytes(normalize(current_db / filename))
entrypoint = working / "docker" / "entrypoint.sh"
entrypoint.parent.mkdir(parents=True)
entrypoint.write_bytes(normalize(ROOT / "docker" / "entrypoint.sh"))
entrypoint.chmod(0o644)
(working / "docker-compose.yml").write_bytes(normalize(ROOT / "docker-compose.yml"))
(working / "Dockerfile").write_bytes(legacy_dockerfile)
def git(*args):
return subprocess.run(
["git", "-C", str(working), *args],
check=True, capture_output=True, text=True,
).stdout.strip()
git("add", "--all")
git("commit", "-m", "fixed legacy AppOps source")
old_revision = git("rev-parse", "HEAD")
git("push", "origin", "HEAD:refs/heads/main")
old_tree = git("rev-parse", f"{old_revision}^{{tree}}")
old_database_tree = git("rev-parse", f"{old_revision}:backend/app/db")
candidate_init = current_init
if candidate_mutation == "previous_reviewed_seed_tree":
route_seed = forge_seed_rows[2]
assert candidate_init.count(route_seed) == 1
candidate_init = candidate_init.replace(route_seed, b"", 1)
elif candidate_mutation == "legacy_database_unchanged":
candidate_init = old_init
(db / "init_db.py").write_bytes(candidate_init)
if candidate_mutation == "extra_database_file":
(db / "unreviewed.py").write_text("UNREVIEWED = True\n", encoding="utf-8")
elif candidate_mutation == "models_content":
(db / "models.py").write_bytes(normalize(current_db / "models.py") + b"\n# unreviewed\n")
elif candidate_mutation == "session_content":
(db / "session.py").write_bytes(normalize(current_db / "session.py") + b"\n# unreviewed\n")
elif candidate_mutation == "init_db_content":
(db / "init_db.py").write_bytes(candidate_init + b"\n# unreviewed database behavior\n")
elif candidate_mutation == "forge_policy_changed":
changed_init = candidate_init.replace(
b'action="forge.oidc.provision"',
b'action="forge.oidc.provision-unreviewed"',
1,
)
assert changed_init != current_init
(db / "init_db.py").write_bytes(changed_init)
elif candidate_mutation in {
"route_stage_manual_allowed", "route_stage_voice_allowed", "route_stage_risk_lowered",
}:
route_seed = forge_seed_rows[2]
replacements = {
"route_stage_manual_allowed": (b'manual="blocked"', b'manual="allowed"'),
"route_stage_voice_allowed": (b'voice="blocked"', b'voice="allowed"'),
"route_stage_risk_lowered": (b'risk="critical"', b'risk="high"'),
}
old_field, new_field = replacements[candidate_mutation]
assert route_seed.count(old_field) == 1
candidate_init = candidate_init.replace(route_seed, route_seed.replace(old_field, new_field, 1), 1)
(db / "init_db.py").write_bytes(candidate_init)
elif candidate_mutation == "unknown_policy_row":
route_seed = forge_seed_rows[2]
unknown_policy = (
b' Policy(id="policy-forge-route-stage-unreviewed", '
b'action="forge.route.stage.unreviewed", target="itworx-forge", '
b'voice="blocked", manual="blocked", risk="critical", '
b'reason="Unreviewed policy row."),\n'
)
assert candidate_init.count(route_seed) == 1
candidate_init = candidate_init.replace(route_seed, route_seed + unknown_policy, 1)
(db / "init_db.py").write_bytes(candidate_init)
elif candidate_mutation == "database_file_mode":
(db / "session.py").chmod(0o755)
elif candidate_mutation == "entrypoint_content":
entrypoint.write_bytes(entrypoint.read_bytes() + b"# unreviewed entrypoint\n")
elif candidate_mutation not in {
None, "previous_reviewed_seed_tree", "legacy_database_unchanged",
"route_stage_manual_allowed", "route_stage_voice_allowed", "route_stage_risk_lowered",
"unknown_policy_row", "dockerfile_one_byte", "dockerfile_legacy_unchanged",
}:
raise AssertionError("unknown fixture mutation")
if candidate_mutation == "dockerfile_legacy_unchanged":
(working / "Dockerfile").write_bytes(legacy_dockerfile)
else:
candidate_dockerfile = normalize(ROOT / "Dockerfile")
if candidate_mutation == "dockerfile_one_byte":
original_mode = b'mkdir -m 0700 "$lock_root"'
changed_mode = b'mkdir -m 0701 "$lock_root"'
assert candidate_dockerfile.count(original_mode) == 1
candidate_dockerfile = candidate_dockerfile.replace(
original_mode, changed_mode, 1
)
assert len(candidate_dockerfile) == len(normalize(ROOT / "Dockerfile"))
assert sum(left != right for left, right in zip(
candidate_dockerfile, normalize(ROOT / "Dockerfile")
)) == 1
(working / "Dockerfile").write_bytes(candidate_dockerfile)
git("add", "--all")
if candidate_mutation == "database_file_mode":
# Windows does not preserve chmod's executable bit on the worktree;
# stage the real Git mode mutation so this remains a meaningful tree
# identity negative fixture on every platform.
git("update-index", "--chmod=+x", "--", "backend/app/db/session.py")
git("commit", "-m", "candidate source")
candidate_revision = git("rev-parse", "HEAD")
git("push", "origin", "HEAD:refs/heads/main")
candidate_tree = git("rev-parse", f"{candidate_revision}^{{tree}}")
candidate_database_tree = git("rev-parse", f"{candidate_revision}:backend/app/db")
candidate_entrypoint = git("rev-parse", f"{candidate_revision}:docker/entrypoint.sh")
old_dockerfile_blob = git("rev-parse", f"{old_revision}:Dockerfile")
candidate_dockerfile_blob = git("rev-parse", f"{candidate_revision}:Dockerfile")
return {
"repository": repository,
"oldRevision": old_revision,
"oldTree": old_tree,
"oldDatabaseTree": old_database_tree,
"candidateRevision": candidate_revision,
"candidateTree": candidate_tree,
"candidateDatabaseTree": candidate_database_tree,
"candidateEntrypoint": candidate_entrypoint,
"oldDockerfileBlob": old_dockerfile_blob,
"candidateDockerfileBlob": candidate_dockerfile_blob,
}
def _reviewed_appops_source_project(deploy_module):
return {
"_repository_name": "jens/unraid-appops-gateway",
"branch": "main",
"gitea_repository_id": 9,
"source_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"containers": ["Unraid-AppOps-Gateway"],
"verify_oci_revision": True,
"environment": {
"APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH,
"APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST,
"APPOPS_PROXY_NETWORK": "itworx-mcp-hub-npm",
},
}
def test_appops_legacy_source_proof_accepts_the_exact_main_184_dockerfile_update(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(tmp_path)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
assert proof == {
"legacyRevision": history["oldRevision"],
"legacyTree": history["oldTree"],
"databaseTree": "21d29c1799118fdc43946c2b6fd5585ccc2c4db1",
"entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8",
"candidateRevision": history["candidateRevision"],
"composeBlob": "243aee2c6924775bed7706373a5777f4e619e053",
"dockerfileBlob": "aae97717280b8495a93bc277e290eeb515b76862",
}
def test_appops_legacy_source_proof_keeps_the_old_dockerfile_candidate_valid(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation="dockerfile_legacy_unchanged"
)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
assert proof["dockerfileBlob"] == "73e083dcfef25402665ace5a0bf07a083cf37432"
def test_appops_legacy_source_proof_keeps_the_previous_reviewed_seed_tree_valid(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation="previous_reviewed_seed_tree"
)
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
assert history["candidateDatabaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
assert proof["databaseTree"] == "c87c9aff5a0f0e55b9209e2f085cf1663876520a"
def test_appops_legacy_source_proof_rejects_a_one_byte_dockerfile_change(
deploy_module, monkeypatch, tmp_path
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation="dockerfile_one_byte"
)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
with pytest.raises(deploy_module.DeployError, match="fixed legacy source"):
deploy_module.verify_appops_legacy_source(
history["repository"],
_reviewed_appops_source_project(deploy_module),
history["candidateRevision"],
)
def test_appops_legacy_source_bindings_remain_fixed_for_the_new_reviewed_tree(
deploy_module, monkeypatch, tmp_path,
):
history = _build_reviewed_appops_seed_history(tmp_path)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
mutations = (
("_repository_name", "jens/other-repository"),
("branch", "release"),
("gitea_repository_id", 10),
("source_dir", "/tmp/unreviewed-source"),
("compose_project", "unreviewed-project"),
("compose_files", ["other-compose.yml"]),
("containers", ["OtherContainer"]),
("verify_oci_revision", False),
)
for field, value in mutations:
project = _reviewed_appops_source_project(deploy_module)
project[field] = value
with pytest.raises(deploy_module.DeployError, match="fixed legacy source or resolved Compose inputs"):
deploy_module.verify_appops_legacy_source(
history["repository"], project, history["candidateRevision"],
)
for field, value in (
("APPOPS_DATA_PATH", "/tmp/other-data"),
("APPOPS_PUBLISH_HOST", "127.0.0.1"),
("APPOPS_PROXY_NETWORK", "unreviewed-network"),
):
project = _reviewed_appops_source_project(deploy_module)
project["environment"][field] = value
with pytest.raises(deploy_module.DeployError, match="fixed legacy source or resolved Compose inputs"):
deploy_module.verify_appops_legacy_source(
history["repository"], project, history["candidateRevision"],
)
@pytest.mark.parametrize(
("candidate_mutation", "expected_database_tree"),
[
(None, "21d29c1799118fdc43946c2b6fd5585ccc2c4db1"),
("previous_reviewed_seed_tree", "c87c9aff5a0f0e55b9209e2f085cf1663876520a"),
("legacy_database_unchanged", "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"),
],
)
def test_appops_schema_compatibility_accepts_only_the_two_reviewed_pairs_or_same_tree(
deploy_module, monkeypatch, tmp_path, candidate_mutation, expected_database_tree,
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation=candidate_mutation,
)
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
assert history["candidateDatabaseTree"] == expected_database_tree
assert history["candidateEntrypoint"] == "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8"
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
proof = deploy_module.verify_appops_schema_compatibility(
history["repository"], history["candidateRevision"]
)
assert proof == {
"legacyRevision": history["oldRevision"],
"legacyTree": history["oldTree"],
"databaseTree": expected_database_tree,
"entrypointBlob": "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8",
"candidateRevision": history["candidateRevision"],
}
@pytest.mark.parametrize(
"mutation",
[
"extra_database_file",
"models_content",
"session_content",
"init_db_content",
"forge_policy_changed",
"unknown_policy_row",
"route_stage_manual_allowed",
"route_stage_voice_allowed",
"route_stage_risk_lowered",
"database_file_mode",
"entrypoint_content",
],
)
def test_appops_seed_tree_exception_rejects_every_unreviewed_tree_or_entrypoint_change(
deploy_module, monkeypatch, tmp_path, mutation
):
history = _build_reviewed_appops_seed_history(
tmp_path, candidate_mutation=mutation
)
assert history["oldDatabaseTree"] == "d7f8a6188f6c6e0c69bce0e547df31d36ed84a28"
if mutation == "entrypoint_content":
assert history["candidateDatabaseTree"] == "21d29c1799118fdc43946c2b6fd5585ccc2c4db1"
assert history["candidateEntrypoint"] != "4f71ff74e67d7f15e3cf6b3e79b654158dfea2f8"
else:
assert history["candidateDatabaseTree"] not in {
"c87c9aff5a0f0e55b9209e2f085cf1663876520a",
"21d29c1799118fdc43946c2b6fd5585ccc2c4db1",
}
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", history["oldRevision"])
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_TREE", history["oldTree"])
with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"):
deploy_module.verify_appops_schema_compatibility(
history["repository"], history["candidateRevision"]
)
def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint(
deploy_module, tmp_path, monkeypatch
):
repository = tmp_path / "source.git"
working = tmp_path / "work"
working.mkdir()
subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True)
subprocess.run(["git", "init", str(working)], check=True, capture_output=True)
subprocess.run(["git", "-C", str(working), "config", "user.name", "Fixture"], check=True)
subprocess.run(["git", "-C", str(working), "config", "user.email", "fixture@example.invalid"], check=True)
db = working / "backend" / "app" / "db"
db.mkdir(parents=True)
entrypoint = working / "docker" / "entrypoint.sh"
entrypoint.parent.mkdir(parents=True)
(db / "models.py").write_text("MODEL = 1\n")
(db / "init_db.py").write_text("INIT = 1\n")
(db / "session.py").write_text("SESSION = 1\n")
entrypoint.write_text("#!/bin/sh\nexec app\n")
subprocess.run(["git", "-C", str(working), "add", "."], check=True)
subprocess.run(["git", "-C", str(working), "commit", "-m", "legacy"], check=True, capture_output=True)
old_revision = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip()
subprocess.run(["git", "-C", str(working), "remote", "add", "origin", str(repository)], check=True)
subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True)
(working / "backend" / "app" / "core").mkdir(parents=True)
(working / "backend" / "app" / "core" / "config.py").write_text("STATS_KEY = 1\n")
subprocess.run(["git", "-C", str(working), "add", "."], check=True)
subprocess.run(["git", "-C", str(working), "commit", "-m", "non-db stats config"], check=True, capture_output=True)
candidate = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip()
subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True)
monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", old_revision)
monkeypatch.setattr(
deploy_module,
"APPOPS_LEGACY_OLD_TREE",
subprocess.check_output(
["git", "-C", str(working), "rev-parse", f"{old_revision}^{{tree}}"],
text=True,
).strip(),
)
assert deploy_module.verify_appops_schema_compatibility(repository, candidate)["databaseTree"]
(db / "migrations").mkdir()
(db / "migrations" / "001.sql").write_text("DROP TABLE app;\n")
subprocess.run(["git", "-C", str(working), "add", "."], check=True)
subprocess.run(["git", "-C", str(working), "commit", "-m", "schema migration"], check=True, capture_output=True)
changed = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip()
subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True)
with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"):
deploy_module.verify_appops_schema_compatibility(repository, changed)
def _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path, *, parent_source="typed_self_upgrade",
parent_status="validation_timeout", sibling_source=None, include_sibling=True,
):
revision = "c" * 40
identity = {
"gitea_repository_id": 9,
"ref": "refs/heads/main",
"revision": revision,
}
semantic = hashlib.sha256(
json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
state_root = tmp_path / "state"
inbox = state_root / "webhook-inbox"
inbox.mkdir(parents=True, mode=0o700)
inbox.chmod(0o700)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
delivery = "12345678-1234-4234-8234-123456789012"
body_sha256 = "a" * 64
required_contexts = ["Managed validation / full (push)"]
parent = {
"schema": "gitea-webhook-inbox.v2",
"semantic_sha256": semantic,
"body_sha256": body_sha256,
"delivery": delivery,
**identity,
"repository": "jens/unraid-appops-gateway",
"source": parent_source,
"status": parent_status,
"reason": parent_status,
"validation": {
"validation_gate_version": 1,
"required_contexts": required_contexts,
"timeout_seconds": 1800,
"poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
"deadline_at": 1900,
"next_check_at": 1900,
"attempts": 360,
"observed": [],
"last_result": (
"timeout" if parent_status == "validation_timeout" else "reject"
),
},
"audit": [
{"stage": "awaiting_validation"},
{"stage": parent_status, "reason": parent_status},
],
}
parent_marker = inbox / f"{semantic}.json"
parent_marker.write_text(json.dumps(parent), encoding="utf-8")
parent_marker.chmod(0o600)
sibling = {
"schema": "gitea-webhook-revalidation.v1",
"semantic_sha256": semantic,
"revalidation_of": semantic,
"body_sha256": body_sha256,
"delivery": delivery,
**identity,
"repository": "jens/unraid-appops-gateway",
"status": "failed",
"reason": "deployment_failed",
"validation": {
"validation_gate_version": 1,
"required_contexts": required_contexts,
"timeout_seconds": 1800,
"poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
"deadline_at": 2800,
"next_check_at": 2500,
"attempts": 1,
"observed": [{
"context": "Managed validation / full (push)",
"status": "success",
}],
"last_result": "pass",
},
"audit": [
{"stage": "revalidation_requested"},
{"stage": "validation_passed"},
{"stage": "deployment_started"},
{"stage": "deployment_failed", "reason": "deployment_failed"},
],
}
if sibling_source is not None:
sibling["source"] = sibling_source
sibling_marker = inbox / f"{semantic}.revalidation.json"
if include_sibling:
sibling_marker.write_text(json.dumps(sibling), encoding="utf-8")
sibling_marker.chmod(0o600)
return {
"semantic": semantic,
"revision": revision,
"parent": parent,
"sibling": sibling,
"parent_marker": parent_marker,
"sibling_marker": sibling_marker,
}
@pytest.mark.parametrize("parent_source", ["gitea_webhook", "typed_self_upgrade"])
@pytest.mark.parametrize("parent_status", ["validation_timeout", "validation_failed"])
def test_appops_failed_delivery_accepts_only_a_chained_failed_revalidation(
deploy_module, monkeypatch, tmp_path, parent_source, parent_status
):
pair = _write_appops_failed_revalidation_pair(
deploy_module,
monkeypatch,
tmp_path,
parent_source=parent_source,
parent_status=parent_status,
)
original_bytes = pair["parent_marker"].read_bytes()
sibling_bytes = pair["sibling_marker"].read_bytes()
proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"])
assert proof["revision"] == pair["revision"]
assert proof["source"] == parent_source
assert proof["delivery"] == "12345678-1234-4234-8234-123456789012"
assert proof["bodySha256"] == "a" * 64
assert proof["deliveryStatus"] == "failed"
assert proof["deliveryReason"] == "deployment_failed"
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_green_delivery(pair["revision"])
assert pair["parent_marker"].read_bytes() == original_bytes
assert pair["sibling_marker"].read_bytes() == sibling_bytes
@pytest.mark.parametrize(
("which", "field", "value"),
[
("parent", "repository", "attacker/repo"),
("parent", "gitea_repository_id", 8),
("parent", "ref", "refs/heads/other"),
("parent", "revision", "d" * 40),
("parent", "semantic_sha256", "b" * 64),
("parent", "body_sha256", "b" * 64),
("parent", "delivery", "87654321-4321-4321-8321-210987654321"),
("parent", "schema", "gitea-webhook-inbox.v1"),
("parent", "status", "failed"),
("parent", "source", "unknown_source"),
("parent", "reason", "validation_failed"),
("parent", "validation.timeout_seconds", 900),
("parent", "validation.required_contexts", ["Managed validation / pull_request"]),
("parent", "audit", [{"stage": "validation_timeout", "reason": "deployment_failed"}]),
("sibling", "repository", "attacker/repo"),
("sibling", "gitea_repository_id", 8),
("sibling", "ref", "refs/heads/other"),
("sibling", "revision", "d" * 40),
("sibling", "semantic_sha256", "b" * 64),
("sibling", "body_sha256", "b" * 64),
("sibling", "delivery", "87654321-4321-4321-8321-210987654321"),
("sibling", "schema", "gitea-webhook-inbox.v2"),
("sibling", "revalidation_of", "b" * 64),
("sibling", "source", "typed_self_upgrade"),
("sibling", "source", "gitea_webhook"),
("sibling", "status", "deployed"),
("sibling", "reason", "deployment_interrupted"),
("sibling", "validation.timeout_seconds", 900),
("sibling", "validation.validation_gate_version", True),
("sibling", "validation.required_contexts", ["Managed validation / pull_request"]),
("sibling", "validation.observed", [{
"context": "Managed validation / full (push)", "status": "pending",
}]),
("sibling", "audit", [{"stage": "validation_passed"}]),
("sibling", "audit", [
{"stage": "revalidation_requested"},
{"stage": "deployment_started"},
{"stage": "validation_passed"},
{"stage": "deployment_failed", "reason": "deployment_failed"},
]),
],
)
def test_appops_failed_delivery_rejects_an_unbound_or_incomplete_revalidation(
deploy_module, monkeypatch, tmp_path, which, field, value
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path
)
target = pair[which]
keys = field.split(".")
for key in keys[:-1]:
target = target[key]
target[keys[-1]] = value
marker = pair[f"{which}_marker"]
marker.write_text(json.dumps(pair[which]), encoding="utf-8")
marker.chmod(0o600)
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
def test_appops_failed_delivery_does_not_use_a_revalidation_without_its_original(
deploy_module, monkeypatch, tmp_path
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path
)
pair["parent_marker"].unlink()
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
def test_appops_failed_delivery_does_not_reopen_an_original_validation_timeout(
deploy_module, monkeypatch, tmp_path
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path, include_sibling=False
)
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_failed_green_delivery(pair["revision"])
def test_appops_failed_delivery_keeps_the_original_primary_failure_path(
deploy_module, monkeypatch, tmp_path
):
pair = _write_appops_failed_revalidation_pair(
deploy_module, monkeypatch, tmp_path, include_sibling=False
)
primary = pair["parent"]
primary["status"] = "failed"
primary["reason"] = "deployment_failed"
primary["validation"]["observed"] = [{
"context": "Managed validation / full (push)", "status": "success",
}]
primary["validation"]["last_result"] = "pass"
primary["audit"] = [
{"stage": "validation_passed"},
{"stage": "deployment_started"},
{"stage": "deployment_failed", "reason": "deployment_failed"},
]
pair["parent_marker"].write_text(json.dumps(primary), encoding="utf-8")
pair["parent_marker"].chmod(0o600)
proof = deploy_module.verify_appops_failed_green_delivery(pair["revision"])
assert proof["revision"] == pair["revision"]
assert proof["source"] == "typed_self_upgrade"
assert proof["deliveryStatus"] == "failed"
def test_appops_migration_requires_private_exact_full_green_delivery(
deploy_module, monkeypatch, tmp_path
):
revision = "c" * 40
state_root = tmp_path / "state"
inbox = state_root / "webhook-inbox"
inbox.mkdir(parents=True, mode=0o700)
inbox.chmod(0o700)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
semantic = deploy_module._canonical_json_sha256({
"gitea_repository_id": 9, "ref": "refs/heads/main", "revision": revision,
})
record = {
"schema": "gitea-webhook-inbox.v2", "semantic_sha256": semantic,
"body_sha256": "a" * 64, "delivery": "12345678-1234-4234-8234-123456789012",
"gitea_repository_id": 9, "ref": "refs/heads/main",
"repository": "jens/unraid-appops-gateway", "revision": revision,
"source": "typed_self_upgrade", "status": "deploying",
"validation": {
"validation_gate_version": 1,
"required_contexts": ["Managed validation / full (push)"],
"timeout_seconds": 1800, "poll_interval_seconds": 5,
"max_poll_interval_seconds": 30, "last_result": "pass",
"observed": [{"context": "Managed validation / full (push)", "status": "success"}],
},
"audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"}],
}
marker = inbox / f"{semantic}.json"
marker.write_text(json.dumps(record), encoding="utf-8")
marker.chmod(0o600)
proof = deploy_module.verify_appops_green_delivery(revision)
assert proof["validationContext"] == "Managed validation / full (push)"
# The webhook worker atomically makes this same immutable record terminal
# after deploy.py has already committed the success state. A restarted
# broker must still be able to finish the protected intent checkpoint.
record["status"] = "deployed"
marker.write_text(json.dumps(record), encoding="utf-8")
marker.chmod(0o600)
assert deploy_module.verify_appops_green_delivery(revision) == proof
record["validation"]["observed"][0]["status"] = "pending"
marker.write_text(json.dumps(record), encoding="utf-8")
marker.chmod(0o600)
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_green_delivery(revision)
record["status"] = "failed"
record["validation"]["observed"][0]["status"] = "success"
marker.write_text(json.dumps(record), encoding="utf-8")
marker.chmod(0o600)
with pytest.raises(deploy_module.DeployError, match="protected full-green"):
deploy_module.verify_appops_green_delivery(revision)
marker.unlink()
marker.symlink_to(tmp_path / "untrusted.json")
with pytest.raises(deploy_module.DeployError, match="unsafe"):
deploy_module.verify_appops_green_delivery(revision)
def test_appops_retry_completes_pending_intent_after_delivery_becomes_deployed(
deploy_module, monkeypatch, tmp_path
):
repository = "jens/unraid-appops-gateway"
revision = "c" * 40
tree = "d" * 40
metadata = {
"id": revision[:12], "repository": repository, "revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-10-02T10:00:00.000Z",
}
protected_main = {"revision": revision, "delivery": "green-main-delivery"}
source_proof = {"candidateRevision": revision, "databaseTree": "1" * 40}
snapshot = {"revision": deploy_module.APPOPS_LEGACY_OLD_REVISION}
receipt = {"repository": repository, "revision": revision, "tree": tree, "build": metadata}
state = {
"repository": repository, "live_revision": revision, "live_status": "healthy",
"live_legacy_provenance": False, "live_receipt": receipt,
"rollback_snapshot": snapshot,
}
state_sha256 = "a" * 64
intent = {
"status": "deploying", "targetRevision": revision, "targetMetadata": metadata,
"migrationProof": {"protectedMain": protected_main, "source": source_proof},
"rollbackSnapshot": snapshot, "dataBackup": {"sha256": "b" * 64},
}
project = _project(tmp_path)
project.update({"_repository_name": repository, "branch": "main"})
intent_writes = []
monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "c" * 64))
monkeypatch.setattr(
deploy_module, "read_appops_state_preimage",
lambda _project: (state, b"controller-committed-success", state_sha256),
)
monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main)
monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof)
monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None)
monkeypatch.setattr(
deploy_module, "write_appops_legacy_intent",
lambda payload, **kwargs: intent_writes.append((payload, kwargs)) or "d" * 64,
)
# The state-write/intention-write crash leaves the original deploying
# intent, while the webhook worker subsequently marks its durable green
# record deployed. The current-receipt retry must only close that gap.
deploy_module.reconcile_appops_deployed_intent(
project, repository, tmp_path, revision, metadata, receipt,
)
assert len(intent_writes) == 1
assert intent_writes[0][0]["status"] == "deployed"
assert intent_writes[0][0]["stateObservedSha256"] == state_sha256
assert intent_writes[0][1]["expected_sha256"] == "c" * 64
def _workmesh_product_project(tmp_path: Path) -> dict[str, object]:
return {
"slug": "workmesh",
"source_dir": str(tmp_path / "source"),
"compose_project": "workmesh",
"compose_files": ["compose.yaml"],
"containers": ["workmesh-postgres-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1"],
"retired_compose_containers": ["workmesh-mock-ats-1"],
"_repository_name": "jens/workmesh",
}
def _workmesh_legacy_reattest_setup(tmp_path: Path):
revision = "a" * 40
project = _workmesh_product_project(tmp_path)
project.update({
"branch": "main",
"containers": [
"workmesh-postgres-1",
"workmesh-mock-ats-1",
"workmesh-api-1",
"workmesh-n8n-1",
"workmesh-workmesh-1",
],
"compose_files": ["compose.yaml"],
"exact_source": True,
"verify_oci_revision": False,
})
project.pop("retired_compose_containers")
receipt = _receipt(project, revision)
for index, name in enumerate(project["containers"], start=1):
receipt["containers"][name]["container_id"] = f"{index:x}" * 64
receipt["containers"][name]["container_revision"] = ""
for field in ("image_revision", "image_source", "image_tree", "image_version", "image_created"):
receipt["containers"][name][field] = ""
receipt_containers = receipt["containers"]
artifacts = {
"containers": {
name: dict(item)
for name, item in receipt_containers.items()
},
"compose_sha256": "c" * 64,
}
state = {
"schema_version": 3,
"repository": "jens/workmesh",
"slug": "workmesh",
"compose_project": "workmesh",
"revision": revision,
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": False,
"live_artifacts": copy.deepcopy(artifacts),
"live_receipt": copy.deepcopy(receipt),
"rollback_snapshot": {},
"status": "success",
"message": "verified live WorkMesh deployment",
"updated_at_epoch": 100,
"artifacts": copy.deepcopy(artifacts),
"last_attempt": {
"revision": revision,
"status": "success",
"message": "verified live WorkMesh deployment",
"updated_at_epoch": 100,
"metadata": copy.deepcopy(receipt["build"]),
},
"previous_revision": "",
"rollback_restored_revision": "",
}
configured = copy.deepcopy(project)
configured.pop("_repository_name")
project.update({
"_repository_name": "jens/workmesh",
"_repository_path": str(tmp_path),
"_deployment_metadata": copy.deepcopy(receipt["build"]),
})
projects = {"jens/workmesh": configured}
state_root = tmp_path / "state"
state_root.mkdir()
state_path = state_root / "workmesh.json"
original_bytes = (json.dumps(state, indent=2) + "\n").encode("utf-8")
state_path.write_bytes(original_bytes)
return revision, project, projects, receipt, artifacts, state_path, original_bytes
def _install_workmesh_reattest_runtime(
deploy_module, monkeypatch, receipt, artifacts, state_path, *, drift=None
):
observed_receipt = copy.deepcopy(receipt)
observed_artifacts = copy.deepcopy(artifacts["containers"])
observed_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64
observed_artifacts["workmesh-n8n-1"]["container_id"] = "f" * 64
if drift:
name, field, value = drift
observed_receipt["containers"][name][field] = value
observed_artifacts[name][field] = value
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
monkeypatch.setattr(
deploy_module, "validate_commit", lambda _repo, _branch, revision: revision
)
monkeypatch.setattr(
deploy_module,
"deployment_metadata",
lambda _repo, _revision, _repository, *, built_at=None: copy.deepcopy(receipt["build"]),
)
monkeypatch.setattr(
deploy_module,
"git_revision_compose_sha256",
lambda *_args: str(artifacts["compose_sha256"]),
)
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: set(deploy_module.WORKMESH_LEGACY_CONTAINERS),
)
monkeypatch.setattr(
deploy_module,
"verify_deployment",
lambda _project, _build, **_kwargs: copy.deepcopy(observed_receipt),
)
monkeypatch.setattr(
deploy_module,
"deployment_container_artifacts",
lambda _project: copy.deepcopy(observed_artifacts),
)
monkeypatch.setattr(deploy_module, "_fsync_directory", lambda _path: None)
reverified = []
def verify_after_write(repository, _project, revision, tree):
reverified.append((repository, revision, tree))
return json.loads(state_path.read_text(encoding="utf-8"))["live_receipt"]
monkeypatch.setattr(deploy_module, "verified_current_receipt", verify_after_write)
return reverified
def test_workmesh_n8n_recreation_reattests_only_container_id(
deploy_module, tmp_path, monkeypatch
):
(
revision,
project,
projects,
receipt,
artifacts,
state_path,
original_bytes,
) = _workmesh_legacy_reattest_setup(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent)
reverified = _install_workmesh_reattest_runtime(
deploy_module, monkeypatch, receipt, artifacts, state_path
)
result = deploy_module.reattest_workmesh_n8n_receipt(
projects, project, tmp_path, revision
)
persisted = json.loads(state_path.read_text(encoding="utf-8"))
expected_receipt = copy.deepcopy(receipt)
expected_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64
expected_artifacts = copy.deepcopy(artifacts)
expected_artifacts["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64
assert result == {
"revision": revision,
"container": "workmesh-n8n-1",
"status": "reattested",
}
assert persisted["live_receipt"] == expected_receipt
assert persisted["live_artifacts"] == expected_artifacts
assert persisted["artifacts"] == expected_artifacts
assert reverified == [("jens/workmesh", revision, receipt["tree"])]
backup = state_path.parent / (
f"workmesh-before-n8n-reattest-{revision[:12]}-{'4' * 12}.json"
)
assert backup.read_bytes() == original_bytes
if os.name != "nt":
assert stat.S_IMODE(backup.stat().st_mode) == 0o600
assert persisted["status"] == "success"
assert persisted["last_attempt"] == json.loads(original_bytes)["last_attempt"]
@pytest.mark.parametrize(
("name", "field", "value"),
[
("workmesh-api-1", "image_id", "sha256:" + "9" * 64),
("workmesh-api-1", "image_tree", "d" * 40),
("workmesh-api-1", "container_id", "e" * 64),
],
)
def test_workmesh_n8n_reattest_rejects_other_runtime_drift(
deploy_module, tmp_path, monkeypatch, name, field, value
):
(
revision,
project,
projects,
receipt,
artifacts,
state_path,
original_bytes,
) = _workmesh_legacy_reattest_setup(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent)
_install_workmesh_reattest_runtime(
deploy_module,
monkeypatch,
receipt,
artifacts,
state_path,
drift=(name, field, value),
)
with pytest.raises(deploy_module.DeployError, match="runtime drift"):
deploy_module.reattest_workmesh_n8n_receipt(
projects, project, tmp_path, revision
)
assert state_path.read_bytes() == original_bytes
assert list(state_path.parent.glob("workmesh-before-n8n-reattest-*.json")) == []
def test_workmesh_n8n_reattest_requires_compose_digest_from_live_revision(
deploy_module, tmp_path, monkeypatch
):
(
revision,
project,
projects,
receipt,
artifacts,
state_path,
original_bytes,
) = _workmesh_legacy_reattest_setup(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent)
_install_workmesh_reattest_runtime(
deploy_module, monkeypatch, receipt, artifacts, state_path
)
monkeypatch.setattr(
deploy_module,
"git_revision_compose_sha256",
lambda *_args: "d" * 64,
)
with pytest.raises(deploy_module.DeployError, match="Compose digest"):
deploy_module.reattest_workmesh_n8n_receipt(
projects, project, tmp_path, revision
)
assert state_path.read_bytes() == original_bytes
assert list(state_path.parent.glob("workmesh-before-n8n-reattest-*.json")) == []
def test_workmesh_n8n_reattest_rejects_non_root_controller(
deploy_module, tmp_path, monkeypatch
):
revision, project, projects, _receipt_value, _artifacts, state_path, original_bytes = (
_workmesh_legacy_reattest_setup(tmp_path)
)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent)
monkeypatch.setattr(
deploy_module,
"require_host_root",
lambda: (_ for _ in ()).throw(
deploy_module.DeployError("WorkMesh live receipt reattestation is host-root-only")
),
)
with pytest.raises(deploy_module.DeployError, match="host-root-only"):
deploy_module.reattest_workmesh_n8n_receipt(
projects, project, tmp_path, revision
)
assert state_path.read_bytes() == original_bytes
def test_workmesh_n8n_mode_dispatches_under_both_controller_locks(
deploy_module, tmp_path, monkeypatch, capsys
):
revision = "a" * 40
project_config = {
"slug": "workmesh",
"branch": "main",
"source_dir": str(tmp_path / "source"),
"compose_project": "workmesh",
"compose_files": ["compose.yaml"],
"containers": list(deploy_module.WORKMESH_LEGACY_CONTAINERS),
"exact_source": True,
"verify_oci_revision": True,
}
projects = {"jens/workmesh": project_config}
events = []
@contextmanager
def project_lock(path, _description):
events.append(("project-lock", path))
yield
@contextmanager
def global_lock():
events.append(("global-lock",))
yield
monkeypatch.setattr(sys, "argv", [
"deploy.py", "reattest-workmesh-n8n", "jens/workmesh", revision,
])
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(
deploy_module, "validate_commit", lambda _repo, _branch, requested: requested
)
monkeypatch.setattr(
deploy_module, "deployment_metadata", lambda *_args: {"tree": "b" * 40}
)
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock)
def reattest(configured_projects, project, repo, requested_revision):
assert configured_projects == projects
assert project["_repository_name"] == "jens/workmesh"
assert repo == tmp_path
assert requested_revision == revision
events.append(("reattest",))
return {"revision": revision, "container": "workmesh-n8n-1", "status": "reattested"}
monkeypatch.setattr(deploy_module, "reattest_workmesh_n8n_receipt", reattest)
assert deploy_module.main() == 0
assert events == [
("project-lock", deploy_module.STATE_ROOT / "locks" / "workmesh.lock"),
("global-lock",),
("reattest",),
]
assert json.loads(capsys.readouterr().out) == {
"repository": "jens/workmesh",
"revision": revision,
"container": "workmesh-n8n-1",
"status": "reattested",
}
def _workmesh_product_reattest_setup(deploy_module, tmp_path: Path):
revision = "a" * 40
project = {
"slug": "workmesh",
"branch": "main",
"source_dir": (deploy_module.EXACT_SOURCE_ROOT / "workmesh").as_posix(),
"env_source": "/srv/example/appdata/workmesh/.env",
"compose_project": "workmesh",
"compose_files": ["compose.yaml"],
"containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS),
"retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER],
"exact_source": True,
"verify_oci_revision": False,
"health_timeout": 300,
}
receipt = _receipt({**project, "_repository_name": "jens/workmesh"}, revision)
for index, name in enumerate(project["containers"], start=1):
receipt["containers"][name]["container_id"] = f"{index:x}" * 64
receipt["containers"][name]["container_revision"] = ""
receipt["containers"][name]["image_revision"] = ""
receipt["containers"][name]["image_source"] = ""
receipt["containers"][name]["image_tree"] = ""
receipt["containers"][name]["image_version"] = ""
receipt["containers"][name]["image_created"] = ""
artifacts = {
"containers": copy.deepcopy(receipt["containers"]),
"compose_sha256": "c" * 64,
}
candidate_revision = "d" * 40
state = {
"schema_version": 3,
"repository": "jens/workmesh",
"slug": "workmesh",
"compose_project": "workmesh",
"revision": revision,
"live_revision": revision,
"live_status": "unknown",
"live_legacy_provenance": False,
"live_artifacts": copy.deepcopy(artifacts),
"live_receipt": copy.deepcopy(receipt),
"rollback_snapshot": {},
"status": "failed",
"message": "candidate deploy failed; live state needs reconciliation",
"updated_at_epoch": 100,
"artifacts": copy.deepcopy(artifacts),
"last_attempt": {
"revision": candidate_revision,
"status": "failed",
"message": "candidate deploy failed",
"updated_at_epoch": 101,
"metadata": {"revision": candidate_revision},
},
"previous_revision": "",
"rollback_restored_revision": "",
}
configured = copy.deepcopy(project)
project.update({
"_repository_name": "jens/workmesh",
"_repository_path": str(tmp_path),
"_deployment_metadata": copy.deepcopy(receipt["build"]),
})
state_root = tmp_path / "state"
state_root.mkdir()
state_path = state_root / "workmesh.json"
original_bytes = (json.dumps(state, indent=2) + "\n").encode()
state_path.write_bytes(original_bytes)
return revision, project, {"jens/workmesh": configured}, receipt, artifacts, state_path, original_bytes
def test_workmesh_product_n8n_recovery_repairs_only_n8n_id_after_failed_attempt(
deploy_module, tmp_path, monkeypatch
):
(
revision,
project,
projects,
receipt,
artifacts,
state_path,
original_bytes,
) = _workmesh_product_reattest_setup(deploy_module, tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent)
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(
deploy_module,
"exact_source_matches_revision",
lambda _repo, requested, _source: requested == revision,
)
monkeypatch.setattr(
deploy_module,
"deployment_metadata",
lambda _repo, _requested, _repository, *, built_at=None: copy.deepcopy(receipt["build"]),
)
monkeypatch.setattr(
deploy_module,
"git_revision_compose_sha256",
lambda *_args: artifacts["compose_sha256"],
)
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: set(deploy_module.WORKMESH_PRODUCT_CONTAINERS),
)
monkeypatch.setattr(deploy_module, "container_state", lambda _name: (True, "healthy", "workmesh"))
observed_receipt = copy.deepcopy(receipt)
observed_artifacts = copy.deepcopy(artifacts["containers"])
observed_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64
observed_artifacts["workmesh-n8n-1"]["container_id"] = "f" * 64
monkeypatch.setattr(
deploy_module,
"verify_deployment",
lambda *_args, **_kwargs: copy.deepcopy(observed_receipt),
)
monkeypatch.setattr(
deploy_module,
"deployment_container_artifacts",
lambda _project: copy.deepcopy(observed_artifacts),
)
monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: "n8n " + "e" * 64)
monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda name: {
"Id": "f" * 64,
"Image": "sha256:" + "1" * 64,
"Config": {
"Image": "workmesh-n8n-1:production",
"Labels": {
"com.docker.compose.project": "workmesh",
"com.docker.compose.service": "n8n",
"com.docker.compose.config-hash": "e" * 64,
}
},
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Mounts": [
{
"Type": "volume",
"Name": "workmesh_workmesh_n8n_data",
"Destination": "/home/node/.n8n",
"RW": True,
}
],
} if name == "workmesh-n8n-1" else None,
)
monkeypatch.setattr(deploy_module, "_fsync_directory", lambda _path: None)
monkeypatch.setattr(
deploy_module,
"verified_current_receipt",
lambda _repository, _project, _requested, _tree: json.loads(
state_path.read_text(encoding="utf-8")
)["live_receipt"],
)
result = deploy_module.reattest_workmesh_product_n8n_receipt(
projects, project, tmp_path, revision
)
persisted = json.loads(state_path.read_text(encoding="utf-8"))
assert result == {
"revision": revision,
"container": "workmesh-n8n-1",
"status": "reattested",
}
assert persisted["live_status"] == "healthy"
assert persisted["status"] == "failed"
assert persisted["last_attempt"] == json.loads(original_bytes)["last_attempt"]
assert persisted["live_receipt"]["containers"]["workmesh-n8n-1"]["container_id"] == "f" * 64
assert persisted["live_artifacts"]["containers"]["workmesh-n8n-1"]["container_id"] == "f" * 64
for name in deploy_module.WORKMESH_PRODUCT_CONTAINERS:
if name != "workmesh-n8n-1":
assert persisted["live_receipt"]["containers"][name] == receipt["containers"][name]
backup = state_path.parent / (
f"workmesh-product-before-n8n-reattest-{revision[:12]}-{'3' * 12}.json"
)
assert backup.read_bytes() == original_bytes
def test_workmesh_product_n8n_recovery_refuses_volume_or_config_drift(
deploy_module, tmp_path, monkeypatch
):
(
revision,
project,
projects,
receipt,
artifacts,
state_path,
original_bytes,
) = _workmesh_product_reattest_setup(deploy_module, tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent)
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_args: True)
monkeypatch.setattr(
deploy_module,
"deployment_metadata",
lambda *_args, **_kwargs: copy.deepcopy(receipt["build"]),
)
monkeypatch.setattr(deploy_module, "git_revision_compose_sha256", lambda *_args: artifacts["compose_sha256"])
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda _name: set(deploy_module.WORKMESH_PRODUCT_CONTAINERS))
monkeypatch.setattr(deploy_module, "container_state", lambda _name: (True, "healthy", "workmesh"))
observed_receipt = copy.deepcopy(receipt)
observed_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64
observed_artifacts = copy.deepcopy(artifacts["containers"])
observed_artifacts["workmesh-n8n-1"]["container_id"] = "f" * 64
monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args, **_kwargs: copy.deepcopy(observed_receipt))
monkeypatch.setattr(deploy_module, "deployment_container_artifacts", lambda _project: copy.deepcopy(observed_artifacts))
monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: "n8n " + "e" * 64)
monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda _name: {
"Id": "f" * 64,
"Image": "sha256:" + "1" * 64,
"Config": {"Image": "workmesh-n8n-1:production", "Labels": {
"com.docker.compose.project": "workmesh",
"com.docker.compose.service": "n8n",
"com.docker.compose.config-hash": "e" * 64,
}},
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Mounts": [{
"Type": "volume",
"Name": "wrong_workmesh_n8n_data",
"Destination": "/home/node/.n8n",
"RW": True,
}],
},
)
with pytest.raises(deploy_module.DeployError, match="n8n volume"):
deploy_module.reattest_workmesh_product_n8n_receipt(projects, project, tmp_path, revision)
assert state_path.read_bytes() == original_bytes
assert list(state_path.parent.glob("workmesh-product-before-n8n-reattest-*.json")) == []
def test_workmesh_product_n8n_recovery_dispatches_under_both_locks(
deploy_module, tmp_path, monkeypatch, capsys
):
revision = "a" * 40
configured = {
"slug": "workmesh",
"branch": "main",
"source_dir": (deploy_module.EXACT_SOURCE_ROOT / "workmesh").as_posix(),
"env_source": "/srv/example/appdata/workmesh/.env",
"compose_project": "workmesh",
"compose_files": ["compose.yaml"],
"containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS),
"retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER],
"exact_source": True,
"verify_oci_revision": False,
"health_timeout": 300,
}
projects = {"jens/workmesh": configured}
events = []
@contextmanager
def project_lock(path, _description):
events.append(("project-lock", path))
yield
@contextmanager
def global_lock():
events.append(("global-lock",))
yield
monkeypatch.setattr(sys, "argv", [
"deploy.py", "reattest-workmesh-product-n8n", "jens/workmesh", revision,
])
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: {"tree": "b" * 40})
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock)
def reattest(received_projects, project, repo, requested_revision):
assert received_projects == projects
assert project["_repository_name"] == "jens/workmesh"
assert repo == tmp_path
assert requested_revision == revision
events.append(("reattest-product",))
return {"revision": revision, "container": "workmesh-n8n-1", "status": "reattested"}
monkeypatch.setattr(deploy_module, "reattest_workmesh_product_n8n_receipt", reattest)
assert deploy_module.main() == 0
assert events == [
("project-lock", deploy_module.STATE_ROOT / "locks" / "workmesh.lock"),
("global-lock",),
("reattest-product",),
]
assert json.loads(capsys.readouterr().out) == {
"repository": "jens/workmesh",
"revision": revision,
"container": "workmesh-n8n-1",
"status": "reattested",
}
def test_workmesh_topology_transition_requires_exact_registry_and_recorded_receipt(
deploy_module, tmp_path
):
project = _workmesh_product_project(tmp_path)
previous = {"live_receipt": {"containers": {
name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.WORKMESH_LEGACY_CONTAINERS
}}}
assert deploy_module.workmesh_transition_enabled(project)
assert deploy_module.workmesh_rollback_topology(project, previous)["containers"] == deploy_module.WORKMESH_LEGACY_CONTAINERS
assert deploy_module.workmesh_rollback_topology(project, {"live_receipt": {"containers": {}}}) is project
wrong = {**project, "retired_compose_containers": ["some-other-container"]}
assert not deploy_module.workmesh_transition_enabled(wrong)
assert deploy_module.workmesh_rollback_topology(wrong, previous) is wrong
def test_workmesh_product_deploy_removes_only_reviewed_compose_orphan(
deploy_module, tmp_path, monkeypatch
):
project = _workmesh_product_project(tmp_path)
commands = []
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {})
monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "prepare_compose_migration", lambda *_: [])
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS))
monkeypatch.setattr(deploy_module, "run", lambda command, **_: commands.append(command))
deploy_module.deploy_project(project, "a" * 40, "jens/workmesh")
assert commands[-1] == ["docker", "compose", "up", "--detach", "--no-build", "--remove-orphans"]
other = _project(tmp_path)
deploy_module.deploy_project(other, "a" * 40, "jens/unraid-appops-gateway")
assert commands[-1] == ["docker", "compose", "up", "--detach", "--no-build"]
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_PRODUCT_CONTAINERS))
deploy_module.deploy_project(project, "a" * 40, "jens/workmesh")
assert commands[-1] == ["docker", "compose", "up", "--detach", "--no-build"]
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS) | {"unreviewed-sidecar"})
with pytest.raises(deploy_module.DeployError, match="topology changed"):
deploy_module.deploy_project(project, "a" * 40, "jens/workmesh")
def test_workmesh_registry_transition_requires_live_proof_and_fixed_candidate(
deploy_module, tmp_path, monkeypatch
):
configured = _workmesh_product_project(tmp_path)
configured["containers"] = deploy_module.WORKMESH_LEGACY_CONTAINERS
configured.pop("retired_compose_containers")
configured["exact_source"] = True
projects = {"jens/workmesh": configured}
path = tmp_path / "projects.json"
path.write_text(json.dumps(projects), encoding="utf-8")
monkeypatch.setattr(deploy_module, "CONFIG_PATH", path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path)
project = {**configured, "_repository_name": "jens/workmesh"}
live_revision = "a" * 40
revision = "b" * 40
monkeypatch.setattr(deploy_module, "read_state", lambda *_: {
"live_revision": live_revision,
"live_status": "healthy",
"live_receipt": {"build": {"tree": "c" * 40}},
})
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: {"verified": True})
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS))
monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_: True)
monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {})
monkeypatch.setattr(deploy_module, "output", lambda *_ , **__: "postgres\napi\nn8n\nworkmesh")
deploy_module.transition_workmesh_registry(projects, project, tmp_path, revision)
changed = json.loads(path.read_text(encoding="utf-8"))["jens/workmesh"]
assert changed["containers"] == deploy_module.WORKMESH_PRODUCT_CONTAINERS
assert changed["retired_compose_containers"] == ["workmesh-mock-ats-1"]
backup = tmp_path / f"workmesh-registry-before-product-{revision[:12]}.json"
assert json.loads(backup.read_text(encoding="utf-8")) == projects
with pytest.raises(deploy_module.DeployError, match="changed before topology transition"):
deploy_module.transition_workmesh_registry(projects, project, tmp_path, revision)
def test_workmesh_registry_transition_rejects_unverified_source_or_service(
deploy_module, tmp_path, monkeypatch
):
configured = _workmesh_product_project(tmp_path)
configured["containers"] = deploy_module.WORKMESH_LEGACY_CONTAINERS
configured.pop("retired_compose_containers")
configured["exact_source"] = True
projects = {"jens/workmesh": configured}
path = tmp_path / "projects.json"
path.write_text(json.dumps(projects), encoding="utf-8")
monkeypatch.setattr(deploy_module, "CONFIG_PATH", path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path)
project = {**configured, "_repository_name": "jens/workmesh"}
monkeypatch.setattr(deploy_module, "read_state", lambda *_: {
"live_revision": "a" * 40,
"live_status": "healthy",
"live_receipt": {"build": {"tree": "c" * 40}},
})
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: None)
with pytest.raises(deploy_module.DeployError, match="live receipt"):
deploy_module.transition_workmesh_registry(projects, project, tmp_path, "b" * 40)
monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: {"verified": True})
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS))
monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_: True)
monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {})
monkeypatch.setattr(deploy_module, "output", lambda *_ , **__: "postgres\napi\nn8n\nworkmesh\nmock-ats")
with pytest.raises(deploy_module.DeployError, match="four-service"):
deploy_module.transition_workmesh_registry(projects, project, tmp_path, "b" * 40)
assert json.loads(path.read_text(encoding="utf-8")) == projects
def test_failed_workmesh_product_deploy_retains_five_container_artifacts(
deploy_module, tmp_path, monkeypatch
):
project = _workmesh_product_project(tmp_path)
previous = {"live_revision": "a" * 40, "live_artifacts": {"containers": {}},
"live_receipt": {"containers": {
name: {"image_id": "sha256:" + "1" * 64}
for name in deploy_module.WORKMESH_LEGACY_CONTAINERS
}}}
observed = []
monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda target: (
observed.append(list(target["containers"])) or {"containers": {
name: {"image_id": "sha256:" + "1" * 64} for name in target["containers"]
}}
))
result = deploy_module.resolve_failed_live_state(
previous,
{"revision": "a" * 40, "receipt": previous["live_receipt"]},
project,
production_started=True,
failure_stage="deployment",
restore_error=None,
)
assert observed == [deploy_module.WORKMESH_LEGACY_CONTAINERS]
assert set(result["live_artifacts"]["containers"]) == set(deploy_module.WORKMESH_LEGACY_CONTAINERS)
def test_workmesh_registry_rollback_accepts_only_exact_saved_baseline(
deploy_module, tmp_path, monkeypatch
):
original = _workmesh_product_project(tmp_path)
original["containers"] = deploy_module.WORKMESH_LEGACY_CONTAINERS
original.pop("retired_compose_containers")
current = _workmesh_product_project(tmp_path)
projects = {"jens/workmesh": current, "jens/other": {"slug": "other"}}
original_projects = {"jens/workmesh": original, "jens/other": {"slug": "other"}}
revision = "b" * 40
path = tmp_path / "projects.json"
path.write_text(json.dumps(projects), encoding="utf-8")
(tmp_path / f"workmesh-registry-before-product-{revision[:12]}.json").write_text(
json.dumps(original_projects), encoding="utf-8"
)
monkeypatch.setattr(deploy_module, "CONFIG_PATH", path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path)
backup = deploy_module.workmesh_registry_rollback_backup(projects, revision)
deploy_module.restore_workmesh_registry(projects, backup)
assert json.loads(path.read_text(encoding="utf-8")) == original_projects
path.write_text(json.dumps({**projects, "jens/other": {"slug": "changed"}}), encoding="utf-8")
with pytest.raises(deploy_module.DeployError, match="Registry changed"):
deploy_module.workmesh_registry_rollback_backup(projects, revision)
def test_workmesh_product_snapshot_retains_authorized_five_container_rollback(
deploy_module, tmp_path, monkeypatch
):
project = _workmesh_product_project(tmp_path)
previous = {**project, "containers": deploy_module.WORKMESH_LEGACY_CONTAINERS}
old_revision, new_revision = "a" * 40, "b" * 40
old_receipt = _receipt(previous, old_revision)
snapshot = {
"revision": old_revision,
"receipt": old_receipt,
"containers": [{
"name": name,
"image_id": "sha256:" + "1" * 64,
"image_ref": name + ":production",
"rollback_ref": f"gitea-deploy-rollback/workmesh-{index}:{old_revision[:12]}",
} for index, name in enumerate(deploy_module.WORKMESH_LEGACY_CONTAINERS)],
}
state_root = tmp_path / "state"
state_root.mkdir()
(state_root / "workmesh.json").write_text(json.dumps({
"live_revision": new_revision,
"live_status": "healthy",
"rollback_snapshot": snapshot,
}), encoding="utf-8")
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "inspect_image", lambda image_id: {"Id": image_id})
assert deploy_module.durable_rollback_snapshot(project, "jens/workmesh", new_revision) == snapshot
def test_deployment_environment_exports_verified_build_date(deploy_module, tmp_path):
project = _project(tmp_path)
repository = "jens/unraid-appops-gateway"
revision = "a" * 40
built_at = "2026-09-23T10:15:00.000Z"
metadata = {"repository": repository, "revision": revision, "builtAt": built_at}
environment = deploy_module.deployment_environment(project, revision, repository, metadata)
assert environment["APPOPS_BUILD_DATE"] == built_at
def test_deployment_environment_binds_workmesh_build_arg_to_exact_revision(
deploy_module, tmp_path, monkeypatch
):
revision = "a" * 40
monkeypatch.delenv("WORKMESH_BUILD_REVISION", raising=False)
project = _workmesh_product_project(tmp_path)
project["environment"] = {"WORKMESH_BUILD_REVISION": "operator-override"}
environment = deploy_module.deployment_environment(project, revision, "jens/workmesh")
assert environment["WORKMESH_BUILD_REVISION"] == revision
other_environment = deploy_module.deployment_environment(
_project(tmp_path), revision, "jens/unraid-appops-gateway"
)
assert "WORKMESH_BUILD_REVISION" not in other_environment
def test_deployment_environment_clears_unverified_build_date(deploy_module, tmp_path, monkeypatch):
project = _project(tmp_path)
repository = "jens/unraid-appops-gateway"
revision = "a" * 40
monkeypatch.setenv("APPOPS_BUILD_DATE", "ambient-unverified-value")
metadata = {
"repository": "jens/another-repository",
"revision": revision,
"builtAt": "2026-09-23T10:15:00.000Z",
}
environment = deploy_module.deployment_environment(project, revision, repository, metadata)
assert environment["APPOPS_BUILD_DATE"] == ""
def test_deployment_environment_allows_unverified_project_build_date_override(deploy_module, tmp_path):
project = _project(tmp_path)
project["verify_oci_revision"] = False
project["environment"] = {"APPOPS_BUILD_DATE": "operator-configured-value"}
repository = "jens/unraid-appops-gateway"
revision = "a" * 40
metadata = {
"repository": repository,
"revision": revision,
"builtAt": "2026-09-23T10:15:00.000Z",
}
environment = deploy_module.deployment_environment(project, revision, repository, metadata)
assert environment["APPOPS_BUILD_DATE"] == "operator-configured-value"
def _receipt(
project: dict[str, object],
revision: str,
*,
tree: str = "b" * 40,
) -> dict[str, object]:
repository = str(project["_repository_name"])
built_at = "2026-08-30T10:00:00.000Z"
digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
build = {
"id": revision[:12],
"repository": repository,
"revision": revision,
"tree": tree,
"treeDigest": digest,
"builtAt": built_at,
}
containers = {
str(name): {
"container_id": "2" * 64,
"image_id": "sha256:" + "1" * 64,
"image_ref": f"{name}:production",
"image_revision": revision,
"image_source": repository,
"image_tree": tree,
"image_version": revision[:12],
"image_created": built_at,
}
for name in project["containers"]
}
live = None
if project.get("live_probe"):
live = {
"status": "live",
"build": {
key: build[key]
for key in ("id", "revision", "tree", "treeDigest", "builtAt")
},
}
return {
"schema": "gitea-deploy-receipt.v1",
"repository": repository,
"revision": revision,
"tree": tree,
"treeDigest": digest,
"build": build,
"containers": containers,
"live": live,
}
def test_failed_attempts_never_replace_the_verified_live_revision(
deploy_module, tmp_path, monkeypatch
):
revision_a = "a" * 40
revision_b = "b" * 40
revision_c = "c" * 40
project = _project(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state")
state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json"
state_path.parent.mkdir(parents=True)
state_path.write_text(
json.dumps(
{
"schema_version": 2,
"repository": "jens/unraid-appops-gateway",
"revision": revision_a,
"live_revision": revision_a,
"live_status": "healthy",
"live_artifacts": {
"containers": {"Unraid-AppOps-Gateway": {"image_id": "sha256:a"}}
},
"status": "success",
}
),
encoding="utf-8",
)
deploy_module.write_state(
"jens/unraid-appops-gateway", project, revision_b, "failed", "B failed"
)
after_b = json.loads(state_path.read_text(encoding="utf-8"))
assert after_b["live_revision"] == revision_a
assert after_b["revision"] == revision_a
assert after_b["last_attempt"]["revision"] == revision_b
deploy_module.write_state(
"jens/unraid-appops-gateway", project, revision_c, "failed", "C failed"
)
after_c = json.loads(state_path.read_text(encoding="utf-8"))
assert after_c["live_revision"] == revision_a
assert after_c["last_attempt"]["revision"] == revision_c
def test_snapshot_is_optional_only_for_a_true_first_deploy(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state")
monkeypatch.setattr(
deploy_module, "compose_project_container_names", lambda _name: set()
)
monkeypatch.setattr(deploy_module, "inspect_container", lambda _name: None)
assert deploy_module.capture_compose_rollback(project) is None
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda _name: {
"Id": "container-id",
"Image": "sha256:" + "1" * 64,
"Config": {
"Image": "unraid-appops-gateway:production",
"Labels": {"com.docker.compose.project": "unraid-appops-gateway"},
},
"State": {"Running": True},
},
)
with pytest.raises(
deploy_module.DeployError, match="valid healthy live deployment state"
):
deploy_module.capture_compose_rollback(project)
def test_snapshot_rejects_partial_container_sets_before_tagging(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
project["containers"] = ["one", "two"]
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state")
monkeypatch.setattr(
deploy_module, "compose_project_container_names", lambda _name: set()
)
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda name: None if name == "two" else {"Id": "one"},
)
commands: list[list[str]] = []
monkeypatch.setattr(
deploy_module, "run", lambda command, **_kwargs: commands.append(command)
)
with pytest.raises(deploy_module.DeployError, match="containers are missing"):
deploy_module.capture_compose_rollback(project)
assert commands == []
def test_snapshot_rejects_receipt_for_another_revision_before_tagging(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
live_revision = "a" * 40
receipt_revision = "b" * 40
image_id = "sha256:" + "1" * 64
bad_receipt = _receipt(project, receipt_revision)
monkeypatch.setattr(
deploy_module,
"read_state",
lambda _project: {
"schema_version": 3,
"repository": project["_repository_name"],
"live_revision": live_revision,
"live_status": "healthy",
"live_legacy_provenance": False,
"live_artifacts": {
"containers": {"Unraid-AppOps-Gateway": {"image_id": image_id}}
},
"live_receipt": bad_receipt,
},
)
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Unraid-AppOps-Gateway"},
)
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda _name: {
"Id": "2" * 64,
"Image": image_id,
"Config": {
"Image": "unraid-appops-gateway:production",
"Labels": {"com.docker.compose.project": "unraid-appops-gateway"},
},
"State": {"Running": True},
},
)
monkeypatch.setattr(
deploy_module,
"inspect_image",
lambda _image: {
"Id": image_id,
"Config": {
"Labels": {
"org.opencontainers.image.revision": live_revision,
"org.opencontainers.image.source": project["_repository_name"],
}
},
},
)
commands: list[list[str]] = []
monkeypatch.setattr(
deploy_module, "run", lambda command, **_kwargs: commands.append(command)
)
with pytest.raises(deploy_module.DeployError, match="receipt identity"):
deploy_module.capture_compose_rollback(project)
assert commands == []
def test_snapshot_rejects_unrecorded_same_project_containers(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Unraid-AppOps-Gateway", "unrecorded-sidecar"},
)
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda _name: pytest.fail("container inspection must not start"),
)
with pytest.raises(deploy_module.DeployError, match="unrecorded containers"):
deploy_module.capture_compose_rollback(project)
def test_oci_verification_reads_the_immutable_image_not_container_override(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
requested = "b" * 40
image_id = "sha256:" + "1" * 64
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Unraid-AppOps-Gateway"},
)
monkeypatch.setattr(
deploy_module,
"container_state",
lambda _name: (True, "healthy", "unraid-appops-gateway"),
)
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda _name: {
"Image": image_id,
"Config": {"Labels": {"org.opencontainers.image.revision": requested}},
},
)
monkeypatch.setattr(
deploy_module,
"inspect_image",
lambda _image_id: {
"Id": image_id,
"Config": {
"Labels": {
"org.opencontainers.image.revision": "a" * 40,
"org.opencontainers.image.source": "jens/unraid-appops-gateway",
}
},
},
)
with pytest.raises(deploy_module.DeployError, match="Immutable image provenance"):
deploy_module.verify_containers(
project, timeout=30, expected_revision=requested
)
def test_forward_gate_rejects_same_drifted_or_older_revision(
deploy_module, tmp_path, monkeypatch
):
live = "b" * 40
project = _project(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state")
state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json"
state_path.parent.mkdir(parents=True)
state_path.write_text(
json.dumps({"live_revision": live, "live_status": "healthy"}),
encoding="utf-8",
)
with pytest.raises(deploy_module.DeployError, match="runtime verification failed"):
deploy_module.validate_forward_revision(Path("repo.git"), project, live)
monkeypatch.setattr(
deploy_module.subprocess,
"run",
lambda *_args, **_kwargs: types.SimpleNamespace(returncode=1),
)
with pytest.raises(deploy_module.NonForwardDeployError):
deploy_module.validate_forward_revision(Path("repo.git"), project, "a" * 40)
def test_candidate_is_revalidated_against_the_current_branch_under_lock(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
project["branch"] = "main"
revision = "a" * 40
metadata = {
"id": revision[:12],
"repository": project["_repository_name"],
"revision": revision,
"tree": "b" * 40,
"treeDigest": hashlib.sha256(f"git-tree:{'b' * 40}".encode()).hexdigest(),
"builtAt": "2026-08-30T10:00:00.000Z",
}
monkeypatch.setattr(
deploy_module,
"validate_commit",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
deploy_module.DeployError("commit is no longer on the branch")
),
)
with pytest.raises(deploy_module.DeployError, match="no longer on the branch"):
deploy_module.revalidate_deployment_candidate(
Path("repo.git"),
project,
revision,
str(project["_repository_name"]),
metadata,
)
def test_bootstrap_legacy_cannot_downgrade_modern_state(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
revision = "a" * 40
monkeypatch.setattr(
deploy_module,
"read_state",
lambda _project: {
"schema_version": 3,
"repository": project["_repository_name"],
"revision": revision,
"live_revision": revision,
"live_receipt": _receipt(project, revision),
"status": "success",
},
)
monkeypatch.setattr(
deploy_module,
"verify_containers",
lambda *_args, **_kwargs: pytest.fail("modern state must not be re-enrolled"),
)
with pytest.raises(deploy_module.DeployError, match="modern schema"):
deploy_module.bootstrap_legacy_state(
str(project["_repository_name"]), project, revision
)
def test_interrupted_legacy_recovery_requires_recorded_pre_mutation_guard(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
live_revision = "a" * 40
candidate_revision = "b" * 40
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state")
state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json"
state_path.parent.mkdir(parents=True)
state_path.write_text(
json.dumps(
{
"schema_version": 3,
"repository": project["_repository_name"],
"revision": live_revision,
"live_revision": live_revision,
"live_status": "unknown",
"live_legacy_provenance": False,
"live_artifacts": {"containers": {}},
"live_receipt": {},
"status": "failed",
"last_attempt": {
"revision": candidate_revision,
"status": "failed",
"message": (
"Refusing to mutate a Compose project with unrecorded containers: "
"['unrecorded-sidecar']"
),
"updated_at_epoch": 123,
},
}
),
encoding="utf-8",
)
verified = []
monkeypatch.setattr(
deploy_module,
"verify_containers",
lambda value: verified.append(value),
)
artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "legacy"}}}
monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: artifacts)
deploy_module.recover_interrupted_legacy_state(
str(project["_repository_name"]), project, live_revision
)
recovered = json.loads(state_path.read_text(encoding="utf-8"))
assert verified == [project]
assert recovered["status"] == "success"
assert recovered["live_revision"] == live_revision
assert recovered["live_status"] == "healthy"
assert recovered["live_legacy_provenance"] is True
assert recovered["live_artifacts"] == artifacts
assert recovered["last_attempt"]["metadata"] == {
"recovery": "interrupted-legacy-pre-mutation-allowlist-guard",
"reconciled_from_attempt_revision": candidate_revision,
"reconciled_from_attempt_updated_at_epoch": 123,
}
@pytest.mark.parametrize(
"change",
[
{"live_status": "healthy"},
{"live_receipt": {"unverified": True}},
{"last_attempt": {"status": "failed", "message": "source sync failed"}},
{
"last_attempt": {
"status": "success",
"message": "Refusing to mutate a Compose project with unrecorded containers: []",
}
},
],
)
def test_interrupted_legacy_recovery_rejects_any_other_state(
deploy_module, tmp_path, monkeypatch, change
):
project = _project(tmp_path)
revision = "a" * 40
state = {
"schema_version": 3,
"repository": project["_repository_name"],
"revision": revision,
"live_revision": revision,
"live_status": "unknown",
"live_legacy_provenance": False,
"live_artifacts": {"containers": {}},
"live_receipt": {},
"status": "failed",
"last_attempt": {
"revision": "b" * 40,
"status": "failed",
"message": "Refusing to mutate a Compose project with unrecorded containers: ['sidecar']",
},
}
state.update(change)
monkeypatch.setattr(deploy_module, "read_state", lambda _project: state)
monkeypatch.setattr(
deploy_module,
"verify_containers",
lambda *_args, **_kwargs: pytest.fail("untrusted state must not inspect containers"),
)
with pytest.raises(deploy_module.DeployError, match="exact recorded pre-mutation"):
deploy_module.recover_interrupted_legacy_state(
str(project["_repository_name"]), project, revision
)
def test_pre_mutation_failure_preserves_legacy_and_unknown_live_state(
deploy_module, tmp_path
):
project = _project(tmp_path)
revision = "a" * 40
artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "legacy"}}}
previous = {
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": True,
"live_artifacts": artifacts,
"live_receipt": {},
}
candidate_failure = deploy_module.resolve_failed_live_state(
previous,
None,
project,
production_started=False,
failure_stage="candidate-revalidation",
restore_error=None,
)
assert candidate_failure == {
"live_revision": revision,
"live_artifacts": artifacts,
"live_status": "healthy",
"live_legacy_provenance": True,
"live_receipt": {},
}
previous["live_status"] = "unknown"
unknown_failure = deploy_module.resolve_failed_live_state(
previous,
None,
project,
production_started=False,
failure_stage="candidate-revalidation",
restore_error=None,
)
assert unknown_failure["live_status"] == "unknown"
assert unknown_failure["live_legacy_provenance"] is True
def test_snapshot_capture_failure_preserves_evidence_but_marks_live_unknown(
deploy_module, tmp_path
):
project = _project(tmp_path)
revision = "a" * 40
artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "legacy"}}}
result = deploy_module.resolve_failed_live_state(
{
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": True,
"live_artifacts": artifacts,
"live_receipt": {},
},
None,
project,
production_started=False,
failure_stage="snapshot",
restore_error=None,
)
assert result["live_revision"] == revision
assert result["live_status"] == "unknown"
assert result["live_legacy_provenance"] is True
assert result["live_artifacts"] == artifacts
def test_container_without_exact_compose_label_is_rejected(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
monkeypatch.setattr(
deploy_module,
"container_state",
lambda _name: (True, "healthy", ""),
)
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Unraid-AppOps-Gateway"},
)
with pytest.raises(deploy_module.DeployError, match="wrong Compose project"):
deploy_module.verify_containers(project, timeout=30)
def test_post_deploy_verification_rejects_extra_compose_services(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
monkeypatch.setattr(
deploy_module,
"container_state",
lambda _name: (True, "healthy", "unraid-appops-gateway"),
)
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Unraid-AppOps-Gateway", "unreviewed-sidecar"},
)
with pytest.raises(deploy_module.DeployError, match="container set"):
deploy_module.verify_containers(project, timeout=30)
def test_rollback_removes_only_containers_created_after_snapshot(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
snapshot = {
"compose_containers": ["Unraid-AppOps-Gateway"],
"containers": [{"name": "Unraid-AppOps-Gateway"}],
}
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Unraid-AppOps-Gateway", "failed-revision-sidecar"},
)
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda name: {
"Config": {
"Labels": {"com.docker.compose.project": "unraid-appops-gateway"}
}
}
if name == "failed-revision-sidecar"
else None,
)
commands: list[list[str]] = []
monkeypatch.setattr(
deploy_module, "run", lambda command, **_kwargs: commands.append(command)
)
deploy_module.remove_attempt_only_compose_containers(project, snapshot)
assert commands == [["docker", "rm", "--force", "failed-revision-sidecar"]]
def _projectbrain_rollback_fixture(tmp_path):
project = {"slug": "projectbrain", "source_dir": str(tmp_path), "deploy_script": "scripts/deploy-unraid-single.sh",
"containers": ["projectbrain"], "compose_project": "projectbrain",
"_repository_name": "jens/projectbrain"}
binding = {"app_dir": str(tmp_path.resolve()), "data_path": (tmp_path / "data").as_posix(),
"container_name": "projectbrain", "image_ref": "projectbrain-unraid:latest",
"image_id": "sha256:" + "a" * 64}
template = tmp_path / "docker/unraid/projectbrain-container.env.example"
template.parent.mkdir(parents=True)
template.write_text("PROJECTBRAIN_STOP_COMPOSE_APP_SERVICES=true\nPROJECTBRAIN_REMOVE_OLD_POSTGRES=true\n")
values = {"PROJECTBRAIN_IMAGE": binding["image_ref"], "PROJECTBRAIN_CONTAINER_NAME": "projectbrain",
"PROJECTBRAIN_APPDATA_PATH": binding["data_path"],
"PROJECTBRAIN_STOP_COMPOSE_APP_SERVICES": "false", "PROJECTBRAIN_REMOVE_OLD_POSTGRES": "false"}
for key in ("POSTGRES_PASSWORD", "PROJECTBRAIN_OWNER_PASSWORD", "PROJECTBRAIN_AUTH_SECRET",
"PROJECTBRAIN_HUB_SERVICE_TOKEN", "PROJECTBRAIN_GITEA_WEBHOOK_SECRET"):
values[key] = "fixture-only-not-live"
(tmp_path / ".env").write_text("".join(f"{key}={value}\n" for key, value in values.items()))
return project, binding
def test_single_container_rollback_never_starts_legacy_compose(deploy_module, monkeypatch, tmp_path):
project, binding = _projectbrain_rollback_fixture(tmp_path)
calls = []
monkeypatch.setattr(deploy_module, "run", lambda command, **kw: calls.append((command, kw)))
monkeypatch.setattr(deploy_module, "compose_command", lambda _: pytest.fail("wrong topology"))
monkeypatch.setattr(deploy_module, "inspect_image", lambda _: {"Id": binding["image_id"]})
env = {"VITE_PROJECTBRAIN_COMMIT": "a" * 40}
safe = deploy_module.projectbrain_rollback_environment(project, binding, env)
deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, env, binding)
assert calls == [(["sh", str(tmp_path / "scripts/deploy-unraid-single.sh"), "--skip-pull", "--skip-build"],
{"cwd": tmp_path, "env": safe})]
@pytest.mark.parametrize("assignment", [
"PROJECTBRAIN_DEPLOY_SKIP_BUILD=false", "PROJECTBRAIN_DEPLOY_SKIP_PULL=false",
"PROJECTBRAIN_DEPLOY_REF=main", "PROJECTBRAIN_IMAGE=other:latest",
"PROJECTBRAIN_CONTAINER_NAME=other", "PROJECTBRAIN_APPDATA_PATH=/other",
"PROJECTBRAIN_STOP_COMPOSE_APP_SERVICES=true", "PROJECTBRAIN_REMOVE_OLD_POSTGRES=true",
])
def test_projectbrain_file_control_drift_fails_before_mutation(deploy_module, tmp_path, monkeypatch, assignment):
project, binding = _projectbrain_rollback_fixture(tmp_path)
key = assignment.split("=", 1)[0]
path = tmp_path / ".env"
lines = [line for line in path.read_text().splitlines() if not line.startswith(key + "=")]
path.write_text("\n".join([*lines, assignment]) + "\n")
monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("unsafe mutation"))
with pytest.raises(deploy_module.DeployError, match="conflicts"):
deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, {}, binding)
def test_projectbrain_missing_live_key_cannot_import_unsafe_template_default(deploy_module, tmp_path):
project, binding = _projectbrain_rollback_fixture(tmp_path)
path = tmp_path / ".env"
path.write_text(path.read_text().replace("PROJECTBRAIN_REMOVE_OLD_POSTGRES=false\n", ""))
with pytest.raises(deploy_module.DeployError, match="REMOVE_OLD_POSTGRES"):
deploy_module.projectbrain_rollback_environment(project, binding, {})
def test_projectbrain_inherited_rollback_controls_are_forced_without_shell_evaluation(deploy_module, tmp_path):
project, binding = _projectbrain_rollback_fixture(tmp_path)
inherited = {"PROJECTBRAIN_DEPLOY_REF": "main", "PROJECTBRAIN_DEPLOY_SKIP_BUILD": "false",
"PROJECTBRAIN_DEPLOY_SKIP_PULL": "false", "PROJECTBRAIN_REMOVE_OLD_POSTGRES": "true",
"ENV": "/unreviewed/startup", "BASH_ENV": "/unreviewed/startup"}
result = deploy_module.projectbrain_rollback_environment(project, binding, inherited)
assert result["PROJECTBRAIN_DEPLOY_REF"] == ""
assert result["PROJECTBRAIN_DEPLOY_SKIP_BUILD"] == "true"
assert result["PROJECTBRAIN_DEPLOY_SKIP_PULL"] == "true"
assert result["PROJECTBRAIN_REMOVE_OLD_POSTGRES"] == "false"
assert "ENV" not in result and "BASH_ENV" not in result
assert inherited["PROJECTBRAIN_DEPLOY_REF"] == "main"
@pytest.mark.parametrize("key", ["PROJECTBRAIN_IMAGE", "PROJECTBRAIN_CONTAINER_NAME", "PROJECTBRAIN_APP_DIR", "PROJECTBRAIN_APPDATA_PATH"])
def test_projectbrain_inherited_target_drift_fails_closed(deploy_module, tmp_path, key):
project, binding = _projectbrain_rollback_fixture(tmp_path)
with pytest.raises(deploy_module.DeployError, match="inherited"):
deploy_module.projectbrain_rollback_environment(project, binding, {key: "other"})
@pytest.mark.parametrize("line", ["PROJECTBRAIN_TEST=$(id)", "PROJECTBRAIN_TEST=`id`",
"PROJECTBRAIN_TEST=value; true", "APP_DIR=/other", "PATH=/other"])
def test_projectbrain_environment_never_evaluates_shell_input(deploy_module, tmp_path, line):
project, binding = _projectbrain_rollback_fixture(tmp_path)
path = tmp_path / ".env"
path.write_text(path.read_text() + line + "\n")
with pytest.raises(deploy_module.DeployError):
deploy_module.projectbrain_rollback_environment(project, binding, {})
def test_projectbrain_snapshot_preflight_precedes_image_tag_mutations(deploy_module, tmp_path, monkeypatch):
project, binding = _projectbrain_rollback_fixture(tmp_path)
(tmp_path / ".env").write_text((tmp_path / ".env").read_text().replace("PROJECTBRAIN_REMOVE_OLD_POSTGRES=false", "PROJECTBRAIN_REMOVE_OLD_POSTGRES=true"))
runtime = {"Id": "c" * 64, "Image": binding["image_id"],
"Config": {"Image": binding["image_ref"], "Labels": {"com.docker.compose.project": "projectbrain"}},
"Mounts": [{"Type": "bind", "Source": binding["data_path"], "Destination": "/var/lib/projectbrain", "RW": True}],
"State": {"Running": True}}
state = {"status": "success", "revision": "a" * 40, "live_legacy_provenance": True,
"artifacts": {"containers": {"projectbrain": {"image_id": binding["image_id"]}}}}
monkeypatch.setattr(deploy_module, "read_state", lambda _: state)
monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda _: {"projectbrain"})
monkeypatch.setattr(deploy_module, "inspect_container", lambda _: runtime)
monkeypatch.setattr(deploy_module, "inspect_image", lambda _: {"Id": binding["image_id"]})
monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("snapshot mutation before preflight"))
with pytest.raises(deploy_module.DeployError, match="REMOVE_OLD_POSTGRES"):
deploy_module.capture_compose_rollback(project)
def test_projectbrain_rollback_rechecks_restored_immutable_tag(deploy_module, tmp_path, monkeypatch):
project, binding = _projectbrain_rollback_fixture(tmp_path)
monkeypatch.setattr(deploy_module, "inspect_image", lambda _: {"Id": "wrong"})
monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("wrong image mutation"))
with pytest.raises(deploy_module.DeployError, match="immutable snapshot"):
deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, {}, binding)
@pytest.mark.parametrize("key,value", [("DOCKER_HOST", "tcp://elsewhere.invalid:2376"),
("DOCKER_CONTEXT", "other"), ("DOCKER_TLS_VERIFY", "1")])
def test_projectbrain_rollback_rejects_inherited_docker_target(deploy_module, tmp_path, key, value):
project, binding = _projectbrain_rollback_fixture(tmp_path)
with pytest.raises(deploy_module.DeployError, match="controller-local Docker"):
deploy_module.projectbrain_rollback_environment(project, binding, {key: value})
def test_projectbrain_rollback_revalidates_before_any_restore_mutation(deploy_module, tmp_path, monkeypatch):
project, binding = _projectbrain_rollback_fixture(tmp_path)
path = tmp_path / ".env"
path.write_text(path.read_text() + "PROJECTBRAIN_DEPLOY_SKIP_BUILD=false\n")
monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("mutation before guard"))
monkeypatch.setattr(deploy_module, "sync_source", lambda *args, **kw: pytest.fail("source restore before guard"))
snapshot = {"revision": "a" * 40, "containers": [], "script_binding": binding}
with pytest.raises(deploy_module.DeployError, match="DEPLOY_SKIP_BUILD"):
deploy_module.restore_compose_snapshot(project, snapshot, "jens/projectbrain", "b" * 40)
def test_projectbrain_mount_binding_rejects_wrong_or_additional_data_targets(deploy_module, tmp_path):
project, binding = _projectbrain_rollback_fixture(tmp_path)
mount = {"Type": "bind", "Source": binding["data_path"], "Destination": "/wrong", "RW": True}
container = {"Config": {"Image": binding["image_ref"]}, "Image": binding["image_id"], "Mounts": [mount]}
with pytest.raises(deploy_module.DeployError, match="data mount"):
deploy_module.projectbrain_snapshot_binding(project, container)
mount["Destination"] = "/var/lib/projectbrain"
container["Mounts"].append(dict(mount))
with pytest.raises(deploy_module.DeployError, match="data mount"):
deploy_module.projectbrain_snapshot_binding(project, container)
def test_projectbrain_static_parse_error_does_not_echo_sensitive_values(deploy_module, tmp_path):
project, binding = _projectbrain_rollback_fixture(tmp_path)
value = "fixture-sensitive-value-with-$-syntax"
path = tmp_path / ".env"
path.write_text(path.read_text() + f"PROJECTBRAIN_TEST={value}\n")
with pytest.raises(deploy_module.DeployError) as raised:
deploy_module.projectbrain_rollback_environment(project, binding, {})
assert value not in str(raised.value)
@pytest.mark.parametrize("change", [
{"deploy_script": "unknown.sh"},
{"archive_deploy_script": "archive.sh"},
{"containers": ["other"]},
])
def test_unreviewed_script_rollback_fails_closed(deploy_module, monkeypatch, tmp_path, change):
project = {"source_dir": str(tmp_path), "deploy_script": "scripts/deploy-unraid-single.sh",
"containers": ["projectbrain"], **change}
monkeypatch.setattr(deploy_module, "run", lambda *a, **kw: pytest.fail("unreviewed mutation"))
with pytest.raises(deploy_module.DeployError, match="reviewed immutable rollback"):
deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, {})
def test_rollback_image_tag_lifecycle_preserves_only_current_snapshot(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
project["slug"] = "chimera"
current = "gitea-deploy-rollback/chimera-0:aaaaaaaaaaaa"
stale = "gitea-deploy-rollback/chimera-0:bbbbbbbbbbbb"
commands: list[list[str]] = []
def fake_run(command, **_kwargs):
commands.append(command)
if command[:3] == ["docker", "image", "ls"]:
return types.SimpleNamespace(
returncode=0,
stdout=f"{current}\n{stale}\n",
stderr="",
)
return types.SimpleNamespace(returncode=0, stdout="", stderr="")
monkeypatch.setattr(deploy_module.subprocess, "run", fake_run)
deploy_module.cleanup_rollback_image_tags(project, {current})
assert ["docker", "image", "rm", stale] in commands
assert ["docker", "image", "rm", current] not in commands
def test_rollback_image_cleanup_is_project_exact_and_never_raises(
deploy_module, tmp_path, monkeypatch
):
project = _project(tmp_path)
project["slug"] = "chimera"
owned = "gitea-deploy-rollback/chimera-12:aaaaaaaaaaaa"
colliding_project = "gitea-deploy-rollback/chimera-preview-0:bbbbbbbbbbbb"
noncanonical_tag = "gitea-deploy-rollback/chimera-0:latest"
commands: list[list[str]] = []
def fake_run(command, **_kwargs):
commands.append(command)
if command[:3] == ["docker", "image", "ls"]:
return types.SimpleNamespace(
returncode=0,
stdout=f"{owned}\n{colliding_project}\n{noncanonical_tag}\n",
stderr="",
)
return types.SimpleNamespace(returncode=0, stdout="", stderr="")
monkeypatch.setattr(deploy_module.subprocess, "run", fake_run)
deploy_module.cleanup_rollback_image_tags(project)
assert ["docker", "image", "rm", owned] in commands
assert ["docker", "image", "rm", colliding_project] not in commands
assert ["docker", "image", "rm", noncanonical_tag] not in commands
monkeypatch.setattr(
deploy_module.subprocess,
"run",
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("docker unavailable")),
)
deploy_module.cleanup_rollback_image_tags(project)
def test_commit_and_tree_metadata_are_controller_derived(deploy_module, monkeypatch):
revision = "a" * 40
tree = "b" * 40
calls: list[list[str]] = []
def fake_output(command, **_kwargs):
calls.append(command)
if command[-1].endswith("^{commit}"):
return revision
if command[-1].endswith("^{tree}"):
return tree
if command[-1].startswith("refs/heads/"):
return revision
raise AssertionError(command)
monkeypatch.setattr(deploy_module, "output", fake_output)
monkeypatch.setattr(
deploy_module.subprocess,
"run",
lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0),
)
assert deploy_module.validate_commit(Path("repo.git"), "main", revision) == revision
metadata = deploy_module.deployment_metadata(
Path("repo.git"),
revision,
"jens/chimera",
built_at="2026-08-30T10:00:00Z",
)
assert metadata == {
"id": revision[:12],
"repository": "jens/chimera",
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-08-30T10:00:00.000Z",
}
assert any(command[-1] == f"{revision}^{{tree}}" for command in calls)
monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: "c" * 40)
with pytest.raises(
deploy_module.DeployError, match="object itself must be a commit"
):
deploy_module.validate_commit(Path("repo.git"), "main", revision)
def test_exact_source_sync_replaces_dirty_context_from_a_fresh_archive(
deploy_module, tmp_path, monkeypatch
):
exact_root = tmp_path / "managed-sources"
target = exact_root / "chimera"
target.mkdir(parents=True)
(target / "tracked.txt").write_text("old", encoding="utf-8")
(target / "untracked-runtime.js").write_text("dirty", encoding="utf-8")
revision = "a" * 40
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root)
monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"])
def fake_extract(_repo, _revision, destination):
(destination / "tracked.txt").write_text("reviewed", encoding="utf-8")
monkeypatch.setattr(deploy_module, "extract_revision", fake_extract)
deploy_module.sync_source(Path("repo.git"), revision, target, "chimera", exact=True)
assert (target / "tracked.txt").read_text(encoding="utf-8") == "reviewed"
assert not (target / "untracked-runtime.js").exists()
assert not (target / ".gitea-deploy").exists()
assert deploy_module.exact_source_matches_revision(
Path("repo.git"), revision, target
)
outside = tmp_path / "not-controller-owned"
with pytest.raises(deploy_module.DeployError, match="controller-owned source root"):
deploy_module.sync_source(
Path("repo.git"), revision, outside, "chimera", exact=True
)
def test_exact_repo_sources_apply_group_read_policy_only_to_fixed_slugs(
deploy_module, tmp_path, monkeypatch
):
exact_root = tmp_path / "managed-sources"
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root)
monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"])
def fake_extract(_repo, _revision, destination):
(destination / "tracked.txt").write_text("reviewed", encoding="utf-8")
monkeypatch.setattr(deploy_module, "extract_revision", fake_extract)
prepared = []
monkeypatch.setattr(
deploy_module,
"_prepare_exact_source_group_read",
lambda target, slug: prepared.append((slug, target.parent.resolve() == exact_root.resolve())),
raising=False,
)
for slug in ("repopulse", "privatemuse", "other-project"):
deploy_module.sync_source(
Path("repo.git"), "a" * 40, exact_root / slug, slug, exact=True,
)
assert prepared == [("repopulse", True), ("privatemuse", True)]
@pytest.mark.parametrize("slug", ["repopulse", "privatemuse"])
def test_exact_source_permission_failure_preserves_the_previous_live_tree(
deploy_module, tmp_path, monkeypatch, slug
):
exact_root = tmp_path / "sources"
target = exact_root / slug
target.mkdir(parents=True)
(target / "tracked.txt").write_text("previous-live-source", encoding="utf-8")
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root)
monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"])
def fake_extract(_repo, _revision, destination):
(destination / "tracked.txt").write_text("candidate-source", encoding="utf-8")
def deny_permission_setup(_target, _slug):
raise deploy_module.DeployError("fixed source permission setup failed")
monkeypatch.setattr(deploy_module, "extract_revision", fake_extract)
monkeypatch.setattr(deploy_module, "_prepare_exact_source_group_read", deny_permission_setup)
with pytest.raises(deploy_module.DeployError, match="permission setup failed"):
deploy_module.sync_source(Path("repo.git"), "a" * 40, target, slug, exact=True)
assert (target / "tracked.txt").read_text(encoding="utf-8") == "previous-live-source"
assert sorted(item.name for item in exact_root.iterdir()) == [slug]
@pytest.mark.skipif(os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="requires root-owned Linux source paths")
def test_fixed_source_group_policy_is_read_only_and_keeps_other_siblings_private(
deploy_module, tmp_path, monkeypatch
):
exact_root = tmp_path / "sources"
exact_root.mkdir()
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root)
readable_sources = []
for slug in ("repopulse", "privatemuse"):
staging = exact_root / f".{slug}-aabbccddeeff-stage"
staging.mkdir(mode=0o700)
nested = staging / "nested"
nested.mkdir(mode=0o700)
ordinary = nested / "module.py"
ordinary.write_text("print('scan')\n", encoding="utf-8")
ordinary.chmod(0o600)
executable = nested / "tool"
executable.write_text("#!/bin/sh\n", encoding="utf-8")
executable.chmod(0o700)
deploy_module._prepare_exact_source_group_read(staging, slug)
readable_sources.append(staging)
private_sibling = exact_root / "other-project"
private_sibling.mkdir(mode=0o700)
private_before = private_sibling.stat()
for staging in readable_sources:
nested = staging / "nested"
assert staging.stat().st_gid == nested.stat().st_gid == 10001
assert stat.S_IMODE(staging.stat().st_mode) == stat.S_IMODE(nested.stat().st_mode) == 0o2750
assert (nested / "module.py").stat().st_gid == 10001
assert stat.S_IMODE((nested / "module.py").stat().st_mode) == 0o640
assert (nested / "tool").stat().st_gid == 10001
assert stat.S_IMODE((nested / "tool").stat().st_mode) == 0o750
assert private_sibling.stat().st_gid == private_before.st_gid
assert stat.S_IMODE(private_sibling.stat().st_mode) == 0o700
def test_controller_source_mode_contract_rejects_private_mode_700(status_module):
private_dir = os.stat_result((stat.S_IFDIR | 0o700, 10, 1, 1, 0, 10001, 0, 0, 0, 0))
readable_dir = os.stat_result((stat.S_IFDIR | 0o2750, 11, 1, 1, 0, 10001, 0, 0, 0, 0))
assert status_module._source_mode_matches_runtime_contract(private_dir) is False
assert status_module._source_mode_matches_runtime_contract(readable_dir) is True
def test_managed_repopulse_render_accepts_only_its_read_only_source_mount(
deploy_module, tmp_path, monkeypatch
):
source = tmp_path / "managed-source" / "repopulse"
project = {
"source_dir": str(source),
"managed_plan_version": "itworx.deployment/v1",
"managed_profile": "repopulse-authenticated",
"managed_application_id": "repopulse",
"managed_repository": "jens/repopulse",
"managed_appdata_path": "/srv/example/appdata/repopulse",
"managed_image_repository": "repopulse-managed",
"managed_image_tag": "production",
"managed_proxy_network": "itworx-mcp-hub-npm",
"managed_internal_port": 8000,
"managed_build_context": ".",
"managed_build_dockerfile": "deploy/managed/Dockerfile",
"managed_runtime_environment_keys": [
"REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN",
],
"managed_runtime_uid": 10001,
"managed_runtime_gid": 10001,
"managed_proxy_hostname": "repopulse.example.com",
"managed_health_path": "/health",
}
rendered = {
"services": {
"repopulse": {
"image": "repopulse-managed:production",
"build": {"context": str(source), "dockerfile": "deploy/managed/Dockerfile"},
"restart": "unless-stopped",
"read_only": True,
"user": "10001:10001",
"security_opt": ["no-new-privileges:true"],
"cap_drop": ["ALL"],
"networks": ["itworx-mcp-hub-npm"],
"environment": {
"REPOPULSE_GITEA_URL": "https://gitea.example.com",
"REPOPULSE_GITEA_TOKEN": "opaque-gitea-value",
"REPOPULSE_MAINTENANCE_TOKEN": "opaque-maintenance-value",
},
"volumes": [
{"type": "bind", "source": "/srv/example/appdata/repopulse/data", "target": "/data", "read_only": False},
{"type": "bind", "source": "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse", "target": "/scan/repopulse", "read_only": True},
{"type": "bind", "source": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse", "target": "/scan/privatemuse", "read_only": True},
],
}
},
"networks": {"itworx-mcp-hub-npm": {"external": True}},
}
monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {})
monkeypatch.setattr(deploy_module, "managed_runtime_environment", lambda _project: rendered["services"]["repopulse"]["environment"])
monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: json.dumps(rendered))
deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/repopulse")
rendered["services"]["repopulse"]["volumes"][2]["source"] = "/srv/example/appdata/repopulse/sources"
with pytest.raises(deploy_module.DeployError):
deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/repopulse")
rendered["services"]["repopulse"]["volumes"][2] = dict(rendered["services"]["repopulse"]["volumes"][1])
with pytest.raises(deploy_module.DeployError):
deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/repopulse")
def test_managed_appdata_roots_are_fixed_and_bound_to_each_registration(
deploy_module, monkeypatch, tmp_path
):
legacy_root = tmp_path / "legacy-appdata"
managed_root = tmp_path / "managed-volume" / "_data"
monkeypatch.setattr(deploy_module, "LEGACY_MANAGED_APPCONFIG_ROOT", legacy_root)
monkeypatch.setattr(deploy_module, "MANAGED_APPCONFIG_ROOT", legacy_root)
monkeypatch.setattr(deploy_module, "MANAGED_APPCONFIG_VOLUME_ROOT", managed_root)
monkeypatch.setattr(deploy_module, "managed_appdata_volume_initialized", lambda: False)
assert deploy_module._registered_managed_appdata_root({}) == legacy_root
assert deploy_module._registered_managed_appdata_root(
{"managed_appdata_root": legacy_root.as_posix()}
) == legacy_root
with pytest.raises(deploy_module.DeployError, match="not initialized and available"):
deploy_module._registered_managed_appdata_root(
{"managed_appdata_root": managed_root.as_posix()}
)
with pytest.raises(deploy_module.DeployError, match="fixed registered roots"):
deploy_module._registered_managed_appdata_root(
{"managed_appdata_root": (tmp_path / "arbitrary").as_posix()}
)
monkeypatch.setattr(deploy_module, "managed_appdata_volume_initialized", lambda: True)
assert deploy_module._registered_managed_appdata_root(
{"managed_appdata_root": managed_root.as_posix()}
) == managed_root
for root, host in ((legacy_root, "legacy.example"), (managed_root, "managed.example")):
env_path = root / "demo" / "deploy.env"
env_path.parent.mkdir(parents=True)
env_path.write_text(f"APPOPS_PUBLIC_URL=https://{host}\n", encoding="utf-8")
if os.name != "nt":
env_path.chmod(0o600)
project = {
"appops_managed_target": True,
"managed_plan_version": "itworx.deployment/v1",
"managed_profile": "internal-webapp",
"managed_application_id": "demo",
"managed_runtime_environment_keys": ["APPOPS_PUBLIC_URL"],
"env_source": (legacy_root / "demo" / "deploy.env").as_posix(),
}
assert deploy_module.managed_runtime_environment(project) == {
"APPOPS_PUBLIC_URL": "https://legacy.example"
}
project["managed_appdata_root"] = managed_root.as_posix()
project["env_source"] = (managed_root / "demo" / "deploy.env").as_posix()
assert deploy_module.managed_runtime_environment(project) == {
"APPOPS_PUBLIC_URL": "https://managed.example"
}
def test_managed_project_path_is_derived_from_its_stored_appdata_root(
deploy_module, monkeypatch
):
monkeypatch.setattr(deploy_module, "managed_appdata_volume_initialized", lambda: True)
monkeypatch.setattr(deploy_module, "git_path_exists", lambda *_args: True)
project = {
"containers": ["demo"],
"source_dir": (deploy_module.EXACT_SOURCE_ROOT / "demo").as_posix(),
"exact_source": True,
"compose_dir": ".",
"compose_files": ["compose.yml"],
"compose_project": "demo",
"managed_plan_version": "itworx.deployment/v1",
"managed_application_id": "demo",
"managed_appdata_path": "/srv/example/appdata/demo",
"managed_image_repository": "demo",
"managed_image_tag": "production",
"managed_proxy_network": "itworx-proxy",
"managed_internal_port": 8000,
"appops_deployment_secret_file": "jens__demo",
}
deploy_module.validate_project_config(Path("repo.git"), project, "a" * 40)
managed = {
**project,
"managed_appdata_root": deploy_module.MANAGED_APPCONFIG_VOLUME_ROOT.as_posix(),
"managed_appdata_path": (
deploy_module.MANAGED_APPCONFIG_VOLUME_ROOT / "demo"
).as_posix(),
}
deploy_module.validate_project_config(Path("repo.git"), managed, "a" * 40)
remapped = {**managed, "managed_appdata_path": "/srv/example/appdata/demo"}
with pytest.raises(deploy_module.DeployError, match="not derived"):
deploy_module.validate_project_config(Path("repo.git"), remapped, "a" * 40)
def test_repopulse_project_registry_requires_its_fixed_build_identity(
deploy_module, monkeypatch
):
source = "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse"
project = {
"branch": "main",
"slug": "repopulse",
"source_dir": str(source),
"exact_source": True,
"compose_dir": str(source),
"compose_files": [".itworx/deployment.yml"],
"compose_project": "repopulse",
"containers": ["repopulse"],
"managed_plan_version": "itworx.deployment/v1",
"managed_profile": "repopulse-authenticated",
"managed_application_id": "repopulse",
"managed_repository": "jens/repopulse",
"managed_appdata_path": "/srv/example/appdata/repopulse",
"managed_image_repository": "repopulse-managed",
"managed_image_tag": "production",
"managed_proxy_network": "itworx-mcp-hub-npm",
"managed_internal_port": 8000,
"managed_build_context": ".",
"managed_build_dockerfile": "deploy/managed/Dockerfile",
"managed_runtime_environment_keys": [
"REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN",
],
"managed_runtime_uid": 10001,
"managed_runtime_gid": 10001,
"managed_proxy_hostname": "repopulse.example.com",
"managed_health_path": "/health",
"appops_deployment_secret_file": "jens__repopulse",
"appops_managed_target": True,
}
monkeypatch.setattr(deploy_module, "git_path_exists", lambda *_args: True)
deploy_module.validate_project_config(Path("repo.git"), project, "a" * 40)
for field, value in (
("managed_repository", "jens/another-app"),
("managed_build_dockerfile", "Dockerfile"),
("managed_runtime_gid", 1000),
("managed_proxy_network", "itworx-npm"),
("source_dir", "/mnt/cache/appdata/gitea-deploy-runners/sources/other"),
):
unsafe = {**project, field: value}
with pytest.raises(deploy_module.DeployError):
deploy_module.validate_project_config(Path("repo.git"), unsafe, "a" * 40)
def test_controller_metadata_path_is_reserved(deploy_module, monkeypatch):
monkeypatch.setattr(
deploy_module,
"output",
lambda *_args, **_kwargs: "compose.yaml\n.gitea-deploy/revision",
)
with pytest.raises(deploy_module.DeployError, match="metadata path"):
deploy_module.tracked_files(Path("repo.git"), "a" * 40)
@pytest.mark.parametrize(
("name", "linkname", "kind"),
[
("../outside.txt", "", "file"),
("/absolute.txt", "", "file"),
("safe/link", "../../outside.txt", "symlink"),
("safe/link", "../outside.txt", "hardlink"),
],
)
def test_git_archive_links_and_paths_cannot_escape_staging(
deploy_module, tmp_path, monkeypatch, name, linkname, kind
):
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w") as archive:
member = tarfile.TarInfo(name)
if kind == "symlink":
member.type = tarfile.SYMTYPE
member.linkname = linkname
archive.addfile(member)
elif kind == "hardlink":
member.type = tarfile.LNKTYPE
member.linkname = linkname
archive.addfile(member)
else:
payload = b"hostile"
member.size = len(payload)
archive.addfile(member, io.BytesIO(payload))
archive_bytes = buffer.getvalue()
def fake_run(_command, **kwargs):
kwargs["stdout"].write(archive_bytes)
return types.SimpleNamespace(returncode=0)
monkeypatch.setattr(deploy_module.subprocess, "run", fake_run)
target = tmp_path / "staging"
target.mkdir()
with pytest.raises(deploy_module.DeployError):
deploy_module.extract_revision(Path("repo.git"), "a" * 40, target)
assert not (tmp_path / "outside.txt").exists()
def test_exact_source_manifest_failure_preserves_previous_tree(
deploy_module, tmp_path, monkeypatch
):
exact_root = tmp_path / "managed-sources"
target = exact_root / "chimera"
target.mkdir(parents=True)
(target / "live.txt").write_text("previous", encoding="utf-8")
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root)
monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"])
def fake_extract(_repo, _revision, destination):
(destination / "tracked.txt").write_text("reviewed", encoding="utf-8")
(destination / "unexpected.txt").write_text("dirty", encoding="utf-8")
monkeypatch.setattr(deploy_module, "extract_revision", fake_extract)
with pytest.raises(deploy_module.DeployError, match="file manifest"):
deploy_module.sync_source(
Path("repo.git"), "a" * 40, target, "chimera", exact=True
)
assert (target / "live.txt").read_text(encoding="utf-8") == "previous"
def test_exact_source_atomic_swap_failure_restores_previous_tree(
deploy_module, tmp_path, monkeypatch
):
exact_root = tmp_path / "managed-sources"
target = exact_root / "chimera"
target.mkdir(parents=True)
(target / "tracked.txt").write_text("previous", encoding="utf-8")
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root)
monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"])
def fake_extract(_repo, _revision, destination):
(destination / "tracked.txt").write_text("reviewed", encoding="utf-8")
monkeypatch.setattr(deploy_module, "extract_revision", fake_extract)
original_replace = Path.replace
def fail_staging_swap(self, destination):
if self.name.startswith(".chimera-") and Path(destination) == target:
raise OSError("simulated atomic swap failure")
return original_replace(self, destination)
monkeypatch.setattr(Path, "replace", fail_staging_swap)
with pytest.raises(OSError, match="simulated atomic swap failure"):
deploy_module.sync_source(
Path("repo.git"), "a" * 40, target, "chimera", exact=True
)
assert (target / "tracked.txt").read_text(encoding="utf-8") == "previous"
def test_exact_source_uses_external_compose_environment(deploy_module, tmp_path):
source = tmp_path / "source"
source.mkdir()
(source / "compose.yaml").write_text("services: {}\n", encoding="utf-8")
env_source = tmp_path / "deploy.env"
env_source.write_text("SECRET=not-copied\n", encoding="utf-8")
project = {
"source_dir": str(source),
"compose_dir": "",
"compose_files": ["compose.yaml"],
"compose_project": "chimera",
"exact_source": True,
"env_source": str(env_source),
}
deploy_module.copy_environment(project, source)
_, command = deploy_module.compose_command(project)
assert not (source / ".env").exists()
assert command[:7] == [
"docker",
"compose",
"--project-name",
"chimera",
"--env-file",
str(env_source),
"--file",
]
def test_managed_nested_manifest_uses_exact_source_as_project_directory(deploy_module, tmp_path):
source = tmp_path / "source"
manifest = source / ".itworx" / "deployment.yml"
manifest.parent.mkdir(parents=True)
manifest.write_text("services: {}\n", encoding="utf-8")
project = {
"source_dir": str(source),
"compose_files": [".itworx/deployment.yml"],
"compose_project": "repopulse",
"exact_source": True,
"managed_plan_version": "itworx.deployment/v1",
}
cwd, command = deploy_module.compose_command(project)
assert cwd == source
assert command == [
"docker", "compose", "--project-name", "repopulse",
"--project-directory", str(source), "--file", str(manifest),
]
def test_immutable_image_and_live_probe_must_match_every_build_field(
deploy_module, tmp_path, monkeypatch
):
revision = "a" * 40
tree = "b" * 40
metadata = {
"id": revision[:12],
"repository": "jens/chimera",
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-08-30T10:00:00.000Z",
}
project = _project(tmp_path)
project.update(
{
"slug": "chimera",
"compose_project": "chimera",
"containers": ["Chimera"],
"_repository_name": "jens/chimera",
"live_probe": {
"type": "chimera-health-live-v1",
"container": "Chimera",
"port": 6967,
"path": "/health/live",
"host": "chimera:6967",
},
}
)
image_id = "sha256:" + "1" * 64
monkeypatch.setattr(
deploy_module,
"compose_project_container_names",
lambda _name: {"Chimera"},
)
labels = {
"org.opencontainers.image.revision": revision,
"org.opencontainers.image.source": "jens/chimera",
"org.opencontainers.image.source-tree": tree,
"org.opencontainers.image.version": revision[:12],
"org.opencontainers.image.created": metadata["builtAt"],
}
monkeypatch.setattr(
deploy_module, "container_state", lambda _name: (True, "healthy", "chimera")
)
monkeypatch.setattr(
deploy_module,
"inspect_container",
lambda _name: {"Image": image_id, "Config": {"Labels": {}}},
)
monkeypatch.setattr(
deploy_module,
"inspect_image",
lambda _image_id: {"Id": image_id, "Config": {"Labels": dict(labels)}},
)
monkeypatch.setattr(
deploy_module.subprocess,
"run",
lambda *_args, **_kwargs: types.SimpleNamespace(
returncode=0,
stdout=json.dumps({"status": "live", "build": metadata}),
stderr="",
),
)
receipt = deploy_module.verify_deployment(project, metadata)
assert receipt["build"] == metadata
assert receipt["containers"]["Chimera"]["image_id"] == image_id
labels["org.opencontainers.image.source-tree"] = "c" * 40
with pytest.raises(deploy_module.DeployError, match="Immutable image provenance"):
deploy_module.verify_deployment(project, metadata)
def test_live_probe_rejects_every_provenance_drift_and_hostile_response(
deploy_module, tmp_path, monkeypatch
):
revision = "a" * 40
tree = "b" * 40
metadata = {
"id": revision[:12],
"repository": "jens/chimera",
"revision": revision,
"tree": tree,
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
"builtAt": "2026-08-30T10:00:00.000Z",
}
project = _project(tmp_path)
project.update(
{
"containers": ["Chimera"],
"live_probe": {
"type": "chimera-health-live-v1",
"container": "Chimera",
"port": 6967,
"path": "/health/live",
"host": "chimera:6967",
},
}
)
bad_outputs = ["not-json", "x" * 65537]
bad_outputs.append(json.dumps({"status": "starting", "build": metadata}))
for key in ("id", "revision", "tree", "treeDigest", "builtAt"):
changed = dict(metadata)
changed[key] = "wrong"
bad_outputs.append(json.dumps({"status": "live", "build": changed}))
for stdout in bad_outputs:
monkeypatch.setattr(
deploy_module.subprocess,
"run",
lambda *_args, _stdout=stdout, **_kwargs: types.SimpleNamespace(
returncode=0,
stdout=_stdout,
stderr="",
),
)
with pytest.raises(deploy_module.DeployError):
deploy_module.verify_live_probe(project, metadata)
monkeypatch.setattr(
deploy_module.subprocess,
"run",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
subprocess.TimeoutExpired("docker", 10)
),
)
with pytest.raises(deploy_module.DeployError, match="timed out"):
deploy_module.verify_live_probe(project, metadata)
def test_same_revision_revalidates_runtime_receipt_before_success(
deploy_module, tmp_path, monkeypatch
):
revision = "a" * 40
tree = "b" * 40
project = _project(tmp_path)
receipt = _receipt(project, revision, tree=tree)
monkeypatch.setattr(
deploy_module,
"read_state",
lambda _project: {
"schema_version": 3,
"repository": "jens/unraid-appops-gateway",
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": False,
"live_receipt": receipt,
},
)
calls = []
monkeypatch.setattr(
deploy_module,
"verify_deployment",
lambda _project, metadata, **_kwargs: calls.append(metadata) or receipt,
)
assert (
deploy_module.verified_current_receipt(
"jens/unraid-appops-gateway", project, revision, tree
)
== receipt
)
assert calls == [receipt["build"]]
monkeypatch.setattr(
deploy_module,
"verify_deployment",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
deploy_module.DeployError("drift")
),
)
assert (
deploy_module.verified_current_receipt(
"jens/unraid-appops-gateway", project, revision, tree
)
is None
)
def _webhook_project(repository: str = "jens/repo-a") -> dict[str, object]:
return {
"branch": "main",
"gitea_repository_id": 77,
"gitea_webhook_secret_file": repository.replace("/", "__"),
"validation": {
"validation_gate_version": 1,
"required": True,
"contexts": ["Managed validation / full (push)"],
"timeout_seconds": 600,
"poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
},
}
def _write_webhook_secret(root: Path, name: str, secret: bytes) -> None:
root.mkdir()
root.chmod(0o700)
path = root / name
path.write_bytes(secret + b"\n")
path.chmod(0o600)
def _push_payload(
revision: str, *, repository: str = "jens/repo-a", repository_id: int = 77
) -> dict[str, object]:
return {
"ref": "refs/heads/main",
"before": "b" * 40,
"after": revision,
"repository": {
"id": repository_id,
"full_name": repository,
"default_branch": "main",
"object_format_name": "sha1",
},
"head_commit": {"id": revision},
}
def test_repository_webhook_secrets_are_name_scoped_and_hmac_bound(
status_module, tmp_path, monkeypatch
):
secret_root = tmp_path / "secrets"
secret = b"repo-a-" + b"x" * 40
_write_webhook_secret(secret_root, "jens__repo-a", secret)
monkeypatch.setattr(status_module, "WEBHOOK_SECRET_ROOT", secret_root)
project = _webhook_project()
body = b'{"repository":{"full_name":"Jens/repo-a"}}'
signature = hmac.new(secret, body, hashlib.sha256).hexdigest()
assert status_module._webhook_secret("jens/repo-a", project) == secret
assert status_module._webhook_secret("jens/repo-b", project) is None
assert status_module._valid_webhook_signature(
body, signature, "jens/repo-a", project
)
assert not status_module._valid_webhook_signature(
body + b" ", signature, "jens/repo-a", project
)
def test_webhook_policy_rejects_legacy_or_mismatched_credentials(status_module):
project = _webhook_project()
assert status_module._webhook_policy("jens/repo-a", project) == (
77,
"main",
"jens__repo-a",
)
for invalid in (
{},
{**project, "gitea_repository_id": True},
{**project, "gitea_repository_id": 0},
{**project, "gitea_webhook_secret_file": "jens__other"},
{**project, "broker_token_file": "jens__repo-a"},
{**project, "legacy_broker_token": True},
):
assert status_module._webhook_policy("jens/repo-a", invalid) is None
def test_webhook_push_identity_binds_repo_id_branch_revision_and_head(status_module):
revision = "a" * 40
project = _webhook_project()
assert (
status_module._webhook_push_identity(
_push_payload(revision), "jens/repo-a", project
)
== revision
)
assert (
status_module._webhook_push_identity(
_push_payload(revision, repository="Jens/Repo-A"),
"jens/repo-a",
project,
)
== revision
)
mutations = (
("ref", "refs/heads/other"),
("before", revision),
("before", "B" * 40),
("after", "0" * 40),
("after", "A" * 40),
(
"repository",
{
"id": 78,
"full_name": "jens/repo-a",
"default_branch": "main",
"object_format_name": "sha1",
},
),
(
"repository",
{
"id": 77,
"full_name": "jens/other",
"default_branch": "main",
"object_format_name": "sha1",
},
),
(
"repository",
{
"id": 77,
"full_name": "jens/repo-a",
"default_branch": "other",
"object_format_name": "sha1",
},
),
(
"repository",
{
"id": 77,
"full_name": "jens/repo-a",
"default_branch": "main",
"object_format_name": "sha256",
},
),
("head_commit", {"id": "b" * 40}),
)
for field, value in mutations:
payload = _push_payload(revision)
payload[field] = value
assert (
status_module._webhook_push_identity(payload, "jens/repo-a", project)
is None
)
def test_webhook_inbox_is_durable_and_replay_keys_the_signed_semantics(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
delivery = "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d"
revision = "a" * 40
ref = "refs/heads/main"
raw_body = json.dumps(
_push_payload(revision, repository="Jens/repo-a"),
separators=(",", ":"),
).encode()
body_digest = hashlib.sha256(raw_body).hexdigest()
semantic_digest = status_module._webhook_semantic_digest(77, ref, revision)
policy = status_module._validation_policy("jens/repo-a", _webhook_project())
assert policy is not None
assert status_module._enqueue_webhook_delivery(
raw_body, delivery, 77, ref, "jens/repo-a", revision, policy
) == ("awaiting_validation", semantic_digest)
marker_path = inbox_root / f"{semantic_digest}.json"
marker = json.loads(marker_path.read_text())
assert marker["schema"] == "gitea-webhook-inbox.v2"
assert marker["semantic_sha256"] == semantic_digest
assert marker["body_sha256"] == body_digest
assert marker["gitea_repository_id"] == 77
assert marker["ref"] == ref
assert marker["repository"] == "jens/repo-a"
assert marker["revision"] == revision
assert marker["status"] == "awaiting_validation"
assert marker["validation"]["required_contexts"] == ["Managed validation / full (push)"]
changed_unsigned_delivery = "145440ab-b7b1-4c74-ae84-996d8d48167e"
assert status_module._enqueue_webhook_delivery(
raw_body,
changed_unsigned_delivery,
77,
ref,
"jens/repo-a",
revision,
policy,
) == ("awaiting_validation", semantic_digest)
assert json.loads(marker_path.read_text())["delivery"] == delivery
same_semantics_different_body = json.dumps(
{**_push_payload(revision), "compare_url": "https://example.invalid/compare"},
separators=(",", ":"),
).encode()
assert hashlib.sha256(same_semantics_different_body).hexdigest() != body_digest
assert status_module._enqueue_webhook_delivery(
same_semantics_different_body,
changed_unsigned_delivery,
77,
ref,
"jens/repo-a",
revision,
policy,
) == ("awaiting_validation", semantic_digest)
assert json.loads(marker_path.read_text())["body_sha256"] == body_digest
assert status_module._enqueue_webhook_delivery(
raw_body, "../../escape", 77, ref, "jens/repo-a", revision, policy
) == ("invalid", "")
def test_terminal_validation_revalidation_creates_a_separate_idempotent_attempt(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True)
revision = "a" * 40
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, semantic_digest = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload(revision), separators=(",", ":")).encode(),
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d",
77,
"refs/heads/main",
"jens/repo-a",
revision,
policy,
)
original_marker = inbox_root / f"{semantic_digest}.json"
original = json.loads(original_marker.read_text())
original["status"] = "validation_failed"
original["reason"] = "validation_failed"
original["validation"]["last_result"] = "reject"
original_marker.write_text(
json.dumps(original, sort_keys=True, separators=(",", ":")) + "\n",
encoding="utf-8",
)
original_bytes = original_marker.read_bytes()
assert status_module._enqueue_terminal_revalidation(
"jens/repo-a", revision, project
) == ("awaiting_validation", semantic_digest)
revalidation_marker = inbox_root / f"{semantic_digest}.revalidation.json"
attempt = json.loads(revalidation_marker.read_text())
assert original_marker.read_bytes() == original_bytes
assert attempt["schema"] == "gitea-webhook-revalidation.v1"
assert attempt["semantic_sha256"] == semantic_digest
assert attempt["revalidation_of"] == semantic_digest
assert attempt["gitea_repository_id"] == 77
assert attempt["ref"] == "refs/heads/main"
assert attempt["repository"] == "jens/repo-a"
assert attempt["revision"] == revision
assert attempt["status"] == "awaiting_validation"
assert attempt["validation"]["attempts"] == 0
assert attempt["validation"]["observed"] == []
assert [entry["stage"] for entry in attempt["audit"]] == [
"revalidation_requested",
"awaiting_validation",
]
attempt_bytes = revalidation_marker.read_bytes()
assert status_module._enqueue_terminal_revalidation(
"jens/repo-a", revision, project
) == ("awaiting_validation", semantic_digest)
assert revalidation_marker.read_bytes() == attempt_bytes
assert len(list(inbox_root.glob("*.json"))) == 2
@pytest.mark.parametrize(
"terminal_status",
["deployed", "failed", "non_forward", "rejected", "superseded"],
)
def test_revalidation_refuses_non_validation_terminal_states(
status_module, tmp_path, monkeypatch, terminal_status
):
inbox_root = tmp_path / "inbox"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True)
revision = "b" * 40
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, semantic_digest = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload(revision), separators=(",", ":")).encode(),
"145440ab-b7b1-4c74-ae84-996d8d48167e",
77,
"refs/heads/main",
"jens/repo-a",
revision,
policy,
)
original_marker = inbox_root / f"{semantic_digest}.json"
original = json.loads(original_marker.read_text())
original["status"] = terminal_status
original_marker.write_text(json.dumps(original), encoding="utf-8")
original_bytes = original_marker.read_bytes()
assert status_module._enqueue_terminal_revalidation(
"jens/repo-a", revision, project
) == ("not_revalidatable", semantic_digest)
assert original_marker.read_bytes() == original_bytes
assert not (inbox_root / f"{semantic_digest}.revalidation.json").exists()
def test_revalidation_refuses_a_revision_that_is_not_the_current_branch_head(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: False)
revision = "c" * 40
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, semantic_digest = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload(revision), separators=(",", ":")).encode(),
"766ea001-aa32-49c4-b765-5ea0a39a7831",
77,
"refs/heads/main",
"jens/repo-a",
revision,
policy,
)
original_marker = inbox_root / f"{semantic_digest}.json"
original = json.loads(original_marker.read_text())
original["status"] = "validation_timeout"
original_marker.write_text(json.dumps(original), encoding="utf-8")
original_bytes = original_marker.read_bytes()
assert status_module._enqueue_terminal_revalidation(
"jens/repo-a", revision, project
) == ("not_current_head", semantic_digest)
assert original_marker.read_bytes() == original_bytes
assert not (inbox_root / f"{semantic_digest}.revalidation.json").exists()
def test_revalidation_attempt_requires_fresh_status_success_and_preserves_original(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
projects_path = tmp_path / "projects.json"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True)
revision = "d" * 40
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, semantic_digest = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload(revision), separators=(",", ":")).encode(),
"92869210-fcd2-47d2-a775-ab3fa47f229f",
77,
"refs/heads/main",
"jens/repo-a",
revision,
policy,
)
original_marker = inbox_root / f"{semantic_digest}.json"
original = json.loads(original_marker.read_text())
original["status"] = "validation_failed"
original["reason"] = "validation_failed"
original_marker.write_text(json.dumps(original), encoding="utf-8")
original_bytes = original_marker.read_bytes()
assert status_module._enqueue_terminal_revalidation(
"jens/repo-a", revision, project
) == ("awaiting_validation", semantic_digest)
attempt_marker = inbox_root / f"{semantic_digest}.revalidation.json"
deploy_calls: list[list[str]] = []
class Result:
returncode = 0
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda args, **_kwargs: deploy_calls.append(args) or Result(),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True})
monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("wait", []))
status_module._process_webhook_record(attempt_marker)
assert json.loads(attempt_marker.read_text())["status"] == "awaiting_validation"
assert deploy_calls == []
assert original_marker.read_bytes() == original_bytes
monkeypatch.setattr(
status_module,
"_fetch_gitea_validation",
lambda *_args: (
"pass",
[{
"context": "Managed validation / full (push)",
"status": "success",
"updated_at": "2026-09-21T18:10:15Z",
}],
),
)
attempt = json.loads(attempt_marker.read_text())
attempt["validation"]["next_check_at"] = 0
attempt_marker.write_text(json.dumps(attempt), encoding="utf-8")
status_module._process_webhook_record(attempt_marker)
assert json.loads(attempt_marker.read_text())["status"] == "deployed"
assert deploy_calls == [[
"python3",
str(status_module.ROOT / "deploy.py"),
"deploy",
"jens/repo-a",
revision,
]]
assert original_marker.read_bytes() == original_bytes
def test_delivery_status_identifies_revalidation_attempts(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
project = _webhook_project()
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True)
monkeypatch.setattr(
status_module,
"_gitea_validation_settings",
lambda: ("https://gitea.example.test/api/v1", b"x" * 40),
)
revision = "e" * 40
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, semantic_digest = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload(revision), separators=(",", ":")).encode(),
"c492f8a1-dd2b-48fe-af98-f071c802b53e",
77,
"refs/heads/main",
"jens/repo-a",
revision,
policy,
)
original_marker = inbox_root / f"{semantic_digest}.json"
original = json.loads(original_marker.read_text())
original["status"] = "validation_failed"
original_marker.write_text(json.dumps(original), encoding="utf-8")
status_module._enqueue_terminal_revalidation("jens/repo-a", revision, project)
payload = status_module._delivery_status_payload()
revalidation = next(item for item in payload["items"] if item.get("attemptType") == "revalidation")
assert revalidation["repository"] == "jens/repo-a"
assert revalidation["revision"] == revision
assert revalidation["revalidationOf"] == semantic_digest
def test_revalidation_rechecks_exact_branch_head_immediately_before_deploy(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
projects_path = tmp_path / "projects.json"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
current_head = iter([True, False])
monkeypatch.setattr(
status_module, "_is_current_branch_head", lambda *_args: next(current_head)
)
revision = "f" * 40
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, semantic_digest = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload(revision), separators=(",", ":")).encode(),
"691f58fa-fbd1-49d7-87c3-d7caa947cb10",
77,
"refs/heads/main",
"jens/repo-a",
revision,
policy,
)
original_marker = inbox_root / f"{semantic_digest}.json"
original = json.loads(original_marker.read_text())
original["status"] = "validation_failed"
original_marker.write_text(json.dumps(original), encoding="utf-8")
original_bytes = original_marker.read_bytes()
assert status_module._enqueue_terminal_revalidation(
"jens/repo-a", revision, project
) == ("awaiting_validation", semantic_digest)
attempt_marker = inbox_root / f"{semantic_digest}.revalidation.json"
monkeypatch.setattr(
status_module,
"_fetch_gitea_validation",
lambda *_args: (
"pass",
[{
"context": "Managed validation / full (push)",
"status": "success",
"updated_at": "2026-09-21T18:10:15Z",
}],
),
)
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda *_args, **_kwargs: pytest.fail("deployment ran after branch head changed"),
)
status_module._process_webhook_record(attempt_marker)
attempt = json.loads(attempt_marker.read_text())
assert attempt["status"] == "superseded"
assert attempt["reason"] == "revision_not_current_head"
assert original_marker.read_bytes() == original_bytes
def test_revalidation_cli_refuses_non_root_linux_operator(status_module, monkeypatch):
monkeypatch.setattr(status_module.os, "name", "posix")
monkeypatch.setattr(status_module.os, "geteuid", lambda: 1000, raising=False)
monkeypatch.setattr(
status_module.sys,
"argv",
[
"status_server.py",
"--revalidate-delivery",
"jens/repo-a",
"a" * 40,
],
)
with pytest.raises(PermissionError, match="root"):
status_module.main()
def test_revalidation_cli_queues_through_local_supported_surface(
status_module, monkeypatch, capsys
):
project = _webhook_project()
revision = "a" * 40
digest = "b" * 64
monkeypatch.setattr(status_module.os, "name", "posix")
monkeypatch.setattr(status_module.os, "geteuid", lambda: 0, raising=False)
monkeypatch.setattr(status_module, "_require_controller_release", dict)
monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/repo-a": project})
requests: list[tuple[str, str, dict[str, object]]] = []
monkeypatch.setattr(
status_module,
"_enqueue_terminal_revalidation",
lambda repository, requested_revision, requested_project: (
requests.append((repository, requested_revision, requested_project))
or ("awaiting_validation", digest)
),
)
monkeypatch.setattr(
status_module.sys,
"argv",
[
"status_server.py",
"--revalidate-delivery",
"jens/repo-a",
revision,
],
)
status_module.main()
assert requests == [("jens/repo-a", revision, project)]
assert json.loads(capsys.readouterr().out) == {
"status": "awaiting_validation",
"repository": "jens/repo-a",
"revision": revision,
"delivery": digest,
}
def test_revalidation_has_no_http_mutation_route(status_module):
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
try:
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/revalidate",
body=b"{}",
headers={"Content-Type": "application/json", "Content-Length": "2"},
)
response = client.getresponse()
response.read()
client.close()
assert response.status == 404
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_workmesh_promotion_client_uses_only_fixed_signed_contract(monkeypatch):
from backend.app.services.deployment_broker import DeploymentBrokerClient
captured = []
class Response:
status_code = 200
@staticmethod
def json():
return {"status": "accepted", "operation": "workmesh_promote"}
class Client:
def __init__(self, **_kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def request(self, method, url, *, content, headers):
captured.append((method, url, content, headers))
return Response()
monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client)
client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True)
secret = "workmesh-controller-secret-" + "x" * 32
revision = "a" * 40
digest = "b" * 64
key = "workmesh-promotion:workmesh-prom-0123456789abcdef"
client.workmesh_promotion_preflight(revision, secret=secret)
client.workmesh_promotion_apply(
revision,
secret=secret,
preflight_sha256=digest,
idempotency_key=key,
)
expected = [
(
"POST",
"https://controller.example/v1/appops/workmesh/promotions/preflight",
{"revision": revision},
),
(
"POST",
"https://controller.example/v1/appops/workmesh/promotions/apply",
{"revision": revision, "preflightSha256": digest, "idempotencyKey": key},
),
]
assert [
(method, url, json.loads(body)) for method, url, body, _headers in captured
] == expected
for method, url, body, headers in captured:
path = url.removeprefix("https://controller.example")
assert headers["X-ITWorx-Deployment-Signature"] == DeploymentBrokerClient._signature(
secret,
method,
path,
headers["X-ITWorx-Deployment-Timestamp"],
headers["X-ITWorx-Deployment-Nonce"],
body,
)
assert client.workmesh_promotion_apply(
"main",
secret=secret,
preflight_sha256=digest,
idempotency_key=key,
) == {"status": "invalid", "reason": "invalid_workmesh_revision"}
assert len(captured) == 2
def test_workmesh_apply_preserves_rejection_identity_without_extra_fields(monkeypatch):
from backend.app.services.deployment_broker import DeploymentBrokerClient
request_id = "a" * 64
class Response:
status_code = 409
@staticmethod
def json():
return {
"status": "preflight_changed",
"request": request_id,
"repository": "jens/workmesh",
"revision": "b" * 40,
"operation": "workmesh_promote",
"reason": "approved_preflight_no_longer_matches",
"secret": "discard-me",
}
class Client:
def __init__(self, **_kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def request(self, *_args, **_kwargs):
return Response()
monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client)
client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True)
result = client.workmesh_promotion_apply(
"b" * 40,
secret="workmesh-controller-secret-" + "x" * 32,
preflight_sha256="c" * 64,
idempotency_key="workmesh-promotion:workmesh-prom-0123456789abcdef",
)
assert result == {
"status": "preflight_changed",
"reason": "approved_preflight_no_longer_matches",
"request": request_id,
"repository": "jens/workmesh",
"revision": "b" * 40,
"operation": "workmesh_promote",
}
def test_health_is_served_while_initial_runner_maintenance_is_blocked(
status_module, monkeypatch
):
maintenance_started = threading.Event()
release_maintenance = threading.Event()
server_created = threading.Event()
serving_started = threading.Event()
server_holder = {}
real_server = status_module.BoundedThreadingHTTPServer
class BlockingFleet:
def automatic_cycle(self):
maintenance_started.set()
assert release_maintenance.wait(timeout=5)
class ObservedServer(real_server):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
server_holder["server"] = self
server_created.set()
def serve_forever(self, *args, **kwargs):
serving_started.set()
return super().serve_forever(*args, **kwargs)
monkeypatch.setattr(status_module, "RunnerFleetService", BlockingFleet)
monkeypatch.setattr(status_module, "runner_fleet_specialization_state", lambda: "legacy")
monkeypatch.setattr(status_module, "BoundedThreadingHTTPServer", ObservedServer)
monkeypatch.setattr(
status_module,
"_controller_release",
lambda: {
"apiVersion": "itworx-controller-release.v1",
"controllerVersion": "2.1.3",
"sourceRepository": "jens/Unraid-AppOps-Gateway",
"sourceRevision": "a" * 40,
"sourceTree": "b" * 40,
"artifactSha256": "c" * 64,
"builtAt": "2026-09-21T20:58:36Z",
},
)
monkeypatch.setattr(
status_module,
"_delivery_status_payload",
lambda: {"status": "ok", "health": "healthy", "summary": {}},
)
monkeypatch.setattr(
status_module.sys,
"argv",
["status_server.py", "--listen", "127.0.0.1", "--port", "0"],
)
main_thread = threading.Thread(target=status_module.main, daemon=True)
main_thread.start()
try:
assert maintenance_started.wait(timeout=2)
assert server_created.wait(timeout=1), "server must bind before startup maintenance"
assert serving_started.wait(timeout=1), "server must serve before startup maintenance"
server = server_holder["server"]
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request("GET", "/health")
response = client.getresponse()
payload = json.loads(response.read())
client.close()
assert response.status == 200
assert payload["status"] == "ok"
assert payload["transport"] == "http"
assert not release_maintenance.is_set()
finally:
release_maintenance.set()
if serving_started.wait(timeout=2):
server_holder["server"].shutdown()
main_thread.join(timeout=3)
assert not main_thread.is_alive()
def test_webhook_worker_deploys_once_and_requires_verified_receipt(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
projects_path = tmp_path / "projects.json"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
revision = "a" * 40
raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode()
delivery = "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d"
_, request_digest = status_module._enqueue_webhook_delivery(
raw_body,
delivery,
77,
"refs/heads/main",
"jens/repo-a",
revision,
status_module._validation_policy("jens/repo-a", project),
)
marker = inbox_root / f"{request_digest}.json"
calls = []
class Result:
returncode = 0
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda arguments, **_kwargs: calls.append(arguments) or Result(),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"ok": True})
monkeypatch.setattr(
status_module,
"_fetch_gitea_validation",
lambda *_args: ("pass", [{"context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-14T00:00:00Z"}]),
)
status_module._process_webhook_record(marker)
assert json.loads(marker.read_text())["status"] == "deployed"
assert calls == [
[
"python3",
str(status_module.ROOT / "deploy.py"),
"deploy",
"jens/repo-a",
revision,
]
]
status_module._process_webhook_record(marker)
assert len(calls) == 1
second_body = json.dumps(_push_payload("c" * 40), separators=(",", ":")).encode()
_, second_digest = status_module._enqueue_webhook_delivery(
second_body,
"145440ab-b7b1-4c74-ae84-996d8d48167e",
77,
"refs/heads/main",
"jens/repo-a",
"c" * 40,
status_module._validation_policy("jens/repo-a", project),
)
second_marker = inbox_root / f"{second_digest}.json"
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None)
status_module._process_webhook_record(second_marker)
assert json.loads(second_marker.read_text())["status"] == "failed"
def test_webhook_worker_recovers_interrupted_deployment_from_verified_receipt(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
projects_path = tmp_path / "projects.json"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
revision = "a" * 40
raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode()
_, request_digest = status_module._enqueue_webhook_delivery(
raw_body,
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d",
77,
"refs/heads/main",
"jens/repo-a",
revision,
status_module._validation_policy("jens/repo-a", project),
)
marker = inbox_root / f"{request_digest}.json"
record = json.loads(marker.read_text())
record["status"] = "deploying"
marker.write_text(json.dumps(record), encoding="utf-8")
receipt = {"schema": "gitea-deploy-receipt.v1", "revision": revision}
calls = []
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda arguments, **_kwargs: calls.append(arguments),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: receipt)
status_module._process_webhook_record(marker)
recovered = json.loads(marker.read_text())
assert recovered["status"] == "deployed"
assert recovered["receipt_sha256"] == hashlib.sha256(
json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
assert recovered["audit"][-1]["stage"] == "deployed"
assert calls == []
def test_webhook_worker_fails_interrupted_deployment_without_verified_receipt(
status_module, tmp_path, monkeypatch
):
inbox_root = tmp_path / "inbox"
projects_path = tmp_path / "projects.json"
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
revision = "a" * 40
raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode()
_, request_digest = status_module._enqueue_webhook_delivery(
raw_body,
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d",
77,
"refs/heads/main",
"jens/repo-a",
revision,
status_module._validation_policy("jens/repo-a", project),
)
marker = inbox_root / f"{request_digest}.json"
record = json.loads(marker.read_text())
record["status"] = "deploying"
marker.write_text(json.dumps(record), encoding="utf-8")
calls = []
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda arguments, **_kwargs: calls.append(arguments),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None)
status_module._process_webhook_record(marker)
recovered = json.loads(marker.read_text())
assert recovered["status"] == "failed"
assert recovered["reason"] == "deployment_interrupted"
assert recovered["audit"][-1] == {
"at": recovered["audit"][-1]["at"],
"stage": "deployment_failed",
"reason": "deployment_interrupted",
}
assert calls == []
def test_gitea_validation_uses_latest_exact_sha_push_context_only(status_module, monkeypatch):
revision = "a" * 40
policy = status_module._validation_policy("jens/repo-a", _webhook_project())
assert policy is not None
class Response:
status = 200
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def read(self, _limit):
return json.dumps([
{
"id": 1,
"context": "Managed validation / full (push)",
"status": "success",
"url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}",
"target_url": "/Jens/Repo-A/actions/runs/100/jobs/200",
"updated_at": "2026-09-14T00:00:00Z",
},
{
"id": 2,
"context": "Managed validation / full (push)",
"status": "failure",
"url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}",
"target_url": "/Jens/Repo-A/actions/runs/100/jobs/200",
"updated_at": "2026-09-14T00:01:00Z",
},
{
"id": 3,
"context": "Managed validation / full (push)",
"status": "success",
"url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{'b' * 40}",
"target_url": "/Jens/Repo-A/actions/runs/100/jobs/200",
"updated_at": "2026-09-14T00:02:00Z",
},
{
"id": 4,
"context": "lint (push)",
"status": "success",
"url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}",
"target_url": "/Jens/Repo-A/actions/runs/100/jobs/200",
"updated_at": "2026-09-14T00:03:00Z",
},
{
"id": 5,
"context": "Managed validation / full (push)",
"status": "success",
"url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}",
"target_url": "/Jens/other-repo/actions/runs/100/jobs/200",
"updated_at": "2026-09-14T00:04:00Z",
},
]).encode()
class Opener:
def open(self, _request, timeout):
assert timeout == status_module.GITEA_STATUS_TIMEOUT_SECONDS
return Response()
monkeypatch.setattr(
status_module,
"_gitea_validation_settings",
lambda: ("https://gitea.example.test/api/v1", b"x" * 40),
)
monkeypatch.setattr(status_module, "build_opener", lambda *_args: Opener())
outcome, observed = status_module._fetch_gitea_validation("jens/repo-a", revision, policy)
assert outcome == "reject"
assert observed == [{
"context": "Managed validation / full (push)",
"status": "failure",
"updated_at": "2026-09-14T00:01:00Z",
}]
def test_gitea_validation_api_probe_uses_only_required_repository_read_scope(status_module, monkeypatch):
class Response:
status = 200
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def read(self, _limit):
return b'{"full_name":"Jens/Unraid-AppOps-Gateway"}'
class Opener:
def open(self, request, timeout):
assert request.full_url == "https://gitea.example.test/api/v1/repos/jens/unraid-appops-gateway"
assert timeout == status_module.GITEA_STATUS_TIMEOUT_SECONDS
return Response()
monkeypatch.setattr(
status_module,
"_gitea_validation_settings",
lambda: ("https://gitea.example.test/api/v1", b"x" * 40),
)
monkeypatch.setattr(
status_module,
"_controller_release",
lambda: {"sourceRepository": "jens/unraid-appops-gateway"},
)
monkeypatch.setattr(status_module, "build_opener", lambda *_args: Opener())
assert status_module._gitea_validation_api_reachable()
def test_validation_gate_never_deploys_before_exact_status_success(status_module, tmp_path, monkeypatch):
inbox = tmp_path / "inbox"
projects = tmp_path / "projects.json"
project = {**_webhook_project(), "slug": "repo-a", "containers": ["RepoA"]}
projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, request_id = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(),
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d",
77,
"refs/heads/main",
"jens/repo-a",
"a" * 40,
policy,
)
marker = inbox / f"{request_id}.json"
deploy_calls: list[list[str]] = []
class Result:
returncode = 0
monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: deploy_calls.append(args) or Result())
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True})
monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("wait", []))
status_module._process_webhook_record(marker)
assert json.loads(marker.read_text())["status"] == "awaiting_validation"
assert deploy_calls == []
monkeypatch.setattr(
status_module,
"_fetch_gitea_validation",
lambda *_args: ("pass", [{"context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-14T00:00:00Z"}]),
)
record = json.loads(marker.read_text())
record["validation"]["next_check_at"] = 0
marker.write_text(json.dumps(record), encoding="utf-8")
status_module._process_webhook_record(marker)
assert json.loads(marker.read_text())["status"] == "deployed"
assert len(deploy_calls) == 1
@pytest.mark.parametrize(
("outcome", "expected_state"),
[
("reject", "validation_failed"),
("unavailable", "awaiting_validation"),
],
)
def test_validation_failure_or_gitea_outage_fails_closed(
status_module, tmp_path, monkeypatch, outcome, expected_state
):
inbox = tmp_path / "inbox"
projects = tmp_path / "projects.json"
project = {**_webhook_project(), "slug": "repo-a", "containers": ["RepoA"]}
projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, request_id = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(),
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d",
77,
"refs/heads/main",
"jens/repo-a",
"a" * 40,
policy,
)
marker = inbox / f"{request_id}.json"
deploy_calls: list[list[str]] = []
monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: deploy_calls.append(args))
monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: (outcome, []))
status_module._process_webhook_record(marker)
record = json.loads(marker.read_text())
assert record["status"] == expected_state
assert deploy_calls == []
if outcome == "unavailable":
assert record["reason"] == "gitea_validation_unavailable"
def test_newer_validated_descendant_supersedes_old_undeployed_delivery(status_module, tmp_path, monkeypatch):
inbox = tmp_path / "inbox"
projects = tmp_path / "projects.json"
project = {**_webhook_project(), "slug": "repo-a", "containers": ["RepoA"]}
projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
policy = status_module._validation_policy("jens/repo-a", project)
assert policy is not None
_, old_id = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(),
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", "a" * 40, policy,
)
_, new_id = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload("b" * 40), separators=(",", ":")).encode(),
"145440ab-b7b1-4c74-ae84-996d8d48167e", 77, "refs/heads/main", "jens/repo-a", "b" * 40, policy,
)
old_marker = inbox / f"{old_id}.json"
new_marker = inbox / f"{new_id}.json"
monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("pass", []))
monkeypatch.setattr(status_module, "_is_forward_descendant", lambda *_args: True)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True})
monkeypatch.setattr(status_module.subprocess, "run", lambda *_args, **_kwargs: type("Result", (), {"returncode": 0})())
status_module._process_webhook_record(new_marker)
assert json.loads(old_marker.read_text())["status"] == "superseded"
assert json.loads(new_marker.read_text())["status"] == "deployed"
def test_delivery_status_projection_is_bounded_and_secret_free(status_module, tmp_path, monkeypatch):
inbox = tmp_path / "inbox"
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
policy = status_module._validation_policy("jens/repo-a", _webhook_project())
assert policy is not None
_, request_id = status_module._enqueue_webhook_delivery(
json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(),
"5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", "a" * 40, policy,
)
marker = inbox / f"{request_id}.json"
record = json.loads(marker.read_text())
record["webhook_secret"] = "must-not-be-returned"
record["authorization"] = "must-not-be-returned"
marker.write_text(json.dumps(record), encoding="utf-8")
monkeypatch.setattr(status_module, "_gitea_validation_settings", lambda: ("https://gitea.example.test/api/v1", b"x" * 40))
payload = status_module._delivery_status_payload()
serialized = json.dumps(payload)
assert payload["summary"]["pendingValidation"] == 1
assert payload["items"][0]["repository"] == "jens/repo-a"
assert "must-not-be-returned" not in serialized
def test_delivery_status_marks_an_unwritable_inbox_unhealthy(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "inbox"
inbox.mkdir()
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
monkeypatch.setattr(
status_module,
"_gitea_validation_settings",
lambda: ("https://gitea.example.test/api/v1", b"x" * 40),
)
monkeypatch.setattr(
status_module,
"_controller_release",
lambda: {"sourceRepository": "jens/unraid-appops-gateway"},
)
def no_space_left(*_args, **_kwargs):
raise OSError(28, "No space left on device")
monkeypatch.setattr(status_module.os, "open", no_space_left)
payload = status_module._delivery_status_payload()
assert payload["status"] == "unhealthy"
assert payload["health"]["inbox"] == "unhealthy"
assert list(inbox.iterdir()) == []
def test_inbox_health_probe_cleans_up_when_the_create_directory_fsync_fails(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "inbox"
inbox.mkdir()
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
fsync_calls = 0
def fail_the_first_directory_fsync():
nonlocal fsync_calls
fsync_calls += 1
if fsync_calls == 1:
raise OSError(5, "Directory fsync failed")
monkeypatch.setattr(status_module, "_fsync_inbox_root", fail_the_first_directory_fsync)
with pytest.raises(OSError, match="Directory fsync failed"):
status_module._probe_inbox_writable()
assert fsync_calls == 2
assert list(inbox.iterdir()) == []
def test_delivery_status_client_is_read_only_and_does_not_send_a_secret(monkeypatch):
from backend.app.core.config import settings
from backend.app.services import deployment_broker
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_URL", "https://controller.example.test")
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", True)
captured: dict[str, object] = {}
class Response:
status_code = 200
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def iter_bytes(self):
return iter([b'{"status":"healthy","items":[]}'])
class Client:
def __init__(self, **kwargs):
captured["client"] = kwargs
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def stream(self, method, url, headers):
captured.update({"method": method, "url": url, "headers": headers})
return Response()
monkeypatch.setattr(deployment_broker.httpx, "Client", Client)
assert deployment_broker.DeploymentBrokerClient().delivery_status() == {"status": "healthy", "items": []}
assert captured["method"] == "GET"
assert captured["url"] == "https://controller.example.test/v1/gitea/deliveries"
assert captured["headers"] == {"Accept": "application/json"}
assert captured["client"] == {"timeout": 10.0, "verify": True, "follow_redirects": False}
def _write_controller_tls_pair(tmp_path: Path) -> tuple[Path, Path]:
from datetime import datetime, timedelta, timezone
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "controller.test")])
now = datetime.now(timezone.utc).replace(tzinfo=None)
certificate = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - timedelta(minutes=1))
.not_valid_after(now + timedelta(days=1))
.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
.add_extension(
x509.SubjectAlternativeName(
[x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]
),
critical=False,
)
.add_extension(
x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False
)
.sign(key, hashes.SHA256())
)
certificate_path = tmp_path / "controller-test.crt"
private_key_path = tmp_path / "controller-test.key"
certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
private_key_path.write_bytes(
key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
)
private_key_path.chmod(0o600)
return certificate_path, private_key_path
def test_private_tls_configuration_requires_a_complete_private_listener_record(
status_module, tmp_path, monkeypatch
):
monkeypatch.setattr(status_module, "os", types.SimpleNamespace(name="nt"))
certificate, private_key = _write_controller_tls_pair(tmp_path)
config_path = tmp_path / "private-tls.json"
monkeypatch.setattr(status_module, "PRIVATE_TLS_CONFIG_PATH", config_path)
assert status_module._private_tls_configuration() is None
with pytest.raises(RuntimeError, match="missing"):
status_module._private_tls_configuration(required=True)
payload = {
"schema": "itworx.controller-private-tls/v1",
"listen_ip": "198.18.10.50",
"port": 1300,
"certificate_file": str(certificate),
"private_key_file": str(private_key),
}
config_path.write_text(json.dumps(payload), encoding="utf-8")
config_path.chmod(0o600)
incomplete = dict(payload)
incomplete.pop("private_key_file")
config_path.write_text(json.dumps(incomplete), encoding="utf-8")
with pytest.raises(RuntimeError, match="configuration is invalid"):
status_module._private_tls_configuration(required=True)
config_path.write_text(json.dumps(payload), encoding="utf-8")
configuration = status_module._private_tls_configuration(required=True)
assert configuration is not None
assert configuration["listen_ip"] == "198.18.10.50"
assert configuration["port"] == 1300
context = status_module._private_tls_context(configuration)
assert context.minimum_version == ssl.TLSVersion.TLSv1_2
payload["listen_ip"] = "0.0.0.0"
config_path.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(RuntimeError, match="explicit private IPv4"):
status_module._private_tls_configuration(required=True)
payload["listen_ip"] = "fd00::1"
config_path.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(RuntimeError, match="explicit private IPv4"):
status_module._private_tls_configuration(required=True)
def test_private_tls_listener_serves_health_over_verified_tls(
status_module, tmp_path, monkeypatch
):
certificate, private_key = _write_controller_tls_pair(tmp_path)
monkeypatch.setattr(
status_module,
"_controller_release",
lambda: {"controllerVersion": "test"},
)
monkeypatch.setattr(
status_module,
"_delivery_status_payload",
lambda: {"status": "ok", "health": "healthy", "summary": {}},
)
server = status_module._private_tls_server(
{
"listen_ip": "127.0.0.1",
"port": 0,
"certificate_file": certificate,
"private_key_file": private_key,
}
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
try:
client = http.client.HTTPSConnection(
*server.server_address,
context=ssl.create_default_context(cafile=str(certificate)),
timeout=2,
)
client.request("GET", "/health")
response = client.getresponse()
payload = json.loads(response.read())
client.close()
assert response.status == 200
assert payload["status"] == "ok"
assert payload["transport"] == "https"
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_deployment_broker_uses_private_ca_and_rejects_disabled_https_verification(
monkeypatch, tmp_path
):
from backend.app.core.config import settings
from backend.app.services import deployment_broker
monkeypatch.setattr(deployment_broker, "os", types.SimpleNamespace(name="nt"))
ca_file = tmp_path / "private-ca.pem"
ca_file.write_text("test CA bundle", encoding="utf-8")
ca_file.chmod(0o644)
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_URL", "https://controller.private")
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", True)
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_CA_FILE", str(ca_file))
captured: dict[str, object] = {}
class Response:
status_code = 200
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def iter_bytes(self):
return iter([b'{"status":"healthy","items":[]}'])
class Client:
def __init__(self, **kwargs):
captured["client"] = kwargs
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def stream(self, method, url, headers):
captured.update({"method": method, "url": url, "headers": headers})
return Response()
monkeypatch.setattr(deployment_broker.httpx, "Client", Client)
assert deployment_broker.DeploymentBrokerClient().delivery_status() == {
"status": "healthy",
"items": [],
}
assert captured["client"] == {
"timeout": 10.0,
"verify": str(ca_file),
"follow_redirects": False,
}
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", False)
insecure = deployment_broker.DeploymentBrokerClient()
assert insecure.delivery_status() == {
"status": "missing_config",
"reason": "deployment_broker_tls_verification_required",
}
assert (
insecure.configuration_error()
== "deployment_broker_tls_verification_required"
)
monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", True)
monkeypatch.setitem(
settings.apis,
"DEPLOYMENT_BROKER_CA_FILE",
str(tmp_path / "missing-ca.pem"),
)
invalid_ca = deployment_broker.DeploymentBrokerClient()
assert invalid_ca.delivery_status() == {
"status": "missing_config",
"reason": "deployment_broker_ca_file_invalid",
}
def test_deployment_broker_ca_file_must_be_owned_by_the_appops_process(
monkeypatch, tmp_path
):
from backend.app.services import deployment_broker
ca_file = tmp_path / "private-ca.pem"
owner = 1234
file_info = types.SimpleNamespace(
st_mode=stat.S_IFREG | 0o640,
st_size=16,
st_uid=owner,
)
monkeypatch.setattr(deployment_broker.Path, "lstat", lambda _path: file_info)
monkeypatch.setattr(
deployment_broker,
"os",
types.SimpleNamespace(
name="posix",
R_OK=4,
geteuid=lambda: owner,
access=lambda _path, _mode: True,
),
)
assert deployment_broker.DeploymentBrokerClient._trusted_ca_file_is_safe(
str(ca_file)
)
monkeypatch.setattr(
deployment_broker,
"os",
types.SimpleNamespace(
name="posix",
R_OK=4,
geteuid=lambda: -1,
access=lambda _path, _mode: True,
),
)
assert not deployment_broker.DeploymentBrokerClient._trusted_ca_file_is_safe(
str(ca_file)
)
file_info.st_mode = stat.S_IFREG | 0o660
monkeypatch.setattr(
deployment_broker,
"os",
types.SimpleNamespace(
name="posix",
R_OK=4,
geteuid=lambda: owner,
access=lambda _path, _mode: True,
),
)
assert not deployment_broker.DeploymentBrokerClient._trusted_ca_file_is_safe(
str(ca_file)
)
def test_status_receipt_must_match_the_authenticated_request(status_module):
revision = "a" * 40
project = {
"containers": ["Chimera"],
"verify_oci_revision": True,
"live_probe": {
"type": "chimera-health-live-v1",
},
}
project["_repository_name"] = "jens/chimera"
receipt = _receipt(project, revision)
state = {
"schema_version": 3,
"repository": "jens/chimera",
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": False,
"live_receipt": receipt,
}
assert (
status_module._verified_receipt(state, "jens/chimera", revision, project)
== receipt
)
assert (
status_module._verified_receipt(state, "jens/chimera", "d" * 40, project)
is None
)
assert (
status_module._verified_receipt(state, "jens/other", revision, project) is None
)
def test_status_receipt_rejects_corrupt_state_container_and_live_proof(status_module):
revision = "a" * 40
project = {
"containers": ["Chimera"],
"verify_oci_revision": True,
"live_probe": {"type": "chimera-health-live-v1"},
"_repository_name": "jens/chimera",
}
receipt = _receipt(project, revision)
state = {
"schema_version": 3,
"repository": "jens/chimera",
"live_revision": revision,
"live_status": "healthy",
"live_legacy_provenance": False,
"live_receipt": receipt,
}
corruptions = []
bad = copy.deepcopy(state)
bad["live_legacy_provenance"] = True
corruptions.append(bad)
bad = copy.deepcopy(state)
bad["live_receipt"]["treeDigest"] = "0" * 64
corruptions.append(bad)
bad = copy.deepcopy(state)
bad["live_receipt"]["build"]["id"] = "wrong"
corruptions.append(bad)
bad = copy.deepcopy(state)
bad["live_receipt"]["containers"] = {"bogus": {}}
corruptions.append(bad)
bad = copy.deepcopy(state)
bad["live_receipt"]["containers"]["Chimera"]["image_id"] = "sha256:short"
corruptions.append(bad)
bad = copy.deepcopy(state)
bad["live_receipt"]["containers"]["Chimera"]["image_source"] = "jens/other"
corruptions.append(bad)
bad = copy.deepcopy(state)
bad["live_receipt"]["live"]["status"] = "starting"
corruptions.append(bad)
for corrupt in corruptions:
assert (
status_module._verified_receipt(corrupt, "jens/chimera", revision, project)
is None
)
def test_repository_webhook_secret_rejects_symlink(
status_module, tmp_path, monkeypatch
):
secret_root = tmp_path / "secrets"
secret_root.mkdir()
real = tmp_path / "real-secret"
real.write_text("x" * 48, encoding="utf-8")
real.chmod(0o600)
link = secret_root / "jens__repo-a"
try:
link.symlink_to(real)
except OSError:
pytest.skip("symbolic links are unavailable on this host")
monkeypatch.setattr(status_module, "WEBHOOK_SECRET_ROOT", secret_root)
assert status_module._webhook_secret("jens/repo-a", _webhook_project()) is None
def test_status_server_bounds_threads_and_times_out_partial_bodies(
status_module, monkeypatch
):
monkeypatch.setattr(status_module, "MAX_REQUEST_THREADS", 2)
monkeypatch.setattr(status_module, "REQUEST_TIMEOUT_SECONDS", 0.1)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(
target=server.serve_forever,
kwargs={"poll_interval": 0.01},
daemon=True,
)
serving.start()
try:
assert server._request_slots.acquire(blocking=False)
assert server._request_slots.acquire(blocking=False)
assert not server._request_slots.acquire(blocking=False)
server._request_slots.release()
server._request_slots.release()
with socket.create_connection(server.server_address, timeout=2) as client:
client.settimeout(2)
client.sendall(
b"POST /v1/gitea/push/jens/repo-a HTTP/1.1\r\n"
b"Host: localhost\r\n"
b"Content-Length: 10\r\n"
b"\r\n"
b"{"
)
chunks: list[bytes] = []
while chunk := client.recv(4096):
chunks.append(chunk)
response = b"".join(chunks)
assert b" 408 " in response
assert b"request_timeout" in response
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_status_server_accepts_only_exact_signed_repository_push(
status_module, tmp_path, monkeypatch
):
secret = b"webhook-" + b"x" * 48
secret_root = tmp_path / "secrets"
inbox_root = tmp_path / "inbox"
projects_path = tmp_path / "projects.json"
_write_webhook_secret(secret_root, "jens__repo-a", secret)
project = {
**_webhook_project(),
"slug": "repo-a",
"containers": ["RepoA"],
}
projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_SECRET_ROOT", secret_root)
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
revision = "a" * 40
raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode()
signature = hmac.new(secret, raw_body, hashlib.sha256).hexdigest()
headers = {
"Content-Type": "application/json",
"X-Gitea-Delivery": "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d",
"X-Gitea-Event": "push",
"X-Gitea-Event-Type": "push",
"X-Gitea-Hook-Installation-Target-Type": "repository",
"X-Gitea-Signature": signature,
}
try:
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/push/jens/repo-a",
body=raw_body,
headers=headers,
)
response = client.getresponse()
body = json.loads(response.read())
client.close()
assert response.status == 202
assert body["status"] == "accepted"
assert body["revision"] == revision
request_digest = body["request"]
replay_body = json.dumps(
{
**_push_payload(revision, repository="Jens/repo-a"),
"compare_url": "https://example.invalid",
},
separators=(",", ":"),
).encode()
replay_headers = {
**headers,
"X-Gitea-Delivery": "145440ab-b7b1-4c74-ae84-996d8d48167e",
"X-Gitea-Signature": hmac.new(
secret, replay_body, hashlib.sha256
).hexdigest(),
}
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/push/jens/repo-a",
body=replay_body,
headers=replay_headers,
)
response = client.getresponse()
replay_response = json.loads(response.read())
client.close()
assert response.status == 202
assert replay_response["request"] == request_digest
assert len(list(inbox_root.glob("*.json"))) == 1
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/push/jens/repo-a",
body=raw_body + b" ",
headers=headers,
)
response = client.getresponse()
assert response.status == 403
assert b"webhook_signature_rejected" in response.read()
client.close()
invalid_json = b"{"
invalid_headers = {**headers, "X-Gitea-Signature": "0" * 64}
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/push/jens/repo-a",
body=invalid_json,
headers=invalid_headers,
)
response = client.getresponse()
assert response.status == 403
assert b"webhook_signature_rejected" in response.read()
client.close()
valid_invalid_headers = {
**headers,
"X-Gitea-Signature": hmac.new(
secret, invalid_json, hashlib.sha256
).hexdigest(),
}
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/push/jens/repo-a",
body=invalid_json,
headers=valid_invalid_headers,
)
response = client.getresponse()
assert response.status == 400
assert b"invalid_json" in response.read()
client.close()
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request(
"POST",
"/v1/gitea/push/Jens/repo-a",
body=raw_body,
headers=headers,
)
response = client.getresponse()
assert response.status == 404
client.close()
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
@pytest.mark.parametrize(
("raw_value", "expected"),
[
("1", 1),
("4096", 4096),
("1048576", 1048576),
("0", None),
("1048577", None),
("00001", None),
("-1", None),
("\N{SUPERSCRIPT TWO}", None),
("9" * 5000, None),
],
)
def test_status_server_content_length_is_bounded_ascii(
status_module, raw_value, expected
):
assert status_module._bounded_content_length(raw_value) == expected
def test_status_server_closes_completed_json_responses(status_module):
handler = status_module.Handler.__new__(status_module.Handler)
headers: list[tuple[str, str]] = []
statuses: list[int] = []
handler.close_connection = False
handler.wfile = io.BytesIO()
handler.send_response = statuses.append
handler.send_header = lambda name, value: headers.append((name, value))
handler.end_headers = lambda: None
handler._send_json(200, b'{"status":"ok"}\n')
assert statuses == [200]
assert ("Connection", "close") in headers
assert handler.close_connection is True
assert handler.wfile.getvalue() == b'{"status":"ok"}\n'
def _appops_project() -> dict[str, object]:
return {
"slug": "repo-a",
"compose_project": "repo-a",
"compose_files": [".itworx/deployment.yml"],
"containers": ["repo-a"],
"managed_plan_version": "itworx.deployment/v1",
"managed_application_id": "repo-a",
"managed_appdata_path": "/srv/example/appdata/repo-a",
"managed_image_repository": "repo-a",
"managed_image_tag": "1.2.3",
"managed_proxy_network": "itworx-npm",
"managed_internal_port": 8080,
"managed_proxy_hostname": None,
"managed_oidc_application_slug": None,
"appops_deployment_secret_file": "jens__repo-a",
}
def _appops_plan(revision: str) -> dict[str, object]:
return {
"api_version": "itworx.deployment/v1",
"application_id": "repo-a",
"repository": "jens/repo-a",
"revision": revision,
"release_version": "1.2.3",
"image_repository": "repo-a",
"image_tag": "1.2.3",
"topology": {
"kind": "compose",
"compose_file": ".itworx/deployment.yml",
"compose_project": "repo-a",
"containers": ["repo-a"],
"proxy_network": "itworx-npm",
"internal_port": 8080,
},
"broker_secret_ref": "REPO_A_DEPLOYMENT_BROKER",
}
def test_appops_plan_root_must_match_the_registered_target_root(status_module):
project = _appops_project()
plan = _appops_plan("a" * 40)
assert status_module._appops_plan({"jens/repo-a": project}, plan) == (
"jens/repo-a", "a" * 40, project,
)
project["managed_appdata_root"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data"
project["managed_appdata_path"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data/repo-a"
alternate_plan = {**plan, "trusted_appdata_root": "/var/lib/docker/volumes/itworx-managed-appdata/_data"}
assert status_module._appops_plan({"jens/repo-a": project}, alternate_plan) == (
"jens/repo-a", "a" * 40, project,
)
assert status_module._appops_plan({"jens/repo-a": project}, plan) is None
drifted = {**project, "managed_appdata_path": "/srv/example/appdata/repo-a"}
assert status_module._appops_plan({"jens/repo-a": drifted}, alternate_plan) is None
def _write_appops_secret(root: Path, name: str, value: bytes) -> None:
root.mkdir(mode=0o700)
root.chmod(0o700)
path = root / name
path.write_bytes(value)
path.chmod(0o600)
def test_bootstrap_signatures_bind_endpoint_and_are_single_use(status_module, tmp_path, monkeypatch):
secret = b"bootstrap-" + b"x" * 48
secret_path = tmp_path / "enrollment-bootstrap"
secret_path.write_bytes(secret)
secret_path.chmod(0o600)
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", secret_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
raw = b'{"enrollment":{}}'
timestamp = str(int(status_module.time.time()))
nonce = "3" * 32
signature = status_module._request_signature(
secret,
method="POST",
path="/v1/appops/enrollments/preflight",
timestamp=timestamp,
nonce=nonce,
raw_body=raw,
)
assert not status_module._bootstrap_signature_is_valid(
raw, timestamp, nonce, signature,
method="POST", path="/v1/appops/enrollments/rollback",
)
assert status_module._bootstrap_signature_is_valid(
raw, timestamp, nonce, signature,
method="POST", path="/v1/appops/enrollments/preflight",
)
assert not status_module._bootstrap_signature_is_valid(
raw, timestamp, nonce, signature,
method="POST", path="/v1/appops/enrollments/preflight",
)
def test_repopulse_adoption_capabilities_is_hmac_protected_and_read_only(
status_module, tmp_path, monkeypatch
):
bootstrap_secret = b"adoption-bootstrap-" + b"x" * 48
bootstrap_path = tmp_path / "bootstrap"
bootstrap_path.write_bytes(bootstrap_secret)
bootstrap_path.chmod(0o600)
nonce_root = tmp_path / "nonces"
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root)
monkeypatch.setattr(
status_module._repopulse_adoption_store,
"capability_report",
lambda: {
"repository": "jens/repopulse",
"executionEnabled": True,
"controllerAliasesReady": True,
"registryReady": True,
"secretRootReady": True,
},
)
monkeypatch.setattr(status_module, "_private_tls_configuration", lambda **_kwargs: {})
monkeypatch.setattr(status_module, "_private_tls_context", lambda _configuration: object())
monkeypatch.setattr(status_module, "_repopulse_adoption_alias_mounts_ready", lambda: True)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
body = b"{}"
timestamp = str(int(status_module.time.time()))
nonce = "5" * 32
path = "/v1/appops/repopulse/adoption/capabilities"
signature = status_module._request_signature(
bootstrap_secret,
method="POST",
path=path,
timestamp=timestamp,
nonce=nonce,
raw_body=body,
)
headers = {
"Content-Type": "application/json",
"X-ITWorx-Deployment-Timestamp": timestamp,
"X-ITWorx-Deployment-Nonce": nonce,
"X-ITWorx-Deployment-Signature": signature,
}
try:
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request("POST", path, body=body, headers=headers)
response = client.getresponse()
payload = json.loads(response.read())
client.close()
assert response.status == 200
assert payload == {
"apiVersion": "itworx.repopulse-adoption-capabilities/v1",
"status": "ready",
"repository": "jens/repopulse",
"adoptionStore": {"executionEnabled": True},
"privateTls": {"status": "ready"},
"controller": {
"aliases": "ready",
"registry": "ready",
"secretRoot": "ready",
},
}
assert bootstrap_secret.decode("ascii") not in json.dumps(payload)
assert nonce_root.is_dir()
assert len(list(nonce_root.iterdir())) == 1
assert not (tmp_path / "repopulse-adoption").exists()
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request("POST", path, body=body, headers=headers)
replay = client.getresponse()
replay_payload = json.loads(replay.read())
client.close()
assert replay.status == 403
assert replay_payload == {"status": "authentication_rejected"}
assert len(list(nonce_root.iterdir())) == 1
for unavailable_path in (
"/v1/appops/repopulse/adoption/execute",
"/v1/appops/repopulse/adoption/rollback",
):
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request("POST", unavailable_path, body=body)
unsupported = client.getresponse()
unsupported.read()
client.close()
assert unsupported.status == 404
assert len(list(nonce_root.iterdir())) == 1
monkeypatch.setattr(
status_module,
"_private_tls_configuration",
lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("unsafe path detail")),
)
tls_nonce = "6" * 32
tls_timestamp = str(int(status_module.time.time()))
tls_signature = status_module._request_signature(
bootstrap_secret,
method="POST",
path=path,
timestamp=tls_timestamp,
nonce=tls_nonce,
raw_body=body,
)
tls_headers = {
**headers,
"X-ITWorx-Deployment-Timestamp": tls_timestamp,
"X-ITWorx-Deployment-Nonce": tls_nonce,
"X-ITWorx-Deployment-Signature": tls_signature,
}
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request("POST", path, body=body, headers=tls_headers)
tls_unavailable = client.getresponse()
tls_payload = json.loads(tls_unavailable.read())
client.close()
assert tls_unavailable.status == 503
assert tls_payload["privateTls"] == {"status": "unavailable"}
assert tls_payload["controller"]["aliases"] == "ready"
assert "unsafe path detail" not in json.dumps(tls_payload)
assert len(list(nonce_root.iterdir())) == 2
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def _configure_repopulse_adoption_store_for_http_test(
status_module, tmp_path, monkeypatch, *, enabled
):
store_module = status_module._repopulse_adoption_store
controller = tmp_path / "controller"
controller.mkdir()
controller.chmod(0o755)
projects = {
"jens/repopulse": store_module._legacy_registry_record(),
"jens/other": {"slug": "other"},
}
projects_path = controller / "projects.json"
projects_path.write_text(json.dumps(projects, sort_keys=True) + "\n", encoding="utf-8")
projects_path.chmod(0o600)
secrets = controller / "appops-deployment-secrets"
secrets.mkdir()
secrets.chmod(0o700)
state = tmp_path / "state"
state.mkdir()
state.chmod(0o755)
locks = state / "locks"
locks.mkdir()
locks.chmod(0o755)
state_store = state / "repopulse-adoption"
project_lock = locks / "repopulse.lock"
global_lock = tmp_path / "deploy.lock"
for name, value in {
"CONTROLLER_READONLY_ROOT": controller,
"CONTROLLER_RW_ROOT": controller,
"STATE_ROOT": state,
"GLOBAL_DEPLOY_LOCK": global_lock,
"PROJECT_LOCK": project_lock,
"ADOPTION_STATE_ROOT": state_store,
"EXECUTION_ENABLED": enabled,
"_IS_POSIX_CONTROLLER": True,
}.items():
monkeypatch.setattr(store_module, name, value)
test_uid = controller.stat().st_uid
monkeypatch.setattr(store_module, "_ROOT_UID", test_uid)
monkeypatch.setattr(store_module, "_effective_uid", lambda: test_uid)
if os.name != "posix":
monkeypatch.setattr(
store_module,
"_mode_bits",
lambda path, info: (
0o700
if path.name in {"appops-deployment-secrets", "repopulse-adoption"}
else 0o755
if stat.S_ISDIR(info.st_mode)
else 0o600
),
)
monkeypatch.setattr(store_module, "_directory_fsync", lambda _path: None)
return store_module, state_store, project_lock, global_lock, controller
def _adoption_bootstrap_headers(status_module, secret, *, method, path, body, nonce):
timestamp = str(int(status_module.time.time()))
signature = status_module._request_signature(
secret,
method=method,
path=path,
timestamp=timestamp,
nonce=nonce,
raw_body=body,
)
return {
"X-ITWorx-Deployment-Timestamp": timestamp,
"X-ITWorx-Deployment-Nonce": nonce,
"X-ITWorx-Deployment-Signature": signature,
}
def _adoption_http_request(server, method, path, *, body=None, headers=None):
client = http.client.HTTPConnection(*server.server_address, timeout=2)
if body is None:
client.request(method, path, headers=headers or {})
else:
client.request(method, path, body=body, headers=headers or {})
response = client.getresponse()
status = response.status
payload = json.loads(response.read())
client.close()
return status, payload
def _valid_repopulse_reserve_body(status_module, *, seed="reserve-seed"):
idempotency_sha256 = hashlib.sha256(seed.encode("ascii")).hexdigest()
payload = {
"intentId": f"rpa-{idempotency_sha256[:20]}",
"idempotencySha256": idempotency_sha256,
"preflightSha256": hashlib.sha256(b"fixed-preflight").hexdigest(),
}
return json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8"), payload
def test_repopulse_reserve_is_hmac_protected_strict_and_disabled_before_store_write(
status_module, tmp_path, monkeypatch
):
store_module, state_store, project_lock, global_lock, _controller = (
_configure_repopulse_adoption_store_for_http_test(
status_module, tmp_path, monkeypatch, enabled=False
)
)
secret = b"reserve-bootstrap-secret-" + b"x" * 40
bootstrap_path = tmp_path / "bootstrap"
bootstrap_path.write_bytes(secret)
bootstrap_path.chmod(0o600)
nonce_root = tmp_path / "nonces"
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root)
transaction_calls = []
def forbidden_transaction(*_args, **_kwargs):
transaction_calls.append(True)
raise AssertionError("disabled reserve must not enter store transaction")
monkeypatch.setattr(
store_module.RepoPulseAdoptionStore, "transaction", forbidden_transaction
)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
reserve_path = status_module.REPOPULSE_ADOPTION_RESERVE_PATH
raw_body, payload = _valid_repopulse_reserve_body(status_module)
try:
invalid_payloads = [
{**payload, "path": "/tmp/attacker"},
{key: value for key, value in payload.items() if key != "preflightSha256"},
{**payload, "idempotencySha256": True},
{**payload, "intentId": "rpa-" + "f" * 20},
{**payload, "preflightSha256": "A" * 64},
]
for invalid in invalid_payloads:
body = json.dumps(invalid, separators=(",", ":")).encode("utf-8")
response_status, response = _adoption_http_request(
server, "POST", reserve_path, body=body
)
assert response_status == 400
assert response == {"status": "invalid_reserve_request"}
# Duplicate keys are rejected before auth and never interpreted as a
# different request from the one the caller intended to sign.
duplicate_body = (
b'{"intentId":"'
+ payload["intentId"].encode("ascii")
+ b'","intentId":"'
+ payload["intentId"].encode("ascii")
+ b'","idempotencySha256":"'
+ payload["idempotencySha256"].encode("ascii")
+ b'","preflightSha256":"'
+ payload["preflightSha256"].encode("ascii")
+ b'"}'
)
duplicate_status, duplicate_response = _adoption_http_request(
server, "POST", reserve_path, body=duplicate_body
)
assert duplicate_status == 400
assert duplicate_response == {"status": "invalid_json"}
# Even a valid contract cannot reach the store without the HMAC.
unauthenticated_status, unauthenticated = _adoption_http_request(
server, "POST", reserve_path, body=raw_body
)
assert unauthenticated_status == 403
assert unauthenticated == {"status": "authentication_rejected"}
headers = _adoption_bootstrap_headers(
status_module,
secret,
method="POST",
path=reserve_path,
body=raw_body,
nonce="1" * 32,
)
disabled_status, disabled = _adoption_http_request(
server, "POST", reserve_path, body=raw_body, headers=headers
)
assert disabled_status == 503
assert disabled == {
"apiVersion": "itworx.repopulse-adoption-reserve/v1",
"status": "execution_disabled",
}
replay_status, replay = _adoption_http_request(
server, "POST", reserve_path, body=raw_body, headers=headers
)
assert replay_status == 403
assert replay == {"status": "authentication_rejected"}
status_path = status_module.REPOPULSE_ADOPTION_STATUS_PATH
status_headers = _adoption_bootstrap_headers(
status_module,
secret,
method="GET",
path=status_path,
body=b"",
nonce="2" * 32,
)
status_status, status_payload = _adoption_http_request(
server, "GET", status_path, headers=status_headers
)
assert status_status == 200
assert status_payload == {
"apiVersion": "itworx.repopulse-adoption-status/v1",
"status": "not_reserved",
"phase": None,
"hashes": {},
}
assert not state_store.exists()
assert not project_lock.exists()
assert not global_lock.exists()
replay_status, replay = _adoption_http_request(
server, "GET", status_path, headers=status_headers
)
assert replay_status == 403
assert replay == {"status": "authentication_rejected"}
state_store.write_text("unsafe status state", encoding="utf-8")
unsafe_headers = _adoption_bootstrap_headers(
status_module,
secret,
method="GET",
path=status_path,
body=b"",
nonce="7" * 32,
)
unsafe_status, unsafe_payload = _adoption_http_request(
server, "GET", status_path, headers=unsafe_headers
)
assert unsafe_status == 503
assert unsafe_payload == {
"apiVersion": "itworx.repopulse-adoption-status/v1",
"status": "unavailable",
}
assert state_store.read_text(encoding="utf-8") == "unsafe status state"
assert "unsafe status state" not in json.dumps(unsafe_payload)
assert len(list(nonce_root.iterdir())) == 3
assert transaction_calls == []
assert not project_lock.exists()
assert not global_lock.exists()
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_repopulse_reserve_is_idempotent_locked_and_status_redacts_credential(
status_module, tmp_path, monkeypatch
):
store_module, state_store, project_lock, global_lock, controller = (
_configure_repopulse_adoption_store_for_http_test(
status_module, tmp_path, monkeypatch, enabled=True
)
)
secret = b"reserve-bootstrap-secret-" + b"y" * 40
bootstrap_path = tmp_path / "bootstrap"
bootstrap_path.write_bytes(secret)
bootstrap_path.chmod(0o600)
nonce_root = tmp_path / "nonces"
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root)
credential = b"very-high-entropy-existing-controller-credential" * 2
credential_path = controller / "appops-deployment-secrets" / "jens__repopulse"
credential_path.write_bytes(credential)
credential_path.chmod(0o600)
lock_events = []
original_lock = store_module._exclusive_flock
@contextmanager
def tracked_lock(path):
with original_lock(path):
lock_events.append(path)
yield
monkeypatch.setattr(store_module, "_exclusive_flock", tracked_lock)
original_reserve = store_module.RepoPulseAdoptionStore.reserve_intent
def checked_reserve(store, **kwargs):
assert status_module._enrollment_write_lock.locked()
assert lock_events[-2:] == [project_lock, global_lock]
return original_reserve(store, **kwargs)
monkeypatch.setattr(
store_module.RepoPulseAdoptionStore, "reserve_intent", checked_reserve
)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
reserve_path = status_module.REPOPULSE_ADOPTION_RESERVE_PATH
raw_body, payload = _valid_repopulse_reserve_body(status_module)
try:
first_headers = _adoption_bootstrap_headers(
status_module,
secret,
method="POST",
path=reserve_path,
body=raw_body,
nonce="3" * 32,
)
first_status, first = _adoption_http_request(
server, "POST", reserve_path, body=raw_body, headers=first_headers
)
second_headers = _adoption_bootstrap_headers(
status_module,
secret,
method="POST",
path=reserve_path,
body=raw_body,
nonce="4" * 32,
)
second_status, second = _adoption_http_request(
server, "POST", reserve_path, body=raw_body, headers=second_headers
)
assert first_status == second_status == 200
assert first == second
assert first["apiVersion"] == "itworx.repopulse-adoption-reserve/v1"
assert first["status"] == "reserved"
assert first["phase"] == "reserved"
assert first["intentId"] == payload["intentId"]
assert len(first["intentSha256"]) == 64
assert lock_events == [project_lock, global_lock, project_lock, global_lock]
assert not status_module._enrollment_write_lock.locked()
assert state_store.is_dir()
assert (state_store / "intent.json").is_file()
conflict_body, _conflict_payload = _valid_repopulse_reserve_body(
status_module, seed="different-reservation"
)
conflict_headers = _adoption_bootstrap_headers(
status_module,
secret,
method="POST",
path=reserve_path,
body=conflict_body,
nonce="5" * 32,
)
conflict_status, conflict = _adoption_http_request(
server, "POST", reserve_path, body=conflict_body, headers=conflict_headers
)
assert conflict_status == 409
assert conflict == {
"apiVersion": "itworx.repopulse-adoption-reserve/v1",
"status": "conflict",
}
assert lock_events == [
project_lock,
global_lock,
project_lock,
global_lock,
project_lock,
global_lock,
]
status_path = status_module.REPOPULSE_ADOPTION_STATUS_PATH
status_headers = _adoption_bootstrap_headers(
status_module,
secret,
method="GET",
path=status_path,
body=b"",
nonce="6" * 32,
)
read_status, status_payload = _adoption_http_request(
server, "GET", status_path, headers=status_headers
)
assert read_status == 200
assert status_payload["status"] == "in_progress"
assert status_payload["phase"] == "reserved"
assert status_payload["hashes"]["idempotencySha256"] == payload["idempotencySha256"]
assert status_payload["hashes"]["preflightSha256"] == payload["preflightSha256"]
encoded_status = json.dumps(status_payload)
assert credential.decode("ascii") not in encoded_status
assert hashlib.sha256(credential).hexdigest() not in encoded_status
assert "expectedTargetCredentialSha256" not in encoded_status
assert len(list(nonce_root.iterdir())) == 4
assert not status_module._enrollment_write_lock.locked()
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_repopulse_enrollment_rollback_reloads_registry_under_project_global_then_write_lock(
status_module, tmp_path, monkeypatch
):
store_module, _state_store, project_lock, global_lock, _controller = (
_configure_repopulse_adoption_store_for_http_test(
status_module, tmp_path, monkeypatch, enabled=False
)
)
secret = b"rollback-bootstrap-secret-" + b"x" * 40
bootstrap_path = tmp_path / "bootstrap"
bootstrap_path.write_bytes(secret)
bootstrap_path.chmod(0o600)
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
lock_events = []
original_lock = store_module._exclusive_flock
project_lock_waiting = threading.Event()
allow_project_lock = threading.Event()
@contextmanager
def tracked_lock(path):
if path == project_lock:
project_lock_waiting.set()
if not allow_project_lock.wait(timeout=5):
raise TimeoutError("test did not release the project lock gate")
with original_lock(path):
lock_events.append(path)
yield
monkeypatch.setattr(store_module, "_exclusive_flock", tracked_lock)
class TrackedEnrollmentLock:
def __init__(self):
self._lock = threading.Lock()
def __enter__(self):
lock_events.append("enrollment_lock")
self._lock.acquire()
return self
def __exit__(self, *_args):
self._lock.release()
def locked(self):
return self._lock.locked()
monkeypatch.setattr(status_module, "_enrollment_write_lock", TrackedEnrollmentLock())
load_calls = 0
def load_projects():
nonlocal load_calls
load_calls += 1
if load_calls == 1:
return {"jens/repopulse": {"revision": "stale"}}
assert lock_events[-3:] == [project_lock, global_lock, "enrollment_lock"]
assert status_module._enrollment_write_lock.locked()
assert (
store_module._transaction_owner(store_module._lock_identity()) is None
)
return {"jens/repopulse": {"revision": "fresh"}}
observed_targets = []
def rollback(projects, target):
assert lock_events[-3:] == [project_lock, global_lock, "enrollment_lock"]
assert status_module._enrollment_write_lock.locked()
assert (
store_module._transaction_owner(store_module._lock_identity()) is None
)
assert projects["jens/repopulse"]["revision"] == "fresh"
observed_targets.append(target)
return "rolled_back", {
"enrollmentId": target["enrollment_id"],
"applicationId": target["application_id"],
}
monkeypatch.setattr(status_module, "_load_projects", load_projects)
monkeypatch.setattr(status_module, "_rollback_enrollment_target", rollback)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
path = "/v1/appops/enrollments/rollback"
target = {
"enrollment_id": "enr-" + "c" * 20,
"application_id": "repopulse",
"repository": "jens/repopulse",
}
raw_body = json.dumps({"target": target}, separators=(",", ":")).encode("utf-8")
headers = {
"Content-Type": "application/json",
**_adoption_bootstrap_headers(
status_module,
secret,
method="POST",
path=path,
body=raw_body,
nonce="8" * 32,
),
}
response_values = []
def send_rollback_request():
response_values.append(
_adoption_http_request(server, "POST", path, body=raw_body, headers=headers)
)
request_thread = threading.Thread(target=send_rollback_request)
try:
request_thread.start()
assert project_lock_waiting.wait(timeout=5)
assert not status_module._enrollment_write_lock.locked(), (
"rollback must not hold the enrollment lock while waiting for the project flock"
)
allow_project_lock.set()
request_thread.join(timeout=5)
assert not request_thread.is_alive()
response_status, response = response_values[0]
assert response_status == 200
assert response == {
"status": "rolled_back",
"enrollmentId": target["enrollment_id"],
"applicationId": target["application_id"],
}
assert load_calls == 2
assert observed_targets == [target]
assert lock_events == [project_lock, global_lock, "enrollment_lock"]
assert not status_module._enrollment_write_lock.locked()
finally:
allow_project_lock.set()
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_repopulse_adoption_capabilities_client_uses_fixed_bootstrap_hmac(
monkeypatch,
):
from backend.app.services.deployment_broker import DeploymentBrokerClient
captured = {}
class Response:
status_code = 200
@staticmethod
def json():
return {"status": "ready", "repository": "jens/repopulse"}
class Client:
def __init__(self, **kwargs):
captured["client"] = kwargs
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def request(self, method, url, *, content, headers):
captured.update(
{"method": method, "url": url, "body": content, "headers": headers}
)
return Response()
monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client)
secret = "bootstrap-client-secret-value" * 2
client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True)
assert client.repopulse_adoption_capabilities(secret=secret) == {
"status": "ready",
"repository": "jens/repopulse",
}
assert captured["method"] == "POST"
assert captured["client"]["verify"] is True
assert captured["url"] == (
"https://controller.example/v1/appops/repopulse/adoption/capabilities"
)
assert captured["body"] == b"{}"
headers = captured["headers"]
assert headers["X-ITWorx-Deployment-Signature"] == DeploymentBrokerClient._signature(
secret,
"POST",
"/v1/appops/repopulse/adoption/capabilities",
headers["X-ITWorx-Deployment-Timestamp"],
headers["X-ITWorx-Deployment-Nonce"],
b"{}",
)
def test_repopulse_adoption_client_binds_commands_to_fixed_paths(monkeypatch):
from backend.app.services.deployment_broker import DeploymentBrokerClient
requests = []
class Response:
status_code = 200
@staticmethod
def json():
return {"status": "ok"}
class Client:
def __init__(self, **_kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def request(self, method, url, *, content, headers):
requests.append((method, url, content, headers))
return Response()
monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client)
secret = "bootstrap-client-secret-value" * 2
client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True)
digest = "a" * 64
assert client.repopulse_adoption_reserve(
secret=secret,
intent_id=f"rpa-{digest[:20]}",
idempotency_sha256=digest,
preflight_sha256="b" * 64,
) == {"status": "ok"}
assert client.repopulse_adoption_status(secret=secret) == {"status": "ok"}
assert client.repopulse_adoption_execute(
secret=secret,
intent_id=f"rpa-{digest[:20]}",
idempotency_sha256=digest,
preflight_sha256="b" * 64,
preflight_projection={"repository": "jens/repopulse"},
target_credential="target-secret-value" * 4,
) == {"status": "ok"}
assert [(method, url.rsplit("/", 1)[-1]) for method, url, _, _ in requests] == [
("POST", "reserve"),
("GET", "status"),
("POST", "execute"),
]
assert json.loads(requests[0][2]) == {
"idempotencySha256": digest,
"intentId": f"rpa-{digest[:20]}",
"preflightSha256": "b" * 64,
}
assert requests[1][2] == b""
assert json.loads(requests[2][2]) == {
"idempotencySha256": digest,
"intentId": f"rpa-{digest[:20]}",
"preflightProjection": {"repository": "jens/repopulse"},
"preflightSha256": "b" * 64,
"targetCredential": "target-secret-value" * 4,
}
for method, url, body, headers in requests:
path = url.removeprefix("https://controller.example")
assert headers["X-ITWorx-Deployment-Signature"] == DeploymentBrokerClient._signature(
secret,
method,
path,
headers["X-ITWorx-Deployment-Timestamp"],
headers["X-ITWorx-Deployment-Nonce"],
body,
)
assert DeploymentBrokerClient(
base_url="http://127.0.0.1:1300", verify_tls=False
).repopulse_adoption_execute(
secret=secret,
intent_id=f"rpa-{digest[:20]}",
idempotency_sha256=digest,
preflight_sha256="b" * 64,
preflight_projection={"repository": "jens/repopulse"},
target_credential="target-secret-value" * 4,
) == {
"status": "missing_config",
"reason": "repopulse_adoption_private_tls_required",
}
assert len(requests) == 3
def test_controller_rejects_a_group_writable_projects_registry(status_module, tmp_path, monkeypatch):
if status_module.os.name == "nt":
pytest.skip("POSIX ownership and permissions are enforced by the controller")
projects = tmp_path / "projects.json"
projects.write_text("{}", encoding="utf-8")
projects.chmod(0o666)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
assert status_module._projects_registry_is_safe() is False
with pytest.raises(OSError, match="unsafe"):
status_module._load_projects()
def test_repopulse_capability_requires_fixed_readonly_and_rw_mount_aliases(status_module, monkeypatch):
# Exercise the installed aliases, rather than the shared source-tree fallback.
readonly_root = Path("/controller-config")
rw_root = Path("/controller-hub-releases/controller-config")
monkeypatch.setattr(status_module._repopulse_adoption_store, "CONTROLLER_READONLY_ROOT", readonly_root)
monkeypatch.setattr(status_module._repopulse_adoption_store, "CONTROLLER_RW_ROOT", rw_root)
valid_mountinfo = (
f"25 1 0:2 / {readonly_root.as_posix()} ro - ext4 /dev/test ro\n"
f"26 1 0:2 / {rw_root.as_posix()} rw - ext4 /dev/test rw\n"
)
assert status_module._repopulse_alias_mounts_match_contract(valid_mountinfo) is True
readonly_changed_to_rw = valid_mountinfo.replace(
f"{readonly_root.as_posix()} ro", f"{readonly_root.as_posix()} rw"
)
assert status_module._repopulse_alias_mounts_match_contract(readonly_changed_to_rw) is False
rw_changed_to_readonly = valid_mountinfo.replace(
f"{rw_root.as_posix()} rw", f"{rw_root.as_posix()} ro"
)
assert status_module._repopulse_alias_mounts_match_contract(rw_changed_to_readonly) is False
assert status_module._repopulse_alias_mounts_match_contract("") is False
def test_controller_release_identity_is_content_bound(status_module, tmp_path, monkeypatch, capsys):
renderer_spec = importlib.util.spec_from_file_location(
"prepare_controller_artifact",
Path(__file__).resolve().parents[2] / "scripts" / "prepare_controller_artifact.py",
)
assert renderer_spec is not None and renderer_spec.loader is not None
renderer = importlib.util.module_from_spec(renderer_spec)
renderer_spec.loader.exec_module(renderer)
assert renderer._ARTIFACT_FILES == status_module._CONTROLLER_ARTIFACT_FILES
candidate_helper = "prepare_mcp_hub_candidate.py"
assert candidate_helper in status_module._CONTROLLER_ARTIFACT_FILES
forge_first_install_index = status_module._CONTROLLER_ARTIFACT_FILES.index(
"forge_first_install.py"
)
assert status_module._CONTROLLER_ARTIFACT_FILES[forge_first_install_index + 1] == (
"forge_database_credentials.py"
)
artifact_root = tmp_path / "controller"
artifact_root.mkdir()
for filename in status_module._CONTROLLER_ARTIFACT_FILES:
source = (
Path(__file__).resolve().parents[2] / "scripts" / filename
if filename == "run_repopulse_adoption.py"
else status_module.ROOT / filename
)
(artifact_root / filename).write_bytes(source.read_bytes())
assert (artifact_root / candidate_helper).read_bytes() == (
status_module.ROOT / candidate_helper
).read_bytes()
monkeypatch.setattr(status_module, "ROOT", artifact_root)
monkeypatch.setattr(status_module, "CONTROLLER_RELEASE_PATH", artifact_root / "controller-release.json")
digest = status_module._controller_artifact_digest()
assert digest is not None
release = {
"apiVersion": "itworx.controller-artifact/v1",
"controllerVersion": "2.0.0",
"sourceRepository": "jens/unraid-appops-gateway",
"sourceRevision": "a" * 40,
"sourceTree": "b" * 40,
"artifactSha256": digest,
"builtAt": "2026-09-14T01:00:00.000Z",
}
status_module.CONTROLLER_RELEASE_PATH.write_text(json.dumps(release), encoding="utf-8")
assert status_module._controller_release() == release
monkeypatch.setattr(status_module, "_load_projects", lambda: (_ for _ in ()).throw(AssertionError("legacy config must not be read")))
monkeypatch.setattr(status_module.sys, "argv", ["status_server.py", "--check-artifact"])
status_module.main()
assert "validated controller artifact 2.0.0" in capsys.readouterr().out
(artifact_root / candidate_helper).write_text("tampered", encoding="utf-8")
assert status_module._controller_release() is None
(artifact_root / candidate_helper).unlink()
assert status_module._controller_release() is None
def test_appops_deployment_contract_is_pre_registered_and_hmac_bound(status_module, tmp_path, monkeypatch):
from backend.app.services.deployment_broker import DeploymentBrokerClient
secret_root = tmp_path / "appops-secrets"
secret = b"appops-" + b"x" * 48
_write_appops_secret(secret_root, "jens__repo-a", secret)
project = _appops_project()
plan = _appops_plan("a" * 40)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root)
assert status_module._appops_plan({"jens/repo-a": project}, plan) == (
"jens/repo-a",
"a" * 40,
project,
)
raw = json.dumps({"plan": plan}, sort_keys=True, separators=(",", ":")).encode()
timestamp = str(int(status_module.time.time()))
nonce = "1" * 32
nonce_root = tmp_path / "nonces"
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root)
signature = status_module._request_signature(
secret,
method="POST",
path="/v1/appops/deployments/apply",
timestamp=timestamp,
nonce=nonce,
raw_body=raw,
)
expected = hmac.new(
secret,
(
"itworx-controller-request/v2\nPOST\n/v1/appops/deployments/apply\n"
f"{timestamp}\n{nonce}\n{hashlib.sha256(raw).hexdigest()}"
).encode("ascii"),
hashlib.sha256,
).hexdigest()
assert expected == signature
# A captured apply request cannot be retargeted to rollback before its
# nonce is consumed: the endpoint path is part of the authenticated input.
assert not status_module._appops_signature_is_valid(
raw, timestamp, nonce, signature, "jens/repo-a", project,
method="POST", path="/v1/appops/deployments/rollback",
)
assert status_module._appops_signature_is_valid(
raw, timestamp, nonce, signature, "jens/repo-a", project,
method="POST", path="/v1/appops/deployments/apply",
)
# The same authenticated request is rejected after its atomic nonce write.
assert not status_module._appops_signature_is_valid(
raw, timestamp, nonce, signature, "jens/repo-a", project,
method="POST", path="/v1/appops/deployments/apply",
)
assert not status_module._appops_signature_is_valid(
raw, timestamp, "2" * 32, "0" * 64, "jens/repo-a", project,
method="POST", path="/v1/appops/deployments/apply",
)
legacy_signature = hmac.new(
secret,
f"{timestamp}\n{'4' * 32}\n{hashlib.sha256(raw).hexdigest()}".encode(),
hashlib.sha256,
).hexdigest()
assert DeploymentBrokerClient._signature(
secret.decode(), "POST", "/v1/appops/deployments/apply", timestamp, nonce, raw
) == signature
assert not status_module._appops_signature_is_valid(
raw, timestamp, "4" * 32, legacy_signature, "jens/repo-a", project,
method="POST", path="/v1/appops/deployments/apply",
)
tampered = {**plan, "image_repository": "other"}
assert status_module._appops_plan({"jens/repo-a": project}, tampered) is None
def test_appops_request_is_idempotent_and_runs_only_fixed_deploy(status_module, tmp_path, monkeypatch):
inbox = tmp_path / "appops-inbox"
projects = tmp_path / "projects.json"
project = _appops_project()
projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
plan = _appops_plan("a" * 40)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
status, request_id = status_module._enqueue_appops_request(
repository="jens/repo-a",
revision="a" * 40,
operation="apply",
idempotency_key="appops-repo-a-0001",
plan=plan,
)
assert status == "pending"
assert status_module._enqueue_appops_request(
repository="jens/repo-a",
revision="a" * 40,
operation="apply",
idempotency_key="appops-repo-a-0001",
plan=plan,
) == ("pending", request_id)
marker = inbox / f"{request_id}.json"
calls = []
class Result:
returncode = 0
monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: calls.append(args) or Result())
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True})
status_module._process_appops_record(marker)
assert json.loads(marker.read_text())["status"] == "deployed"
assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", "jens/repo-a", "a" * 40]]
@pytest.mark.parametrize("operation", ["apply", "rollback"])
def test_generic_appops_hub_request_is_rejected_before_inbox_creation(
status_module, tmp_path, monkeypatch, operation
):
inbox = tmp_path / "appops-inbox"
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
status, request_id = status_module._enqueue_appops_request(
repository="Jens/ITWorx_MCP_Hub",
revision="a" * 40,
operation=operation,
idempotency_key=f"appops-hub-{operation}-0001",
plan={},
)
assert (status, request_id) == ("invalid", "")
assert not inbox.exists()
@pytest.mark.parametrize("operation", ["apply", "rollback"])
@pytest.mark.parametrize(
("prior_status", "expected_status"),
[("pending", "failed"), ("running", "outcome_unknown")],
)
def test_direct_appops_hub_record_is_denied_before_project_or_deploy_access(
status_module, tmp_path, monkeypatch, operation, prior_status, expected_status
):
inbox = tmp_path / "appops-inbox"
inbox.mkdir()
request_id = "a" * 64
marker = inbox / f"{request_id}.json"
marker.write_text(
json.dumps(
{
"schema": "appops-deployment-request.v1",
"request": request_id,
"repository": "jens/itworx_mcp_hub",
"revision": "b" * 40,
"operation": operation,
"plan_sha256": "c" * 64,
"status": prior_status,
"received_at": 1,
"updated_at": 1,
}
),
encoding="utf-8",
)
monkeypatch.setattr(
status_module,
"_load_projects",
lambda: pytest.fail("generic Hub records must not inspect deployment projects"),
)
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda *_args, **_kwargs: pytest.fail("generic Hub records must not launch deploy.py"),
)
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == expected_status
assert record["reason"] == "canonical_hub_push_required"
def test_hub_protected_push_waits_for_candidate_before_fixed_deploy_path(
status_module, tmp_path, monkeypatch
):
repository = "jens/itworx_mcp_hub"
inbox = tmp_path / "webhook-inbox"
projects = tmp_path / "projects.json"
project = {
**_webhook_project(repository),
"slug": "itworx-mcp-hub",
"containers": ["itworx-mcp-hub"],
}
projects.write_text(json.dumps({repository: project}), encoding="utf-8")
monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
revision = "d" * 40
raw_body = json.dumps(
_push_payload(revision, repository=repository), separators=(",", ":")
).encode()
_, request_id = status_module._enqueue_webhook_delivery(
raw_body,
"145440ab-b7b1-4c74-ae84-996d8d48167e",
77,
"refs/heads/main",
repository,
revision,
status_module._validation_policy(repository, project),
)
marker = inbox / f"{request_id}.json"
calls = []
class Result:
returncode = 0
monkeypatch.setattr(
status_module.subprocess,
"run",
lambda arguments, **_kwargs: calls.append(arguments) or Result(),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"ok": True})
monkeypatch.setattr(
status_module,
"_fetch_gitea_validation",
lambda *_args: (
"pass",
[{
"context": "Managed validation / full (push)",
"status": "success",
"updated_at": "2026-09-14T00:00:00Z",
}],
),
)
monkeypatch.setattr(status_module, "_prepare_hub_candidate_delivery", lambda *_args: False)
status_module._process_webhook_record(marker)
waiting = json.loads(marker.read_text(encoding="utf-8"))
assert waiting["status"] == "validation_passed"
assert "deployment_started" not in [entry["stage"] for entry in waiting["audit"]]
assert calls == []
monkeypatch.setattr(status_module, "_prepare_hub_candidate_delivery", lambda *_args: True)
status_module._process_webhook_record(marker)
assert json.loads(marker.read_text(encoding="utf-8"))["status"] == "deployed"
assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", repository, revision]]
def test_durable_rollback_snapshot_is_receipt_bound_and_not_caller_selected(deploy_module, tmp_path, monkeypatch):
previous_revision = "a" * 40
current_revision = "b" * 40
project = _project(tmp_path)
monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state")
previous_receipt = _receipt(project, previous_revision)
current_receipt = _receipt(project, current_revision)
snapshot = {
"revision": previous_revision,
"legacy_provenance": False,
"receipt": previous_receipt,
"compose_containers": ["Unraid-AppOps-Gateway"],
"containers": [{
"name": "Unraid-AppOps-Gateway",
"container_id": "container-a",
"image_id": "sha256:" + "1" * 64,
"image_ref": "Unraid-AppOps-Gateway:production",
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:aaaaaaaaaaaa",
"was_running": True,
}],
}
state = {
"schema_version": 3,
"repository": "jens/unraid-appops-gateway",
"live_revision": current_revision,
"revision": current_revision,
"live_status": "healthy",
"live_receipt": current_receipt,
"rollback_snapshot": snapshot,
}
state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json"
state_path.parent.mkdir(parents=True)
state_path.write_text(json.dumps(state), encoding="utf-8")
monkeypatch.setattr(
deploy_module,
"inspect_image",
lambda image_id: {"Id": image_id},
)
assert deploy_module.durable_rollback_snapshot(
project, "jens/unraid-appops-gateway", current_revision
) == snapshot
with pytest.raises(deploy_module.DeployError, match="verified live deployment"):
deploy_module.durable_rollback_snapshot(
project, "jens/unraid-appops-gateway", "c" * 40
)
def test_managed_runtime_contract_rejects_published_ports_and_accepts_owned_hardened_compose(deploy_module, tmp_path, monkeypatch):
project = _project(tmp_path)
project.update({
"managed_plan_version": "itworx.deployment/v1",
"managed_application_id": "unraid-appops-gateway",
"managed_appdata_path": "/srv/example/appdata/unraid-appops-gateway",
"managed_image_repository": "unraid-appops-gateway",
"managed_image_tag": "1.5.0",
"managed_proxy_network": "itworx-npm",
"managed_internal_port": 1216,
"appops_deployment_secret_file": "jens__unraid-appops-gateway",
})
rendered = {
"services": {
"unraid-appops-gateway": {
"image": "unraid-appops-gateway:1.5.0",
"restart": "unless-stopped",
"read_only": True,
"security_opt": ["no-new-privileges:true"],
"cap_drop": ["ALL"],
"networks": ["itworx-npm"],
"volumes": [{
"type": "bind",
"source": "/srv/example/appdata/unraid-appops-gateway/data",
"target": "/data",
}],
}
},
"networks": {"itworx-npm": {"external": True}},
}
monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {})
monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: json.dumps(rendered))
deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/unraid-appops-gateway")
rendered["services"]["unraid-appops-gateway"]["ports"] = [{"published": "1216"}]
with pytest.raises(deploy_module.DeployError, match="published ports"):
deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/unraid-appops-gateway")
def test_managed_runtime_contract_rejects_appdata_path_traversal_and_controller_env_inheritance(deploy_module, tmp_path, monkeypatch):
project = _project(tmp_path)
project.update({
"managed_plan_version": "itworx.deployment/v1",
"managed_application_id": "unraid-appops-gateway",
"managed_appdata_path": "/srv/example/appdata/unraid-appops-gateway",
"managed_image_repository": "unraid-appops-gateway",
"managed_image_tag": "1.5.0",
"managed_proxy_network": "itworx-npm",
"managed_internal_port": 1216,
"appops_deployment_secret_file": "jens__unraid-appops-gateway",
"environment": {"PORTAL_ADMIN_KEY": "managed-secret"},
})
monkeypatch.setenv("UNRELATED_CONTROLLER_SECRET", "must-not-reach-compose")
environment = deploy_module.deployment_environment(project, "a" * 40, "jens/unraid-appops-gateway")
assert environment["PORTAL_ADMIN_KEY"] == "managed-secret"
assert "UNRELATED_CONTROLLER_SECRET" not in environment
rendered = {
"services": {
"unraid-appops-gateway": {
"image": "unraid-appops-gateway:1.5.0",
"restart": "unless-stopped",
"read_only": True,
"security_opt": ["no-new-privileges:true"],
"cap_drop": ["ALL"],
"environment": {"PORTAL_ADMIN_KEY": "managed-secret"},
"networks": ["itworx-npm"],
"volumes": [{
"type": "bind",
"source": "/srv/example/appdata/unraid-appops-gateway/../../other-app",
"target": "/data",
}],
}
},
"networks": {"itworx-npm": {"external": True}},
}
monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"]))
monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: json.dumps(rendered))
with pytest.raises(deploy_module.DeployError, match="appdata root"):
deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/unraid-appops-gateway")
def test_appops_rollback_runs_only_receipt_bound_controller_mode(status_module, tmp_path, monkeypatch):
inbox = tmp_path / "appops-inbox"
projects = tmp_path / "projects.json"
project = _appops_project()
projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8")
plan = _appops_plan("a" * 40)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects)
status, request_id = status_module._enqueue_appops_request(
repository="jens/repo-a",
revision="a" * 40,
operation="rollback",
idempotency_key="rollback-repo-a-0001",
plan=plan,
)
assert status == "pending"
marker = inbox / f"{request_id}.json"
calls = []
class Result:
returncode = 0
monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: calls.append(args) or Result())
monkeypatch.setattr(
status_module,
"_deployment_state",
lambda _slug: {"rollback_restored_revision": "b" * 40},
)
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True})
status_module._process_appops_record(marker)
assert json.loads(marker.read_text())["status"] == "rolled_back"
assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "rollback", "jens/repo-a", "a" * 40]]
def test_webhook_broker_has_no_runner_or_job_secret_artifacts():
assert not (BROKER_ROOT / "docker-host").exists()
assert not (BROKER_ROOT / "deploy-runner.config.example.yaml").exists()
assert not (BROKER_ROOT / "deploy-runner.compose.example.yaml").exists()
source = (BROKER_ROOT / "status_server.py").read_text(encoding="utf-8")
assert "BROKER_TOKEN_ROOT" not in source
assert "GITEA_DEPLOY_BROKER_TOKEN" not in source
assert "UNRAID_DEPLOY_BROKER_TOKEN" not in source
assert "X-Gitea-Signature" in source
assert "gitea-webhook-inbox.v1" in source
def test_appops_uses_push_validation_without_a_deploy_runner_workflow():
workflow_root = ROOT / ".gitea" / "workflows"
assert not (workflow_root / "unraid-deploy.yml").exists()
managed_validation = (workflow_root / "managed-validation.yml").read_text(encoding="utf-8")
assert "push:" in managed_validation
assert "branches: [main]" in managed_validation
assert "runs-on: linux-validation" in managed_validation
assert "unraid-deploy" not in managed_validation
def test_chimera_example_requires_exact_source_webhook_auth_and_live_proof():
projects = json.loads(
(BROKER_ROOT / "projects.example.json").read_text(encoding="utf-8")
)
chimera = projects["jens/chimera"]
assert chimera["exact_source"] is True
assert chimera["source_dir"].startswith(
"/mnt/cache/appdata/gitea-deploy-runners/sources/"
)
assert chimera["env_source"] == "/srv/example/appdata/chimera/deploy.env"
assert chimera["verify_oci_revision"] is True
assert chimera["gitea_repository_id"] == 1234
assert chimera["gitea_webhook_secret_file"] == "jens__chimera"
assert "broker_token_file" not in chimera
assert "legacy_broker_token" not in chimera
assert chimera["live_probe"] == {
"type": "chimera-health-live-v1",
"container": "Chimera",
"port": 6967,
"path": "/health/live",
"host": "chimera:6967",
}
assert chimera["environment"] == {
"CHIMERA_BUILD_ID": "{build_id}",
"CHIMERA_SOURCE_REPOSITORY": "{repository}",
"CHIMERA_SOURCE_REVISION": "{revision}",
"CHIMERA_SOURCE_TREE": "{tree}",
"CHIMERA_BUILD_DATE": "{build_date}",
}
def _controller_enrollment_payload(*, credential: bool = False) -> dict[str, object]:
value: dict[str, object] = {
"api_version": "itworx.controller-enrollment/v1",
"enrollment_id": "enr-" + "a" * 20,
"application_id": "wsbc-portal",
"repository": "jens/windows-support-bundle-collector",
"revision": "b" * 40,
"profile": "internal-webapp-oidc",
"container_name": "wsbc-portal",
"compose_file": ".itworx/deployment.yml",
"trusted_appdata_root": "/srv/example/appdata",
"appdata_path": "/srv/example/appdata/wsbc-portal",
"data_path": "/srv/example/appdata/wsbc-portal/data",
"runtime_uid": 1654,
"runtime_gid": 1654,
"network": "itworx-npm",
"internal_port": 8080,
"proxy_hostname": "wsbc.example.com",
"image_repository": "wsbc-portal",
"image_tag": "0.23.0",
"health_path": "/health",
"runtime_environment_keys": [
"APPOPS_PUBLIC_URL",
"OIDC_CLIENT_ID",
"OIDC_CLIENT_SECRET",
"OIDC_ISSUER",
"OIDC_ROLE_CLAIM",
],
}
if credential:
value["target_credential"] = "x" * 64
return value
def _repopulse_controller_enrollment_payload(*, credential: bool = False) -> dict[str, object]:
value: dict[str, object] = {
"api_version": "itworx.controller-enrollment/v1",
"enrollment_id": "enr-" + "c" * 20,
"application_id": "repopulse",
"repository": "jens/repopulse",
"revision": "d" * 40,
"profile": "repopulse-authenticated",
"container_name": "repopulse",
"compose_file": ".itworx/deployment.yml",
"trusted_appdata_root": "/srv/example/appdata",
"appdata_path": "/srv/example/appdata/repopulse",
"data_path": "/srv/example/appdata/repopulse/data",
"runtime_uid": 10001,
"runtime_gid": 10001,
"network": "itworx-mcp-hub-npm",
"internal_port": 8000,
"proxy_hostname": "repopulse.example.com",
"image_repository": "repopulse-managed",
"image_tag": "production",
"health_path": "/health",
"runtime_environment_keys": [
"REPOPULSE_GITEA_URL",
"REPOPULSE_GITEA_TOKEN",
"REPOPULSE_MAINTENANCE_TOKEN",
],
}
if credential:
value["target_credential"] = "x" * 64
return value
def test_controller_repopulse_profile_is_fixed_and_registers_managed_build(status_module):
payload = _repopulse_controller_enrollment_payload()
enrollment = status_module._controller_enrollment(payload, require_credential=False)
assert enrollment is not None
project = status_module._enrollment_project(enrollment)
assert project["managed_build_context"] == "."
assert project["managed_build_dockerfile"] == "deploy/managed/Dockerfile"
assert project["managed_repository"] == "jens/repopulse"
assert project["managed_runtime_environment_keys"] == [
"REPOPULSE_GITEA_URL",
"REPOPULSE_GITEA_TOKEN",
"REPOPULSE_MAINTENANCE_TOKEN",
]
for field, value in (
("application_id", "another-app"),
("repository", "jens/another-app"),
("network", "other-network"),
("runtime_uid", 1000),
("internal_port", 8080),
("image_tag", "1.5.0"),
("proxy_hostname", "another.example.com"),
):
unsafe = {**payload, field: value}
assert status_module._controller_enrollment(unsafe, require_credential=False) is None
def test_controller_repopulse_runtime_seal_fixes_gitea_url_and_hides_values(status_module, tmp_path, monkeypatch):
appdata_root = tmp_path / "appdata"
app_root = appdata_root / "repopulse"
app_root.mkdir(parents=True)
legacy_env_path = app_root / "deploy.env"
legacy_env_path.write_text("REPOPULSE_OPERATOR_TOKEN=legacy-preserved\n", encoding="utf-8")
env_path = app_root / "deploy.managed.env"
env_path.write_text("", encoding="utf-8")
monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", appdata_root)
monkeypatch.setattr(status_module, "LEGACY_APPDATA_ROOT", appdata_root)
monkeypatch.setattr(status_module, "EXACT_SOURCE_ROOT", tmp_path / "sources")
payload = _repopulse_controller_enrollment_payload(credential=True)
payload["trusted_appdata_root"] = appdata_root.as_posix()
payload["appdata_path"] = app_root.as_posix()
payload["data_path"] = (app_root / "data").as_posix()
enrollment = status_module._controller_enrollment(payload, require_credential=True)
assert enrollment is not None
project = status_module._enrollment_project(enrollment)
assert project["env_source"] == env_path.as_posix()
projects = {"jens/repopulse": project}
runtime = {
"enrollment_id": enrollment["enrollment_id"],
"application_id": "repopulse",
"repository": "jens/repopulse",
"values": {
"REPOPULSE_GITEA_URL": "https://gitea.example.com",
"REPOPULSE_GITEA_TOKEN": "write-only-gitea-token",
"REPOPULSE_MAINTENANCE_TOKEN": "write-only-maintenance-token",
},
}
status, evidence = status_module._configure_enrollment_runtime(projects, runtime)
assert status == "configured"
assert evidence == {"enrollmentId": enrollment["enrollment_id"], "applicationId": "repopulse"}
assert "write-only-gitea-token" in env_path.read_text(encoding="utf-8")
assert "write-only-maintenance-token" in env_path.read_text(encoding="utf-8")
assert legacy_env_path.read_text(encoding="utf-8") == "REPOPULSE_OPERATOR_TOKEN=legacy-preserved\n"
assert "write-only-gitea-token" not in str(evidence)
assert "write-only-maintenance-token" not in str(evidence)
unsafe = {**runtime, "values": {**runtime["values"], "REPOPULSE_GITEA_URL": "https://attacker.invalid"}}
status, evidence = status_module._configure_enrollment_runtime(projects, unsafe)
assert status == "runtime_environment_rejected"
assert evidence == {}
def test_repopulse_preflight_blocks_existing_private_source_tree(status_module, monkeypatch):
payload = _repopulse_controller_enrollment_payload()
enrollment = status_module._controller_enrollment(payload, require_credential=False)
assert enrollment is not None
monkeypatch.setattr(status_module, "_repopulse_controller_storage_ready", lambda: (True, "preflight_ok"))
monkeypatch.setattr(status_module, "_repopulse_sources_readable", lambda: False)
code, detail = status_module._safe_enrollment_preflight({}, enrollment)
assert (code, detail) == (
"source_permissions_unavailable",
"fixed_local_sources_are_not_readable_by_runtime_gid",
)
@pytest.mark.skipif(
os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0,
reason="requires Linux root-owned source trees",
)
def test_repopulse_preflight_blocks_an_existing_mode_700_live_source_tree(
status_module, deploy_module, tmp_path, monkeypatch
):
sources_root = tmp_path / "sources"
sources_root.mkdir()
for slug in ("repopulse", "privatemuse"):
source = sources_root / slug
source.mkdir(mode=0o700)
os.chown(source, 0, 10001)
source.chmod(0o700)
monkeypatch.setattr(status_module, "EXACT_SOURCE_ROOT", sources_root)
monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", sources_root)
monkeypatch.setattr(status_module, "_repopulse_controller_storage_ready", lambda: (True, "preflight_ok"))
payload = _repopulse_controller_enrollment_payload()
enrollment = status_module._controller_enrollment(payload, require_credential=False)
assert enrollment is not None
code, detail = status_module._safe_enrollment_preflight({}, enrollment)
assert (code, detail) == (
"source_permissions_unavailable",
"fixed_local_sources_are_not_readable_by_runtime_gid",
)
for slug in ("repopulse", "privatemuse"):
source = sources_root / slug
staging = sources_root / f".{slug}-aabbccddeeff-stage"
source.rename(staging)
deploy_module._prepare_exact_source_group_read(staging, slug)
staging.rename(source)
assert status_module._repopulse_sources_readable() is True
def test_repopulse_preflight_blocks_read_only_controller_mount(status_module, monkeypatch):
payload = _repopulse_controller_enrollment_payload()
enrollment = status_module._controller_enrollment(payload, require_credential=False)
assert enrollment is not None
monkeypatch.setattr(
status_module,
"_repopulse_controller_storage_ready",
lambda: (False, "controller_storage_read_only"),
)
code, detail = status_module._safe_enrollment_preflight({}, enrollment)
assert (code, detail) == (
"controller_storage_read_only",
"controller_storage_not_writable_for_repopulse_profile",
)
def test_controller_mount_read_only_check_uses_the_most_specific_mount(status_module):
mountinfo = (
"30 20 0:30 / /opt/gitea-deploy ro,relatime - ext4 /dev/root ro\n"
"31 20 0:31 / /mnt/cache/appdata/gitea-deploy-runners/sources rw,relatime - ext4 /dev/cache rw\n"
)
assert status_module._mount_is_read_only_for("/opt/gitea-deploy/status_server.py", mountinfo) is True
assert status_module._mount_is_read_only_for(
"/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse", mountinfo
) is False
def test_controller_repopulse_rollback_refuses_legacy_env_collision(status_module, tmp_path, monkeypatch):
appdata_root = tmp_path / "appdata"
app_root = appdata_root / "repopulse"
data_path = app_root / "data"
data_path.mkdir(parents=True)
managed_env = app_root / "deploy.managed.env"
managed_env.write_text("REPOPULSE_GITEA_URL=https://gitea.example.com\n", encoding="utf-8")
legacy_env = app_root / "deploy.env"
legacy_env.write_text("legacy configuration\n", encoding="utf-8")
state_root = tmp_path / "state"
state_root.mkdir()
secret_root = tmp_path / "target-secrets"
secret_root.mkdir()
secret = secret_root / "jens__repopulse"
secret.write_text("x" * 64, encoding="ascii")
monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", appdata_root)
monkeypatch.setattr(status_module, "STATE_ROOT", state_root)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root)
writes = []
monkeypatch.setattr(status_module, "_write_projects", lambda _projects: writes.append("write"))
enrollment_id = "enr-" + "c" * 20
project = {
"appops_managed_target": True,
"managed_enrollment_id": enrollment_id,
"managed_application_id": "repopulse",
"managed_profile": "repopulse-authenticated",
}
status, evidence = status_module._rollback_enrollment_target(
{"jens/repopulse": project},
{"enrollment_id": enrollment_id, "application_id": "repopulse", "repository": "jens/repopulse"},
)
assert status == "rollback_failed"
assert evidence == {}
assert writes == []
assert managed_env.is_file() and legacy_env.is_file() and secret.is_file()
assert data_path.is_dir()
def test_controller_enrollment_contract_rejects_arbitrary_compose_and_paths(status_module):
value = _controller_enrollment_payload()
assert status_module._controller_enrollment(value, require_credential=False) is not None
unsafe = dict(value)
unsafe["compose"] = "services: {}"
assert status_module._controller_enrollment(unsafe, require_credential=False) is None
unsafe = dict(value)
unsafe["data_path"] = "/etc"
assert status_module._controller_enrollment(unsafe, require_credential=False) is None
unsafe = dict(value)
unsafe["proxy_hostname"] = "*.example.com"
assert status_module._controller_enrollment(unsafe, require_credential=False) is None
unsafe = _controller_enrollment_payload(credential=True)
unsafe["target_credential"] = "short"
assert status_module._controller_enrollment(unsafe, require_credential=True) is None
def test_controller_enrollment_derives_paths_from_the_selected_fixed_root(status_module, monkeypatch):
managed_root = "/var/lib/docker/volumes/itworx-managed-appdata/_data"
monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", Path(managed_root))
monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda root: root.as_posix() == managed_root)
alternate = _controller_enrollment_payload()
alternate["trusted_appdata_root"] = managed_root
alternate["appdata_path"] = managed_root + "/wsbc-portal"
alternate["data_path"] = managed_root + "/wsbc-portal/data"
enrollment = status_module._controller_enrollment(alternate, require_credential=False)
assert enrollment is not None
project = status_module._enrollment_project(enrollment)
assert project["managed_appdata_root"] == managed_root
assert project["managed_appdata_path"] == managed_root + "/wsbc-portal"
unsafe_path = {**alternate, "data_path": "/srv/example/appdata/wsbc-portal/data"}
assert status_module._controller_enrollment(unsafe_path, require_credential=False) is None
unsupported_root = {**alternate, "trusted_appdata_root": "/srv/example/other"}
assert status_module._controller_enrollment(unsupported_root, require_credential=False) is None
def test_controller_accepts_legacy_requests_after_managed_marker_is_initialized(status_module, monkeypatch):
managed_root = "/var/lib/docker/volumes/itworx-managed-appdata/_data"
monkeypatch.setattr(status_module, "_refresh_appdata_root_selection", lambda: Path(managed_root))
monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda root: root.as_posix() == managed_root)
legacy = _controller_enrollment_payload()
assert legacy["trusted_appdata_root"] == "/srv/example/appdata"
enrollment = status_module._controller_enrollment(legacy, require_credential=False)
assert enrollment is not None
assert enrollment["trusted_appdata_root"] == "/srv/example/appdata"
assert "managed_appdata_root" not in status_module._enrollment_project(enrollment)
def test_controller_accepts_legacy_v1_payload_without_root_field(status_module):
legacy = _controller_enrollment_payload()
legacy.pop("trusted_appdata_root")
enrollment = status_module._controller_enrollment(legacy, require_credential=False)
assert enrollment is not None
assert enrollment["trusted_appdata_root"] == "/srv/example/appdata"
assert "managed_appdata_root" not in status_module._enrollment_project(enrollment)
def test_repopulse_enrollment_remains_bound_to_legacy_appdata_root(status_module, monkeypatch):
monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", Path("/var/lib/docker/volumes/itworx-managed-appdata/_data"))
payload = _repopulse_controller_enrollment_payload()
payload["trusted_appdata_root"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data"
payload["appdata_path"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data/repopulse"
payload["data_path"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data/repopulse/data"
assert status_module._controller_enrollment(payload, require_credential=False) is None
def test_fixed_repopulse_profile_stays_legacy_with_managed_marker(status_module, monkeypatch):
managed_root = "/var/lib/docker/volumes/itworx-managed-appdata/_data"
monkeypatch.setattr(status_module, "_refresh_appdata_root_selection", lambda: Path(managed_root))
monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda root: root.as_posix() == managed_root)
enrollment = status_module._controller_enrollment(
_repopulse_controller_enrollment_payload(),
require_credential=False,
)
assert enrollment is not None
assert enrollment["trusted_appdata_root"] == "/srv/example/appdata"
def test_controller_runtime_seal_accepts_only_profile_owned_keys(status_module, tmp_path, monkeypatch):
appdata_root = tmp_path / "appdata"
app_root = appdata_root / "wsbc-portal"
app_root.mkdir(parents=True)
env_path = app_root / "deploy.env"
env_path.write_text("", encoding="utf-8")
monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", appdata_root)
monkeypatch.setattr(status_module, "LEGACY_APPDATA_ROOT", appdata_root)
monkeypatch.setattr(status_module, "EXACT_SOURCE_ROOT", tmp_path / "sources")
payload = _controller_enrollment_payload(credential=True)
payload["trusted_appdata_root"] = appdata_root.as_posix()
payload["appdata_path"] = app_root.as_posix()
payload["data_path"] = (app_root / "data").as_posix()
enrollment = status_module._controller_enrollment(payload, require_credential=True)
assert enrollment is not None
project = status_module._enrollment_project(enrollment)
projects = {"jens/windows-support-bundle-collector": project}
runtime = {
"enrollment_id": enrollment["enrollment_id"],
"application_id": "wsbc-portal",
"repository": "jens/windows-support-bundle-collector",
"values": {
"APPOPS_PUBLIC_URL": "https://wsbc.example.com",
"OIDC_CLIENT_ID": "wsbc-portal",
"OIDC_CLIENT_SECRET": "opaque-write-only-value",
"OIDC_ISSUER": "https://auth.example.com/application/o/wsbc-portal",
"OIDC_ROLE_CLAIM": "roles",
},
}
status, evidence = status_module._configure_enrollment_runtime(projects, runtime)
assert status == "configured"
assert evidence == {"enrollmentId": enrollment["enrollment_id"], "applicationId": "wsbc-portal"}
assert "opaque-write-only-value" in env_path.read_text(encoding="utf-8")
unsafe = {**runtime, "values": {**runtime["values"], "ARBITRARY_COMMAND": "id"}}
status, _evidence = status_module._configure_enrollment_runtime(projects, unsafe)
assert status == "runtime_environment_rejected"
def test_controller_enrollment_replay_requires_the_existing_target_credential(status_module, tmp_path, monkeypatch):
projects_path = tmp_path / "projects.json"
secrets_root = tmp_path / "target-secrets"
credential_path = secrets_root / "jens__windows-support-bundle-collector"
secrets_root.mkdir()
credential_path.write_text("x" * 64, encoding="ascii")
enrollment = _controller_enrollment_payload(credential=True)
project = status_module._enrollment_project(enrollment)
projects_path.write_text(json.dumps({"jens/windows-support-bundle-collector": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secrets_root)
monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda _root: True)
enrollment["target_credential"] = "y" * 64
status, _evidence = status_module._enroll_controller_target(enrollment)
assert status == "target_credential_conflict"
def _workmesh_promotion_project(status_module) -> dict[str, object]:
return {
"slug": "workmesh",
"branch": "main",
"source_dir": (status_module.EXACT_SOURCE_ROOT / "workmesh").as_posix(),
"exact_source": True,
"env_source": "/srv/example/appdata/workmesh/.env",
"compose_files": ["compose.yaml"],
"compose_project": "workmesh",
"containers": [
"workmesh-postgres-1",
"workmesh-api-1",
"workmesh-n8n-1",
"workmesh-workmesh-1",
],
"retired_compose_containers": ["workmesh-mock-ats-1"],
"verify_oci_revision": False,
"health_timeout": 300,
"appops_deployment_secret_file": "jens__workmesh",
"validation": {
"validation_gate_version": 1,
"required": True,
"contexts": ["WorkMesh CI (push)"],
"timeout_seconds": 600,
"poll_interval_seconds": 10,
"max_poll_interval_seconds": 30,
},
}
def test_workmesh_promotion_policy_accepts_only_root_registered_product_topology(status_module):
project = _workmesh_promotion_project(status_module)
assert status_module._workmesh_promotion_policy("jens/workmesh", project) == (
"workmesh",
"jens__workmesh",
)
assert status_module._workmesh_promotion_policy("jens/other", project) is None
unsafe = {**project, "source_dir": "/tmp/workmesh"}
assert status_module._workmesh_promotion_policy("jens/workmesh", unsafe) is None
unsafe = {**project, "containers": [*project["containers"], "sidecar"]}
assert status_module._workmesh_promotion_policy("jens/workmesh", unsafe) is None
unsafe_oci_mode = {**project, "verify_oci_revision": True}
assert status_module._workmesh_promotion_policy("jens/workmesh", unsafe_oci_mode) is None
def test_workmesh_preflight_binds_exact_main_validation_and_controller_check(
status_module, monkeypatch
):
project = _workmesh_promotion_project(status_module)
revision = "a" * 40
validation = [{"context": "WorkMesh CI (push)", "status": "success"}]
calls: list[list[str]] = []
class Result:
returncode = 0
stdout = json.dumps({
"status": "preflight-ok",
"repository": "jens/workmesh",
"revision": "a" * 40,
"backupInputsSha256": "c" * 64,
}).encode()
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True)
monkeypatch.setattr(status_module, "_validation_policy", lambda *_args: {"contexts": ["WorkMesh CI (push)"]})
monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("pass", validation))
monkeypatch.setattr(status_module, "_controller_release", lambda: {
"controllerVersion": "2.1.4",
"sourceRevision": "c" * 40,
"sourceTree": "d" * 40,
"artifactSha256": "e" * 64,
})
monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: calls.append(args) or Result())
result = status_module._workmesh_promotion_preflight("jens/workmesh", revision, project)
assert result["status"] == "preflight_ok"
assert result["repository"] == "jens/workmesh"
assert result["revision"] == revision
assert len(result["preflightSha256"]) == 64
assert result["backupInputsSha256"] == "c" * 64
assert result["checks"] == {
"exactMain": "passed",
"pushValidation": "passed",
"controllerPreflight": "passed",
"dataBackup": "passed",
}
assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "workmesh-preflight", "jens/workmesh", revision]]
def test_workmesh_preflight_blocks_branch_drift_before_controller_command(
status_module, monkeypatch
):
project = _workmesh_promotion_project(status_module)
monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: False)
monkeypatch.setattr(status_module.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("controller must not run"))
result = status_module._workmesh_promotion_preflight("jens/workmesh", "b" * 40, project)
assert result["status"] == "preflight_failed"
assert result["checks"]["exactMain"] == "blocked"
assert result["checks"]["pushValidation"] == "unknown"
assert result["checks"]["controllerPreflight"] == "unknown"
def test_workmesh_promotion_queue_deploys_only_after_digest_recheck_and_receipt(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "appops-inbox"
projects_path = tmp_path / "projects.json"
project = _workmesh_promotion_project(status_module)
projects_path.write_text(json.dumps({"jens/workmesh": project}), encoding="utf-8")
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: {
"status": "preflight_ok",
"repository": "jens/workmesh",
"revision": "a" * 40,
"preflightSha256": "f" * 64,
"backupInputsSha256": "e" * 64,
})
status, request_id = status_module._enqueue_appops_request(
repository="jens/workmesh",
revision="a" * 40,
operation="workmesh_promote",
idempotency_key="workmesh-approval-001",
plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64},
)
assert status == "pending"
assert status_module._enqueue_appops_request(
repository="jens/workmesh",
revision="a" * 40,
operation="workmesh_promote",
idempotency_key="workmesh-approval-001",
plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64},
) == ("pending", request_id)
calls: list[list[str]] = []
monkeypatch.setattr(
status_module,
"_run_workmesh_promotion_deploy",
lambda args: calls.append(args) or status_module.subprocess.CompletedProcess(args, 0),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "healthy"})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"revision": "a" * 40})
monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: "b" * 64)
marker = inbox / f"{request_id}.json"
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == "deployed"
assert record["revision"] == "a" * 40
assert record["receipt_sha256"] == hashlib.sha256(
b'{"revision":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}'
).hexdigest()
assert record["backup_receipt_sha256"] == "b" * 64
assert calls == [[
"python3", str(status_module.ROOT / "deploy.py"), "workmesh-promote",
"jens/workmesh", "a" * 40, request_id, "f" * 64, "e" * 64,
]]
@pytest.mark.parametrize(
("observed_preflight", "observed_backup_inputs"),
[("e" * 64, "e" * 64), ("f" * 64, "9" * 64)],
)
def test_workmesh_promotion_worker_refuses_changed_preflight_without_deploy(
status_module, tmp_path, monkeypatch, observed_preflight, observed_backup_inputs
):
inbox = tmp_path / "appops-inbox"
inbox.mkdir()
marker = inbox / ("a" * 64 + ".json")
marker.write_text(json.dumps({
"schema": "appops-deployment-request.v1",
"request": "a" * 64,
"repository": "jens/workmesh",
"revision": "b" * 40,
"operation": "workmesh_promote",
"preflight_sha256": "f" * 64,
"backup_inputs_sha256": "e" * 64,
"status": "pending",
}), encoding="utf-8")
project = _workmesh_promotion_project(status_module)
monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project})
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: {
"status": "preflight_ok",
"preflightSha256": observed_preflight,
"backupInputsSha256": observed_backup_inputs,
})
monkeypatch.setattr(
status_module,
"_run_workmesh_promotion_deploy",
lambda *_args: pytest.fail("changed approval must not deploy"),
)
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == "failed"
assert record["reason"] == "workmesh_preflight_changed"
def test_workmesh_backup_capture_rejects_changed_inputs_before_stopping_services(
deploy_module, monkeypatch, tmp_path
):
inputs = {
"apiVersion": "itworx.workmesh-backup-inputs/v1",
"repository": "jens/workmesh",
"target": "workmesh",
"candidateRevision": "b" * 40,
"priorLiveRevision": "a" * 40,
"priorReceiptSha256": "c" * 64,
"backupToolSha256": "d" * 64,
"environmentSha256": "e" * 64,
"externalRecoveryReceiptSha256": "f" * 64,
"n8nImageId": "sha256:" + "1" * 64,
"n8nImageRefSha256": "2" * 64,
}
calls: list[str] = []
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(
deploy_module, "workmesh_backup_preflight_inputs", lambda *_args: inputs
)
monkeypatch.setattr(
deploy_module,
"_workmesh_docker",
lambda *args, **_kwargs: calls.append(str(args[0]))
or pytest.fail("input drift must be detected before any Docker operation"),
)
with pytest.raises(deploy_module.WorkMeshPromotionNoMutation, match="preflight changed"):
deploy_module.capture_workmesh_data_backup(
{"slug": "workmesh"},
tmp_path,
"b" * 40,
request_id="3" * 64,
preflight_sha256="4" * 64,
backup_inputs_sha256="9" * 64,
)
assert calls == []
@pytest.mark.parametrize("failure_point", ["backup_input_drift", "backup_restore_gate"])
@pytest.mark.parametrize("source_restore_succeeds", [True, False])
def test_workmesh_promotion_restores_prior_source_before_no_mutation_classification(
deploy_module, monkeypatch, tmp_path, failure_point, source_restore_succeeds
):
live_revision = "a" * 40
candidate_revision = "b" * 40
source_dir = tmp_path / "source"
source_dir.mkdir()
(source_dir / "source-revision").write_text("prior", encoding="utf-8")
request_id = "c" * 64
preflight_sha256 = "d" * 64
backup_inputs = {
"apiVersion": "itworx.workmesh-backup-inputs/v1",
"repository": "jens/workmesh",
"target": "workmesh",
"candidateRevision": candidate_revision,
"priorLiveRevision": live_revision,
"priorReceiptSha256": "e" * 64,
"backupToolSha256": "f" * 64,
"environmentSha256": "1" * 64,
"externalRecoveryReceiptSha256": "2" * 64,
"n8nImageId": "sha256:" + "3" * 64,
"n8nImageRefSha256": "4" * 64,
}
approved_backup_inputs = (
deploy_module.workmesh_backup_inputs_sha256(backup_inputs)
if failure_point == "backup_restore_gate"
else "9" * 64
)
project = {
"slug": "workmesh",
"branch": "main",
"source_dir": str(source_dir),
"env_source": "/srv/example/appdata/workmesh/.env",
"compose_project": "workmesh",
"compose_files": ["compose.yaml"],
"containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS),
"retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER],
"exact_source": True,
"health_timeout": 300,
"verify_oci_revision": False,
}
projects = {"jens/workmesh": project}
previous_state = {
"schema_version": 3,
"repository": "jens/workmesh",
"live_revision": live_revision,
"live_status": "healthy",
"live_receipt": {"containers": {}},
}
state_writes: list[dict[str, object]] = []
backup_calls: list[str] = []
@contextmanager
def no_lock(*_args):
yield
monkeypatch.setattr(sys, "argv", [
"deploy.py", "workmesh-promote", "jens/workmesh", candidate_revision,
request_id, preflight_sha256, approved_backup_inputs,
])
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "read_state", lambda _project: previous_state)
monkeypatch.setattr(
deploy_module, "validate_commit", lambda _repo, _branch, value: value
)
monkeypatch.setattr(
deploy_module, "deployment_metadata", lambda *_args: {"tree": "5" * 40}
)
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_workmesh_require_product_target", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock)
monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock)
monkeypatch.setattr(
deploy_module,
"_workmesh_current_runtime",
lambda *_args: (previous_state, previous_state["live_receipt"], {}),
)
def sync_source(*_args, **_kwargs):
(source_dir / "source-revision").write_text("candidate", encoding="utf-8")
def restore_source(_project, _repository, requested_revision):
assert requested_revision == live_revision
if source_restore_succeeds:
(source_dir / "source-revision").write_text("prior", encoding="utf-8")
monkeypatch.setattr(deploy_module, "sync_source", sync_source)
monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None)
monkeypatch.setattr(deploy_module, "restore_live_source", restore_source)
monkeypatch.setattr(
deploy_module,
"exact_source_matches_revision",
lambda _repo, revision, target: revision == live_revision
and (target / "source-revision").read_text(encoding="utf-8") == "prior",
)
monkeypatch.setattr(
deploy_module, "compose_command", lambda *_args: (tmp_path, ["docker", "compose"])
)
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {})
monkeypatch.setattr(deploy_module, "run", lambda *_args, **_kwargs: None)
monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None)
monkeypatch.setattr(
deploy_module, "workmesh_backup_preflight_inputs", lambda *_args: backup_inputs
)
monkeypatch.setattr(
deploy_module,
"verified_current_receipt",
lambda *_args: None,
)
monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args: None)
monkeypatch.setattr(
deploy_module,
"write_state",
lambda *_args, **kwargs: state_writes.append(kwargs),
)
monkeypatch.setattr(deploy_module, "validate_managed_v1_runtime", lambda *_args: None)
monkeypatch.setattr(deploy_module, "ensure_docker_capacity", lambda: None)
def capture_backup(*_args, **_kwargs):
backup_calls.append("capture")
raise deploy_module.WorkMeshPromotionNoMutation("isolated backup restore rejected")
monkeypatch.setattr(deploy_module, "capture_workmesh_data_backup", capture_backup)
monkeypatch.setattr(
deploy_module,
"deploy_project",
lambda *_args: pytest.fail("no-mutation path must not invoke deployment"),
)
expected_exception = (
deploy_module.WorkMeshPromotionNoMutation
if source_restore_succeeds
else deploy_module.DeployError
)
with pytest.raises(expected_exception):
deploy_module.main()
if source_restore_succeeds:
assert (source_dir / "source-revision").read_text(encoding="utf-8") == "prior"
assert state_writes == []
else:
assert (source_dir / "source-revision").read_text(encoding="utf-8") == "candidate"
assert state_writes
assert state_writes[-1]["live_status"] == "unknown"
assert backup_calls == ([] if failure_point == "backup_input_drift" else ["capture"])
@pytest.mark.parametrize("source_restore_succeeds", [True, False])
def test_workmesh_preflight_restores_prior_source_before_returning(
deploy_module, monkeypatch, tmp_path, capsys, source_restore_succeeds
):
live_revision = "a" * 40
candidate_revision = "b" * 40
source_dir = tmp_path / "source"
source_dir.mkdir()
(source_dir / "source-revision").write_text("prior", encoding="utf-8")
project = {
"slug": "workmesh",
"branch": "main",
"source_dir": str(source_dir),
"env_source": "/srv/example/appdata/workmesh/.env",
"compose_project": "workmesh",
"compose_files": ["compose.yaml"],
"containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS),
"retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER],
"exact_source": True,
"health_timeout": 300,
"verify_oci_revision": False,
}
projects = {"jens/workmesh": project}
previous_state = {
"schema_version": 3,
"repository": "jens/workmesh",
"live_revision": live_revision,
"live_status": "healthy",
"live_receipt": {"containers": {}},
}
state_writes: list[dict[str, object]] = []
@contextmanager
def no_lock(*_args):
yield
monkeypatch.setattr(sys, "argv", [
"deploy.py", "workmesh-preflight", "jens/workmesh", candidate_revision,
])
monkeypatch.setattr(deploy_module, "require_host_root", lambda: None)
monkeypatch.setattr(deploy_module, "load_projects", lambda: projects)
monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path)
monkeypatch.setattr(deploy_module, "read_state", lambda _project: previous_state)
monkeypatch.setattr(
deploy_module, "validate_commit", lambda _repo, _branch, value: value
)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: {"tree": "5" * 40})
monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None)
monkeypatch.setattr(deploy_module, "_workmesh_require_product_target", lambda *_args: None)
monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock)
monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: (
source_dir / "source-revision"
).write_text("candidate", encoding="utf-8"))
monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None)
monkeypatch.setattr(
deploy_module, "compose_command", lambda *_args: (tmp_path, ["docker", "compose"])
)
monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {})
monkeypatch.setattr(deploy_module, "run", lambda *_args, **_kwargs: None)
monkeypatch.setattr(
deploy_module,
"workmesh_backup_preflight_inputs",
lambda *_args: (
{"observed": "candidate"}
if (source_dir / "source-revision").read_text(encoding="utf-8") == "candidate"
else pytest.fail("backup preflight must inspect the prepared candidate")
),
)
def restore_source(_project, _repository, requested_revision):
assert requested_revision == live_revision
if source_restore_succeeds:
(source_dir / "source-revision").write_text("prior", encoding="utf-8")
monkeypatch.setattr(deploy_module, "restore_live_source", restore_source)
monkeypatch.setattr(
deploy_module,
"exact_source_matches_revision",
lambda _repo, revision, target: revision == live_revision
and (target / "source-revision").read_text(encoding="utf-8") == "prior",
)
monkeypatch.setattr(
deploy_module,
"write_state",
lambda *_args, **kwargs: state_writes.append(kwargs),
)
if source_restore_succeeds:
assert deploy_module.main() == 0
response = json.loads(capsys.readouterr().out)
assert response["status"] == "preflight-ok"
assert response["revision"] == candidate_revision
assert (source_dir / "source-revision").read_text(encoding="utf-8") == "prior"
assert state_writes == []
else:
with pytest.raises(deploy_module.DeployError, match="source restoration"):
deploy_module.main()
assert (source_dir / "source-revision").read_text(encoding="utf-8") == "candidate"
assert state_writes
assert state_writes[-1]["live_status"] == "unknown"
def test_workmesh_promotion_worker_keeps_failed_mutation_without_receipt_ambiguous(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "appops-inbox"
project = _workmesh_promotion_project(status_module)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project})
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: {
"status": "preflight_ok",
"preflightSha256": "f" * 64,
"backupInputsSha256": "e" * 64,
})
monkeypatch.setattr(
status_module,
"_run_workmesh_promotion_deploy",
lambda args: status_module.subprocess.CompletedProcess(args, 1),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "degraded"})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None)
_, request_id = status_module._enqueue_appops_request(
repository="jens/workmesh",
revision="a" * 40,
operation="workmesh_promote",
idempotency_key="workmesh-mutation-failure-01",
plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64},
)
marker = inbox / f"{request_id}.json"
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == "outcome_unknown"
assert record["reason"] == "controller_deployment_failed"
def test_workmesh_promotion_worker_reconciles_recovered_request_without_redeploy(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "appops-inbox"
inbox.mkdir()
request_id = "a" * 64
marker = inbox / f"{request_id}.json"
marker.write_text(json.dumps({
"schema": "appops-deployment-request.v1",
"request": request_id,
"repository": "jens/workmesh",
"revision": "b" * 40,
"operation": "workmesh_promote",
"preflight_sha256": "f" * 64,
"backup_inputs_sha256": "e" * 64,
"status": "running",
}), encoding="utf-8")
project = _workmesh_promotion_project(status_module)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project})
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "healthy"})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None)
monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: None)
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: {
"status": "preflight_ok",
"preflightSha256": "f" * 64,
"backupInputsSha256": "e" * 64,
})
monkeypatch.setattr(status_module.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("recovered request must not redeploy"))
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == "outcome_unknown"
assert record["reason"] == "controller_recovered_without_receipt"
receipt = {"repository": "jens/workmesh", "revision": "b" * 40, "state": "deployed"}
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: receipt)
monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: "b" * 64)
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == "deployed"
assert record["receipt_sha256"] == hashlib.sha256(
json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
assert record["backup_receipt_sha256"] == "b" * 64
def test_workmesh_promotion_timeout_is_bounded_and_reconciles_without_retry(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "appops-inbox"
project = _workmesh_promotion_project(status_module)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project})
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: {
"status": "preflight_ok",
"preflightSha256": "f" * 64,
"backupInputsSha256": "e" * 64,
})
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "degraded"})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None)
calls: list[tuple[list[str], dict[str, object]]] = []
wait_timeouts: list[float] = []
group_stops = []
class Process:
pid = 45678
def wait(self, *, timeout):
wait_timeouts.append(timeout)
raise status_module.subprocess.TimeoutExpired("deploy.py", timeout)
def timeout_popen(args, **kwargs):
calls.append((args, kwargs))
return Process()
monkeypatch.setattr(status_module, "_workmesh_process_group_supported", lambda: True)
monkeypatch.setattr(status_module, "_stop_workmesh_process_group", lambda process: group_stops.append(process))
monkeypatch.setattr(status_module.subprocess, "Popen", timeout_popen)
_, request_id = status_module._enqueue_appops_request(
repository="jens/workmesh",
revision="a" * 40,
operation="workmesh_promote",
idempotency_key="workmesh-timeout-contract-01",
plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64},
)
marker = inbox / f"{request_id}.json"
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert calls[0][1].get("start_new_session") is True
assert wait_timeouts == [1800]
assert len(group_stops) == 1
assert record["status"] == "outcome_unknown"
status_module._process_appops_record(marker)
assert len(calls) == 1
assert json.loads(marker.read_text(encoding="utf-8"))["status"] == "outcome_unknown"
def test_workmesh_timeout_stops_child_group_before_releasing_lock_owner(status_module):
leader = 12345
members = [(leader, "T"), (23456, "S"), (34567, "S")]
events = []
def read_members(pgid):
assert pgid == leader
return list(members)
def send_group_signal(pgid, signum):
events.append(("group_signal", pgid, signum))
assert signum == 19
members[:] = [(pid, "T") for pid, _state in members]
def send_signal(pid, signum):
events.append(("signal", pid, signum))
if pid != leader and signum == 9:
assert all(state == "T" for _member_pid, state in members)
members[:] = [item for item in members if item[0] != pid]
elif pid == leader and signum == 9:
assert not any(member_pid != leader for member_pid, state in members if state not in {"Z", "X"})
class Process:
pid = leader
def wait(self):
events.append(("wait",))
assert events[-2] == ("signal", leader, 9)
status_module._stop_workmesh_process_group(
Process(),
members_reader=read_members,
group_signal_sender=send_group_signal,
signal_sender=send_signal,
stop_signal=19,
kill_signal=9,
sleep_fn=lambda _seconds: None,
cleanup_timeout_seconds=1,
)
assert events == [
("group_signal", leader, 19),
("signal", 23456, 9),
("signal", 34567, 9),
("signal", leader, 9),
("wait",),
]
def test_workmesh_promotion_http_contract_binds_exact_sha_and_preflight_digest(
status_module, tmp_path, monkeypatch
):
secret = b"workmesh-appops-" + b"x" * 48
secrets_root = tmp_path / "appops-secrets"
_write_appops_secret(secrets_root, "jens__workmesh", secret)
projects_path = tmp_path / "projects.json"
project = _workmesh_promotion_project(status_module)
projects_path.write_text(json.dumps({"jens/workmesh": project}), encoding="utf-8")
inbox = tmp_path / "inbox"
monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secrets_root)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda repository, revision, _project: {
"apiVersion": "itworx.workmesh-promotion/v2",
"status": "preflight_ok",
"repository": repository,
"revision": revision,
"preflightSha256": "f" * 64,
"backupInputsSha256": "e" * 64,
"checks": {"exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed"},
})
server = status_module.BoundedThreadingHTTPServer(("127.0.0.1", 0), status_module.Handler)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
def post(path: str, payload: dict[str, object], nonce: str) -> tuple[int, dict[str, object]]:
body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
timestamp = str(int(status_module.time.time()))
signature = status_module._request_signature(
secret, method="POST", path=path, timestamp=timestamp, nonce=nonce, raw_body=body
)
client = http.client.HTTPConnection(*server.server_address, timeout=2)
client.request("POST", path, body=body, headers={
"Content-Type": "application/json",
"X-ITWorx-Deployment-Timestamp": timestamp,
"X-ITWorx-Deployment-Nonce": nonce,
"X-ITWorx-Deployment-Signature": signature,
})
response = client.getresponse()
decoded = json.loads(response.read())
client.close()
return response.status, decoded
try:
revision = "a" * 40
preflight_path = status_module.WORKMESH_PROMOTION_PREFLIGHT_PATH
preflight_status, preflight = post(preflight_path, {"revision": revision}, "1" * 32)
assert preflight_status == 200
assert preflight["preflightSha256"] == "f" * 64
apply_path = status_module.WORKMESH_PROMOTION_APPLY_PATH
rejected_status, rejected = post(apply_path, {
"revision": revision,
"preflightSha256": "e" * 64,
"idempotencyKey": "workmesh-approval-001",
}, "2" * 32)
assert rejected_status == 409
assert rejected["status"] == "preflight_changed"
assert rejected["request"] == status_module._appops_request_digest(
"jens/workmesh", revision, "workmesh_promote", "workmesh-approval-001"
)
assert rejected["repository"] == "jens/workmesh"
assert rejected["revision"] == revision
assert rejected["operation"] == "workmesh_promote"
assert rejected["reason"] == "approved_preflight_no_longer_matches"
assert not inbox.exists() or not list(inbox.iterdir())
accepted_status, accepted = post(apply_path, {
"revision": revision,
"preflightSha256": "f" * 64,
"idempotencyKey": "workmesh-approval-002",
}, "3" * 32)
assert accepted_status == 202
assert accepted["status"] == "accepted"
assert accepted["repository"] == "jens/workmesh"
assert accepted["revision"] == revision
queued = json.loads((inbox / f"{accepted['request']}.json").read_text(encoding="utf-8"))
assert queued["operation"] == "workmesh_promote"
assert queued["preflight_sha256"] == "f" * 64
assert queued["backup_inputs_sha256"] == "e" * 64
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_workmesh_isolated_n8n_restore_uses_exact_image_and_no_network(
deploy_module, monkeypatch, tmp_path
):
backup_dir = tmp_path / "backup"
backup_dir.mkdir()
state_root = tmp_path / "state"
state_root.mkdir()
calls: list[tuple[str, ...]] = []
secret = "synthetic-n8n-key-" + "x" * 40
n8n_id = "a" * 64
def docker(*args, timeout=None):
command = tuple(str(item) for item in args)
calls.append(command)
if command[:2] == ("run", "--detach"):
env_file = Path(command[command.index("--env-file") + 1])
if os.name != "nt":
assert env_file.stat().st_mode & 0o777 == 0o600
assert f"N8N_ENCRYPTION_KEY={secret}" in env_file.read_text(encoding="utf-8")
assert secret not in " ".join(command)
return subprocess.CompletedProcess(command, 0, stdout=(n8n_id + "\n").encode())
if "volume" in command and "create" in command:
return subprocess.CompletedProcess(command, 0, stdout=b"workmesh-restore-volume\n")
return subprocess.CompletedProcess(command, 0, stdout=b"")
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "_workmesh_docker", docker)
monkeypatch.setattr(
deploy_module,
"_workmesh_inspect_container",
lambda _name: {"Id": n8n_id, "State": {"Running": True}},
)
monkeypatch.setattr(deploy_module.time, "sleep", lambda _seconds: None)
evidence = deploy_module.verify_workmesh_n8n_backup_restore(
backup_dir=backup_dir,
request_id="b" * 64,
image_id="sha256:" + "c" * 64,
n8n_encryption_key=secret,
)
assert evidence == {
"status": "verified",
"imageId": "sha256:" + "c" * 64,
"health": "healthy",
"network": "none",
}
run_commands = [command for command in calls if command[:1] == ("run",)]
assert len(run_commands) == 3
assert all(command[command.index("--network") + 1] == "none" for command in run_commands)
extractor = next(command for command in run_commands if "-xf" in command)
assert extractor[extractor.index("--user") + 1] == "0:0"
database_probe = next(
command for command in run_commands
if any("database.sqlite" in argument for argument in command)
)
assert database_probe[database_probe.index("--entrypoint") + 1] == "node"
assert database_probe[database_probe.index("--user") + 1] == "1000:1000"
assert any(command.endswith("target=/home/node/.n8n,readonly") for command in database_probe)
n8n_command = next(command for command in run_commands if "start" in command)
assert "sha256:" + "c" * 64 in n8n_command
assert "N8N_USER_FOLDER=/home/node/.n8n" in n8n_command
assert calls[-2][:2] == ("rm", "-f")
assert calls[-1][:2] == ("volume", "rm")
def test_workmesh_n8n_restore_rejects_missing_nested_sqlite_and_cleans_scratch_volume(
deploy_module, monkeypatch, tmp_path
):
backup_dir = tmp_path / "backup"
backup_dir.mkdir()
state_root = tmp_path / "state"
state_root.mkdir()
secret = "synthetic-n8n-key-" + "y" * 40
calls: list[tuple[str, ...]] = []
def docker(*args, timeout=None):
command = tuple(str(item) for item in args)
calls.append(command)
if "--entrypoint" in command:
raise deploy_module.DeployError("isolated SQLite path/header probe failed")
return subprocess.CompletedProcess(command, 0, stdout=b"")
monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root)
monkeypatch.setattr(deploy_module, "_workmesh_docker", docker)
with pytest.raises(deploy_module.DeployError, match="SQLite path/header probe"):
deploy_module.verify_workmesh_n8n_backup_restore(
backup_dir=backup_dir,
request_id="b" * 64,
image_id="sha256:" + "c" * 64,
n8n_encryption_key=secret,
)
assert not any(command[:2] == ("run", "--detach") for command in calls)
assert calls[-1][:2] == ("volume", "rm")
assert not list(state_root.glob(".workmesh-n8n-restore-*"))
def test_workmesh_backup_restart_uses_existing_container_ids_without_compose_recreate(
deploy_module, monkeypatch
):
revision = "a" * 40
names = ["workmesh-postgres-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1"]
ids = {name: f"{index:064x}" for index, name in enumerate(names, start=1)}
image = "sha256:" + "b" * 64
receipt = {
"revision": revision,
"containers": {
name: {"container_id": ids[name], "image_id": image}
for name in names
},
}
prior_state = {
"live_revision": revision,
"live_status": "healthy",
"live_receipt": receipt,
}
containers = {
name: {
"Id": ids[name],
"Image": image,
"State": {
"Running": name != "workmesh-n8n-1",
"Health": {"Status": "healthy"},
},
"Config": {
"Labels": {
"com.docker.compose.project": "workmesh",
"com.docker.compose.service": name.removeprefix("workmesh-").removesuffix("-1"),
"com.docker.compose.config-hash": "c" * 64,
}
},
}
for name in names
}
calls: list[tuple[str, ...]] = []
def docker(*args, timeout=None):
calls.append(tuple(args))
if args[:2] == ("ps", "--all"):
return subprocess.CompletedProcess(args, 0, stdout=("\n".join(names) + "\n").encode())
if args[0] == "start":
containers["workmesh-n8n-1"]["State"]["Running"] = True
return subprocess.CompletedProcess(args, 0, stdout=b"")
return subprocess.CompletedProcess(args, 0, stdout=b"")
monkeypatch.setattr(deploy_module, "_workmesh_docker", docker)
monkeypatch.setattr(deploy_module, "_workmesh_inspect_container", lambda name: containers.get(name))
monkeypatch.setattr(deploy_module, "read_state", lambda _project: prior_state)
deploy_module._workmesh_restart_previous_runtime(
{"slug": "workmesh"}, prior_state, receipt, revision
)
assert calls[0] == ("start", ids["workmesh-n8n-1"])
assert not any("compose" in call or call[0] in {"run", "up", "create"} for call in calls)
assert containers["workmesh-n8n-1"]["Id"] == ids["workmesh-n8n-1"]
assert containers["workmesh-n8n-1"]["Config"]["Labels"]["com.docker.compose.config-hash"] == "c" * 64
def test_workmesh_promotion_success_requires_bound_data_backup_receipt(
status_module, tmp_path, monkeypatch
):
inbox = tmp_path / "appops-inbox"
inbox.mkdir()
marker = inbox / ("d" * 64 + ".json")
marker.write_text(json.dumps({
"schema": "appops-deployment-request.v1",
"request": "d" * 64,
"repository": "jens/workmesh",
"revision": "a" * 40,
"operation": "workmesh_promote",
"preflight_sha256": "f" * 64,
"backup_inputs_sha256": "e" * 64,
"status": "pending",
}), encoding="utf-8")
project = _workmesh_promotion_project(status_module)
monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox)
monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project})
monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: {
"status": "preflight_ok",
"preflightSha256": "f" * 64,
"backupInputsSha256": "e" * 64,
})
monkeypatch.setattr(
status_module,
"_run_workmesh_promotion_deploy",
lambda args: status_module.subprocess.CompletedProcess(args, 0),
)
monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "healthy"})
monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"revision": "a" * 40})
monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: None)
status_module._process_appops_record(marker)
record = json.loads(marker.read_text(encoding="utf-8"))
assert record["status"] == "outcome_unknown"
assert record["reason"] == "verified_deployment_or_backup_receipt_unavailable"
assert "backup_receipt_sha256" not in record
def test_workmesh_external_recovery_receipt_binds_environment_and_both_encryption_keys(
deploy_module, monkeypatch, tmp_path
):
path = tmp_path / "workmesh-external-recovery.json"
now = int(deploy_module.time.time())
n8n_key = "synthetic-n8n-key-" + "n" * 40
workmesh_key = "synthetic-workmesh-key-" + "w" * 40
payload = {
"schema": "itworx.workmesh-external-recovery/v1",
"repository": "jens/workmesh",
"environmentSha256": "a" * 64,
"keySha256": {
"N8N_ENCRYPTION_KEY": hashlib.sha256(n8n_key.encode()).hexdigest(),
"WORKMESH_SECRET_ENCRYPTION_KEY": hashlib.sha256(workmesh_key.encode()).hexdigest(),
},
"externalBackupSha256": "b" * 64,
"restoreVerifiedAtEpoch": now,
}
path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
if os.name != "nt":
path.chmod(0o600)
monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True)
receipt_sha = deploy_module.verify_workmesh_external_recovery_receipt(
path,
environment_sha256="a" * 64,
encryption_keys={
"N8N_ENCRYPTION_KEY": n8n_key,
"WORKMESH_SECRET_ENCRYPTION_KEY": workmesh_key,
},
)
assert receipt_sha == hashlib.sha256(path.read_bytes()).hexdigest()
payload["keySha256"]["N8N_ENCRYPTION_KEY"] = "c" * 64
path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
with pytest.raises(deploy_module.DeployError, match="external recovery receipt"):
deploy_module.verify_workmesh_external_recovery_receipt(
path,
environment_sha256="a" * 64,
encryption_keys={
"N8N_ENCRYPTION_KEY": n8n_key,
"WORKMESH_SECRET_ENCRYPTION_KEY": workmesh_key,
},
)
def test_workmesh_backup_manifest_binds_exact_prior_revision_and_file_digests(
deploy_module, tmp_path
):
backup = tmp_path / "backup"
backup.mkdir()
postgres = b"private pg dump bytes"
n8n = b"private n8n volume archive"
(backup / "postgres.dump").write_bytes(postgres)
(backup / "n8n.tar").write_bytes(n8n)
manifest = {
"format": "workmesh-backup/v1",
"created_at": "2026-09-27T12:00:00+00:00",
"project": "workmesh",
"declared_live_revision": "a" * 40,
"postgres": {
"file": "postgres.dump",
"bytes": len(postgres),
"sha256": hashlib.sha256(postgres).hexdigest(),
},
"n8n": {
"file": "n8n.tar",
"bytes": len(n8n),
"sha256": hashlib.sha256(n8n).hexdigest(),
},
"encryption_root": "external; not included",
}
manifest_path = backup / "manifest.json"
manifest_path.write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8")
evidence = deploy_module.verify_workmesh_backup_manifest(backup, "a" * 40)
assert evidence["manifestSha256"] == hashlib.sha256(manifest_path.read_bytes()).hexdigest()
assert evidence["postgresSha256"] == hashlib.sha256(postgres).hexdigest()
assert evidence["n8nSha256"] == hashlib.sha256(n8n).hexdigest()
with pytest.raises(deploy_module.DeployError, match="revision"):
deploy_module.verify_workmesh_backup_manifest(backup, "c" * 40)
(backup / "n8n.tar").write_bytes(b"changed")
with pytest.raises(deploy_module.DeployError, match="checksum or size"):
deploy_module.verify_workmesh_backup_manifest(backup, "a" * 40)
def test_workmesh_backup_receipt_requires_bound_manifest_and_verified_restore_bytes(
status_module, tmp_path, monkeypatch
):
request_id = "d" * 64
candidate = "b" * 40
prior = "a" * 40
preflight = "f" * 64
inputs = "e" * 64
backup_root = tmp_path / "backups"
backup_dir = backup_root / request_id
backup_dir.mkdir(parents=True)
receipt_root = tmp_path / "receipts"
receipt_root.mkdir()
postgres = b"synthetic PostgreSQL dump"
n8n = b"synthetic n8n volume archive"
postgres_sha = hashlib.sha256(postgres).hexdigest()
n8n_sha = hashlib.sha256(n8n).hexdigest()
(backup_dir / "postgres.dump").write_bytes(postgres)
(backup_dir / "n8n.tar").write_bytes(n8n)
manifest = {
"format": "workmesh-backup/v1",
"created_at": "2026-09-27T12:00:00+00:00",
"project": "workmesh",
"declared_live_revision": prior,
"postgres": {"file": "postgres.dump", "bytes": len(postgres), "sha256": postgres_sha},
"n8n": {"file": "n8n.tar", "bytes": len(n8n), "sha256": n8n_sha},
"encryption_root": "external; not included",
}
manifest_bytes = json.dumps(manifest, sort_keys=True, indent=2).encode()
(backup_dir / "manifest.json").write_bytes(manifest_bytes)
payload = {
"schema": "itworx.workmesh-backup-receipt/v1",
"request": request_id,
"repository": "jens/workmesh",
"operation": "workmesh_promote",
"candidateRevision": candidate,
"priorLiveRevision": prior,
"preflightSha256": preflight,
"backupInputsSha256": inputs,
"backupToolSha256": "1" * 64,
"externalRecoveryReceiptSha256": "2" * 64,
"environmentSha256": "3" * 64,
"manifestSha256": hashlib.sha256(manifest_bytes).hexdigest(),
"postgres": {
"sha256": postgres_sha,
"bytes": len(postgres),
"restore": {
"status": "verified",
"alembicRevision": "0022",
"tableCount": 22,
"network": "none",
},
},
"n8n": {
"sha256": n8n_sha,
"bytes": len(n8n),
"restore": {
"status": "verified",
"imageId": "sha256:" + "4" * 64,
"health": "healthy",
"network": "none",
},
},
"automaticDataRollback": False,
"createdAt": "2026-09-27T12:01:00+00:00",
}
receipt_sha = hashlib.sha256(
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
(receipt_root / f"{request_id}.json").write_text(
json.dumps({"receipt": payload, "receiptSha256": receipt_sha}), encoding="utf-8"
)
if os.name != "nt":
for directory in (backup_root, backup_dir, receipt_root):
directory.chmod(0o700)
for path in (
backup_dir / "manifest.json",
backup_dir / "postgres.dump",
backup_dir / "n8n.tar",
receipt_root / f"{request_id}.json",
):
path.chmod(0o600)
monkeypatch.setattr(status_module, "WORKMESH_BACKUP_ROOT", backup_root)
monkeypatch.setattr(status_module, "WORKMESH_BACKUP_RECEIPT_ROOT", receipt_root)
assert status_module._verified_workmesh_backup_receipt(
request_id, candidate, preflight, inputs
) == receipt_sha
assert status_module._verified_workmesh_backup_receipt(
request_id, candidate, "9" * 64, inputs
) is None
(backup_dir / "n8n.tar").write_bytes(b"changed")
assert status_module._verified_workmesh_backup_receipt(
request_id, candidate, preflight, inputs
) is None
def _managed_storage_http_request(
status_module,
server,
*,
body: bytes,
nonce: str,
secret: bytes | None,
signature: str | None = None,
signed_path: str | None = None,
):
path = status_module.MANAGED_APPDATA_INITIALIZE_PATH
timestamp = str(int(status_module.time.time()))
headers = {"Content-Type": "application/json", "Content-Length": str(len(body))}
if nonce:
headers["X-ITWorx-Deployment-Nonce"] = nonce
if timestamp:
headers["X-ITWorx-Deployment-Timestamp"] = timestamp
if signature is None and secret is not None:
signature = status_module._request_signature(
secret,
method="POST",
path=signed_path or path,
timestamp=timestamp,
nonce=nonce,
raw_body=body,
)
if signature is not None:
headers["X-ITWorx-Deployment-Signature"] = signature
connection = http.client.HTTPConnection(*server.server_address, timeout=2)
connection.request("POST", path, body=body, headers=headers)
response = connection.getresponse()
status = response.status
payload = json.loads(response.read())
connection.close()
return status, payload, headers
def test_managed_appdata_initializer_http_requires_exact_signed_one_time_request(
status_module, tmp_path, monkeypatch
):
secret = b"managed-storage-bootstrap-" + b"x" * 48
secret_path = tmp_path / "bootstrap"
secret_path.write_bytes(secret)
secret_path.chmod(0o600)
nonce_root = tmp_path / "nonces"
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", secret_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root)
monkeypatch.setattr(
status_module.Handler,
"_managed_storage_initialization_prerequisites",
lambda _handler: True,
)
initialized = []
monkeypatch.setattr(
status_module,
"ensure_managed_appdata_root",
lambda: initialized.append(True) or "initialized",
)
monkeypatch.setattr(
status_module,
"_refresh_appdata_root_selection",
lambda: status_module.MANAGED_APPDATA_ROOT,
)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
body = json.dumps(
{
"apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION,
"target": status_module.MANAGED_APPDATA_INITIALIZE_TARGET,
},
separators=(",", ":"),
).encode("utf-8")
try:
success_status, success_payload, success_headers = _managed_storage_http_request(
status_module,
server,
body=body,
nonce="a" * 32,
secret=secret,
)
assert success_status == 200
assert success_payload == {
"status": "initialized",
"trustedAppdataRoot": status_module.MANAGED_APPDATA_ROOT.as_posix(),
"backupLocation": "dockerRootDir",
}
assert initialized == [True]
assert secret.decode("ascii") not in json.dumps(success_payload)
replay = http.client.HTTPConnection(*server.server_address, timeout=2)
replay.request(
"POST",
status_module.MANAGED_APPDATA_INITIALIZE_PATH,
body=body,
headers=success_headers,
)
replay_response = replay.getresponse()
replay_payload = json.loads(replay_response.read())
replay.close()
assert replay_response.status == 403
assert replay_payload == {"status": "authentication_rejected"}
assert initialized == [True]
invalid_requests = [
_managed_storage_http_request(
status_module,
server,
body=body,
nonce="b" * 32,
secret=secret,
signature="0" * 64,
),
_managed_storage_http_request(
status_module,
server,
body=body,
nonce="c" * 32,
secret=secret,
signed_path="/v1/appops/enrollments/capabilities",
),
_managed_storage_http_request(
status_module,
server,
body=body,
nonce="d" * 32,
secret=None,
),
]
assert [status for status, _payload, _headers in invalid_requests] == [403, 403, 403]
assert initialized == [True]
for index, payload in enumerate(
[
{
"apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION,
"target": "tower/other-controller",
},
{
"apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION,
"target": status_module.MANAGED_APPDATA_INITIALIZE_TARGET,
"path": "C:/private/sentinel",
},
],
start=1,
):
raw_body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
rejected_status, rejected_payload, _headers = _managed_storage_http_request(
status_module,
server,
body=raw_body,
nonce=f"{index:032x}",
secret=secret,
)
assert rejected_status == 400
assert rejected_payload == {"status": "invalid_managed_storage_request"}
assert "C:/private/sentinel" not in json.dumps(rejected_payload)
oversized_status, oversized_payload, _headers = _managed_storage_http_request(
status_module,
server,
body=b"x" * 1025,
nonce="e" * 32,
secret=None,
)
assert oversized_status == 413
assert oversized_payload == {"status": "invalid_body_size"}
assert initialized == [True]
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
def test_managed_appdata_initializer_http_prerequisite_and_refusal_are_sanitized(
status_module, tmp_path, monkeypatch
):
secret = b"managed-storage-bootstrap-" + b"y" * 48
secret_path = tmp_path / "bootstrap"
secret_path.write_bytes(secret)
secret_path.chmod(0o600)
monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", secret_path)
monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
prerequisites = {"ready": False}
monkeypatch.setattr(
status_module.Handler,
"_managed_storage_initialization_prerequisites",
lambda _handler: prerequisites["ready"],
)
calls = []
monkeypatch.setattr(
status_module,
"ensure_managed_appdata_root",
lambda: calls.append(True) or "initialized",
)
monkeypatch.setattr(
status_module,
"_refresh_appdata_root_selection",
lambda: status_module.MANAGED_APPDATA_ROOT,
)
server = status_module.BoundedThreadingHTTPServer(
("127.0.0.1", 0), status_module.Handler
)
serving = threading.Thread(target=server.serve_forever, daemon=True)
serving.start()
body = json.dumps(
{
"apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION,
"target": status_module.MANAGED_APPDATA_INITIALIZE_TARGET,
},
separators=(",", ":"),
).encode("utf-8")
try:
status, payload, _headers = _managed_storage_http_request(
status_module, server, body=body, nonce="f" * 32, secret=secret
)
assert status == 503
assert payload == {"status": "managed_storage_prerequisite_unavailable"}
assert calls == []
prerequisites["ready"] = True
def refuse_with_sensitive_diagnostic():
raise status_module.UnsafeManagedAppdataRoot(
"secret/path sentinel C:/private/appdata"
)
monkeypatch.setattr(
status_module, "ensure_managed_appdata_root", refuse_with_sensitive_diagnostic
)
status, payload, _headers = _managed_storage_http_request(
status_module, server, body=body, nonce="1" * 32, secret=secret
)
assert status == 409
assert payload == {"status": "managed_storage_initialization_refused"}
assert "secret/path sentinel" not in json.dumps(payload)
assert "C:/private/appdata" not in json.dumps(payload)
assert secret.decode("ascii") not in json.dumps(payload)
finally:
server.shutdown()
server.server_close()
serving.join(timeout=2)
Reference in new issue View Git Blame Copy Permalink