from __future__ import annotations import copy import hashlib import hmac import http.client import importlib.util import io import ipaddress import json import os import socket import ssl import stat import subprocess import sys import tarfile import threading import types from contextlib import contextmanager from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] BROKER_ROOT = ROOT / "ops" / "gitea-deploy-broker" def _load_module(name: str, path: Path): if "fcntl" not in sys.modules: fcntl = types.ModuleType("fcntl") fcntl.LOCK_EX = 2 fcntl.LOCK_UN = 8 fcntl.flock = lambda *_args: None sys.modules["fcntl"] = fcntl spec = importlib.util.spec_from_file_location(name, path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module @pytest.fixture def deploy_module(): return _load_module("appops_deploy_broker_test", BROKER_ROOT / "deploy.py") @pytest.fixture def status_module(): return _load_module("appops_deploy_status_test", BROKER_ROOT / "status_server.py") def test_controller_state_uses_the_physical_cache_path(deploy_module, status_module): expected_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") assert deploy_module.STATE_ROOT == expected_root assert deploy_module.GLOBAL_DEPLOY_LOCK == expected_root.parent / "deploy.lock" assert status_module.STATE_ROOT == expected_root assert status_module.WEBHOOK_INBOX_ROOT == expected_root / "webhook-inbox" def test_normal_deploy_still_takes_its_own_lock(deploy_module, monkeypatch, tmp_path): project_lock = tmp_path / "repopulse.lock" other_lock = tmp_path / "other.lock" monkeypatch.setitem( sys.modules, "repopulse_adoption_store", types.SimpleNamespace(transaction_lock_held=lambda path: path == project_lock), ) with deploy_module.exclusive_lock(project_lock, "project repopulse"): assert project_lock.exists() with deploy_module.exclusive_lock(other_lock, "other project"): assert other_lock.exists() def test_adoption_deploy_requires_both_held_locks_before_entering_core( deploy_module, monkeypatch ): monkeypatch.setitem( sys.modules, "repopulse_adoption_store", types.SimpleNamespace(transaction_lock_held=lambda _path: False), ) monkeypatch.setattr( deploy_module, "main", lambda _argv: pytest.fail("deploy core must not run without the transaction"), ) with pytest.raises(deploy_module.DeployError, match="requires both fixed transaction locks"): deploy_module.deploy_repopulse_under_adoption_transaction("a" * 40) def test_adoption_deploy_rechecks_exact_receipt_after_existing_core( deploy_module, monkeypatch, tmp_path ): revision = "a" * 40 fixed_locks = { deploy_module.STATE_ROOT / "locks" / "repopulse.lock", deploy_module.GLOBAL_DEPLOY_LOCK, } monkeypatch.setitem( sys.modules, "repopulse_adoption_store", types.SimpleNamespace(transaction_lock_held=lambda path: path in fixed_locks), ) observed = [] project = { "managed_profile": "repopulse-authenticated", "branch": "main", "source_dir": str(tmp_path), "slug": "repopulse", "exact_source": True, } monkeypatch.setattr( deploy_module, "load_projects", lambda: {"jens/repopulse": project}, ) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda *_args: revision) monkeypatch.setattr( deploy_module, "deployment_metadata", lambda *_args: {"tree": "b" * 40}, ) for name in ( "validate_project_config", "validate_forward_revision", "sync_source", "copy_environment", "revalidate_deployment_candidate", "validate_managed_v1_runtime", "ensure_docker_capacity", "deploy_project", "write_state", ): monkeypatch.setattr( deploy_module, name, lambda *_args, _name=name, **_kwargs: observed.append(_name), ) receipt = {"repository": "jens/repopulse", "revision": revision} monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt) monkeypatch.setattr( deploy_module, "verified_current_receipt", lambda _repo, _project, _revision, _tree: receipt, ) assert deploy_module.deploy_repopulse_under_adoption_transaction(revision) == receipt assert observed == [ "validate_project_config", "validate_forward_revision", "sync_source", "copy_environment", "revalidate_deployment_candidate", "validate_managed_v1_runtime", "ensure_docker_capacity", "deploy_project", "write_state", ] def test_global_lock_uses_only_the_physical_cache_lock(deploy_module, monkeypatch): events: list[tuple[str, Path] | tuple[str]] = [] @contextmanager def observed_lock(path: Path, _description: str): events.append(("enter", path)) try: yield finally: events.append(("exit", path)) monkeypatch.setattr(deploy_module, "exclusive_lock", observed_lock) with deploy_module.exclusive_global_deploy_lock(): events.append(("body",)) physical = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") assert events == [ ("enter", physical), ("body",), ("exit", physical), ] def test_static_rollback_environment_accepts_application_database_url(deploy_module, tmp_path): path = tmp_path / ".env" path.write_text("DATABASE_URL='postgresql://fixture:fixture@localhost:5432/projectbrain'\n") assert deploy_module.static_projectbrain_environment(path) == { "DATABASE_URL": "postgresql://fixture:fixture@localhost:5432/projectbrain" } @pytest.mark.parametrize("changed", [False, True]) def test_project_config_is_rechecked_after_lock_before_work(deploy_module, monkeypatch, tmp_path, changed): repository = "jens/unraid-appops-gateway" config = {"slug": "unraid-appops-gateway", "branch": "main", "source_dir": str(tmp_path / "source"), "compose_project": "appops", "containers": ["Unraid-AppOps-Gateway"]} locked = False reads = 0 @contextmanager def lock(*_args): nonlocal locked locked = True try: yield finally: locked = False def load(): nonlocal reads reads += 1 if reads == 2: assert locked, "configuration must be rechecked inside the project lock" value = copy.deepcopy(config) if reads == 2 and changed: value["source_dir"] = str(tmp_path / "migrated-source") return {repository: value} monkeypatch.setattr(sys, "argv", ["deploy.py", "check", repository, "a" * 40]) monkeypatch.setattr(deploy_module, "load_projects", load) monkeypatch.setattr(deploy_module, "repository_path", lambda *_: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda *_: "a" * 40) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_: {}) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_: None) monkeypatch.setattr(deploy_module, "exclusive_lock", lock) if changed: with pytest.raises(deploy_module.DeployError, match="configuration changed"): deploy_module.main() else: assert deploy_module.main() == 0 assert reads == 2 def _project(tmp_path: Path) -> dict[str, object]: return { "slug": "unraid-appops-gateway", "source_dir": str(tmp_path / "source"), "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, "_repository_name": "jens/unraid-appops-gateway", } def _appops_known_legacy_state() -> dict[str, object]: old_revision = "fe0772fdf5eb7ad772312c9263560d70ebddb536" failed_revision = "034c9f844d2455850ba37b31379b7edaa786583e" image_id = "sha256:2b15f07e5be8e0e51075e282354ca8f9feeda1a4146c1f55313a7dd9e9b74fbe" stale_container_id = "6632964b26056cb05832bd52cc19a32cb654cdc1e68ec9d569553271e88b1aea" stale_container = { "container_id": stale_container_id, "image_id": image_id, "image_ref": "unraid-appops-gateway:production", "container_revision": old_revision, "image_revision": old_revision, "image_source": "jens/unraid-appops-gateway", "image_tree": "89fae3afc35222c2f176cf50f1fcb19e535e2539", "image_version": "1.8.0-platform-fe0772fdf5eb", "image_created": "2026-09-29T00:00:00Z", } stale_artifacts = {"containers": {"Unraid-AppOps-Gateway": stale_container}} stale_receipt = { "schema": "gitea-deploy-receipt.v1", "repository": "jens/unraid-appops-gateway", "revision": old_revision, "tree": stale_container["image_tree"], "treeDigest": "657e133a772e839b1d917f081b5135f2b985f774aeb825cc7a00f29091b242a9", "build": {"revision": old_revision, "tree": stale_container["image_tree"]}, "containers": {"Unraid-AppOps-Gateway": stale_container}, "live": None, } failure = "Existing production containers require a valid healthy live deployment state" attempt_metadata = { "builtAt": "2026-10-01T18:38:29Z", "id": failed_revision[:12], "repository": "jens/unraid-appops-gateway", "revision": failed_revision, "tree": "a" * 40, "treeDigest": hashlib.sha256(f"git-tree:{'a' * 40}".encode()).hexdigest(), } return { "schema_version": 3, "repository": "jens/unraid-appops-gateway", "slug": "unraid-appops-gateway", "compose_project": "unraid-appops-gateway", "revision": old_revision, "live_revision": old_revision, "live_status": "unknown", "live_legacy_provenance": False, "live_artifacts": copy.deepcopy(stale_artifacts), "live_receipt": copy.deepcopy(stale_receipt), "rollback_snapshot": { "revision": old_revision, "legacy_provenance": False, "receipt": copy.deepcopy(stale_receipt), }, "status": "failed", "message": failure, "updated_at_epoch": 1, "artifacts": copy.deepcopy(stale_artifacts), "last_attempt": { "revision": failed_revision, "status": "failed", "message": failure, "updated_at_epoch": 1, "metadata": attempt_metadata, }, "previous_revision": "", "rollback_restored_revision": "", } def test_appops_legacy_state_anchor_accepts_only_known_preimage_or_current_green_attempt( deploy_module, monkeypatch ): state = _appops_known_legacy_state() raw = (json.dumps(state, indent=2) + "\n").encode() monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", hashlib.sha256(json.dumps( state["last_attempt"]["metadata"], sort_keys=True, separators=(",", ":") ).encode()).hexdigest(), ) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_STATE_SHA256", hashlib.sha256(raw).hexdigest(), ) assert deploy_module.appops_legacy_state_anchor( state, raw, "034c9f844d2455850ba37b31379b7edaa786583e", protected_main_green=False, ) current = "c" * 40 rewritten = copy.deepcopy(state) rewritten["message"] = deploy_module.APPOPS_LEGACY_FAILURE_MESSAGE rewritten["updated_at_epoch"] = 2 candidate_tree = "c" * 40 rewritten["last_attempt"].update({ "revision": current, "updated_at_epoch": 2, "metadata": { "builtAt": "2026-10-02T10:00:00Z", "id": current[:12], "repository": "jens/unraid-appops-gateway", "revision": current, "tree": candidate_tree, "treeDigest": hashlib.sha256(f"git-tree:{candidate_tree}".encode()).hexdigest(), }, }) expected = copy.deepcopy(rewritten) expected.pop("updated_at_epoch") expected.pop("last_attempt") projection = hashlib.sha256( json.dumps(expected, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_PROJECTION_SHA256", projection) rewritten_raw = (json.dumps(rewritten, indent=2) + "\n").encode() assert deploy_module.appops_legacy_state_anchor( rewritten, rewritten_raw, current, candidate_metadata=rewritten["last_attempt"]["metadata"], protected_main_green=True, ) assert not deploy_module.appops_legacy_state_anchor( rewritten, rewritten_raw, current, protected_main_green=False, ) tampered = copy.deepcopy(rewritten) tampered["rollback_snapshot"]["revision"] = "d" * 40 tampered_raw = (json.dumps(tampered, indent=2) + "\n").encode() assert not deploy_module.appops_legacy_state_anchor( tampered, tampered_raw, current, candidate_metadata=rewritten["last_attempt"]["metadata"], protected_main_green=True, ) def test_appops_legacy_runtime_requires_pinned_source_image_and_resolved_model( deploy_module, tmp_path ): project = _project(tmp_path) project.update({ "branch": "main", "gitea_repository_id": 9, "compose_project": "unraid-appops-gateway", "containers": ["Unraid-AppOps-Gateway"], "compose_files": ["docker-compose.yml"], "environment": { "APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216", "APPOPS_PUBLISH_HOST": "198.18.10.150", }, }) source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" old_revision = "5553c04ce497c08a35ed301052dc5e0b02e5d9b4" image_id = "sha256:9b5eddf32c8eb5361106fc35cc0e056c49d287a20d1fb26490662cc8447f09d4" container = { "Id": "72daa9f638defc4f8897d8aad23c4181054b2a1bc04c993078a5ac9597eeb0ec", "Image": image_id, "State": {"Running": True, "Health": {"Status": "healthy"}}, "Config": { "Image": "unraid-appops-gateway:platform-5553c04ce497", "Hostname": "unraid-appops-gateway", "Labels": { "com.docker.compose.project": "unraid-appops-gateway", "com.docker.compose.service": "appops-gateway", "com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source", "com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml", "org.opencontainers.image.source": source, "org.opencontainers.image.revision": old_revision, "org.opencontainers.image.source-tree": "1624e1748053f998d8427c8c8a4203277d22f13c", "org.opencontainers.image.version": "1.8.0-platform-5553c04ce497", "org.opencontainers.image.created": "2026-10-01T16:32:01Z", }, }, "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, "Mounts": [ {"Type": "bind", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "Destination": "/data", "RW": True}, {"Type": "bind", "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models", "Destination": "/mnt/privatemuse-comfyui-models", "RW": True}, ], "NetworkSettings": { "Ports": {"1216/tcp": [{"HostIp": "198.18.10.150", "HostPort": "1216"}]}, "Networks": { "itworx-mcp-hub-npm": {}, "unraid-appops-gateway_default": {}, }, }, } image = { "Id": image_id, "Config": {"Labels": container["Config"]["Labels"]}, } observed = deploy_module.verify_appops_legacy_runtime(project, container, image) assert observed["imageId"] == image_id assert observed["imageRef"] == "unraid-appops-gateway:platform-5553c04ce497" assert observed["sourceRepository"] == source wrong = copy.deepcopy(container) wrong["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "127.0.0.1" with pytest.raises(deploy_module.DeployError, match="fixed AppOps legacy runtime"): deploy_module.verify_appops_legacy_runtime(project, wrong, image) wrong_image = copy.deepcopy(image) wrong_image["Id"] = "sha256:" + "0" * 64 with pytest.raises(deploy_module.DeployError, match="fixed AppOps legacy runtime"): deploy_module.verify_appops_legacy_runtime(project, container, wrong_image) def test_appops_legacy_intent_is_root_private_and_compare_and_swap_bound( deploy_module, monkeypatch, tmp_path ): root = tmp_path / "controller-config" / "appops-legacy-upgrade" monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", root) monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_PATH", root / "unraid-appops-gateway.json") revision = "c" * 40 tree = "d" * 40 metadata = { "id": revision[:12], "repository": "jens/unraid-appops-gateway", "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-10-02T10:00:00.000Z", } source_proof = { "legacyRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "candidateRevision": revision, "composeBlob": "3" * 40, "dockerfileBlob": "4" * 40, } protected_main = { "semanticSha256": "5" * 64, "revision": revision, "source": "gitea_webhook", "validationContext": "Managed validation / full (push)", "validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000", "bodySha256": "6" * 64, "validationSha256": "7" * 64, } runtime_proof = { "resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64, "runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64, "healthSha256": "c" * 64, } backup_receipt = { "schema": "itworx.appops-data-backup/v1", "source": deploy_module.APPOPS_LEGACY_DATA_PATH, "archive": "/srv/example/appdata/unraid-appops-gateway-backups/one/appdata.tar.gz", "sha256": "d" * 64, "bytes": 12, "requestedRevision": revision, "previousRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "createdAt": "2026-10-02T10:00:00.000Z", } backup = { **backup_receipt, "receiptSha256": deploy_module._canonical_json_sha256(backup_receipt), } snapshot = { "revision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "legacy_provenance": True, "receipt": {}, "compose_containers": ["Unraid-AppOps-Gateway"], "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, "image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF, "restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", "was_running": True, }], "captured_at_epoch": 1790884709, } payload = deploy_module._appops_legacy_intent_payload( revision=revision, metadata=metadata, state_sha256="e" * 64, observed_state_sha256="e" * 64, source_proof=source_proof, protected_main=protected_main, runtime_proof=runtime_proof, runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID}, backup=backup, snapshot=snapshot, ) payload["status"] = "prepared" digest = deploy_module.write_appops_legacy_intent(payload) loaded, observed_digest = deploy_module.read_appops_legacy_intent() assert loaded == payload assert observed_digest == digest assert root.stat().st_mode & 0o777 == 0o700 assert deploy_module.APPOPS_LEGACY_INTENT_PATH.stat().st_mode & 0o777 == 0o600 captured_snapshot = copy.deepcopy(payload["rollbackSnapshot"]) updated = deploy_module._appops_legacy_rebind_observed_state(payload, "f" * 64) assert updated["stateObservedSha256"] == "f" * 64 assert updated["rollbackSnapshot"] == captured_snapshot updated["status"] = "snapshot_ready" next_digest = deploy_module.write_appops_legacy_intent(updated, expected_sha256=digest) assert next_digest != digest with pytest.raises(deploy_module.DeployError, match="compare-and-swap"): deploy_module.write_appops_legacy_intent(payload, expected_sha256=digest) loaded, _ = deploy_module.read_appops_legacy_intent() assert loaded["stateObservedSha256"] == "f" * 64 assert loaded["rollbackSnapshot"] == captured_snapshot tampered = copy.deepcopy(loaded) tampered["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] = "f" * 64 with pytest.raises(deploy_module.DeployError, match="intent and rollback snapshot differ"): deploy_module.write_appops_legacy_intent(tampered, expected_sha256=next_digest) hardlink = root / "hardlink.json" os.link(deploy_module.APPOPS_LEGACY_INTENT_PATH, hardlink) with pytest.raises(deploy_module.DeployError, match="unsafe"): deploy_module.read_appops_legacy_intent() def test_appops_legacy_main_dispatch_uses_prepared_snapshot_backup_and_state_cas( deploy_module, monkeypatch, tmp_path, capsys ): repository = "jens/unraid-appops-gateway" revision = "c" * 40 project_config = { "slug": "unraid-appops-gateway", "branch": "main", "source_dir": str(tmp_path / "source"), "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": ["Unraid-AppOps-Gateway"], } projects = {repository: project_config} metadata = { "id": revision[:12], "repository": repository, "revision": revision, "tree": "d" * 40, "treeDigest": hashlib.sha256(f"git-tree:{'d' * 40}".encode()).hexdigest(), "builtAt": "2026-10-02T10:00:00.000Z", } legacy_state = _appops_known_legacy_state() raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode() state_sha256 = hashlib.sha256(raw_state).hexdigest() post_state = {"repository": repository, "live_revision": revision, "live_status": "healthy"} events = [] snapshot = { "revision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "legacy_provenance": True, "receipt": {}, "containers": [{"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497"}], } backup = {"sha256": "e" * 64, "receiptSha256": "f" * 64} intent = {"status": "deploying"} migration = { "intent": intent, "intent_sha256": "1" * 64, "state_sha256": state_sha256, "state": legacy_state, "raw_state": raw_state, "snapshot": snapshot, "backup": backup, } @contextmanager def project_lock(path, _description): events.append(("project-lock", path)) yield @contextmanager def global_lock(): events.append(("global-lock",)) yield monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision]) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: metadata) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None) monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state) monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) monkeypatch.setattr(deploy_module, "validate_managed_v1_runtime", lambda *_args: None) monkeypatch.setattr(deploy_module, "ensure_docker_capacity", lambda: None) monkeypatch.setattr(deploy_module, "capture_compose_rollback", lambda *_args, **_kwargs: pytest.fail("migration snapshot must be reused")) monkeypatch.setattr(deploy_module, "capture_appops_data_backup", lambda *_args: pytest.fail("migration backup must be reused")) def read_preimage(_project): events.append(("state-preimage",)) return (legacy_state, raw_state, state_sha256) if events.count(("state-preimage",)) == 1 else ( post_state, b"post-state", "2" * 64 ) def prepare(project, _repository, _repo, _revision, _metadata, state, raw, digest): assert events[-1] == ("state-preimage",) assert state == legacy_state and raw == raw_state and digest == state_sha256 assert events.count(("global-lock",)) == 1 project["_appops_legacy_migration"] = migration events.append(("prepare",)) return migration monkeypatch.setattr(deploy_module, "read_appops_state_preimage", read_preimage) monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None) monkeypatch.setattr(deploy_module, "_appops_legacy_reconcile_pending_rollback", lambda *_args, **_kwargs: False) monkeypatch.setattr(deploy_module, "_appops_legacy_recover_interrupted_candidate", lambda *_args: False) monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare) monkeypatch.setattr(deploy_module, "write_appops_progress", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({}, "a" * 64, "b" * 64)) monkeypatch.setattr(deploy_module, "deploy_project", lambda *_args: events.append(("deploy",))) receipt = {"repository": repository, "revision": revision, "tree": metadata["tree"]} monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt) monkeypatch.setattr(deploy_module, "finish_compose_migration", lambda *_args: None) def write_state(*_args, **kwargs): events.append(("write-state", kwargs)) def write_intent(payload, *, expected_sha256=None): events.append(("write-intent", payload["status"], expected_sha256)) return "3" * 64 monkeypatch.setattr(deploy_module, "write_state", write_state) monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", write_intent) monkeypatch.setattr(deploy_module, "cleanup_rollback_image_tags", lambda *_args, **_kwargs: None) assert deploy_module.main() == 0 assert ("prepare",) in events and ("deploy",) in events state_event = next(event for event in events if event[0] == "write-state") assert state_event[1]["expected_state_sha256"] == state_sha256 assert state_event[1]["rollback_snapshot"] == snapshot assert events.index(("global-lock",)) < events.index(("prepare",)) < events.index(("deploy",)) assert ("write-intent", "deployed", "1" * 64) in events assert json.loads(capsys.readouterr().out)["status"] == "deployed" def _appops_rollback_intent_fixture(deploy_module, revision: str, capture_sha256: str): repository = "jens/unraid-appops-gateway" tree = "d" * 40 old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION metadata = { "id": revision[:12], "repository": repository, "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-10-02T10:00:00.000Z", } source_proof = { "legacyRevision": old_revision, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "candidateRevision": revision, "composeBlob": "3" * 40, "dockerfileBlob": "4" * 40, } protected_main = { "semanticSha256": "5" * 64, "revision": revision, "source": "gitea_webhook", "validationContext": "Managed validation / full (push)", "validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000", "bodySha256": "6" * 64, "validationSha256": "7" * 64, } runtime_proof = { "resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64, "runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64, "healthSha256": "c" * 64, } backup_receipt = { "schema": "itworx.appops-data-backup/v1", "source": deploy_module.APPOPS_LEGACY_DATA_PATH, "archive": "/tmp/appops-rollback-fixture/appdata.tar.gz", "sha256": "d" * 64, "bytes": 12, "requestedRevision": revision, "previousRevision": old_revision, "createdAt": "2026-10-02T10:00:00.000Z", } backup = { **backup_receipt, "receiptSha256": deploy_module._canonical_json_sha256(backup_receipt), } snapshot = { "revision": old_revision, "legacy_provenance": True, "receipt": {}, "compose_containers": ["Unraid-AppOps-Gateway"], "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, "image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF, "restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, "rollback_ref": ( "gitea-deploy-rollback/unraid-appops-gateway-0:" + old_revision[:12] ), "was_running": True, }], "captured_at_epoch": 1790884709, } intent = deploy_module._appops_legacy_intent_payload( revision=revision, metadata=metadata, state_sha256=capture_sha256, observed_state_sha256=capture_sha256, source_proof=source_proof, protected_main=protected_main, runtime_proof=runtime_proof, runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID}, backup=backup, snapshot=snapshot, status="deploying", ) return repository, metadata, source_proof, protected_main, backup, intent def test_appops_cli_rollback_retries_after_state_write_crash_without_synthetic_receipt( deploy_module, monkeypatch, tmp_path, capsys ): repository = "jens/unraid-appops-gateway" revision = "c" * 40 old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION state_root = tmp_path / "state" state_root.mkdir() intent_root = tmp_path / "appops-legacy-intent" monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json" ) capture_sha256 = hashlib.sha256(b"immutable legacy capture preimage\n").hexdigest() repository, metadata, source_proof, protected_main, backup, intent = ( _appops_rollback_intent_fixture(deploy_module, revision, capture_sha256) ) backup_root = tmp_path / "backups" backup_dir = backup_root / "one" backup_dir.mkdir(parents=True, mode=0o700) backup_dir.chmod(0o700) backup_root.chmod(0o700) archive_path = backup_dir / "appdata.tar.gz" archive_bytes = b"verified AppOps rollback backup fixture" archive_path.write_bytes(archive_bytes) archive_path.chmod(0o600) monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", backup_root) backup["archive"] = str(archive_path) backup["sha256"] = hashlib.sha256(archive_bytes).hexdigest() backup["bytes"] = len(archive_bytes) backup_receipt = {key: value for key, value in backup.items() if key != "receiptSha256"} backup["receiptSha256"] = deploy_module._canonical_json_sha256(backup_receipt) receipt_path = backup_dir / "receipt.json" receipt_path.write_text(json.dumps(backup_receipt, sort_keys=True), encoding="utf-8") receipt_path.chmod(0o600) intent["dataBackup"] = backup intent["rollbackSnapshot"]["appops_legacy"]["dataBackupSha256"] = backup["sha256"] snapshot = intent["rollbackSnapshot"] project_config = { "slug": "unraid-appops-gateway", "branch": "main", "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": ["Unraid-AppOps-Gateway"], "gitea_repository_id": 9, "verify_oci_revision": True, "environment": { "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, }, } projects = {repository: project_config} receipt = { "repository": repository, "revision": revision, "tree": metadata["tree"], "build": metadata, } candidate_artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "sha256:" + "e" * 64}}} candidate_state = { "schema_version": 3, "repository": repository, "slug": "unraid-appops-gateway", "compose_project": "unraid-appops-gateway", "revision": revision, "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_artifacts": candidate_artifacts, "live_receipt": receipt, "rollback_snapshot": snapshot, "status": "success", "message": "verified candidate", "updated_at_epoch": 100, "artifacts": candidate_artifacts, "last_attempt": { "revision": revision, "status": "success", "message": "verified candidate", "updated_at_epoch": 100, "metadata": metadata, }, "previous_revision": old_revision, "rollback_restored_revision": "", } state_path = state_root / "unraid-appops-gateway.json" state_path.write_text(json.dumps(candidate_state, indent=2) + "\n", encoding="utf-8") state_path.chmod(0o644) original_snapshot = copy.deepcopy(snapshot) deploy_module.write_appops_legacy_intent(intent) @contextmanager def no_lock(*_args): yield monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", repository, revision]) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: receipt) monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) monkeypatch.setattr( deploy_module, "validate_recorded_receipt", lambda _project, observed, observed_repository, observed_revision: ( metadata if observed == receipt and observed_repository == repository and observed_revision == revision else pytest.fail("rollback receipt did not match the strict candidate fixture") ), ) monkeypatch.setattr( deploy_module, "inspect_image", lambda _ref: {"Id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID}, ) monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: { "containers": {"Unraid-AppOps-Gateway": { "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, "image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, "container_revision": old_revision, "image_revision": old_revision, "image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0], "image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE, "image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION, "image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED, }}, }) restored = {"runtime": False} def restore(project, _snapshot, _repository, _failed_revision): assert _snapshot == original_snapshot restored["runtime"] = True project["_appops_legacy_restored"] = True def inspect_restored_runtime(_project, *, require_original_container): assert require_original_container is False assert restored["runtime"] is True return ({}, {}, {}, {}) runtime_kinds = [] def classify_runtime(_project, observed_intent): assert observed_intent["rollbackSnapshot"] == original_snapshot runtime_kinds.append("candidate") return "candidate" monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", classify_runtime) monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored_runtime) monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) original_verify_backup = deploy_module.verify_appops_data_backup def verify_backup(observed, requested, previous): assert observed == backup assert requested == revision and previous == old_revision original_verify_backup(observed, requested, previous) monkeypatch.setattr(deploy_module, "verify_appops_data_backup", verify_backup) monkeypatch.setattr( deploy_module, "_appops_legacy_tag_snapshot", lambda observed: observed == original_snapshot or pytest.fail("rollback snapshot changed"), ) monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore) original_write_intent = deploy_module.write_appops_legacy_intent fail_after_state_write = {"enabled": True} def interrupt_completion(payload, **kwargs): if payload.get("status") == "rolled_back" and fail_after_state_write["enabled"]: fail_after_state_write["enabled"] = False raise OSError("simulated crash after the state CAS") return original_write_intent(payload, **kwargs) monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", interrupt_completion) # Reconcile the earlier state-write/intent-write gap against the real # private intent file. Its rollback snapshot remains the original capture. monkeypatch.setattr(deploy_module, "read_appops_state_preimage", deploy_module.read_appops_state_preimage) deploy_module.reconcile_appops_deployed_intent( _project(tmp_path) | {"_repository_name": repository, "branch": "main"}, repository, tmp_path, revision, metadata, receipt, ) deployed_intent, _ = deploy_module.read_appops_legacy_intent() candidate_preimage = deploy_module.read_appops_state_preimage(_project(tmp_path))[2] assert deployed_intent["status"] == "deployed" assert deployed_intent["stateObservedSha256"] == candidate_preimage assert deployed_intent["rollbackSnapshot"] == original_snapshot assert deployed_intent["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] == capture_sha256 candidate_raw = state_path.read_bytes() drifted_state = json.loads(candidate_raw.decode("utf-8")) drifted_state["message"] = "unexpected external state edit" state_path.write_text(json.dumps(drifted_state, indent=2) + "\n", encoding="utf-8") state_path.chmod(0o644) with pytest.raises(deploy_module.DeployError, match="live state changed"): deploy_module.main() still_deployed, _ = deploy_module.read_appops_legacy_intent() assert still_deployed["status"] == "deployed" assert runtime_kinds == [] state_path.write_bytes(candidate_raw) state_path.chmod(0o644) # Verify backup assertions on rollback calls; deployment checkpoint # reconciliation above is intentionally outside the rollback fixture. monkeypatch.setattr( deploy_module, "verify_appops_data_backup", lambda observed, requested, previous: ( observed == backup and requested == revision and previous == old_revision ) or pytest.fail("rollback data backup did not match its immutable intent"), ) with pytest.raises(OSError, match="simulated crash after the state CAS"): deploy_module.main() pending_intent, _ = deploy_module.read_appops_legacy_intent() restored_state, restored_raw, restored_sha256 = deploy_module.read_appops_state_preimage( _project(tmp_path) ) assert pending_intent["status"] == "rollback_pending" assert restored_state["live_status"] == "healthy" assert restored_state["live_revision"] == old_revision assert restored_state["live_legacy_provenance"] is True assert restored_state["live_receipt"] == {} assert restored_state["rollback_snapshot"] == {} assert pending_intent["stateObservedSha256"] == candidate_preimage assert pending_intent["rollbackSnapshot"] == original_snapshot assert hashlib.sha256(restored_raw).hexdigest() == restored_sha256 # The actual next CLI retry recognizes the exact controller-written # legacy state and only closes the durable intent checkpoint. assert deploy_module.main() == 0 completed_intent, _ = deploy_module.read_appops_legacy_intent() final_state, final_raw, final_sha256 = deploy_module.read_appops_state_preimage(_project(tmp_path)) assert completed_intent["status"] == "rolled_back" assert completed_intent["stateObservedSha256"] == final_sha256 assert completed_intent["rollbackSnapshot"] == original_snapshot assert completed_intent["rollbackSnapshot"]["appops_legacy"]["stateObservedSha256"] == capture_sha256 assert final_state["live_receipt"] == {} assert final_state["rollback_snapshot"] == {} assert hashlib.sha256(final_raw).hexdigest() == final_sha256 assert runtime_kinds == ["candidate"] assert json.loads(capsys.readouterr().out)["status"] == "rolled-back" @pytest.mark.parametrize( "runtime_kind", ["candidate", "absent", "legacy_partial", "unknown"] ) def test_appops_candidate_crash_recovers_before_prepare_and_fails_forward( deploy_module, monkeypatch, tmp_path, runtime_kind ): repository = "jens/unraid-appops-gateway" revision = "c" * 40 old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION state_root = tmp_path / "state" state_root.mkdir() intent_root = tmp_path / "appops-legacy-intent" intent_root.mkdir(mode=0o700) intent_root.chmod(0o700) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json" ) legacy_state = _appops_known_legacy_state() raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode() state_sha256 = hashlib.sha256(raw_state).hexdigest() monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", deploy_module._canonical_json_sha256(legacy_state["last_attempt"]["metadata"]), ) repository, metadata, source_proof, protected_main, backup, intent = ( _appops_rollback_intent_fixture(deploy_module, revision, state_sha256) ) intent["status"] = "deploying" state_path = state_root / "unraid-appops-gateway.json" state_path.write_bytes(raw_state) state_path.chmod(0o644) deploy_module.write_appops_legacy_intent(intent) project_config = { "slug": "unraid-appops-gateway", "branch": "main", "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": ["Unraid-AppOps-Gateway"], "gitea_repository_id": 9, "verify_oci_revision": True, "environment": { "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, }, } projects = {repository: project_config} original_snapshot = copy.deepcopy(intent["rollbackSnapshot"]) restored = {"runtime": False, "snapshot": None} classification_calls = [] @contextmanager def no_lock(*_args): yield def classify(_project, observed_intent): assert observed_intent["targetRevision"] == revision assert observed_intent["rollbackSnapshot"] == original_snapshot classification_calls.append(runtime_kind) return runtime_kind def restore(project, observed_snapshot, _repository, _failed_revision): assert observed_snapshot == original_snapshot restored["runtime"] = True restored["snapshot"] = observed_snapshot project["_appops_legacy_restored"] = True def inspect_restored(_project, *, require_original_container): assert require_original_container is False assert restored["runtime"] is True return ({}, {}, {}, {}) old_artifacts = {"containers": {"Unraid-AppOps-Gateway": { "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, "image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, "container_revision": old_revision, "image_revision": old_revision, "image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0], "image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE, "image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION, "image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED, }}} monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision]) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None) monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state) monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None) monkeypatch.setattr( deploy_module, "prepare_appops_legacy_migration", lambda *_args: pytest.fail("partial AppOps runtime must recover before migration preparation"), ) monkeypatch.setattr(deploy_module, "write_appops_progress", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: old_artifacts) monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", classify) monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored) monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda observed: ( observed == original_snapshot or pytest.fail("recovery snapshot was not intent-owned") )) monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore) if runtime_kind == "unknown": with pytest.raises(deploy_module.AppOpsLegacyNoMutation): deploy_module.main() untouched_intent, _ = deploy_module.read_appops_legacy_intent() untouched_state, untouched_raw, untouched_sha256 = deploy_module.read_appops_state_preimage( _project(tmp_path) ) assert untouched_intent["status"] == "deploying" assert untouched_state == legacy_state assert untouched_raw == raw_state assert untouched_sha256 == state_sha256 assert restored["runtime"] is False assert classification_calls == [runtime_kind] return with pytest.raises(deploy_module.AppOpsLegacyRecoveryComplete): deploy_module.main() rolled_state, rolled_raw, rolled_sha256 = deploy_module.read_appops_state_preimage( _project(tmp_path) ) recovered_intent, _ = deploy_module.read_appops_legacy_intent() assert classification_calls == [runtime_kind, runtime_kind] assert restored["snapshot"] == original_snapshot assert recovered_intent["status"] == "rolled_back" assert recovered_intent["stateObservedSha256"] == rolled_sha256 assert recovered_intent["rollbackSnapshot"] == original_snapshot assert rolled_state["live_revision"] == old_revision assert rolled_state["live_status"] == "healthy" assert rolled_state["live_legacy_provenance"] is True assert rolled_state["live_receipt"] == {} assert rolled_state["rollback_snapshot"] == {} assert hashlib.sha256(rolled_raw).hexdigest() == rolled_sha256 retry_prepared = [] runtime_proof = dict(recovered_intent["migrationProof"]["runtime"]) def prepare_same_candidate(*_args): retry_prepared.append(True) raise deploy_module.AppOpsLegacyNoMutation("same candidate reached safe preparation") monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({}, "a" * 64, "b" * 64)) monkeypatch.setattr(deploy_module, "appops_legacy_runtime_hashes", lambda *_args: runtime_proof) monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare_same_candidate) monkeypatch.setattr(deploy_module, "restore_live_source", lambda *_args: None) with pytest.raises(deploy_module.AppOpsLegacyNoMutation, match="same candidate reached safe preparation"): deploy_module.main() assert retry_prepared == [True] def test_appops_forward_failure_rollback_is_pending_before_state_commit_and_resumes( deploy_module, monkeypatch, tmp_path ): repository = "jens/unraid-appops-gateway" revision = "c" * 40 old_revision = deploy_module.APPOPS_LEGACY_OLD_REVISION missing_docker_root = tmp_path / "missing-docker-root" assert not missing_docker_root.exists() monkeypatch.setattr(deploy_module, "DOCKER_ROOT", missing_docker_root) state_root = tmp_path / "state" state_root.mkdir() intent_root = tmp_path / "appops-legacy-intent" intent_root.mkdir(mode=0o700) intent_root.chmod(0o700) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_INTENT_ROOT", intent_root) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_INTENT_PATH", intent_root / "unraid-appops-gateway.json" ) legacy_state = _appops_known_legacy_state() raw_state = (json.dumps(legacy_state, indent=2) + "\n").encode() state_sha256 = hashlib.sha256(raw_state).hexdigest() monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", deploy_module._canonical_json_sha256(legacy_state["last_attempt"]["metadata"]), ) repository, metadata, source_proof, protected_main, backup, intent = ( _appops_rollback_intent_fixture(deploy_module, revision, state_sha256) ) intent["status"] = "deploying" state_path = state_root / "unraid-appops-gateway.json" state_path.write_bytes(raw_state) state_path.chmod(0o644) intent_digest = deploy_module.write_appops_legacy_intent(intent) project_config = { "slug": "unraid-appops-gateway", "branch": "main", "source_dir": "/srv/example/appdata/unraid-appops-gateway/source", "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": ["Unraid-AppOps-Gateway"], "gitea_repository_id": 9, "verify_oci_revision": True, "environment": { "APPOPS_DATA_PATH": deploy_module.APPOPS_LEGACY_DATA_PATH, "APPOPS_PUBLISH_HOST": deploy_module.APPOPS_LEGACY_PUBLISH_HOST, }, } projects = {repository: project_config} snapshot = copy.deepcopy(intent["rollbackSnapshot"]) old_artifacts = {"containers": {"Unraid-AppOps-Gateway": { "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, "image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, "container_revision": old_revision, "image_revision": old_revision, "image_source": sorted(deploy_module.APPOPS_LEGACY_SOURCE_ALIASES)[0], "image_tree": deploy_module.APPOPS_LEGACY_OLD_TREE, "image_version": deploy_module.APPOPS_LEGACY_OLD_VERSION, "image_created": deploy_module.APPOPS_LEGACY_OLD_CREATED, }}} runtime_kinds = iter(["legacy", "candidate"]) restored = {"runtime": False} progress = [] @contextmanager def no_lock(*_args): yield def prepare(project, _repository, _repo, _revision, _metadata, state, raw, digest): assert digest == state_sha256 assert raw == raw_state and state == legacy_state migration = { "intent": intent, "intent_sha256": intent_digest, "state_sha256": digest, "state": state, "raw_state": raw, "snapshot": snapshot, "backup": backup, } project["_appops_legacy_migration"] = migration project["_compose_rollback"] = snapshot project["_appops_data_backup"] = backup return migration def restore(project, observed_snapshot, _repository, _failed_revision): assert observed_snapshot == snapshot pending_intent, _ = deploy_module.read_appops_legacy_intent() assert pending_intent["status"] == "rollback_pending" restored["runtime"] = True project["_rollback_restored_revision"] = old_revision project["_appops_legacy_restored"] = True def inspect_restored(_project, *, require_original_container): assert require_original_container is False assert restored["runtime"] is True return ({}, {}, {}, {}) monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", repository, revision]) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args, **_kwargs: metadata) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_args: None) monkeypatch.setattr(deploy_module, "read_state", lambda _project: legacy_state) monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) monkeypatch.setattr(deploy_module, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) monkeypatch.setattr(deploy_module, "reconcile_appops_legacy_candidate", lambda *_args: None) monkeypatch.setattr(deploy_module, "prepare_appops_legacy_migration", prepare) monkeypatch.setattr(deploy_module, "write_appops_progress", lambda _revision, phase, **_kwargs: progress.append(phase)) monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: old_artifacts) monkeypatch.setattr(deploy_module, "_appops_legacy_recovery_runtime_kind", lambda *_args: next(runtime_kinds)) monkeypatch.setattr(deploy_module, "_appops_legacy_inspect_runtime", inspect_restored) monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda observed: ( observed == snapshot or pytest.fail("forward failure did not use the intent-owned snapshot") )) monkeypatch.setattr(deploy_module, "restore_compose_snapshot", restore) capacity_checks = [] def skip_docker_capacity_for_rollback_test(): assert not deploy_module.DOCKER_ROOT.exists() capacity_checks.append(True) monkeypatch.setattr(deploy_module, "ensure_docker_capacity", skip_docker_capacity_for_rollback_test) deploy_calls = [] def fail_deploy(*_args): deploy_calls.append(revision) raise RuntimeError("simulated candidate failure") monkeypatch.setattr(deploy_module, "deploy_project", fail_deploy) original_write_intent = deploy_module.write_appops_legacy_intent def crash_before_terminal_intent(payload, **kwargs): if payload.get("status") == "rolled_back": observed_state, _raw, _digest = deploy_module.read_appops_state_preimage( _project(tmp_path) ) assert observed_state["status"] == "success" assert observed_state["live_revision"] == old_revision raise OSError("simulated crash after rollback state CAS") return original_write_intent(payload, **kwargs) monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", crash_before_terminal_intent) with pytest.raises(deploy_module.AppOpsLegacyRecoveryPending): deploy_module.main() assert capacity_checks == [True] assert deploy_calls == [revision] pending_intent, _ = deploy_module.read_appops_legacy_intent() pending_state, pending_raw, pending_sha256 = deploy_module.read_appops_state_preimage( _project(tmp_path) ) assert pending_intent["status"] == "rollback_pending" assert pending_intent["stateObservedSha256"] == state_sha256 assert pending_intent["rollbackSnapshot"] == snapshot assert pending_state["status"] == "success" assert pending_state["live_revision"] == old_revision assert pending_state["live_status"] == "healthy" assert pending_state["live_receipt"] == {} assert hashlib.sha256(pending_raw).hexdigest() == pending_sha256 assert progress[-2:] == ["rolling_back", "outcome_unknown"] monkeypatch.setattr(deploy_module, "write_appops_legacy_intent", original_write_intent) with pytest.raises(deploy_module.AppOpsLegacyRecoveryComplete): deploy_module.main() completed_intent, _ = deploy_module.read_appops_legacy_intent() assert completed_intent["status"] == "rolled_back" assert completed_intent["stateObservedSha256"] == pending_sha256 def test_appops_durable_rollback_requires_completed_intent_and_verified_backup( deploy_module, monkeypatch, tmp_path ): revision = "c" * 40 tree = "d" * 40 repository = "jens/unraid-appops-gateway" metadata = { "id": revision[:12], "repository": repository, "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-10-02T10:00:00.000Z", } source_proof = { "legacyRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "legacyTree": deploy_module.APPOPS_LEGACY_OLD_TREE, "databaseTree": "1" * 40, "entrypointBlob": "2" * 40, "candidateRevision": revision, "composeBlob": "3" * 40, "dockerfileBlob": "4" * 40, } protected_main = { "semanticSha256": "5" * 64, "revision": revision, "source": "gitea_webhook", "validationContext": "Managed validation / full (push)", "validationStatus": "success", "delivery": "123e4567-e89b-42d3-a456-426614174000", "bodySha256": "6" * 64, "validationSha256": "7" * 64, } runtime_proof = { "resolvedComposeSha256": "8" * 64, "resolvedEnvironmentSha256": "9" * 64, "runtimeConfigSha256": "a" * 64, "mountsSha256": "b" * 64, "healthSha256": "c" * 64, } backup_receipt = { "schema": "itworx.appops-data-backup/v1", "source": deploy_module.APPOPS_LEGACY_DATA_PATH, "archive": "/srv/example/appdata/unraid-appops-gateway-backups/one/appdata.tar.gz", "sha256": "d" * 64, "bytes": 12, "requestedRevision": revision, "previousRevision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "createdAt": "2026-10-02T10:00:00.000Z", } backup = { **backup_receipt, "receiptSha256": deploy_module._canonical_json_sha256(backup_receipt), } snapshot = { "revision": deploy_module.APPOPS_LEGACY_OLD_REVISION, "legacy_provenance": True, "receipt": {}, "compose_containers": ["Unraid-AppOps-Gateway"], "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID, "image_id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID, "image_ref": deploy_module.APPOPS_LEGACY_OLD_IMAGE_REF, "restore_image_ref": deploy_module.APPOPS_SOURCE_COMPOSE_IMAGE_REF, "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", "was_running": True, }], "captured_at_epoch": 1790884709, } intent = deploy_module._appops_legacy_intent_payload( revision=revision, metadata=metadata, state_sha256="e" * 64, observed_state_sha256="e" * 64, source_proof=source_proof, protected_main=protected_main, runtime_proof=runtime_proof, runtime={"containerId": deploy_module.APPOPS_LEGACY_OLD_CONTAINER_ID}, backup=backup, snapshot=snapshot, status="deployed", ) project = _project(tmp_path) project.update({"branch": "main", "_repository_name": repository}) current_receipt = { "repository": repository, "revision": revision, "tree": tree, "build": metadata, } state = { "schema_version": 3, "repository": repository, "revision": revision, "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": current_receipt, "rollback_snapshot": intent["rollbackSnapshot"], } monkeypatch.setattr(deploy_module, "read_state", lambda _project: state) monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "1" * 64)) monkeypatch.setattr(deploy_module, "validate_recorded_receipt", lambda *_args: metadata) backup_checks = [] monkeypatch.setattr( deploy_module, "verify_appops_data_backup", lambda observed, requested, previous: backup_checks.append((observed, requested, previous)), ) monkeypatch.setattr( deploy_module, "inspect_image", lambda _ref: {"Id": deploy_module.APPOPS_LEGACY_OLD_IMAGE_ID}, ) assert deploy_module.durable_rollback_snapshot(project, repository, revision) == intent["rollbackSnapshot"] assert backup_checks == [(backup, revision, deploy_module.APPOPS_LEGACY_OLD_REVISION)] intent["status"] = "rolled_back" with pytest.raises(deploy_module.DeployError, match="completed migration intent"): deploy_module.durable_rollback_snapshot(project, repository, revision) def test_appops_candidate_crash_reconcile_writes_only_from_exact_anchored_state( deploy_module, monkeypatch, tmp_path ): repository = "jens/unraid-appops-gateway" revision = "c" * 40 tree = "d" * 40 metadata = { "id": revision[:12], "repository": repository, "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-10-02T10:00:00.000Z", } state = _appops_known_legacy_state() raw = (json.dumps(state, indent=2) + "\n").encode() state_sha256 = hashlib.sha256(raw).hexdigest() monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_STATE_SHA256", state_sha256) monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_ATTEMPT_METADATA_SHA256", hashlib.sha256( json.dumps(state["last_attempt"]["metadata"], sort_keys=True, separators=(",", ":")).encode() ).hexdigest()) protected_main = {"revision": revision, "delivery": "same-green-main"} source_proof = {"candidateRevision": revision, "databaseTree": "1" * 40} intent = { "status": "deploying", "targetRevision": revision, "targetMetadata": metadata, "stateObservedSha256": state_sha256, "migrationProof": {"protectedMain": protected_main, "source": source_proof}, "rollbackSnapshot": {}, "dataBackup": {}, } project = _project(tmp_path) project["_repository_path"] = str(tmp_path) receipt = { "repository": repository, "revision": revision, "tree": tree, "build": metadata, } writes = [] intent_writes = [] monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "a" * 64)) monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) monkeypatch.setattr(deploy_module, "appops_legacy_compose_model", lambda *_args: ({"name": project["compose_project"]}, "b" * 64, "c" * 64)) monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) monkeypatch.setattr(deploy_module, "_validate_appops_legacy_snapshot", lambda *_args: None) monkeypatch.setattr(deploy_module, "_appops_legacy_tag_snapshot", lambda *_args: None) monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args: receipt) monkeypatch.setattr(deploy_module, "write_state", lambda *args, **kwargs: writes.append((args, kwargs))) monkeypatch.setattr(deploy_module, "read_appops_state_preimage", lambda _project: (state, raw, state_sha256)) monkeypatch.setattr( deploy_module, "write_appops_legacy_intent", lambda payload, **kwargs: intent_writes.append((payload, kwargs)) or "d" * 64, ) observed = deploy_module.reconcile_appops_legacy_candidate( project, repository, tmp_path, revision, metadata, state, raw, state_sha256, ) assert observed == receipt assert len(writes) == 1 and writes[0][1]["expected_state_sha256"] == state_sha256 assert writes[0][1]["rollback_snapshot"] == {} assert intent_writes[0][0]["status"] == "deployed" writes.clear() changed_raw = raw + b" " assert deploy_module.reconcile_appops_legacy_candidate( project, repository, tmp_path, revision, metadata, state, changed_raw, hashlib.sha256(changed_raw).hexdigest(), ) is None assert writes == [] def test_appops_state_write_rejects_compare_and_swap_drift(deploy_module, monkeypatch, tmp_path): state_root = tmp_path / "state" state_root.mkdir() monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) project = _project(tmp_path) path = state_root / "unraid-appops-gateway.json" original = json.dumps({"schema_version": 3, "repository": "jens/unraid-appops-gateway"}).encode() path.write_bytes(original) path.chmod(0o644) with pytest.raises(deploy_module.DeployError, match="compare-and-swap"): deploy_module.write_state( "jens/unraid-appops-gateway", project, "c" * 40, "failed", "blocked", expected_state_sha256="0" * 64, ) assert path.read_bytes() == original def test_appops_schema_compatibility_compares_full_database_tree_and_entrypoint( deploy_module, tmp_path, monkeypatch ): repository = tmp_path / "source.git" working = tmp_path / "work" working.mkdir() subprocess.run(["git", "init", "--bare", str(repository)], check=True, capture_output=True) subprocess.run(["git", "init", str(working)], check=True, capture_output=True) subprocess.run(["git", "-C", str(working), "config", "user.name", "Fixture"], check=True) subprocess.run(["git", "-C", str(working), "config", "user.email", "fixture@example.invalid"], check=True) db = working / "backend" / "app" / "db" db.mkdir(parents=True) entrypoint = working / "docker" / "entrypoint.sh" entrypoint.parent.mkdir(parents=True) (db / "models.py").write_text("MODEL = 1\n") (db / "init_db.py").write_text("INIT = 1\n") (db / "session.py").write_text("SESSION = 1\n") entrypoint.write_text("#!/bin/sh\nexec app\n") subprocess.run(["git", "-C", str(working), "add", "."], check=True) subprocess.run(["git", "-C", str(working), "commit", "-m", "legacy"], check=True, capture_output=True) old_revision = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip() subprocess.run(["git", "-C", str(working), "remote", "add", "origin", str(repository)], check=True) subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True) (working / "backend" / "app" / "core").mkdir(parents=True) (working / "backend" / "app" / "core" / "config.py").write_text("STATS_KEY = 1\n") subprocess.run(["git", "-C", str(working), "add", "."], check=True) subprocess.run(["git", "-C", str(working), "commit", "-m", "non-db stats config"], check=True, capture_output=True) candidate = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip() subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True) monkeypatch.setattr(deploy_module, "APPOPS_LEGACY_OLD_REVISION", old_revision) monkeypatch.setattr( deploy_module, "APPOPS_LEGACY_OLD_TREE", subprocess.check_output( ["git", "-C", str(working), "rev-parse", f"{old_revision}^{{tree}}"], text=True, ).strip(), ) assert deploy_module.verify_appops_schema_compatibility(repository, candidate)["databaseTree"] (db / "migrations").mkdir() (db / "migrations" / "001.sql").write_text("DROP TABLE app;\n") subprocess.run(["git", "-C", str(working), "add", "."], check=True) subprocess.run(["git", "-C", str(working), "commit", "-m", "schema migration"], check=True, capture_output=True) changed = subprocess.check_output(["git", "-C", str(working), "rev-parse", "HEAD"], text=True).strip() subprocess.run(["git", "-C", str(working), "push", "origin", "HEAD:refs/heads/main"], check=True, capture_output=True) with pytest.raises(deploy_module.DeployError, match="database schema or entrypoint"): deploy_module.verify_appops_schema_compatibility(repository, changed) def test_appops_migration_requires_private_exact_full_green_delivery( deploy_module, monkeypatch, tmp_path ): revision = "c" * 40 state_root = tmp_path / "state" inbox = state_root / "webhook-inbox" inbox.mkdir(parents=True, mode=0o700) inbox.chmod(0o700) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) semantic = deploy_module._canonical_json_sha256({ "gitea_repository_id": 9, "ref": "refs/heads/main", "revision": revision, }) record = { "schema": "gitea-webhook-inbox.v2", "semantic_sha256": semantic, "body_sha256": "a" * 64, "delivery": "12345678-1234-4234-8234-123456789012", "gitea_repository_id": 9, "ref": "refs/heads/main", "repository": "jens/unraid-appops-gateway", "revision": revision, "source": "typed_self_upgrade", "status": "deploying", "validation": { "validation_gate_version": 1, "required_contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, "last_result": "pass", "observed": [{"context": "Managed validation / full (push)", "status": "success"}], }, "audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"}], } marker = inbox / f"{semantic}.json" marker.write_text(json.dumps(record), encoding="utf-8") marker.chmod(0o600) proof = deploy_module.verify_appops_green_delivery(revision) assert proof["validationContext"] == "Managed validation / full (push)" # The webhook worker atomically makes this same immutable record terminal # after deploy.py has already committed the success state. A restarted # broker must still be able to finish the protected intent checkpoint. record["status"] = "deployed" marker.write_text(json.dumps(record), encoding="utf-8") marker.chmod(0o600) assert deploy_module.verify_appops_green_delivery(revision) == proof record["validation"]["observed"][0]["status"] = "pending" marker.write_text(json.dumps(record), encoding="utf-8") marker.chmod(0o600) with pytest.raises(deploy_module.DeployError, match="protected full-green"): deploy_module.verify_appops_green_delivery(revision) record["status"] = "failed" record["validation"]["observed"][0]["status"] = "success" marker.write_text(json.dumps(record), encoding="utf-8") marker.chmod(0o600) with pytest.raises(deploy_module.DeployError, match="protected full-green"): deploy_module.verify_appops_green_delivery(revision) marker.unlink() marker.symlink_to(tmp_path / "untrusted.json") with pytest.raises(deploy_module.DeployError, match="unsafe"): deploy_module.verify_appops_green_delivery(revision) def test_appops_retry_completes_pending_intent_after_delivery_becomes_deployed( deploy_module, monkeypatch, tmp_path ): repository = "jens/unraid-appops-gateway" revision = "c" * 40 tree = "d" * 40 metadata = { "id": revision[:12], "repository": repository, "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-10-02T10:00:00.000Z", } protected_main = {"revision": revision, "delivery": "green-main-delivery"} source_proof = {"candidateRevision": revision, "databaseTree": "1" * 40} snapshot = {"revision": deploy_module.APPOPS_LEGACY_OLD_REVISION} receipt = {"repository": repository, "revision": revision, "tree": tree, "build": metadata} state = { "repository": repository, "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": receipt, "rollback_snapshot": snapshot, } state_sha256 = "a" * 64 intent = { "status": "deploying", "targetRevision": revision, "targetMetadata": metadata, "migrationProof": {"protectedMain": protected_main, "source": source_proof}, "rollbackSnapshot": snapshot, "dataBackup": {"sha256": "b" * 64}, } project = _project(tmp_path) project.update({"_repository_name": repository, "branch": "main"}) intent_writes = [] monkeypatch.setattr(deploy_module, "read_appops_legacy_intent", lambda: (intent, "c" * 64)) monkeypatch.setattr( deploy_module, "read_appops_state_preimage", lambda _project: (state, b"controller-committed-success", state_sha256), ) monkeypatch.setattr(deploy_module, "verify_appops_green_delivery", lambda _revision: protected_main) monkeypatch.setattr(deploy_module, "verify_appops_legacy_source", lambda *_args: source_proof) monkeypatch.setattr(deploy_module, "verify_appops_data_backup", lambda *_args: None) monkeypatch.setattr( deploy_module, "write_appops_legacy_intent", lambda payload, **kwargs: intent_writes.append((payload, kwargs)) or "d" * 64, ) # The state-write/intention-write crash leaves the original deploying # intent, while the webhook worker subsequently marks its durable green # record deployed. The current-receipt retry must only close that gap. deploy_module.reconcile_appops_deployed_intent( project, repository, tmp_path, revision, metadata, receipt, ) assert len(intent_writes) == 1 assert intent_writes[0][0]["status"] == "deployed" assert intent_writes[0][0]["stateObservedSha256"] == state_sha256 assert intent_writes[0][1]["expected_sha256"] == "c" * 64 def _workmesh_product_project(tmp_path: Path) -> dict[str, object]: return { "slug": "workmesh", "source_dir": str(tmp_path / "source"), "compose_project": "workmesh", "compose_files": ["compose.yaml"], "containers": ["workmesh-postgres-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1"], "retired_compose_containers": ["workmesh-mock-ats-1"], "_repository_name": "jens/workmesh", } def _workmesh_legacy_reattest_setup(tmp_path: Path): revision = "a" * 40 project = _workmesh_product_project(tmp_path) project.update({ "branch": "main", "containers": [ "workmesh-postgres-1", "workmesh-mock-ats-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1", ], "compose_files": ["compose.yaml"], "exact_source": True, "verify_oci_revision": False, }) project.pop("retired_compose_containers") receipt = _receipt(project, revision) for index, name in enumerate(project["containers"], start=1): receipt["containers"][name]["container_id"] = f"{index:x}" * 64 receipt["containers"][name]["container_revision"] = "" for field in ("image_revision", "image_source", "image_tree", "image_version", "image_created"): receipt["containers"][name][field] = "" receipt_containers = receipt["containers"] artifacts = { "containers": { name: dict(item) for name, item in receipt_containers.items() }, "compose_sha256": "c" * 64, } state = { "schema_version": 3, "repository": "jens/workmesh", "slug": "workmesh", "compose_project": "workmesh", "revision": revision, "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_artifacts": copy.deepcopy(artifacts), "live_receipt": copy.deepcopy(receipt), "rollback_snapshot": {}, "status": "success", "message": "verified live WorkMesh deployment", "updated_at_epoch": 100, "artifacts": copy.deepcopy(artifacts), "last_attempt": { "revision": revision, "status": "success", "message": "verified live WorkMesh deployment", "updated_at_epoch": 100, "metadata": copy.deepcopy(receipt["build"]), }, "previous_revision": "", "rollback_restored_revision": "", } configured = copy.deepcopy(project) configured.pop("_repository_name") project.update({ "_repository_name": "jens/workmesh", "_repository_path": str(tmp_path), "_deployment_metadata": copy.deepcopy(receipt["build"]), }) projects = {"jens/workmesh": configured} state_root = tmp_path / "state" state_root.mkdir() state_path = state_root / "workmesh.json" original_bytes = (json.dumps(state, indent=2) + "\n").encode("utf-8") state_path.write_bytes(original_bytes) return revision, project, projects, receipt, artifacts, state_path, original_bytes def _install_workmesh_reattest_runtime( deploy_module, monkeypatch, receipt, artifacts, state_path, *, drift=None ): observed_receipt = copy.deepcopy(receipt) observed_artifacts = copy.deepcopy(artifacts["containers"]) observed_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64 observed_artifacts["workmesh-n8n-1"]["container_id"] = "f" * 64 if drift: name, field, value = drift observed_receipt["containers"][name][field] = value observed_artifacts[name][field] = value monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True) monkeypatch.setattr( deploy_module, "validate_commit", lambda _repo, _branch, revision: revision ) monkeypatch.setattr( deploy_module, "deployment_metadata", lambda _repo, _revision, _repository, *, built_at=None: copy.deepcopy(receipt["build"]), ) monkeypatch.setattr( deploy_module, "git_revision_compose_sha256", lambda *_args: str(artifacts["compose_sha256"]), ) monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: set(deploy_module.WORKMESH_LEGACY_CONTAINERS), ) monkeypatch.setattr( deploy_module, "verify_deployment", lambda _project, _build, **_kwargs: copy.deepcopy(observed_receipt), ) monkeypatch.setattr( deploy_module, "deployment_container_artifacts", lambda _project: copy.deepcopy(observed_artifacts), ) monkeypatch.setattr(deploy_module, "_fsync_directory", lambda _path: None) reverified = [] def verify_after_write(repository, _project, revision, tree): reverified.append((repository, revision, tree)) return json.loads(state_path.read_text(encoding="utf-8"))["live_receipt"] monkeypatch.setattr(deploy_module, "verified_current_receipt", verify_after_write) return reverified def test_workmesh_n8n_recreation_reattests_only_container_id( deploy_module, tmp_path, monkeypatch ): ( revision, project, projects, receipt, artifacts, state_path, original_bytes, ) = _workmesh_legacy_reattest_setup(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent) reverified = _install_workmesh_reattest_runtime( deploy_module, monkeypatch, receipt, artifacts, state_path ) result = deploy_module.reattest_workmesh_n8n_receipt( projects, project, tmp_path, revision ) persisted = json.loads(state_path.read_text(encoding="utf-8")) expected_receipt = copy.deepcopy(receipt) expected_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64 expected_artifacts = copy.deepcopy(artifacts) expected_artifacts["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64 assert result == { "revision": revision, "container": "workmesh-n8n-1", "status": "reattested", } assert persisted["live_receipt"] == expected_receipt assert persisted["live_artifacts"] == expected_artifacts assert persisted["artifacts"] == expected_artifacts assert reverified == [("jens/workmesh", revision, receipt["tree"])] backup = state_path.parent / ( f"workmesh-before-n8n-reattest-{revision[:12]}-{'4' * 12}.json" ) assert backup.read_bytes() == original_bytes if os.name != "nt": assert stat.S_IMODE(backup.stat().st_mode) == 0o600 assert persisted["status"] == "success" assert persisted["last_attempt"] == json.loads(original_bytes)["last_attempt"] @pytest.mark.parametrize( ("name", "field", "value"), [ ("workmesh-api-1", "image_id", "sha256:" + "9" * 64), ("workmesh-api-1", "image_tree", "d" * 40), ("workmesh-api-1", "container_id", "e" * 64), ], ) def test_workmesh_n8n_reattest_rejects_other_runtime_drift( deploy_module, tmp_path, monkeypatch, name, field, value ): ( revision, project, projects, receipt, artifacts, state_path, original_bytes, ) = _workmesh_legacy_reattest_setup(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent) _install_workmesh_reattest_runtime( deploy_module, monkeypatch, receipt, artifacts, state_path, drift=(name, field, value), ) with pytest.raises(deploy_module.DeployError, match="runtime drift"): deploy_module.reattest_workmesh_n8n_receipt( projects, project, tmp_path, revision ) assert state_path.read_bytes() == original_bytes assert list(state_path.parent.glob("workmesh-before-n8n-reattest-*.json")) == [] def test_workmesh_n8n_reattest_requires_compose_digest_from_live_revision( deploy_module, tmp_path, monkeypatch ): ( revision, project, projects, receipt, artifacts, state_path, original_bytes, ) = _workmesh_legacy_reattest_setup(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent) _install_workmesh_reattest_runtime( deploy_module, monkeypatch, receipt, artifacts, state_path ) monkeypatch.setattr( deploy_module, "git_revision_compose_sha256", lambda *_args: "d" * 64, ) with pytest.raises(deploy_module.DeployError, match="Compose digest"): deploy_module.reattest_workmesh_n8n_receipt( projects, project, tmp_path, revision ) assert state_path.read_bytes() == original_bytes assert list(state_path.parent.glob("workmesh-before-n8n-reattest-*.json")) == [] def test_workmesh_n8n_reattest_rejects_non_root_controller( deploy_module, tmp_path, monkeypatch ): revision, project, projects, _receipt_value, _artifacts, state_path, original_bytes = ( _workmesh_legacy_reattest_setup(tmp_path) ) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent) monkeypatch.setattr( deploy_module, "require_host_root", lambda: (_ for _ in ()).throw( deploy_module.DeployError("WorkMesh live receipt reattestation is host-root-only") ), ) with pytest.raises(deploy_module.DeployError, match="host-root-only"): deploy_module.reattest_workmesh_n8n_receipt( projects, project, tmp_path, revision ) assert state_path.read_bytes() == original_bytes def test_workmesh_n8n_mode_dispatches_under_both_controller_locks( deploy_module, tmp_path, monkeypatch, capsys ): revision = "a" * 40 project_config = { "slug": "workmesh", "branch": "main", "source_dir": str(tmp_path / "source"), "compose_project": "workmesh", "compose_files": ["compose.yaml"], "containers": list(deploy_module.WORKMESH_LEGACY_CONTAINERS), "exact_source": True, "verify_oci_revision": True, } projects = {"jens/workmesh": project_config} events = [] @contextmanager def project_lock(path, _description): events.append(("project-lock", path)) yield @contextmanager def global_lock(): events.append(("global-lock",)) yield monkeypatch.setattr(sys, "argv", [ "deploy.py", "reattest-workmesh-n8n", "jens/workmesh", revision, ]) monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr( deploy_module, "validate_commit", lambda _repo, _branch, requested: requested ) monkeypatch.setattr( deploy_module, "deployment_metadata", lambda *_args: {"tree": "b" * 40} ) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock) def reattest(configured_projects, project, repo, requested_revision): assert configured_projects == projects assert project["_repository_name"] == "jens/workmesh" assert repo == tmp_path assert requested_revision == revision events.append(("reattest",)) return {"revision": revision, "container": "workmesh-n8n-1", "status": "reattested"} monkeypatch.setattr(deploy_module, "reattest_workmesh_n8n_receipt", reattest) assert deploy_module.main() == 0 assert events == [ ("project-lock", deploy_module.STATE_ROOT / "locks" / "workmesh.lock"), ("global-lock",), ("reattest",), ] assert json.loads(capsys.readouterr().out) == { "repository": "jens/workmesh", "revision": revision, "container": "workmesh-n8n-1", "status": "reattested", } def _workmesh_product_reattest_setup(deploy_module, tmp_path: Path): revision = "a" * 40 project = { "slug": "workmesh", "branch": "main", "source_dir": (deploy_module.EXACT_SOURCE_ROOT / "workmesh").as_posix(), "env_source": "/srv/example/appdata/workmesh/.env", "compose_project": "workmesh", "compose_files": ["compose.yaml"], "containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS), "retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER], "exact_source": True, "verify_oci_revision": False, "health_timeout": 300, } receipt = _receipt({**project, "_repository_name": "jens/workmesh"}, revision) for index, name in enumerate(project["containers"], start=1): receipt["containers"][name]["container_id"] = f"{index:x}" * 64 receipt["containers"][name]["container_revision"] = "" receipt["containers"][name]["image_revision"] = "" receipt["containers"][name]["image_source"] = "" receipt["containers"][name]["image_tree"] = "" receipt["containers"][name]["image_version"] = "" receipt["containers"][name]["image_created"] = "" artifacts = { "containers": copy.deepcopy(receipt["containers"]), "compose_sha256": "c" * 64, } candidate_revision = "d" * 40 state = { "schema_version": 3, "repository": "jens/workmesh", "slug": "workmesh", "compose_project": "workmesh", "revision": revision, "live_revision": revision, "live_status": "unknown", "live_legacy_provenance": False, "live_artifacts": copy.deepcopy(artifacts), "live_receipt": copy.deepcopy(receipt), "rollback_snapshot": {}, "status": "failed", "message": "candidate deploy failed; live state needs reconciliation", "updated_at_epoch": 100, "artifacts": copy.deepcopy(artifacts), "last_attempt": { "revision": candidate_revision, "status": "failed", "message": "candidate deploy failed", "updated_at_epoch": 101, "metadata": {"revision": candidate_revision}, }, "previous_revision": "", "rollback_restored_revision": "", } configured = copy.deepcopy(project) project.update({ "_repository_name": "jens/workmesh", "_repository_path": str(tmp_path), "_deployment_metadata": copy.deepcopy(receipt["build"]), }) state_root = tmp_path / "state" state_root.mkdir() state_path = state_root / "workmesh.json" original_bytes = (json.dumps(state, indent=2) + "\n").encode() state_path.write_bytes(original_bytes) return revision, project, {"jens/workmesh": configured}, receipt, artifacts, state_path, original_bytes def test_workmesh_product_n8n_recovery_repairs_only_n8n_id_after_failed_attempt( deploy_module, tmp_path, monkeypatch ): ( revision, project, projects, receipt, artifacts, state_path, original_bytes, ) = _workmesh_product_reattest_setup(deploy_module, tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent) monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr( deploy_module, "exact_source_matches_revision", lambda _repo, requested, _source: requested == revision, ) monkeypatch.setattr( deploy_module, "deployment_metadata", lambda _repo, _requested, _repository, *, built_at=None: copy.deepcopy(receipt["build"]), ) monkeypatch.setattr( deploy_module, "git_revision_compose_sha256", lambda *_args: artifacts["compose_sha256"], ) monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: set(deploy_module.WORKMESH_PRODUCT_CONTAINERS), ) monkeypatch.setattr(deploy_module, "container_state", lambda _name: (True, "healthy", "workmesh")) observed_receipt = copy.deepcopy(receipt) observed_artifacts = copy.deepcopy(artifacts["containers"]) observed_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64 observed_artifacts["workmesh-n8n-1"]["container_id"] = "f" * 64 monkeypatch.setattr( deploy_module, "verify_deployment", lambda *_args, **_kwargs: copy.deepcopy(observed_receipt), ) monkeypatch.setattr( deploy_module, "deployment_container_artifacts", lambda _project: copy.deepcopy(observed_artifacts), ) monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: "n8n " + "e" * 64) monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"])) monkeypatch.setattr( deploy_module, "inspect_container", lambda name: { "Id": "f" * 64, "Image": "sha256:" + "1" * 64, "Config": { "Image": "workmesh-n8n-1:production", "Labels": { "com.docker.compose.project": "workmesh", "com.docker.compose.service": "n8n", "com.docker.compose.config-hash": "e" * 64, } }, "State": {"Running": True, "Health": {"Status": "healthy"}}, "Mounts": [ { "Type": "volume", "Name": "workmesh_workmesh_n8n_data", "Destination": "/home/node/.n8n", "RW": True, } ], } if name == "workmesh-n8n-1" else None, ) monkeypatch.setattr(deploy_module, "_fsync_directory", lambda _path: None) monkeypatch.setattr( deploy_module, "verified_current_receipt", lambda _repository, _project, _requested, _tree: json.loads( state_path.read_text(encoding="utf-8") )["live_receipt"], ) result = deploy_module.reattest_workmesh_product_n8n_receipt( projects, project, tmp_path, revision ) persisted = json.loads(state_path.read_text(encoding="utf-8")) assert result == { "revision": revision, "container": "workmesh-n8n-1", "status": "reattested", } assert persisted["live_status"] == "healthy" assert persisted["status"] == "failed" assert persisted["last_attempt"] == json.loads(original_bytes)["last_attempt"] assert persisted["live_receipt"]["containers"]["workmesh-n8n-1"]["container_id"] == "f" * 64 assert persisted["live_artifacts"]["containers"]["workmesh-n8n-1"]["container_id"] == "f" * 64 for name in deploy_module.WORKMESH_PRODUCT_CONTAINERS: if name != "workmesh-n8n-1": assert persisted["live_receipt"]["containers"][name] == receipt["containers"][name] backup = state_path.parent / ( f"workmesh-product-before-n8n-reattest-{revision[:12]}-{'3' * 12}.json" ) assert backup.read_bytes() == original_bytes def test_workmesh_product_n8n_recovery_refuses_volume_or_config_drift( deploy_module, tmp_path, monkeypatch ): ( revision, project, projects, receipt, artifacts, state_path, original_bytes, ) = _workmesh_product_reattest_setup(deploy_module, tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", state_path.parent) monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_args: True) monkeypatch.setattr( deploy_module, "deployment_metadata", lambda *_args, **_kwargs: copy.deepcopy(receipt["build"]), ) monkeypatch.setattr(deploy_module, "git_revision_compose_sha256", lambda *_args: artifacts["compose_sha256"]) monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda _name: set(deploy_module.WORKMESH_PRODUCT_CONTAINERS)) monkeypatch.setattr(deploy_module, "container_state", lambda _name: (True, "healthy", "workmesh")) observed_receipt = copy.deepcopy(receipt) observed_receipt["containers"]["workmesh-n8n-1"]["container_id"] = "f" * 64 observed_artifacts = copy.deepcopy(artifacts["containers"]) observed_artifacts["workmesh-n8n-1"]["container_id"] = "f" * 64 monkeypatch.setattr(deploy_module, "verify_deployment", lambda *_args, **_kwargs: copy.deepcopy(observed_receipt)) monkeypatch.setattr(deploy_module, "deployment_container_artifacts", lambda _project: copy.deepcopy(observed_artifacts)) monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: "n8n " + "e" * 64) monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"])) monkeypatch.setattr( deploy_module, "inspect_container", lambda _name: { "Id": "f" * 64, "Image": "sha256:" + "1" * 64, "Config": {"Image": "workmesh-n8n-1:production", "Labels": { "com.docker.compose.project": "workmesh", "com.docker.compose.service": "n8n", "com.docker.compose.config-hash": "e" * 64, }}, "State": {"Running": True, "Health": {"Status": "healthy"}}, "Mounts": [{ "Type": "volume", "Name": "wrong_workmesh_n8n_data", "Destination": "/home/node/.n8n", "RW": True, }], }, ) with pytest.raises(deploy_module.DeployError, match="n8n volume"): deploy_module.reattest_workmesh_product_n8n_receipt(projects, project, tmp_path, revision) assert state_path.read_bytes() == original_bytes assert list(state_path.parent.glob("workmesh-product-before-n8n-reattest-*.json")) == [] def test_workmesh_product_n8n_recovery_dispatches_under_both_locks( deploy_module, tmp_path, monkeypatch, capsys ): revision = "a" * 40 configured = { "slug": "workmesh", "branch": "main", "source_dir": (deploy_module.EXACT_SOURCE_ROOT / "workmesh").as_posix(), "env_source": "/srv/example/appdata/workmesh/.env", "compose_project": "workmesh", "compose_files": ["compose.yaml"], "containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS), "retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER], "exact_source": True, "verify_oci_revision": False, "health_timeout": 300, } projects = {"jens/workmesh": configured} events = [] @contextmanager def project_lock(path, _description): events.append(("project-lock", path)) yield @contextmanager def global_lock(): events.append(("global-lock",)) yield monkeypatch.setattr(sys, "argv", [ "deploy.py", "reattest-workmesh-product-n8n", "jens/workmesh", revision, ]) monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "validate_commit", lambda _repo, _branch, value: value) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: {"tree": "b" * 40}) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", project_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", global_lock) def reattest(received_projects, project, repo, requested_revision): assert received_projects == projects assert project["_repository_name"] == "jens/workmesh" assert repo == tmp_path assert requested_revision == revision events.append(("reattest-product",)) return {"revision": revision, "container": "workmesh-n8n-1", "status": "reattested"} monkeypatch.setattr(deploy_module, "reattest_workmesh_product_n8n_receipt", reattest) assert deploy_module.main() == 0 assert events == [ ("project-lock", deploy_module.STATE_ROOT / "locks" / "workmesh.lock"), ("global-lock",), ("reattest-product",), ] assert json.loads(capsys.readouterr().out) == { "repository": "jens/workmesh", "revision": revision, "container": "workmesh-n8n-1", "status": "reattested", } def test_workmesh_topology_transition_requires_exact_registry_and_recorded_receipt( deploy_module, tmp_path ): project = _workmesh_product_project(tmp_path) previous = {"live_receipt": {"containers": { name: {"image_id": "sha256:" + "1" * 64} for name in deploy_module.WORKMESH_LEGACY_CONTAINERS }}} assert deploy_module.workmesh_transition_enabled(project) assert deploy_module.workmesh_rollback_topology(project, previous)["containers"] == deploy_module.WORKMESH_LEGACY_CONTAINERS assert deploy_module.workmesh_rollback_topology(project, {"live_receipt": {"containers": {}}}) is project wrong = {**project, "retired_compose_containers": ["some-other-container"]} assert not deploy_module.workmesh_transition_enabled(wrong) assert deploy_module.workmesh_rollback_topology(wrong, previous) is wrong def test_workmesh_product_deploy_removes_only_reviewed_compose_orphan( deploy_module, tmp_path, monkeypatch ): project = _workmesh_product_project(tmp_path) commands = [] monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {}) monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"])) monkeypatch.setattr(deploy_module, "prepare_compose_migration", lambda *_: []) monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS)) monkeypatch.setattr(deploy_module, "run", lambda command, **_: commands.append(command)) deploy_module.deploy_project(project, "a" * 40, "jens/workmesh") assert commands[-1] == ["docker", "compose", "up", "--detach", "--no-build", "--remove-orphans"] other = _project(tmp_path) deploy_module.deploy_project(other, "a" * 40, "jens/unraid-appops-gateway") assert commands[-1] == ["docker", "compose", "up", "--detach", "--no-build"] monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_PRODUCT_CONTAINERS)) deploy_module.deploy_project(project, "a" * 40, "jens/workmesh") assert commands[-1] == ["docker", "compose", "up", "--detach", "--no-build"] monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS) | {"unreviewed-sidecar"}) with pytest.raises(deploy_module.DeployError, match="topology changed"): deploy_module.deploy_project(project, "a" * 40, "jens/workmesh") def test_workmesh_registry_transition_requires_live_proof_and_fixed_candidate( deploy_module, tmp_path, monkeypatch ): configured = _workmesh_product_project(tmp_path) configured["containers"] = deploy_module.WORKMESH_LEGACY_CONTAINERS configured.pop("retired_compose_containers") configured["exact_source"] = True projects = {"jens/workmesh": configured} path = tmp_path / "projects.json" path.write_text(json.dumps(projects), encoding="utf-8") monkeypatch.setattr(deploy_module, "CONFIG_PATH", path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path) project = {**configured, "_repository_name": "jens/workmesh"} live_revision = "a" * 40 revision = "b" * 40 monkeypatch.setattr(deploy_module, "read_state", lambda *_: { "live_revision": live_revision, "live_status": "healthy", "live_receipt": {"build": {"tree": "c" * 40}}, }) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: {"verified": True}) monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS)) monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_: True) monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"])) monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {}) monkeypatch.setattr(deploy_module, "output", lambda *_ , **__: "postgres\napi\nn8n\nworkmesh") deploy_module.transition_workmesh_registry(projects, project, tmp_path, revision) changed = json.loads(path.read_text(encoding="utf-8"))["jens/workmesh"] assert changed["containers"] == deploy_module.WORKMESH_PRODUCT_CONTAINERS assert changed["retired_compose_containers"] == ["workmesh-mock-ats-1"] backup = tmp_path / f"workmesh-registry-before-product-{revision[:12]}.json" assert json.loads(backup.read_text(encoding="utf-8")) == projects with pytest.raises(deploy_module.DeployError, match="changed before topology transition"): deploy_module.transition_workmesh_registry(projects, project, tmp_path, revision) def test_workmesh_registry_transition_rejects_unverified_source_or_service( deploy_module, tmp_path, monkeypatch ): configured = _workmesh_product_project(tmp_path) configured["containers"] = deploy_module.WORKMESH_LEGACY_CONTAINERS configured.pop("retired_compose_containers") configured["exact_source"] = True projects = {"jens/workmesh": configured} path = tmp_path / "projects.json" path.write_text(json.dumps(projects), encoding="utf-8") monkeypatch.setattr(deploy_module, "CONFIG_PATH", path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path) project = {**configured, "_repository_name": "jens/workmesh"} monkeypatch.setattr(deploy_module, "read_state", lambda *_: { "live_revision": "a" * 40, "live_status": "healthy", "live_receipt": {"build": {"tree": "c" * 40}}, }) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: None) with pytest.raises(deploy_module.DeployError, match="live receipt"): deploy_module.transition_workmesh_registry(projects, project, tmp_path, "b" * 40) monkeypatch.setattr(deploy_module, "verified_current_receipt", lambda *_: {"verified": True}) monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda *_: set(deploy_module.WORKMESH_LEGACY_CONTAINERS)) monkeypatch.setattr(deploy_module, "exact_source_matches_revision", lambda *_: True) monkeypatch.setattr(deploy_module, "compose_command", lambda *_: (tmp_path, ["docker", "compose"])) monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_: {}) monkeypatch.setattr(deploy_module, "output", lambda *_ , **__: "postgres\napi\nn8n\nworkmesh\nmock-ats") with pytest.raises(deploy_module.DeployError, match="four-service"): deploy_module.transition_workmesh_registry(projects, project, tmp_path, "b" * 40) assert json.loads(path.read_text(encoding="utf-8")) == projects def test_failed_workmesh_product_deploy_retains_five_container_artifacts( deploy_module, tmp_path, monkeypatch ): project = _workmesh_product_project(tmp_path) previous = {"live_revision": "a" * 40, "live_artifacts": {"containers": {}}, "live_receipt": {"containers": { name: {"image_id": "sha256:" + "1" * 64} for name in deploy_module.WORKMESH_LEGACY_CONTAINERS }}} observed = [] monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda target: ( observed.append(list(target["containers"])) or {"containers": { name: {"image_id": "sha256:" + "1" * 64} for name in target["containers"] }} )) result = deploy_module.resolve_failed_live_state( previous, {"revision": "a" * 40, "receipt": previous["live_receipt"]}, project, production_started=True, failure_stage="deployment", restore_error=None, ) assert observed == [deploy_module.WORKMESH_LEGACY_CONTAINERS] assert set(result["live_artifacts"]["containers"]) == set(deploy_module.WORKMESH_LEGACY_CONTAINERS) def test_workmesh_registry_rollback_accepts_only_exact_saved_baseline( deploy_module, tmp_path, monkeypatch ): original = _workmesh_product_project(tmp_path) original["containers"] = deploy_module.WORKMESH_LEGACY_CONTAINERS original.pop("retired_compose_containers") current = _workmesh_product_project(tmp_path) projects = {"jens/workmesh": current, "jens/other": {"slug": "other"}} original_projects = {"jens/workmesh": original, "jens/other": {"slug": "other"}} revision = "b" * 40 path = tmp_path / "projects.json" path.write_text(json.dumps(projects), encoding="utf-8") (tmp_path / f"workmesh-registry-before-product-{revision[:12]}.json").write_text( json.dumps(original_projects), encoding="utf-8" ) monkeypatch.setattr(deploy_module, "CONFIG_PATH", path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path) backup = deploy_module.workmesh_registry_rollback_backup(projects, revision) deploy_module.restore_workmesh_registry(projects, backup) assert json.loads(path.read_text(encoding="utf-8")) == original_projects path.write_text(json.dumps({**projects, "jens/other": {"slug": "changed"}}), encoding="utf-8") with pytest.raises(deploy_module.DeployError, match="Registry changed"): deploy_module.workmesh_registry_rollback_backup(projects, revision) def test_workmesh_product_snapshot_retains_authorized_five_container_rollback( deploy_module, tmp_path, monkeypatch ): project = _workmesh_product_project(tmp_path) previous = {**project, "containers": deploy_module.WORKMESH_LEGACY_CONTAINERS} old_revision, new_revision = "a" * 40, "b" * 40 old_receipt = _receipt(previous, old_revision) snapshot = { "revision": old_revision, "receipt": old_receipt, "containers": [{ "name": name, "image_id": "sha256:" + "1" * 64, "image_ref": name + ":production", "rollback_ref": f"gitea-deploy-rollback/workmesh-{index}:{old_revision[:12]}", } for index, name in enumerate(deploy_module.WORKMESH_LEGACY_CONTAINERS)], } state_root = tmp_path / "state" state_root.mkdir() (state_root / "workmesh.json").write_text(json.dumps({ "live_revision": new_revision, "live_status": "healthy", "rollback_snapshot": snapshot, }), encoding="utf-8") monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) monkeypatch.setattr(deploy_module, "inspect_image", lambda image_id: {"Id": image_id}) assert deploy_module.durable_rollback_snapshot(project, "jens/workmesh", new_revision) == snapshot def test_deployment_environment_exports_verified_build_date(deploy_module, tmp_path): project = _project(tmp_path) repository = "jens/unraid-appops-gateway" revision = "a" * 40 built_at = "2026-09-23T10:15:00.000Z" metadata = {"repository": repository, "revision": revision, "builtAt": built_at} environment = deploy_module.deployment_environment(project, revision, repository, metadata) assert environment["APPOPS_BUILD_DATE"] == built_at def test_deployment_environment_binds_workmesh_build_arg_to_exact_revision( deploy_module, tmp_path, monkeypatch ): revision = "a" * 40 monkeypatch.delenv("WORKMESH_BUILD_REVISION", raising=False) project = _workmesh_product_project(tmp_path) project["environment"] = {"WORKMESH_BUILD_REVISION": "operator-override"} environment = deploy_module.deployment_environment(project, revision, "jens/workmesh") assert environment["WORKMESH_BUILD_REVISION"] == revision other_environment = deploy_module.deployment_environment( _project(tmp_path), revision, "jens/unraid-appops-gateway" ) assert "WORKMESH_BUILD_REVISION" not in other_environment def test_deployment_environment_clears_unverified_build_date(deploy_module, tmp_path, monkeypatch): project = _project(tmp_path) repository = "jens/unraid-appops-gateway" revision = "a" * 40 monkeypatch.setenv("APPOPS_BUILD_DATE", "ambient-unverified-value") metadata = { "repository": "jens/another-repository", "revision": revision, "builtAt": "2026-09-23T10:15:00.000Z", } environment = deploy_module.deployment_environment(project, revision, repository, metadata) assert environment["APPOPS_BUILD_DATE"] == "" def test_deployment_environment_allows_project_build_date_override(deploy_module, tmp_path): project = _project(tmp_path) project["environment"] = {"APPOPS_BUILD_DATE": "operator-configured-value"} repository = "jens/unraid-appops-gateway" revision = "a" * 40 metadata = { "repository": repository, "revision": revision, "builtAt": "2026-09-23T10:15:00.000Z", } environment = deploy_module.deployment_environment(project, revision, repository, metadata) assert environment["APPOPS_BUILD_DATE"] == "operator-configured-value" def _receipt( project: dict[str, object], revision: str, *, tree: str = "b" * 40, ) -> dict[str, object]: repository = str(project["_repository_name"]) built_at = "2026-08-30T10:00:00.000Z" digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest() build = { "id": revision[:12], "repository": repository, "revision": revision, "tree": tree, "treeDigest": digest, "builtAt": built_at, } containers = { str(name): { "container_id": "2" * 64, "image_id": "sha256:" + "1" * 64, "image_ref": f"{name}:production", "image_revision": revision, "image_source": repository, "image_tree": tree, "image_version": revision[:12], "image_created": built_at, } for name in project["containers"] } live = None if project.get("live_probe"): live = { "status": "live", "build": { key: build[key] for key in ("id", "revision", "tree", "treeDigest", "builtAt") }, } return { "schema": "gitea-deploy-receipt.v1", "repository": repository, "revision": revision, "tree": tree, "treeDigest": digest, "build": build, "containers": containers, "live": live, } def test_failed_attempts_never_replace_the_verified_live_revision( deploy_module, tmp_path, monkeypatch ): revision_a = "a" * 40 revision_b = "b" * 40 revision_c = "c" * 40 project = _project(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state") state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json" state_path.parent.mkdir(parents=True) state_path.write_text( json.dumps( { "schema_version": 2, "repository": "jens/unraid-appops-gateway", "revision": revision_a, "live_revision": revision_a, "live_status": "healthy", "live_artifacts": { "containers": {"Unraid-AppOps-Gateway": {"image_id": "sha256:a"}} }, "status": "success", } ), encoding="utf-8", ) deploy_module.write_state( "jens/unraid-appops-gateway", project, revision_b, "failed", "B failed" ) after_b = json.loads(state_path.read_text(encoding="utf-8")) assert after_b["live_revision"] == revision_a assert after_b["revision"] == revision_a assert after_b["last_attempt"]["revision"] == revision_b deploy_module.write_state( "jens/unraid-appops-gateway", project, revision_c, "failed", "C failed" ) after_c = json.loads(state_path.read_text(encoding="utf-8")) assert after_c["live_revision"] == revision_a assert after_c["last_attempt"]["revision"] == revision_c def test_snapshot_is_optional_only_for_a_true_first_deploy( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state") monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: set() ) monkeypatch.setattr(deploy_module, "inspect_container", lambda _name: None) assert deploy_module.capture_compose_rollback(project) is None monkeypatch.setattr( deploy_module, "inspect_container", lambda _name: { "Id": "container-id", "Image": "sha256:" + "1" * 64, "Config": { "Image": "unraid-appops-gateway:production", "Labels": {"com.docker.compose.project": "unraid-appops-gateway"}, }, "State": {"Running": True}, }, ) with pytest.raises( deploy_module.DeployError, match="valid healthy live deployment state" ): deploy_module.capture_compose_rollback(project) def test_snapshot_rejects_partial_container_sets_before_tagging( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) project["containers"] = ["one", "two"] monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state") monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: set() ) monkeypatch.setattr( deploy_module, "inspect_container", lambda name: None if name == "two" else {"Id": "one"}, ) commands: list[list[str]] = [] monkeypatch.setattr( deploy_module, "run", lambda command, **_kwargs: commands.append(command) ) with pytest.raises(deploy_module.DeployError, match="containers are missing"): deploy_module.capture_compose_rollback(project) assert commands == [] def test_snapshot_rejects_receipt_for_another_revision_before_tagging( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) live_revision = "a" * 40 receipt_revision = "b" * 40 image_id = "sha256:" + "1" * 64 bad_receipt = _receipt(project, receipt_revision) monkeypatch.setattr( deploy_module, "read_state", lambda _project: { "schema_version": 3, "repository": project["_repository_name"], "live_revision": live_revision, "live_status": "healthy", "live_legacy_provenance": False, "live_artifacts": { "containers": {"Unraid-AppOps-Gateway": {"image_id": image_id}} }, "live_receipt": bad_receipt, }, ) monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Unraid-AppOps-Gateway"}, ) monkeypatch.setattr( deploy_module, "inspect_container", lambda _name: { "Id": "2" * 64, "Image": image_id, "Config": { "Image": "unraid-appops-gateway:production", "Labels": {"com.docker.compose.project": "unraid-appops-gateway"}, }, "State": {"Running": True}, }, ) monkeypatch.setattr( deploy_module, "inspect_image", lambda _image: { "Id": image_id, "Config": { "Labels": { "org.opencontainers.image.revision": live_revision, "org.opencontainers.image.source": project["_repository_name"], } }, }, ) commands: list[list[str]] = [] monkeypatch.setattr( deploy_module, "run", lambda command, **_kwargs: commands.append(command) ) with pytest.raises(deploy_module.DeployError, match="receipt identity"): deploy_module.capture_compose_rollback(project) assert commands == [] def test_snapshot_rejects_unrecorded_same_project_containers( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Unraid-AppOps-Gateway", "unrecorded-sidecar"}, ) monkeypatch.setattr( deploy_module, "inspect_container", lambda _name: pytest.fail("container inspection must not start"), ) with pytest.raises(deploy_module.DeployError, match="unrecorded containers"): deploy_module.capture_compose_rollback(project) def test_oci_verification_reads_the_immutable_image_not_container_override( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) requested = "b" * 40 image_id = "sha256:" + "1" * 64 monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Unraid-AppOps-Gateway"}, ) monkeypatch.setattr( deploy_module, "container_state", lambda _name: (True, "healthy", "unraid-appops-gateway"), ) monkeypatch.setattr( deploy_module, "inspect_container", lambda _name: { "Image": image_id, "Config": {"Labels": {"org.opencontainers.image.revision": requested}}, }, ) monkeypatch.setattr( deploy_module, "inspect_image", lambda _image_id: { "Id": image_id, "Config": { "Labels": { "org.opencontainers.image.revision": "a" * 40, "org.opencontainers.image.source": "jens/unraid-appops-gateway", } }, }, ) with pytest.raises(deploy_module.DeployError, match="Immutable image provenance"): deploy_module.verify_containers( project, timeout=30, expected_revision=requested ) def test_forward_gate_rejects_same_drifted_or_older_revision( deploy_module, tmp_path, monkeypatch ): live = "b" * 40 project = _project(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state") state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json" state_path.parent.mkdir(parents=True) state_path.write_text( json.dumps({"live_revision": live, "live_status": "healthy"}), encoding="utf-8", ) with pytest.raises(deploy_module.DeployError, match="runtime verification failed"): deploy_module.validate_forward_revision(Path("repo.git"), project, live) monkeypatch.setattr( deploy_module.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace(returncode=1), ) with pytest.raises(deploy_module.NonForwardDeployError): deploy_module.validate_forward_revision(Path("repo.git"), project, "a" * 40) def test_candidate_is_revalidated_against_the_current_branch_under_lock( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) project["branch"] = "main" revision = "a" * 40 metadata = { "id": revision[:12], "repository": project["_repository_name"], "revision": revision, "tree": "b" * 40, "treeDigest": hashlib.sha256(f"git-tree:{'b' * 40}".encode()).hexdigest(), "builtAt": "2026-08-30T10:00:00.000Z", } monkeypatch.setattr( deploy_module, "validate_commit", lambda *_args, **_kwargs: (_ for _ in ()).throw( deploy_module.DeployError("commit is no longer on the branch") ), ) with pytest.raises(deploy_module.DeployError, match="no longer on the branch"): deploy_module.revalidate_deployment_candidate( Path("repo.git"), project, revision, str(project["_repository_name"]), metadata, ) def test_bootstrap_legacy_cannot_downgrade_modern_state( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) revision = "a" * 40 monkeypatch.setattr( deploy_module, "read_state", lambda _project: { "schema_version": 3, "repository": project["_repository_name"], "revision": revision, "live_revision": revision, "live_receipt": _receipt(project, revision), "status": "success", }, ) monkeypatch.setattr( deploy_module, "verify_containers", lambda *_args, **_kwargs: pytest.fail("modern state must not be re-enrolled"), ) with pytest.raises(deploy_module.DeployError, match="modern schema"): deploy_module.bootstrap_legacy_state( str(project["_repository_name"]), project, revision ) def test_interrupted_legacy_recovery_requires_recorded_pre_mutation_guard( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) live_revision = "a" * 40 candidate_revision = "b" * 40 monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state") state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json" state_path.parent.mkdir(parents=True) state_path.write_text( json.dumps( { "schema_version": 3, "repository": project["_repository_name"], "revision": live_revision, "live_revision": live_revision, "live_status": "unknown", "live_legacy_provenance": False, "live_artifacts": {"containers": {}}, "live_receipt": {}, "status": "failed", "last_attempt": { "revision": candidate_revision, "status": "failed", "message": ( "Refusing to mutate a Compose project with unrecorded containers: " "['unrecorded-sidecar']" ), "updated_at_epoch": 123, }, } ), encoding="utf-8", ) verified = [] monkeypatch.setattr( deploy_module, "verify_containers", lambda value: verified.append(value), ) artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "legacy"}}} monkeypatch.setattr(deploy_module, "deployment_artifacts", lambda _project: artifacts) deploy_module.recover_interrupted_legacy_state( str(project["_repository_name"]), project, live_revision ) recovered = json.loads(state_path.read_text(encoding="utf-8")) assert verified == [project] assert recovered["status"] == "success" assert recovered["live_revision"] == live_revision assert recovered["live_status"] == "healthy" assert recovered["live_legacy_provenance"] is True assert recovered["live_artifacts"] == artifacts assert recovered["last_attempt"]["metadata"] == { "recovery": "interrupted-legacy-pre-mutation-allowlist-guard", "reconciled_from_attempt_revision": candidate_revision, "reconciled_from_attempt_updated_at_epoch": 123, } @pytest.mark.parametrize( "change", [ {"live_status": "healthy"}, {"live_receipt": {"unverified": True}}, {"last_attempt": {"status": "failed", "message": "source sync failed"}}, { "last_attempt": { "status": "success", "message": "Refusing to mutate a Compose project with unrecorded containers: []", } }, ], ) def test_interrupted_legacy_recovery_rejects_any_other_state( deploy_module, tmp_path, monkeypatch, change ): project = _project(tmp_path) revision = "a" * 40 state = { "schema_version": 3, "repository": project["_repository_name"], "revision": revision, "live_revision": revision, "live_status": "unknown", "live_legacy_provenance": False, "live_artifacts": {"containers": {}}, "live_receipt": {}, "status": "failed", "last_attempt": { "revision": "b" * 40, "status": "failed", "message": "Refusing to mutate a Compose project with unrecorded containers: ['sidecar']", }, } state.update(change) monkeypatch.setattr(deploy_module, "read_state", lambda _project: state) monkeypatch.setattr( deploy_module, "verify_containers", lambda *_args, **_kwargs: pytest.fail("untrusted state must not inspect containers"), ) with pytest.raises(deploy_module.DeployError, match="exact recorded pre-mutation"): deploy_module.recover_interrupted_legacy_state( str(project["_repository_name"]), project, revision ) def test_pre_mutation_failure_preserves_legacy_and_unknown_live_state( deploy_module, tmp_path ): project = _project(tmp_path) revision = "a" * 40 artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "legacy"}}} previous = { "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": True, "live_artifacts": artifacts, "live_receipt": {}, } candidate_failure = deploy_module.resolve_failed_live_state( previous, None, project, production_started=False, failure_stage="candidate-revalidation", restore_error=None, ) assert candidate_failure == { "live_revision": revision, "live_artifacts": artifacts, "live_status": "healthy", "live_legacy_provenance": True, "live_receipt": {}, } previous["live_status"] = "unknown" unknown_failure = deploy_module.resolve_failed_live_state( previous, None, project, production_started=False, failure_stage="candidate-revalidation", restore_error=None, ) assert unknown_failure["live_status"] == "unknown" assert unknown_failure["live_legacy_provenance"] is True def test_snapshot_capture_failure_preserves_evidence_but_marks_live_unknown( deploy_module, tmp_path ): project = _project(tmp_path) revision = "a" * 40 artifacts = {"containers": {"Unraid-AppOps-Gateway": {"image_id": "legacy"}}} result = deploy_module.resolve_failed_live_state( { "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": True, "live_artifacts": artifacts, "live_receipt": {}, }, None, project, production_started=False, failure_stage="snapshot", restore_error=None, ) assert result["live_revision"] == revision assert result["live_status"] == "unknown" assert result["live_legacy_provenance"] is True assert result["live_artifacts"] == artifacts def test_container_without_exact_compose_label_is_rejected( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) monkeypatch.setattr( deploy_module, "container_state", lambda _name: (True, "healthy", ""), ) monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Unraid-AppOps-Gateway"}, ) with pytest.raises(deploy_module.DeployError, match="wrong Compose project"): deploy_module.verify_containers(project, timeout=30) def test_post_deploy_verification_rejects_extra_compose_services( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) monkeypatch.setattr( deploy_module, "container_state", lambda _name: (True, "healthy", "unraid-appops-gateway"), ) monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Unraid-AppOps-Gateway", "unreviewed-sidecar"}, ) with pytest.raises(deploy_module.DeployError, match="container set"): deploy_module.verify_containers(project, timeout=30) def test_rollback_removes_only_containers_created_after_snapshot( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) snapshot = { "compose_containers": ["Unraid-AppOps-Gateway"], "containers": [{"name": "Unraid-AppOps-Gateway"}], } monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Unraid-AppOps-Gateway", "failed-revision-sidecar"}, ) monkeypatch.setattr( deploy_module, "inspect_container", lambda name: { "Config": { "Labels": {"com.docker.compose.project": "unraid-appops-gateway"} } } if name == "failed-revision-sidecar" else None, ) commands: list[list[str]] = [] monkeypatch.setattr( deploy_module, "run", lambda command, **_kwargs: commands.append(command) ) deploy_module.remove_attempt_only_compose_containers(project, snapshot) assert commands == [["docker", "rm", "--force", "failed-revision-sidecar"]] def _projectbrain_rollback_fixture(tmp_path): project = {"slug": "projectbrain", "source_dir": str(tmp_path), "deploy_script": "scripts/deploy-unraid-single.sh", "containers": ["projectbrain"], "compose_project": "projectbrain", "_repository_name": "jens/projectbrain"} binding = {"app_dir": str(tmp_path.resolve()), "data_path": (tmp_path / "data").as_posix(), "container_name": "projectbrain", "image_ref": "projectbrain-unraid:latest", "image_id": "sha256:" + "a" * 64} template = tmp_path / "docker/unraid/projectbrain-container.env.example" template.parent.mkdir(parents=True) template.write_text("PROJECTBRAIN_STOP_COMPOSE_APP_SERVICES=true\nPROJECTBRAIN_REMOVE_OLD_POSTGRES=true\n") values = {"PROJECTBRAIN_IMAGE": binding["image_ref"], "PROJECTBRAIN_CONTAINER_NAME": "projectbrain", "PROJECTBRAIN_APPDATA_PATH": binding["data_path"], "PROJECTBRAIN_STOP_COMPOSE_APP_SERVICES": "false", "PROJECTBRAIN_REMOVE_OLD_POSTGRES": "false"} for key in ("POSTGRES_PASSWORD", "PROJECTBRAIN_OWNER_PASSWORD", "PROJECTBRAIN_AUTH_SECRET", "PROJECTBRAIN_HUB_SERVICE_TOKEN", "PROJECTBRAIN_GITEA_WEBHOOK_SECRET"): values[key] = "fixture-only-not-live" (tmp_path / ".env").write_text("".join(f"{key}={value}\n" for key, value in values.items())) return project, binding def test_single_container_rollback_never_starts_legacy_compose(deploy_module, monkeypatch, tmp_path): project, binding = _projectbrain_rollback_fixture(tmp_path) calls = [] monkeypatch.setattr(deploy_module, "run", lambda command, **kw: calls.append((command, kw))) monkeypatch.setattr(deploy_module, "compose_command", lambda _: pytest.fail("wrong topology")) monkeypatch.setattr(deploy_module, "inspect_image", lambda _: {"Id": binding["image_id"]}) env = {"VITE_PROJECTBRAIN_COMMIT": "a" * 40} safe = deploy_module.projectbrain_rollback_environment(project, binding, env) deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, env, binding) assert calls == [(["sh", str(tmp_path / "scripts/deploy-unraid-single.sh"), "--skip-pull", "--skip-build"], {"cwd": tmp_path, "env": safe})] @pytest.mark.parametrize("assignment", [ "PROJECTBRAIN_DEPLOY_SKIP_BUILD=false", "PROJECTBRAIN_DEPLOY_SKIP_PULL=false", "PROJECTBRAIN_DEPLOY_REF=main", "PROJECTBRAIN_IMAGE=other:latest", "PROJECTBRAIN_CONTAINER_NAME=other", "PROJECTBRAIN_APPDATA_PATH=/other", "PROJECTBRAIN_STOP_COMPOSE_APP_SERVICES=true", "PROJECTBRAIN_REMOVE_OLD_POSTGRES=true", ]) def test_projectbrain_file_control_drift_fails_before_mutation(deploy_module, tmp_path, monkeypatch, assignment): project, binding = _projectbrain_rollback_fixture(tmp_path) key = assignment.split("=", 1)[0] path = tmp_path / ".env" lines = [line for line in path.read_text().splitlines() if not line.startswith(key + "=")] path.write_text("\n".join([*lines, assignment]) + "\n") monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("unsafe mutation")) with pytest.raises(deploy_module.DeployError, match="conflicts"): deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, {}, binding) def test_projectbrain_missing_live_key_cannot_import_unsafe_template_default(deploy_module, tmp_path): project, binding = _projectbrain_rollback_fixture(tmp_path) path = tmp_path / ".env" path.write_text(path.read_text().replace("PROJECTBRAIN_REMOVE_OLD_POSTGRES=false\n", "")) with pytest.raises(deploy_module.DeployError, match="REMOVE_OLD_POSTGRES"): deploy_module.projectbrain_rollback_environment(project, binding, {}) def test_projectbrain_inherited_rollback_controls_are_forced_without_shell_evaluation(deploy_module, tmp_path): project, binding = _projectbrain_rollback_fixture(tmp_path) inherited = {"PROJECTBRAIN_DEPLOY_REF": "main", "PROJECTBRAIN_DEPLOY_SKIP_BUILD": "false", "PROJECTBRAIN_DEPLOY_SKIP_PULL": "false", "PROJECTBRAIN_REMOVE_OLD_POSTGRES": "true", "ENV": "/unreviewed/startup", "BASH_ENV": "/unreviewed/startup"} result = deploy_module.projectbrain_rollback_environment(project, binding, inherited) assert result["PROJECTBRAIN_DEPLOY_REF"] == "" assert result["PROJECTBRAIN_DEPLOY_SKIP_BUILD"] == "true" assert result["PROJECTBRAIN_DEPLOY_SKIP_PULL"] == "true" assert result["PROJECTBRAIN_REMOVE_OLD_POSTGRES"] == "false" assert "ENV" not in result and "BASH_ENV" not in result assert inherited["PROJECTBRAIN_DEPLOY_REF"] == "main" @pytest.mark.parametrize("key", ["PROJECTBRAIN_IMAGE", "PROJECTBRAIN_CONTAINER_NAME", "PROJECTBRAIN_APP_DIR", "PROJECTBRAIN_APPDATA_PATH"]) def test_projectbrain_inherited_target_drift_fails_closed(deploy_module, tmp_path, key): project, binding = _projectbrain_rollback_fixture(tmp_path) with pytest.raises(deploy_module.DeployError, match="inherited"): deploy_module.projectbrain_rollback_environment(project, binding, {key: "other"}) @pytest.mark.parametrize("line", ["PROJECTBRAIN_TEST=$(id)", "PROJECTBRAIN_TEST=`id`", "PROJECTBRAIN_TEST=value; true", "APP_DIR=/other", "PATH=/other"]) def test_projectbrain_environment_never_evaluates_shell_input(deploy_module, tmp_path, line): project, binding = _projectbrain_rollback_fixture(tmp_path) path = tmp_path / ".env" path.write_text(path.read_text() + line + "\n") with pytest.raises(deploy_module.DeployError): deploy_module.projectbrain_rollback_environment(project, binding, {}) def test_projectbrain_snapshot_preflight_precedes_image_tag_mutations(deploy_module, tmp_path, monkeypatch): project, binding = _projectbrain_rollback_fixture(tmp_path) (tmp_path / ".env").write_text((tmp_path / ".env").read_text().replace("PROJECTBRAIN_REMOVE_OLD_POSTGRES=false", "PROJECTBRAIN_REMOVE_OLD_POSTGRES=true")) runtime = {"Id": "c" * 64, "Image": binding["image_id"], "Config": {"Image": binding["image_ref"], "Labels": {"com.docker.compose.project": "projectbrain"}}, "Mounts": [{"Type": "bind", "Source": binding["data_path"], "Destination": "/var/lib/projectbrain", "RW": True}], "State": {"Running": True}} state = {"status": "success", "revision": "a" * 40, "live_legacy_provenance": True, "artifacts": {"containers": {"projectbrain": {"image_id": binding["image_id"]}}}} monkeypatch.setattr(deploy_module, "read_state", lambda _: state) monkeypatch.setattr(deploy_module, "compose_project_container_names", lambda _: {"projectbrain"}) monkeypatch.setattr(deploy_module, "inspect_container", lambda _: runtime) monkeypatch.setattr(deploy_module, "inspect_image", lambda _: {"Id": binding["image_id"]}) monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("snapshot mutation before preflight")) with pytest.raises(deploy_module.DeployError, match="REMOVE_OLD_POSTGRES"): deploy_module.capture_compose_rollback(project) def test_projectbrain_rollback_rechecks_restored_immutable_tag(deploy_module, tmp_path, monkeypatch): project, binding = _projectbrain_rollback_fixture(tmp_path) monkeypatch.setattr(deploy_module, "inspect_image", lambda _: {"Id": "wrong"}) monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("wrong image mutation")) with pytest.raises(deploy_module.DeployError, match="immutable snapshot"): deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, {}, binding) @pytest.mark.parametrize("key,value", [("DOCKER_HOST", "tcp://elsewhere.invalid:2376"), ("DOCKER_CONTEXT", "other"), ("DOCKER_TLS_VERIFY", "1")]) def test_projectbrain_rollback_rejects_inherited_docker_target(deploy_module, tmp_path, key, value): project, binding = _projectbrain_rollback_fixture(tmp_path) with pytest.raises(deploy_module.DeployError, match="controller-local Docker"): deploy_module.projectbrain_rollback_environment(project, binding, {key: value}) def test_projectbrain_rollback_revalidates_before_any_restore_mutation(deploy_module, tmp_path, monkeypatch): project, binding = _projectbrain_rollback_fixture(tmp_path) path = tmp_path / ".env" path.write_text(path.read_text() + "PROJECTBRAIN_DEPLOY_SKIP_BUILD=false\n") monkeypatch.setattr(deploy_module, "run", lambda *args, **kw: pytest.fail("mutation before guard")) monkeypatch.setattr(deploy_module, "sync_source", lambda *args, **kw: pytest.fail("source restore before guard")) snapshot = {"revision": "a" * 40, "containers": [], "script_binding": binding} with pytest.raises(deploy_module.DeployError, match="DEPLOY_SKIP_BUILD"): deploy_module.restore_compose_snapshot(project, snapshot, "jens/projectbrain", "b" * 40) def test_projectbrain_mount_binding_rejects_wrong_or_additional_data_targets(deploy_module, tmp_path): project, binding = _projectbrain_rollback_fixture(tmp_path) mount = {"Type": "bind", "Source": binding["data_path"], "Destination": "/wrong", "RW": True} container = {"Config": {"Image": binding["image_ref"]}, "Image": binding["image_id"], "Mounts": [mount]} with pytest.raises(deploy_module.DeployError, match="data mount"): deploy_module.projectbrain_snapshot_binding(project, container) mount["Destination"] = "/var/lib/projectbrain" container["Mounts"].append(dict(mount)) with pytest.raises(deploy_module.DeployError, match="data mount"): deploy_module.projectbrain_snapshot_binding(project, container) def test_projectbrain_static_parse_error_does_not_echo_sensitive_values(deploy_module, tmp_path): project, binding = _projectbrain_rollback_fixture(tmp_path) value = "fixture-sensitive-value-with-$-syntax" path = tmp_path / ".env" path.write_text(path.read_text() + f"PROJECTBRAIN_TEST={value}\n") with pytest.raises(deploy_module.DeployError) as raised: deploy_module.projectbrain_rollback_environment(project, binding, {}) assert value not in str(raised.value) @pytest.mark.parametrize("change", [ {"deploy_script": "unknown.sh"}, {"archive_deploy_script": "archive.sh"}, {"containers": ["other"]}, ]) def test_unreviewed_script_rollback_fails_closed(deploy_module, monkeypatch, tmp_path, change): project = {"source_dir": str(tmp_path), "deploy_script": "scripts/deploy-unraid-single.sh", "containers": ["projectbrain"], **change} monkeypatch.setattr(deploy_module, "run", lambda *a, **kw: pytest.fail("unreviewed mutation")) with pytest.raises(deploy_module.DeployError, match="reviewed immutable rollback"): deploy_module.restore_snapshot_runtime(project, "jens/projectbrain", "b" * 40, {}) def test_rollback_image_tag_lifecycle_preserves_only_current_snapshot( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) project["slug"] = "chimera" current = "gitea-deploy-rollback/chimera-0:aaaaaaaaaaaa" stale = "gitea-deploy-rollback/chimera-0:bbbbbbbbbbbb" commands: list[list[str]] = [] def fake_run(command, **_kwargs): commands.append(command) if command[:3] == ["docker", "image", "ls"]: return types.SimpleNamespace( returncode=0, stdout=f"{current}\n{stale}\n", stderr="", ) return types.SimpleNamespace(returncode=0, stdout="", stderr="") monkeypatch.setattr(deploy_module.subprocess, "run", fake_run) deploy_module.cleanup_rollback_image_tags(project, {current}) assert ["docker", "image", "rm", stale] in commands assert ["docker", "image", "rm", current] not in commands def test_rollback_image_cleanup_is_project_exact_and_never_raises( deploy_module, tmp_path, monkeypatch ): project = _project(tmp_path) project["slug"] = "chimera" owned = "gitea-deploy-rollback/chimera-12:aaaaaaaaaaaa" colliding_project = "gitea-deploy-rollback/chimera-preview-0:bbbbbbbbbbbb" noncanonical_tag = "gitea-deploy-rollback/chimera-0:latest" commands: list[list[str]] = [] def fake_run(command, **_kwargs): commands.append(command) if command[:3] == ["docker", "image", "ls"]: return types.SimpleNamespace( returncode=0, stdout=f"{owned}\n{colliding_project}\n{noncanonical_tag}\n", stderr="", ) return types.SimpleNamespace(returncode=0, stdout="", stderr="") monkeypatch.setattr(deploy_module.subprocess, "run", fake_run) deploy_module.cleanup_rollback_image_tags(project) assert ["docker", "image", "rm", owned] in commands assert ["docker", "image", "rm", colliding_project] not in commands assert ["docker", "image", "rm", noncanonical_tag] not in commands monkeypatch.setattr( deploy_module.subprocess, "run", lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("docker unavailable")), ) deploy_module.cleanup_rollback_image_tags(project) def test_commit_and_tree_metadata_are_controller_derived(deploy_module, monkeypatch): revision = "a" * 40 tree = "b" * 40 calls: list[list[str]] = [] def fake_output(command, **_kwargs): calls.append(command) if command[-1].endswith("^{commit}"): return revision if command[-1].endswith("^{tree}"): return tree if command[-1].startswith("refs/heads/"): return revision raise AssertionError(command) monkeypatch.setattr(deploy_module, "output", fake_output) monkeypatch.setattr( deploy_module.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0), ) assert deploy_module.validate_commit(Path("repo.git"), "main", revision) == revision metadata = deploy_module.deployment_metadata( Path("repo.git"), revision, "jens/chimera", built_at="2026-08-30T10:00:00Z", ) assert metadata == { "id": revision[:12], "repository": "jens/chimera", "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-08-30T10:00:00.000Z", } assert any(command[-1] == f"{revision}^{{tree}}" for command in calls) monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: "c" * 40) with pytest.raises( deploy_module.DeployError, match="object itself must be a commit" ): deploy_module.validate_commit(Path("repo.git"), "main", revision) def test_exact_source_sync_replaces_dirty_context_from_a_fresh_archive( deploy_module, tmp_path, monkeypatch ): exact_root = tmp_path / "managed-sources" target = exact_root / "chimera" target.mkdir(parents=True) (target / "tracked.txt").write_text("old", encoding="utf-8") (target / "untracked-runtime.js").write_text("dirty", encoding="utf-8") revision = "a" * 40 monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root) monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"]) def fake_extract(_repo, _revision, destination): (destination / "tracked.txt").write_text("reviewed", encoding="utf-8") monkeypatch.setattr(deploy_module, "extract_revision", fake_extract) deploy_module.sync_source(Path("repo.git"), revision, target, "chimera", exact=True) assert (target / "tracked.txt").read_text(encoding="utf-8") == "reviewed" assert not (target / "untracked-runtime.js").exists() assert not (target / ".gitea-deploy").exists() assert deploy_module.exact_source_matches_revision( Path("repo.git"), revision, target ) outside = tmp_path / "not-controller-owned" with pytest.raises(deploy_module.DeployError, match="controller-owned source root"): deploy_module.sync_source( Path("repo.git"), revision, outside, "chimera", exact=True ) def test_exact_repo_sources_apply_group_read_policy_only_to_fixed_slugs( deploy_module, tmp_path, monkeypatch ): exact_root = tmp_path / "managed-sources" monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root) monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"]) def fake_extract(_repo, _revision, destination): (destination / "tracked.txt").write_text("reviewed", encoding="utf-8") monkeypatch.setattr(deploy_module, "extract_revision", fake_extract) prepared = [] monkeypatch.setattr( deploy_module, "_prepare_exact_source_group_read", lambda target, slug: prepared.append((slug, target.parent.resolve() == exact_root.resolve())), raising=False, ) for slug in ("repopulse", "privatemuse", "other-project"): deploy_module.sync_source( Path("repo.git"), "a" * 40, exact_root / slug, slug, exact=True, ) assert prepared == [("repopulse", True), ("privatemuse", True)] @pytest.mark.parametrize("slug", ["repopulse", "privatemuse"]) def test_exact_source_permission_failure_preserves_the_previous_live_tree( deploy_module, tmp_path, monkeypatch, slug ): exact_root = tmp_path / "sources" target = exact_root / slug target.mkdir(parents=True) (target / "tracked.txt").write_text("previous-live-source", encoding="utf-8") monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root) monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"]) def fake_extract(_repo, _revision, destination): (destination / "tracked.txt").write_text("candidate-source", encoding="utf-8") def deny_permission_setup(_target, _slug): raise deploy_module.DeployError("fixed source permission setup failed") monkeypatch.setattr(deploy_module, "extract_revision", fake_extract) monkeypatch.setattr(deploy_module, "_prepare_exact_source_group_read", deny_permission_setup) with pytest.raises(deploy_module.DeployError, match="permission setup failed"): deploy_module.sync_source(Path("repo.git"), "a" * 40, target, slug, exact=True) assert (target / "tracked.txt").read_text(encoding="utf-8") == "previous-live-source" assert sorted(item.name for item in exact_root.iterdir()) == [slug] @pytest.mark.skipif(os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="requires root-owned Linux source paths") def test_fixed_source_group_policy_is_read_only_and_keeps_other_siblings_private( deploy_module, tmp_path, monkeypatch ): exact_root = tmp_path / "sources" exact_root.mkdir() monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root) readable_sources = [] for slug in ("repopulse", "privatemuse"): staging = exact_root / f".{slug}-aabbccddeeff-stage" staging.mkdir(mode=0o700) nested = staging / "nested" nested.mkdir(mode=0o700) ordinary = nested / "module.py" ordinary.write_text("print('scan')\n", encoding="utf-8") ordinary.chmod(0o600) executable = nested / "tool" executable.write_text("#!/bin/sh\n", encoding="utf-8") executable.chmod(0o700) deploy_module._prepare_exact_source_group_read(staging, slug) readable_sources.append(staging) private_sibling = exact_root / "other-project" private_sibling.mkdir(mode=0o700) private_before = private_sibling.stat() for staging in readable_sources: nested = staging / "nested" assert staging.stat().st_gid == nested.stat().st_gid == 10001 assert stat.S_IMODE(staging.stat().st_mode) == stat.S_IMODE(nested.stat().st_mode) == 0o2750 assert (nested / "module.py").stat().st_gid == 10001 assert stat.S_IMODE((nested / "module.py").stat().st_mode) == 0o640 assert (nested / "tool").stat().st_gid == 10001 assert stat.S_IMODE((nested / "tool").stat().st_mode) == 0o750 assert private_sibling.stat().st_gid == private_before.st_gid assert stat.S_IMODE(private_sibling.stat().st_mode) == 0o700 def test_controller_source_mode_contract_rejects_private_mode_700(status_module): private_dir = os.stat_result((stat.S_IFDIR | 0o700, 10, 1, 1, 0, 10001, 0, 0, 0, 0)) readable_dir = os.stat_result((stat.S_IFDIR | 0o2750, 11, 1, 1, 0, 10001, 0, 0, 0, 0)) assert status_module._source_mode_matches_runtime_contract(private_dir) is False assert status_module._source_mode_matches_runtime_contract(readable_dir) is True def test_managed_repopulse_render_accepts_only_its_read_only_source_mount( deploy_module, tmp_path, monkeypatch ): source = tmp_path / "managed-source" / "repopulse" project = { "source_dir": str(source), "managed_plan_version": "itworx.deployment/v1", "managed_profile": "repopulse-authenticated", "managed_application_id": "repopulse", "managed_repository": "jens/repopulse", "managed_appdata_path": "/srv/example/appdata/repopulse", "managed_image_repository": "repopulse-managed", "managed_image_tag": "production", "managed_proxy_network": "itworx-mcp-hub-npm", "managed_internal_port": 8000, "managed_build_context": ".", "managed_build_dockerfile": "deploy/managed/Dockerfile", "managed_runtime_environment_keys": [ "REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN", ], "managed_runtime_uid": 10001, "managed_runtime_gid": 10001, "managed_proxy_hostname": "repopulse.example.com", "managed_health_path": "/health", } rendered = { "services": { "repopulse": { "image": "repopulse-managed:production", "build": {"context": str(source), "dockerfile": "deploy/managed/Dockerfile"}, "restart": "unless-stopped", "read_only": True, "user": "10001:10001", "security_opt": ["no-new-privileges:true"], "cap_drop": ["ALL"], "networks": ["itworx-mcp-hub-npm"], "environment": { "REPOPULSE_GITEA_URL": "https://gitea.example.com", "REPOPULSE_GITEA_TOKEN": "opaque-gitea-value", "REPOPULSE_MAINTENANCE_TOKEN": "opaque-maintenance-value", }, "volumes": [ {"type": "bind", "source": "/srv/example/appdata/repopulse/data", "target": "/data", "read_only": False}, {"type": "bind", "source": "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse", "target": "/scan/repopulse", "read_only": True}, {"type": "bind", "source": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse", "target": "/scan/privatemuse", "read_only": True}, ], } }, "networks": {"itworx-mcp-hub-npm": {"external": True}}, } monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"])) monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {}) monkeypatch.setattr(deploy_module, "managed_runtime_environment", lambda _project: rendered["services"]["repopulse"]["environment"]) monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: json.dumps(rendered)) deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/repopulse") rendered["services"]["repopulse"]["volumes"][2]["source"] = "/srv/example/appdata/repopulse/sources" with pytest.raises(deploy_module.DeployError): deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/repopulse") rendered["services"]["repopulse"]["volumes"][2] = dict(rendered["services"]["repopulse"]["volumes"][1]) with pytest.raises(deploy_module.DeployError): deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/repopulse") def test_managed_appdata_roots_are_fixed_and_bound_to_each_registration( deploy_module, monkeypatch, tmp_path ): legacy_root = tmp_path / "legacy-appdata" managed_root = tmp_path / "managed-volume" / "_data" monkeypatch.setattr(deploy_module, "LEGACY_MANAGED_APPCONFIG_ROOT", legacy_root) monkeypatch.setattr(deploy_module, "MANAGED_APPCONFIG_ROOT", legacy_root) monkeypatch.setattr(deploy_module, "MANAGED_APPCONFIG_VOLUME_ROOT", managed_root) monkeypatch.setattr(deploy_module, "managed_appdata_volume_initialized", lambda: False) assert deploy_module._registered_managed_appdata_root({}) == legacy_root assert deploy_module._registered_managed_appdata_root( {"managed_appdata_root": legacy_root.as_posix()} ) == legacy_root with pytest.raises(deploy_module.DeployError, match="not initialized and available"): deploy_module._registered_managed_appdata_root( {"managed_appdata_root": managed_root.as_posix()} ) with pytest.raises(deploy_module.DeployError, match="fixed registered roots"): deploy_module._registered_managed_appdata_root( {"managed_appdata_root": (tmp_path / "arbitrary").as_posix()} ) monkeypatch.setattr(deploy_module, "managed_appdata_volume_initialized", lambda: True) assert deploy_module._registered_managed_appdata_root( {"managed_appdata_root": managed_root.as_posix()} ) == managed_root for root, host in ((legacy_root, "legacy.example"), (managed_root, "managed.example")): env_path = root / "demo" / "deploy.env" env_path.parent.mkdir(parents=True) env_path.write_text(f"APPOPS_PUBLIC_URL=https://{host}\n", encoding="utf-8") if os.name != "nt": env_path.chmod(0o600) project = { "appops_managed_target": True, "managed_plan_version": "itworx.deployment/v1", "managed_profile": "internal-webapp", "managed_application_id": "demo", "managed_runtime_environment_keys": ["APPOPS_PUBLIC_URL"], "env_source": (legacy_root / "demo" / "deploy.env").as_posix(), } assert deploy_module.managed_runtime_environment(project) == { "APPOPS_PUBLIC_URL": "https://legacy.example" } project["managed_appdata_root"] = managed_root.as_posix() project["env_source"] = (managed_root / "demo" / "deploy.env").as_posix() assert deploy_module.managed_runtime_environment(project) == { "APPOPS_PUBLIC_URL": "https://managed.example" } def test_managed_project_path_is_derived_from_its_stored_appdata_root( deploy_module, monkeypatch ): monkeypatch.setattr(deploy_module, "managed_appdata_volume_initialized", lambda: True) monkeypatch.setattr(deploy_module, "git_path_exists", lambda *_args: True) project = { "containers": ["demo"], "source_dir": (deploy_module.EXACT_SOURCE_ROOT / "demo").as_posix(), "exact_source": True, "compose_dir": ".", "compose_files": ["compose.yml"], "compose_project": "demo", "managed_plan_version": "itworx.deployment/v1", "managed_application_id": "demo", "managed_appdata_path": "/srv/example/appdata/demo", "managed_image_repository": "demo", "managed_image_tag": "production", "managed_proxy_network": "itworx-proxy", "managed_internal_port": 8000, "appops_deployment_secret_file": "jens__demo", } deploy_module.validate_project_config(Path("repo.git"), project, "a" * 40) managed = { **project, "managed_appdata_root": deploy_module.MANAGED_APPCONFIG_VOLUME_ROOT.as_posix(), "managed_appdata_path": ( deploy_module.MANAGED_APPCONFIG_VOLUME_ROOT / "demo" ).as_posix(), } deploy_module.validate_project_config(Path("repo.git"), managed, "a" * 40) remapped = {**managed, "managed_appdata_path": "/srv/example/appdata/demo"} with pytest.raises(deploy_module.DeployError, match="not derived"): deploy_module.validate_project_config(Path("repo.git"), remapped, "a" * 40) def test_repopulse_project_registry_requires_its_fixed_build_identity( deploy_module, monkeypatch ): source = "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse" project = { "branch": "main", "slug": "repopulse", "source_dir": str(source), "exact_source": True, "compose_dir": str(source), "compose_files": [".itworx/deployment.yml"], "compose_project": "repopulse", "containers": ["repopulse"], "managed_plan_version": "itworx.deployment/v1", "managed_profile": "repopulse-authenticated", "managed_application_id": "repopulse", "managed_repository": "jens/repopulse", "managed_appdata_path": "/srv/example/appdata/repopulse", "managed_image_repository": "repopulse-managed", "managed_image_tag": "production", "managed_proxy_network": "itworx-mcp-hub-npm", "managed_internal_port": 8000, "managed_build_context": ".", "managed_build_dockerfile": "deploy/managed/Dockerfile", "managed_runtime_environment_keys": [ "REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN", ], "managed_runtime_uid": 10001, "managed_runtime_gid": 10001, "managed_proxy_hostname": "repopulse.example.com", "managed_health_path": "/health", "appops_deployment_secret_file": "jens__repopulse", "appops_managed_target": True, } monkeypatch.setattr(deploy_module, "git_path_exists", lambda *_args: True) deploy_module.validate_project_config(Path("repo.git"), project, "a" * 40) for field, value in ( ("managed_repository", "jens/another-app"), ("managed_build_dockerfile", "Dockerfile"), ("managed_runtime_gid", 1000), ("managed_proxy_network", "itworx-npm"), ("source_dir", "/mnt/cache/appdata/gitea-deploy-runners/sources/other"), ): unsafe = {**project, field: value} with pytest.raises(deploy_module.DeployError): deploy_module.validate_project_config(Path("repo.git"), unsafe, "a" * 40) def test_controller_metadata_path_is_reserved(deploy_module, monkeypatch): monkeypatch.setattr( deploy_module, "output", lambda *_args, **_kwargs: "compose.yaml\n.gitea-deploy/revision", ) with pytest.raises(deploy_module.DeployError, match="metadata path"): deploy_module.tracked_files(Path("repo.git"), "a" * 40) @pytest.mark.parametrize( ("name", "linkname", "kind"), [ ("../outside.txt", "", "file"), ("/absolute.txt", "", "file"), ("safe/link", "../../outside.txt", "symlink"), ("safe/link", "../outside.txt", "hardlink"), ], ) def test_git_archive_links_and_paths_cannot_escape_staging( deploy_module, tmp_path, monkeypatch, name, linkname, kind ): buffer = io.BytesIO() with tarfile.open(fileobj=buffer, mode="w") as archive: member = tarfile.TarInfo(name) if kind == "symlink": member.type = tarfile.SYMTYPE member.linkname = linkname archive.addfile(member) elif kind == "hardlink": member.type = tarfile.LNKTYPE member.linkname = linkname archive.addfile(member) else: payload = b"hostile" member.size = len(payload) archive.addfile(member, io.BytesIO(payload)) archive_bytes = buffer.getvalue() def fake_run(_command, **kwargs): kwargs["stdout"].write(archive_bytes) return types.SimpleNamespace(returncode=0) monkeypatch.setattr(deploy_module.subprocess, "run", fake_run) target = tmp_path / "staging" target.mkdir() with pytest.raises(deploy_module.DeployError): deploy_module.extract_revision(Path("repo.git"), "a" * 40, target) assert not (tmp_path / "outside.txt").exists() def test_exact_source_manifest_failure_preserves_previous_tree( deploy_module, tmp_path, monkeypatch ): exact_root = tmp_path / "managed-sources" target = exact_root / "chimera" target.mkdir(parents=True) (target / "live.txt").write_text("previous", encoding="utf-8") monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root) monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"]) def fake_extract(_repo, _revision, destination): (destination / "tracked.txt").write_text("reviewed", encoding="utf-8") (destination / "unexpected.txt").write_text("dirty", encoding="utf-8") monkeypatch.setattr(deploy_module, "extract_revision", fake_extract) with pytest.raises(deploy_module.DeployError, match="file manifest"): deploy_module.sync_source( Path("repo.git"), "a" * 40, target, "chimera", exact=True ) assert (target / "live.txt").read_text(encoding="utf-8") == "previous" def test_exact_source_atomic_swap_failure_restores_previous_tree( deploy_module, tmp_path, monkeypatch ): exact_root = tmp_path / "managed-sources" target = exact_root / "chimera" target.mkdir(parents=True) (target / "tracked.txt").write_text("previous", encoding="utf-8") monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", exact_root) monkeypatch.setattr(deploy_module, "tracked_files", lambda *_args: ["tracked.txt"]) def fake_extract(_repo, _revision, destination): (destination / "tracked.txt").write_text("reviewed", encoding="utf-8") monkeypatch.setattr(deploy_module, "extract_revision", fake_extract) original_replace = Path.replace def fail_staging_swap(self, destination): if self.name.startswith(".chimera-") and Path(destination) == target: raise OSError("simulated atomic swap failure") return original_replace(self, destination) monkeypatch.setattr(Path, "replace", fail_staging_swap) with pytest.raises(OSError, match="simulated atomic swap failure"): deploy_module.sync_source( Path("repo.git"), "a" * 40, target, "chimera", exact=True ) assert (target / "tracked.txt").read_text(encoding="utf-8") == "previous" def test_exact_source_uses_external_compose_environment(deploy_module, tmp_path): source = tmp_path / "source" source.mkdir() (source / "compose.yaml").write_text("services: {}\n", encoding="utf-8") env_source = tmp_path / "deploy.env" env_source.write_text("SECRET=not-copied\n", encoding="utf-8") project = { "source_dir": str(source), "compose_dir": "", "compose_files": ["compose.yaml"], "compose_project": "chimera", "exact_source": True, "env_source": str(env_source), } deploy_module.copy_environment(project, source) _, command = deploy_module.compose_command(project) assert not (source / ".env").exists() assert command[:7] == [ "docker", "compose", "--project-name", "chimera", "--env-file", str(env_source), "--file", ] def test_managed_nested_manifest_uses_exact_source_as_project_directory(deploy_module, tmp_path): source = tmp_path / "source" manifest = source / ".itworx" / "deployment.yml" manifest.parent.mkdir(parents=True) manifest.write_text("services: {}\n", encoding="utf-8") project = { "source_dir": str(source), "compose_files": [".itworx/deployment.yml"], "compose_project": "repopulse", "exact_source": True, "managed_plan_version": "itworx.deployment/v1", } cwd, command = deploy_module.compose_command(project) assert cwd == source assert command == [ "docker", "compose", "--project-name", "repopulse", "--project-directory", str(source), "--file", str(manifest), ] def test_immutable_image_and_live_probe_must_match_every_build_field( deploy_module, tmp_path, monkeypatch ): revision = "a" * 40 tree = "b" * 40 metadata = { "id": revision[:12], "repository": "jens/chimera", "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-08-30T10:00:00.000Z", } project = _project(tmp_path) project.update( { "slug": "chimera", "compose_project": "chimera", "containers": ["Chimera"], "_repository_name": "jens/chimera", "live_probe": { "type": "chimera-health-live-v1", "container": "Chimera", "port": 6967, "path": "/health/live", "host": "chimera:6967", }, } ) image_id = "sha256:" + "1" * 64 monkeypatch.setattr( deploy_module, "compose_project_container_names", lambda _name: {"Chimera"}, ) labels = { "org.opencontainers.image.revision": revision, "org.opencontainers.image.source": "jens/chimera", "org.opencontainers.image.source-tree": tree, "org.opencontainers.image.version": revision[:12], "org.opencontainers.image.created": metadata["builtAt"], } monkeypatch.setattr( deploy_module, "container_state", lambda _name: (True, "healthy", "chimera") ) monkeypatch.setattr( deploy_module, "inspect_container", lambda _name: {"Image": image_id, "Config": {"Labels": {}}}, ) monkeypatch.setattr( deploy_module, "inspect_image", lambda _image_id: {"Id": image_id, "Config": {"Labels": dict(labels)}}, ) monkeypatch.setattr( deploy_module.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace( returncode=0, stdout=json.dumps({"status": "live", "build": metadata}), stderr="", ), ) receipt = deploy_module.verify_deployment(project, metadata) assert receipt["build"] == metadata assert receipt["containers"]["Chimera"]["image_id"] == image_id labels["org.opencontainers.image.source-tree"] = "c" * 40 with pytest.raises(deploy_module.DeployError, match="Immutable image provenance"): deploy_module.verify_deployment(project, metadata) def test_live_probe_rejects_every_provenance_drift_and_hostile_response( deploy_module, tmp_path, monkeypatch ): revision = "a" * 40 tree = "b" * 40 metadata = { "id": revision[:12], "repository": "jens/chimera", "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": "2026-08-30T10:00:00.000Z", } project = _project(tmp_path) project.update( { "containers": ["Chimera"], "live_probe": { "type": "chimera-health-live-v1", "container": "Chimera", "port": 6967, "path": "/health/live", "host": "chimera:6967", }, } ) bad_outputs = ["not-json", "x" * 65537] bad_outputs.append(json.dumps({"status": "starting", "build": metadata})) for key in ("id", "revision", "tree", "treeDigest", "builtAt"): changed = dict(metadata) changed[key] = "wrong" bad_outputs.append(json.dumps({"status": "live", "build": changed})) for stdout in bad_outputs: monkeypatch.setattr( deploy_module.subprocess, "run", lambda *_args, _stdout=stdout, **_kwargs: types.SimpleNamespace( returncode=0, stdout=_stdout, stderr="", ), ) with pytest.raises(deploy_module.DeployError): deploy_module.verify_live_probe(project, metadata) monkeypatch.setattr( deploy_module.subprocess, "run", lambda *_args, **_kwargs: (_ for _ in ()).throw( subprocess.TimeoutExpired("docker", 10) ), ) with pytest.raises(deploy_module.DeployError, match="timed out"): deploy_module.verify_live_probe(project, metadata) def test_same_revision_revalidates_runtime_receipt_before_success( deploy_module, tmp_path, monkeypatch ): revision = "a" * 40 tree = "b" * 40 project = _project(tmp_path) receipt = _receipt(project, revision, tree=tree) monkeypatch.setattr( deploy_module, "read_state", lambda _project: { "schema_version": 3, "repository": "jens/unraid-appops-gateway", "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": receipt, }, ) calls = [] monkeypatch.setattr( deploy_module, "verify_deployment", lambda _project, metadata, **_kwargs: calls.append(metadata) or receipt, ) assert ( deploy_module.verified_current_receipt( "jens/unraid-appops-gateway", project, revision, tree ) == receipt ) assert calls == [receipt["build"]] monkeypatch.setattr( deploy_module, "verify_deployment", lambda *_args, **_kwargs: (_ for _ in ()).throw( deploy_module.DeployError("drift") ), ) assert ( deploy_module.verified_current_receipt( "jens/unraid-appops-gateway", project, revision, tree ) is None ) def _webhook_project(repository: str = "jens/repo-a") -> dict[str, object]: return { "branch": "main", "gitea_repository_id": 77, "gitea_webhook_secret_file": repository.replace("/", "__"), "validation": { "validation_gate_version": 1, "required": True, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 600, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, }, } def _write_webhook_secret(root: Path, name: str, secret: bytes) -> None: root.mkdir() root.chmod(0o700) path = root / name path.write_bytes(secret + b"\n") path.chmod(0o600) def _push_payload( revision: str, *, repository: str = "jens/repo-a", repository_id: int = 77 ) -> dict[str, object]: return { "ref": "refs/heads/main", "before": "b" * 40, "after": revision, "repository": { "id": repository_id, "full_name": repository, "default_branch": "main", "object_format_name": "sha1", }, "head_commit": {"id": revision}, } def test_repository_webhook_secrets_are_name_scoped_and_hmac_bound( status_module, tmp_path, monkeypatch ): secret_root = tmp_path / "secrets" secret = b"repo-a-" + b"x" * 40 _write_webhook_secret(secret_root, "jens__repo-a", secret) monkeypatch.setattr(status_module, "WEBHOOK_SECRET_ROOT", secret_root) project = _webhook_project() body = b'{"repository":{"full_name":"Jens/repo-a"}}' signature = hmac.new(secret, body, hashlib.sha256).hexdigest() assert status_module._webhook_secret("jens/repo-a", project) == secret assert status_module._webhook_secret("jens/repo-b", project) is None assert status_module._valid_webhook_signature( body, signature, "jens/repo-a", project ) assert not status_module._valid_webhook_signature( body + b" ", signature, "jens/repo-a", project ) def test_webhook_policy_rejects_legacy_or_mismatched_credentials(status_module): project = _webhook_project() assert status_module._webhook_policy("jens/repo-a", project) == ( 77, "main", "jens__repo-a", ) for invalid in ( {}, {**project, "gitea_repository_id": True}, {**project, "gitea_repository_id": 0}, {**project, "gitea_webhook_secret_file": "jens__other"}, {**project, "broker_token_file": "jens__repo-a"}, {**project, "legacy_broker_token": True}, ): assert status_module._webhook_policy("jens/repo-a", invalid) is None def test_webhook_push_identity_binds_repo_id_branch_revision_and_head(status_module): revision = "a" * 40 project = _webhook_project() assert ( status_module._webhook_push_identity( _push_payload(revision), "jens/repo-a", project ) == revision ) assert ( status_module._webhook_push_identity( _push_payload(revision, repository="Jens/Repo-A"), "jens/repo-a", project, ) == revision ) mutations = ( ("ref", "refs/heads/other"), ("before", revision), ("before", "B" * 40), ("after", "0" * 40), ("after", "A" * 40), ( "repository", { "id": 78, "full_name": "jens/repo-a", "default_branch": "main", "object_format_name": "sha1", }, ), ( "repository", { "id": 77, "full_name": "jens/other", "default_branch": "main", "object_format_name": "sha1", }, ), ( "repository", { "id": 77, "full_name": "jens/repo-a", "default_branch": "other", "object_format_name": "sha1", }, ), ( "repository", { "id": 77, "full_name": "jens/repo-a", "default_branch": "main", "object_format_name": "sha256", }, ), ("head_commit", {"id": "b" * 40}), ) for field, value in mutations: payload = _push_payload(revision) payload[field] = value assert ( status_module._webhook_push_identity(payload, "jens/repo-a", project) is None ) def test_webhook_inbox_is_durable_and_replay_keys_the_signed_semantics( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) delivery = "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d" revision = "a" * 40 ref = "refs/heads/main" raw_body = json.dumps( _push_payload(revision, repository="Jens/repo-a"), separators=(",", ":"), ).encode() body_digest = hashlib.sha256(raw_body).hexdigest() semantic_digest = status_module._webhook_semantic_digest(77, ref, revision) policy = status_module._validation_policy("jens/repo-a", _webhook_project()) assert policy is not None assert status_module._enqueue_webhook_delivery( raw_body, delivery, 77, ref, "jens/repo-a", revision, policy ) == ("awaiting_validation", semantic_digest) marker_path = inbox_root / f"{semantic_digest}.json" marker = json.loads(marker_path.read_text()) assert marker["schema"] == "gitea-webhook-inbox.v2" assert marker["semantic_sha256"] == semantic_digest assert marker["body_sha256"] == body_digest assert marker["gitea_repository_id"] == 77 assert marker["ref"] == ref assert marker["repository"] == "jens/repo-a" assert marker["revision"] == revision assert marker["status"] == "awaiting_validation" assert marker["validation"]["required_contexts"] == ["Managed validation / full (push)"] changed_unsigned_delivery = "145440ab-b7b1-4c74-ae84-996d8d48167e" assert status_module._enqueue_webhook_delivery( raw_body, changed_unsigned_delivery, 77, ref, "jens/repo-a", revision, policy, ) == ("awaiting_validation", semantic_digest) assert json.loads(marker_path.read_text())["delivery"] == delivery same_semantics_different_body = json.dumps( {**_push_payload(revision), "compare_url": "https://example.invalid/compare"}, separators=(",", ":"), ).encode() assert hashlib.sha256(same_semantics_different_body).hexdigest() != body_digest assert status_module._enqueue_webhook_delivery( same_semantics_different_body, changed_unsigned_delivery, 77, ref, "jens/repo-a", revision, policy, ) == ("awaiting_validation", semantic_digest) assert json.loads(marker_path.read_text())["body_sha256"] == body_digest assert status_module._enqueue_webhook_delivery( raw_body, "../../escape", 77, ref, "jens/repo-a", revision, policy ) == ("invalid", "") def test_terminal_validation_revalidation_creates_a_separate_idempotent_attempt( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True) revision = "a" * 40 policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, semantic_digest = status_module._enqueue_webhook_delivery( json.dumps(_push_payload(revision), separators=(",", ":")).encode(), "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", revision, policy, ) original_marker = inbox_root / f"{semantic_digest}.json" original = json.loads(original_marker.read_text()) original["status"] = "validation_failed" original["reason"] = "validation_failed" original["validation"]["last_result"] = "reject" original_marker.write_text( json.dumps(original, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8", ) original_bytes = original_marker.read_bytes() assert status_module._enqueue_terminal_revalidation( "jens/repo-a", revision, project ) == ("awaiting_validation", semantic_digest) revalidation_marker = inbox_root / f"{semantic_digest}.revalidation.json" attempt = json.loads(revalidation_marker.read_text()) assert original_marker.read_bytes() == original_bytes assert attempt["schema"] == "gitea-webhook-revalidation.v1" assert attempt["semantic_sha256"] == semantic_digest assert attempt["revalidation_of"] == semantic_digest assert attempt["gitea_repository_id"] == 77 assert attempt["ref"] == "refs/heads/main" assert attempt["repository"] == "jens/repo-a" assert attempt["revision"] == revision assert attempt["status"] == "awaiting_validation" assert attempt["validation"]["attempts"] == 0 assert attempt["validation"]["observed"] == [] assert [entry["stage"] for entry in attempt["audit"]] == [ "revalidation_requested", "awaiting_validation", ] attempt_bytes = revalidation_marker.read_bytes() assert status_module._enqueue_terminal_revalidation( "jens/repo-a", revision, project ) == ("awaiting_validation", semantic_digest) assert revalidation_marker.read_bytes() == attempt_bytes assert len(list(inbox_root.glob("*.json"))) == 2 @pytest.mark.parametrize( "terminal_status", ["deployed", "failed", "non_forward", "rejected", "superseded"], ) def test_revalidation_refuses_non_validation_terminal_states( status_module, tmp_path, monkeypatch, terminal_status ): inbox_root = tmp_path / "inbox" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True) revision = "b" * 40 policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, semantic_digest = status_module._enqueue_webhook_delivery( json.dumps(_push_payload(revision), separators=(",", ":")).encode(), "145440ab-b7b1-4c74-ae84-996d8d48167e", 77, "refs/heads/main", "jens/repo-a", revision, policy, ) original_marker = inbox_root / f"{semantic_digest}.json" original = json.loads(original_marker.read_text()) original["status"] = terminal_status original_marker.write_text(json.dumps(original), encoding="utf-8") original_bytes = original_marker.read_bytes() assert status_module._enqueue_terminal_revalidation( "jens/repo-a", revision, project ) == ("not_revalidatable", semantic_digest) assert original_marker.read_bytes() == original_bytes assert not (inbox_root / f"{semantic_digest}.revalidation.json").exists() def test_revalidation_refuses_a_revision_that_is_not_the_current_branch_head( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: False) revision = "c" * 40 policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, semantic_digest = status_module._enqueue_webhook_delivery( json.dumps(_push_payload(revision), separators=(",", ":")).encode(), "766ea001-aa32-49c4-b765-5ea0a39a7831", 77, "refs/heads/main", "jens/repo-a", revision, policy, ) original_marker = inbox_root / f"{semantic_digest}.json" original = json.loads(original_marker.read_text()) original["status"] = "validation_timeout" original_marker.write_text(json.dumps(original), encoding="utf-8") original_bytes = original_marker.read_bytes() assert status_module._enqueue_terminal_revalidation( "jens/repo-a", revision, project ) == ("not_current_head", semantic_digest) assert original_marker.read_bytes() == original_bytes assert not (inbox_root / f"{semantic_digest}.revalidation.json").exists() def test_revalidation_attempt_requires_fresh_status_success_and_preserves_original( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" projects_path = tmp_path / "projects.json" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True) revision = "d" * 40 policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, semantic_digest = status_module._enqueue_webhook_delivery( json.dumps(_push_payload(revision), separators=(",", ":")).encode(), "92869210-fcd2-47d2-a775-ab3fa47f229f", 77, "refs/heads/main", "jens/repo-a", revision, policy, ) original_marker = inbox_root / f"{semantic_digest}.json" original = json.loads(original_marker.read_text()) original["status"] = "validation_failed" original["reason"] = "validation_failed" original_marker.write_text(json.dumps(original), encoding="utf-8") original_bytes = original_marker.read_bytes() assert status_module._enqueue_terminal_revalidation( "jens/repo-a", revision, project ) == ("awaiting_validation", semantic_digest) attempt_marker = inbox_root / f"{semantic_digest}.revalidation.json" deploy_calls: list[list[str]] = [] class Result: returncode = 0 monkeypatch.setattr( status_module.subprocess, "run", lambda args, **_kwargs: deploy_calls.append(args) or Result(), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True}) monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("wait", [])) status_module._process_webhook_record(attempt_marker) assert json.loads(attempt_marker.read_text())["status"] == "awaiting_validation" assert deploy_calls == [] assert original_marker.read_bytes() == original_bytes monkeypatch.setattr( status_module, "_fetch_gitea_validation", lambda *_args: ( "pass", [{ "context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-21T18:10:15Z", }], ), ) attempt = json.loads(attempt_marker.read_text()) attempt["validation"]["next_check_at"] = 0 attempt_marker.write_text(json.dumps(attempt), encoding="utf-8") status_module._process_webhook_record(attempt_marker) assert json.loads(attempt_marker.read_text())["status"] == "deployed" assert deploy_calls == [[ "python3", str(status_module.ROOT / "deploy.py"), "deploy", "jens/repo-a", revision, ]] assert original_marker.read_bytes() == original_bytes def test_delivery_status_identifies_revalidation_attempts( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" project = _webhook_project() monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True) monkeypatch.setattr( status_module, "_gitea_validation_settings", lambda: ("https://gitea.example.test/api/v1", b"x" * 40), ) revision = "e" * 40 policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, semantic_digest = status_module._enqueue_webhook_delivery( json.dumps(_push_payload(revision), separators=(",", ":")).encode(), "c492f8a1-dd2b-48fe-af98-f071c802b53e", 77, "refs/heads/main", "jens/repo-a", revision, policy, ) original_marker = inbox_root / f"{semantic_digest}.json" original = json.loads(original_marker.read_text()) original["status"] = "validation_failed" original_marker.write_text(json.dumps(original), encoding="utf-8") status_module._enqueue_terminal_revalidation("jens/repo-a", revision, project) payload = status_module._delivery_status_payload() revalidation = next(item for item in payload["items"] if item.get("attemptType") == "revalidation") assert revalidation["repository"] == "jens/repo-a" assert revalidation["revision"] == revision assert revalidation["revalidationOf"] == semantic_digest def test_revalidation_rechecks_exact_branch_head_immediately_before_deploy( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" projects_path = tmp_path / "projects.json" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) current_head = iter([True, False]) monkeypatch.setattr( status_module, "_is_current_branch_head", lambda *_args: next(current_head) ) revision = "f" * 40 policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, semantic_digest = status_module._enqueue_webhook_delivery( json.dumps(_push_payload(revision), separators=(",", ":")).encode(), "691f58fa-fbd1-49d7-87c3-d7caa947cb10", 77, "refs/heads/main", "jens/repo-a", revision, policy, ) original_marker = inbox_root / f"{semantic_digest}.json" original = json.loads(original_marker.read_text()) original["status"] = "validation_failed" original_marker.write_text(json.dumps(original), encoding="utf-8") original_bytes = original_marker.read_bytes() assert status_module._enqueue_terminal_revalidation( "jens/repo-a", revision, project ) == ("awaiting_validation", semantic_digest) attempt_marker = inbox_root / f"{semantic_digest}.revalidation.json" monkeypatch.setattr( status_module, "_fetch_gitea_validation", lambda *_args: ( "pass", [{ "context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-21T18:10:15Z", }], ), ) monkeypatch.setattr( status_module.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("deployment ran after branch head changed"), ) status_module._process_webhook_record(attempt_marker) attempt = json.loads(attempt_marker.read_text()) assert attempt["status"] == "superseded" assert attempt["reason"] == "revision_not_current_head" assert original_marker.read_bytes() == original_bytes def test_revalidation_cli_refuses_non_root_linux_operator(status_module, monkeypatch): monkeypatch.setattr(status_module.os, "name", "posix") monkeypatch.setattr(status_module.os, "geteuid", lambda: 1000, raising=False) monkeypatch.setattr( status_module.sys, "argv", [ "status_server.py", "--revalidate-delivery", "jens/repo-a", "a" * 40, ], ) with pytest.raises(PermissionError, match="root"): status_module.main() def test_revalidation_cli_queues_through_local_supported_surface( status_module, monkeypatch, capsys ): project = _webhook_project() revision = "a" * 40 digest = "b" * 64 monkeypatch.setattr(status_module.os, "name", "posix") monkeypatch.setattr(status_module.os, "geteuid", lambda: 0, raising=False) monkeypatch.setattr(status_module, "_require_controller_release", dict) monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/repo-a": project}) requests: list[tuple[str, str, dict[str, object]]] = [] monkeypatch.setattr( status_module, "_enqueue_terminal_revalidation", lambda repository, requested_revision, requested_project: ( requests.append((repository, requested_revision, requested_project)) or ("awaiting_validation", digest) ), ) monkeypatch.setattr( status_module.sys, "argv", [ "status_server.py", "--revalidate-delivery", "jens/repo-a", revision, ], ) status_module.main() assert requests == [("jens/repo-a", revision, project)] assert json.loads(capsys.readouterr().out) == { "status": "awaiting_validation", "repository": "jens/repo-a", "revision": revision, "delivery": digest, } def test_revalidation_has_no_http_mutation_route(status_module): server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() try: client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/revalidate", body=b"{}", headers={"Content-Type": "application/json", "Content-Length": "2"}, ) response = client.getresponse() response.read() client.close() assert response.status == 404 finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_workmesh_promotion_client_uses_only_fixed_signed_contract(monkeypatch): from backend.app.services.deployment_broker import DeploymentBrokerClient captured = [] class Response: status_code = 200 @staticmethod def json(): return {"status": "accepted", "operation": "workmesh_promote"} class Client: def __init__(self, **_kwargs): pass def __enter__(self): return self def __exit__(self, *_args): return None def request(self, method, url, *, content, headers): captured.append((method, url, content, headers)) return Response() monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client) client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True) secret = "workmesh-controller-secret-" + "x" * 32 revision = "a" * 40 digest = "b" * 64 key = "workmesh-promotion:workmesh-prom-0123456789abcdef" client.workmesh_promotion_preflight(revision, secret=secret) client.workmesh_promotion_apply( revision, secret=secret, preflight_sha256=digest, idempotency_key=key, ) expected = [ ( "POST", "https://controller.example/v1/appops/workmesh/promotions/preflight", {"revision": revision}, ), ( "POST", "https://controller.example/v1/appops/workmesh/promotions/apply", {"revision": revision, "preflightSha256": digest, "idempotencyKey": key}, ), ] assert [ (method, url, json.loads(body)) for method, url, body, _headers in captured ] == expected for method, url, body, headers in captured: path = url.removeprefix("https://controller.example") assert headers["X-ITWorx-Deployment-Signature"] == DeploymentBrokerClient._signature( secret, method, path, headers["X-ITWorx-Deployment-Timestamp"], headers["X-ITWorx-Deployment-Nonce"], body, ) assert client.workmesh_promotion_apply( "main", secret=secret, preflight_sha256=digest, idempotency_key=key, ) == {"status": "invalid", "reason": "invalid_workmesh_revision"} assert len(captured) == 2 def test_workmesh_apply_preserves_rejection_identity_without_extra_fields(monkeypatch): from backend.app.services.deployment_broker import DeploymentBrokerClient request_id = "a" * 64 class Response: status_code = 409 @staticmethod def json(): return { "status": "preflight_changed", "request": request_id, "repository": "jens/workmesh", "revision": "b" * 40, "operation": "workmesh_promote", "reason": "approved_preflight_no_longer_matches", "secret": "discard-me", } class Client: def __init__(self, **_kwargs): pass def __enter__(self): return self def __exit__(self, *_args): return None def request(self, *_args, **_kwargs): return Response() monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client) client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True) result = client.workmesh_promotion_apply( "b" * 40, secret="workmesh-controller-secret-" + "x" * 32, preflight_sha256="c" * 64, idempotency_key="workmesh-promotion:workmesh-prom-0123456789abcdef", ) assert result == { "status": "preflight_changed", "reason": "approved_preflight_no_longer_matches", "request": request_id, "repository": "jens/workmesh", "revision": "b" * 40, "operation": "workmesh_promote", } def test_health_is_served_while_initial_runner_maintenance_is_blocked( status_module, monkeypatch ): maintenance_started = threading.Event() release_maintenance = threading.Event() server_created = threading.Event() serving_started = threading.Event() server_holder = {} real_server = status_module.BoundedThreadingHTTPServer class BlockingFleet: def automatic_cycle(self): maintenance_started.set() assert release_maintenance.wait(timeout=5) class ObservedServer(real_server): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) server_holder["server"] = self server_created.set() def serve_forever(self, *args, **kwargs): serving_started.set() return super().serve_forever(*args, **kwargs) monkeypatch.setattr(status_module, "RunnerFleetService", BlockingFleet) monkeypatch.setattr(status_module, "runner_fleet_specialization_state", lambda: "legacy") monkeypatch.setattr(status_module, "BoundedThreadingHTTPServer", ObservedServer) monkeypatch.setattr( status_module, "_controller_release", lambda: { "apiVersion": "itworx-controller-release.v1", "controllerVersion": "2.1.3", "sourceRepository": "jens/Unraid-AppOps-Gateway", "sourceRevision": "a" * 40, "sourceTree": "b" * 40, "artifactSha256": "c" * 64, "builtAt": "2026-09-21T20:58:36Z", }, ) monkeypatch.setattr( status_module, "_delivery_status_payload", lambda: {"status": "ok", "health": "healthy", "summary": {}}, ) monkeypatch.setattr( status_module.sys, "argv", ["status_server.py", "--listen", "127.0.0.1", "--port", "0"], ) main_thread = threading.Thread(target=status_module.main, daemon=True) main_thread.start() try: assert maintenance_started.wait(timeout=2) assert server_created.wait(timeout=1), "server must bind before startup maintenance" assert serving_started.wait(timeout=1), "server must serve before startup maintenance" server = server_holder["server"] client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request("GET", "/health") response = client.getresponse() payload = json.loads(response.read()) client.close() assert response.status == 200 assert payload["status"] == "ok" assert payload["transport"] == "http" assert not release_maintenance.is_set() finally: release_maintenance.set() if serving_started.wait(timeout=2): server_holder["server"].shutdown() main_thread.join(timeout=3) assert not main_thread.is_alive() def test_webhook_worker_deploys_once_and_requires_verified_receipt( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" projects_path = tmp_path / "projects.json" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) revision = "a" * 40 raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode() delivery = "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d" _, request_digest = status_module._enqueue_webhook_delivery( raw_body, delivery, 77, "refs/heads/main", "jens/repo-a", revision, status_module._validation_policy("jens/repo-a", project), ) marker = inbox_root / f"{request_digest}.json" calls = [] class Result: returncode = 0 monkeypatch.setattr( status_module.subprocess, "run", lambda arguments, **_kwargs: calls.append(arguments) or Result(), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"ok": True}) monkeypatch.setattr( status_module, "_fetch_gitea_validation", lambda *_args: ("pass", [{"context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-14T00:00:00Z"}]), ) status_module._process_webhook_record(marker) assert json.loads(marker.read_text())["status"] == "deployed" assert calls == [ [ "python3", str(status_module.ROOT / "deploy.py"), "deploy", "jens/repo-a", revision, ] ] status_module._process_webhook_record(marker) assert len(calls) == 1 second_body = json.dumps(_push_payload("c" * 40), separators=(",", ":")).encode() _, second_digest = status_module._enqueue_webhook_delivery( second_body, "145440ab-b7b1-4c74-ae84-996d8d48167e", 77, "refs/heads/main", "jens/repo-a", "c" * 40, status_module._validation_policy("jens/repo-a", project), ) second_marker = inbox_root / f"{second_digest}.json" monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None) status_module._process_webhook_record(second_marker) assert json.loads(second_marker.read_text())["status"] == "failed" def test_webhook_worker_recovers_interrupted_deployment_from_verified_receipt( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" projects_path = tmp_path / "projects.json" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) revision = "a" * 40 raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode() _, request_digest = status_module._enqueue_webhook_delivery( raw_body, "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", revision, status_module._validation_policy("jens/repo-a", project), ) marker = inbox_root / f"{request_digest}.json" record = json.loads(marker.read_text()) record["status"] = "deploying" marker.write_text(json.dumps(record), encoding="utf-8") receipt = {"schema": "gitea-deploy-receipt.v1", "revision": revision} calls = [] monkeypatch.setattr( status_module.subprocess, "run", lambda arguments, **_kwargs: calls.append(arguments), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: receipt) status_module._process_webhook_record(marker) recovered = json.loads(marker.read_text()) assert recovered["status"] == "deployed" assert recovered["receipt_sha256"] == hashlib.sha256( json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() assert recovered["audit"][-1]["stage"] == "deployed" assert calls == [] def test_webhook_worker_fails_interrupted_deployment_without_verified_receipt( status_module, tmp_path, monkeypatch ): inbox_root = tmp_path / "inbox" projects_path = tmp_path / "projects.json" project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) revision = "a" * 40 raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode() _, request_digest = status_module._enqueue_webhook_delivery( raw_body, "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", revision, status_module._validation_policy("jens/repo-a", project), ) marker = inbox_root / f"{request_digest}.json" record = json.loads(marker.read_text()) record["status"] = "deploying" marker.write_text(json.dumps(record), encoding="utf-8") calls = [] monkeypatch.setattr( status_module.subprocess, "run", lambda arguments, **_kwargs: calls.append(arguments), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None) status_module._process_webhook_record(marker) recovered = json.loads(marker.read_text()) assert recovered["status"] == "failed" assert recovered["reason"] == "deployment_interrupted" assert recovered["audit"][-1] == { "at": recovered["audit"][-1]["at"], "stage": "deployment_failed", "reason": "deployment_interrupted", } assert calls == [] def test_gitea_validation_uses_latest_exact_sha_push_context_only(status_module, monkeypatch): revision = "a" * 40 policy = status_module._validation_policy("jens/repo-a", _webhook_project()) assert policy is not None class Response: status = 200 def __enter__(self): return self def __exit__(self, *_args): return None def read(self, _limit): return json.dumps([ { "id": 1, "context": "Managed validation / full (push)", "status": "success", "url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}", "target_url": "/Jens/Repo-A/actions/runs/100/jobs/200", "updated_at": "2026-09-14T00:00:00Z", }, { "id": 2, "context": "Managed validation / full (push)", "status": "failure", "url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}", "target_url": "/Jens/Repo-A/actions/runs/100/jobs/200", "updated_at": "2026-09-14T00:01:00Z", }, { "id": 3, "context": "Managed validation / full (push)", "status": "success", "url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{'b' * 40}", "target_url": "/Jens/Repo-A/actions/runs/100/jobs/200", "updated_at": "2026-09-14T00:02:00Z", }, { "id": 4, "context": "lint (push)", "status": "success", "url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}", "target_url": "/Jens/Repo-A/actions/runs/100/jobs/200", "updated_at": "2026-09-14T00:03:00Z", }, { "id": 5, "context": "Managed validation / full (push)", "status": "success", "url": f"https://gitea.example.test/api/v1/repos/Jens/Repo-A/statuses/{revision}", "target_url": "/Jens/other-repo/actions/runs/100/jobs/200", "updated_at": "2026-09-14T00:04:00Z", }, ]).encode() class Opener: def open(self, _request, timeout): assert timeout == status_module.GITEA_STATUS_TIMEOUT_SECONDS return Response() monkeypatch.setattr( status_module, "_gitea_validation_settings", lambda: ("https://gitea.example.test/api/v1", b"x" * 40), ) monkeypatch.setattr(status_module, "build_opener", lambda *_args: Opener()) outcome, observed = status_module._fetch_gitea_validation("jens/repo-a", revision, policy) assert outcome == "reject" assert observed == [{ "context": "Managed validation / full (push)", "status": "failure", "updated_at": "2026-09-14T00:01:00Z", }] def test_gitea_validation_api_probe_uses_only_required_repository_read_scope(status_module, monkeypatch): class Response: status = 200 def __enter__(self): return self def __exit__(self, *_args): return None def read(self, _limit): return b'{"full_name":"Jens/Unraid-AppOps-Gateway"}' class Opener: def open(self, request, timeout): assert request.full_url == "https://gitea.example.test/api/v1/repos/jens/unraid-appops-gateway" assert timeout == status_module.GITEA_STATUS_TIMEOUT_SECONDS return Response() monkeypatch.setattr( status_module, "_gitea_validation_settings", lambda: ("https://gitea.example.test/api/v1", b"x" * 40), ) monkeypatch.setattr( status_module, "_controller_release", lambda: {"sourceRepository": "jens/unraid-appops-gateway"}, ) monkeypatch.setattr(status_module, "build_opener", lambda *_args: Opener()) assert status_module._gitea_validation_api_reachable() def test_validation_gate_never_deploys_before_exact_status_success(status_module, tmp_path, monkeypatch): inbox = tmp_path / "inbox" projects = tmp_path / "projects.json" project = {**_webhook_project(), "slug": "repo-a", "containers": ["RepoA"]} projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, request_id = status_module._enqueue_webhook_delivery( json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(), "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", "a" * 40, policy, ) marker = inbox / f"{request_id}.json" deploy_calls: list[list[str]] = [] class Result: returncode = 0 monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: deploy_calls.append(args) or Result()) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True}) monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("wait", [])) status_module._process_webhook_record(marker) assert json.loads(marker.read_text())["status"] == "awaiting_validation" assert deploy_calls == [] monkeypatch.setattr( status_module, "_fetch_gitea_validation", lambda *_args: ("pass", [{"context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-14T00:00:00Z"}]), ) record = json.loads(marker.read_text()) record["validation"]["next_check_at"] = 0 marker.write_text(json.dumps(record), encoding="utf-8") status_module._process_webhook_record(marker) assert json.loads(marker.read_text())["status"] == "deployed" assert len(deploy_calls) == 1 @pytest.mark.parametrize( ("outcome", "expected_state"), [ ("reject", "validation_failed"), ("unavailable", "awaiting_validation"), ], ) def test_validation_failure_or_gitea_outage_fails_closed( status_module, tmp_path, monkeypatch, outcome, expected_state ): inbox = tmp_path / "inbox" projects = tmp_path / "projects.json" project = {**_webhook_project(), "slug": "repo-a", "containers": ["RepoA"]} projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, request_id = status_module._enqueue_webhook_delivery( json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(), "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", "a" * 40, policy, ) marker = inbox / f"{request_id}.json" deploy_calls: list[list[str]] = [] monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: deploy_calls.append(args)) monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: (outcome, [])) status_module._process_webhook_record(marker) record = json.loads(marker.read_text()) assert record["status"] == expected_state assert deploy_calls == [] if outcome == "unavailable": assert record["reason"] == "gitea_validation_unavailable" def test_newer_validated_descendant_supersedes_old_undeployed_delivery(status_module, tmp_path, monkeypatch): inbox = tmp_path / "inbox" projects = tmp_path / "projects.json" project = {**_webhook_project(), "slug": "repo-a", "containers": ["RepoA"]} projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) policy = status_module._validation_policy("jens/repo-a", project) assert policy is not None _, old_id = status_module._enqueue_webhook_delivery( json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(), "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", "a" * 40, policy, ) _, new_id = status_module._enqueue_webhook_delivery( json.dumps(_push_payload("b" * 40), separators=(",", ":")).encode(), "145440ab-b7b1-4c74-ae84-996d8d48167e", 77, "refs/heads/main", "jens/repo-a", "b" * 40, policy, ) old_marker = inbox / f"{old_id}.json" new_marker = inbox / f"{new_id}.json" monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("pass", [])) monkeypatch.setattr(status_module, "_is_forward_descendant", lambda *_args: True) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True}) monkeypatch.setattr(status_module.subprocess, "run", lambda *_args, **_kwargs: type("Result", (), {"returncode": 0})()) status_module._process_webhook_record(new_marker) assert json.loads(old_marker.read_text())["status"] == "superseded" assert json.loads(new_marker.read_text())["status"] == "deployed" def test_delivery_status_projection_is_bounded_and_secret_free(status_module, tmp_path, monkeypatch): inbox = tmp_path / "inbox" monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) policy = status_module._validation_policy("jens/repo-a", _webhook_project()) assert policy is not None _, request_id = status_module._enqueue_webhook_delivery( json.dumps(_push_payload("a" * 40), separators=(",", ":")).encode(), "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", 77, "refs/heads/main", "jens/repo-a", "a" * 40, policy, ) marker = inbox / f"{request_id}.json" record = json.loads(marker.read_text()) record["webhook_secret"] = "must-not-be-returned" record["authorization"] = "must-not-be-returned" marker.write_text(json.dumps(record), encoding="utf-8") monkeypatch.setattr(status_module, "_gitea_validation_settings", lambda: ("https://gitea.example.test/api/v1", b"x" * 40)) payload = status_module._delivery_status_payload() serialized = json.dumps(payload) assert payload["summary"]["pendingValidation"] == 1 assert payload["items"][0]["repository"] == "jens/repo-a" assert "must-not-be-returned" not in serialized def test_delivery_status_marks_an_unwritable_inbox_unhealthy( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "inbox" inbox.mkdir() monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) monkeypatch.setattr( status_module, "_gitea_validation_settings", lambda: ("https://gitea.example.test/api/v1", b"x" * 40), ) monkeypatch.setattr( status_module, "_controller_release", lambda: {"sourceRepository": "jens/unraid-appops-gateway"}, ) def no_space_left(*_args, **_kwargs): raise OSError(28, "No space left on device") monkeypatch.setattr(status_module.os, "open", no_space_left) payload = status_module._delivery_status_payload() assert payload["status"] == "unhealthy" assert payload["health"]["inbox"] == "unhealthy" assert list(inbox.iterdir()) == [] def test_inbox_health_probe_cleans_up_when_the_create_directory_fsync_fails( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "inbox" inbox.mkdir() monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) fsync_calls = 0 def fail_the_first_directory_fsync(): nonlocal fsync_calls fsync_calls += 1 if fsync_calls == 1: raise OSError(5, "Directory fsync failed") monkeypatch.setattr(status_module, "_fsync_inbox_root", fail_the_first_directory_fsync) with pytest.raises(OSError, match="Directory fsync failed"): status_module._probe_inbox_writable() assert fsync_calls == 2 assert list(inbox.iterdir()) == [] def test_delivery_status_client_is_read_only_and_does_not_send_a_secret(monkeypatch): from backend.app.core.config import settings from backend.app.services import deployment_broker monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_URL", "https://controller.example.test") monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", True) captured: dict[str, object] = {} class Response: status_code = 200 def __enter__(self): return self def __exit__(self, *_args): return None def iter_bytes(self): return iter([b'{"status":"healthy","items":[]}']) class Client: def __init__(self, **kwargs): captured["client"] = kwargs def __enter__(self): return self def __exit__(self, *_args): return None def stream(self, method, url, headers): captured.update({"method": method, "url": url, "headers": headers}) return Response() monkeypatch.setattr(deployment_broker.httpx, "Client", Client) assert deployment_broker.DeploymentBrokerClient().delivery_status() == {"status": "healthy", "items": []} assert captured["method"] == "GET" assert captured["url"] == "https://controller.example.test/v1/gitea/deliveries" assert captured["headers"] == {"Accept": "application/json"} assert captured["client"] == {"timeout": 10.0, "verify": True, "follow_redirects": False} def _write_controller_tls_pair(tmp_path: Path) -> tuple[Path, Path]: from datetime import datetime, timedelta, timezone from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID key = rsa.generate_private_key(public_exponent=65537, key_size=2048) name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "controller.test")]) now = datetime.now(timezone.utc).replace(tzinfo=None) certificate = ( x509.CertificateBuilder() .subject_name(name) .issuer_name(name) .public_key(key.public_key()) .serial_number(x509.random_serial_number()) .not_valid_before(now - timedelta(minutes=1)) .not_valid_after(now + timedelta(days=1)) .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) .add_extension( x509.SubjectAlternativeName( [x509.IPAddress(ipaddress.ip_address("127.0.0.1"))] ), critical=False, ) .add_extension( x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False ) .sign(key, hashes.SHA256()) ) certificate_path = tmp_path / "controller-test.crt" private_key_path = tmp_path / "controller-test.key" certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) private_key_path.write_bytes( key.private_bytes( serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption(), ) ) private_key_path.chmod(0o600) return certificate_path, private_key_path def test_private_tls_configuration_requires_a_complete_private_listener_record( status_module, tmp_path, monkeypatch ): monkeypatch.setattr(status_module, "os", types.SimpleNamespace(name="nt")) certificate, private_key = _write_controller_tls_pair(tmp_path) config_path = tmp_path / "private-tls.json" monkeypatch.setattr(status_module, "PRIVATE_TLS_CONFIG_PATH", config_path) assert status_module._private_tls_configuration() is None with pytest.raises(RuntimeError, match="missing"): status_module._private_tls_configuration(required=True) payload = { "schema": "itworx.controller-private-tls/v1", "listen_ip": "198.18.10.50", "port": 1300, "certificate_file": str(certificate), "private_key_file": str(private_key), } config_path.write_text(json.dumps(payload), encoding="utf-8") config_path.chmod(0o600) incomplete = dict(payload) incomplete.pop("private_key_file") config_path.write_text(json.dumps(incomplete), encoding="utf-8") with pytest.raises(RuntimeError, match="configuration is invalid"): status_module._private_tls_configuration(required=True) config_path.write_text(json.dumps(payload), encoding="utf-8") configuration = status_module._private_tls_configuration(required=True) assert configuration is not None assert configuration["listen_ip"] == "198.18.10.50" assert configuration["port"] == 1300 context = status_module._private_tls_context(configuration) assert context.minimum_version == ssl.TLSVersion.TLSv1_2 payload["listen_ip"] = "0.0.0.0" config_path.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(RuntimeError, match="explicit private IPv4"): status_module._private_tls_configuration(required=True) payload["listen_ip"] = "fd00::1" config_path.write_text(json.dumps(payload), encoding="utf-8") with pytest.raises(RuntimeError, match="explicit private IPv4"): status_module._private_tls_configuration(required=True) def test_private_tls_listener_serves_health_over_verified_tls( status_module, tmp_path, monkeypatch ): certificate, private_key = _write_controller_tls_pair(tmp_path) monkeypatch.setattr( status_module, "_controller_release", lambda: {"controllerVersion": "test"}, ) monkeypatch.setattr( status_module, "_delivery_status_payload", lambda: {"status": "ok", "health": "healthy", "summary": {}}, ) server = status_module._private_tls_server( { "listen_ip": "127.0.0.1", "port": 0, "certificate_file": certificate, "private_key_file": private_key, } ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() try: client = http.client.HTTPSConnection( *server.server_address, context=ssl.create_default_context(cafile=str(certificate)), timeout=2, ) client.request("GET", "/health") response = client.getresponse() payload = json.loads(response.read()) client.close() assert response.status == 200 assert payload["status"] == "ok" assert payload["transport"] == "https" finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_deployment_broker_uses_private_ca_and_rejects_disabled_https_verification( monkeypatch, tmp_path ): from backend.app.core.config import settings from backend.app.services import deployment_broker monkeypatch.setattr(deployment_broker, "os", types.SimpleNamespace(name="nt")) ca_file = tmp_path / "private-ca.pem" ca_file.write_text("test CA bundle", encoding="utf-8") ca_file.chmod(0o644) monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_URL", "https://controller.private") monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", True) monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_CA_FILE", str(ca_file)) captured: dict[str, object] = {} class Response: status_code = 200 def __enter__(self): return self def __exit__(self, *_args): return None def iter_bytes(self): return iter([b'{"status":"healthy","items":[]}']) class Client: def __init__(self, **kwargs): captured["client"] = kwargs def __enter__(self): return self def __exit__(self, *_args): return None def stream(self, method, url, headers): captured.update({"method": method, "url": url, "headers": headers}) return Response() monkeypatch.setattr(deployment_broker.httpx, "Client", Client) assert deployment_broker.DeploymentBrokerClient().delivery_status() == { "status": "healthy", "items": [], } assert captured["client"] == { "timeout": 10.0, "verify": str(ca_file), "follow_redirects": False, } monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", False) insecure = deployment_broker.DeploymentBrokerClient() assert insecure.delivery_status() == { "status": "missing_config", "reason": "deployment_broker_tls_verification_required", } assert ( insecure.configuration_error() == "deployment_broker_tls_verification_required" ) monkeypatch.setitem(settings.apis, "DEPLOYMENT_BROKER_VERIFY_TLS", True) monkeypatch.setitem( settings.apis, "DEPLOYMENT_BROKER_CA_FILE", str(tmp_path / "missing-ca.pem"), ) invalid_ca = deployment_broker.DeploymentBrokerClient() assert invalid_ca.delivery_status() == { "status": "missing_config", "reason": "deployment_broker_ca_file_invalid", } def test_deployment_broker_ca_file_must_be_owned_by_the_appops_process( monkeypatch, tmp_path ): from backend.app.services import deployment_broker ca_file = tmp_path / "private-ca.pem" owner = 1234 file_info = types.SimpleNamespace( st_mode=stat.S_IFREG | 0o640, st_size=16, st_uid=owner, ) monkeypatch.setattr(deployment_broker.Path, "lstat", lambda _path: file_info) monkeypatch.setattr( deployment_broker, "os", types.SimpleNamespace( name="posix", R_OK=4, geteuid=lambda: owner, access=lambda _path, _mode: True, ), ) assert deployment_broker.DeploymentBrokerClient._trusted_ca_file_is_safe( str(ca_file) ) monkeypatch.setattr( deployment_broker, "os", types.SimpleNamespace( name="posix", R_OK=4, geteuid=lambda: -1, access=lambda _path, _mode: True, ), ) assert not deployment_broker.DeploymentBrokerClient._trusted_ca_file_is_safe( str(ca_file) ) file_info.st_mode = stat.S_IFREG | 0o660 monkeypatch.setattr( deployment_broker, "os", types.SimpleNamespace( name="posix", R_OK=4, geteuid=lambda: owner, access=lambda _path, _mode: True, ), ) assert not deployment_broker.DeploymentBrokerClient._trusted_ca_file_is_safe( str(ca_file) ) def test_status_receipt_must_match_the_authenticated_request(status_module): revision = "a" * 40 project = { "containers": ["Chimera"], "verify_oci_revision": True, "live_probe": { "type": "chimera-health-live-v1", }, } project["_repository_name"] = "jens/chimera" receipt = _receipt(project, revision) state = { "schema_version": 3, "repository": "jens/chimera", "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": receipt, } assert ( status_module._verified_receipt(state, "jens/chimera", revision, project) == receipt ) assert ( status_module._verified_receipt(state, "jens/chimera", "d" * 40, project) is None ) assert ( status_module._verified_receipt(state, "jens/other", revision, project) is None ) def test_status_receipt_rejects_corrupt_state_container_and_live_proof(status_module): revision = "a" * 40 project = { "containers": ["Chimera"], "verify_oci_revision": True, "live_probe": {"type": "chimera-health-live-v1"}, "_repository_name": "jens/chimera", } receipt = _receipt(project, revision) state = { "schema_version": 3, "repository": "jens/chimera", "live_revision": revision, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": receipt, } corruptions = [] bad = copy.deepcopy(state) bad["live_legacy_provenance"] = True corruptions.append(bad) bad = copy.deepcopy(state) bad["live_receipt"]["treeDigest"] = "0" * 64 corruptions.append(bad) bad = copy.deepcopy(state) bad["live_receipt"]["build"]["id"] = "wrong" corruptions.append(bad) bad = copy.deepcopy(state) bad["live_receipt"]["containers"] = {"bogus": {}} corruptions.append(bad) bad = copy.deepcopy(state) bad["live_receipt"]["containers"]["Chimera"]["image_id"] = "sha256:short" corruptions.append(bad) bad = copy.deepcopy(state) bad["live_receipt"]["containers"]["Chimera"]["image_source"] = "jens/other" corruptions.append(bad) bad = copy.deepcopy(state) bad["live_receipt"]["live"]["status"] = "starting" corruptions.append(bad) for corrupt in corruptions: assert ( status_module._verified_receipt(corrupt, "jens/chimera", revision, project) is None ) def test_repository_webhook_secret_rejects_symlink( status_module, tmp_path, monkeypatch ): secret_root = tmp_path / "secrets" secret_root.mkdir() real = tmp_path / "real-secret" real.write_text("x" * 48, encoding="utf-8") real.chmod(0o600) link = secret_root / "jens__repo-a" try: link.symlink_to(real) except OSError: pytest.skip("symbolic links are unavailable on this host") monkeypatch.setattr(status_module, "WEBHOOK_SECRET_ROOT", secret_root) assert status_module._webhook_secret("jens/repo-a", _webhook_project()) is None def test_status_server_bounds_threads_and_times_out_partial_bodies( status_module, monkeypatch ): monkeypatch.setattr(status_module, "MAX_REQUEST_THREADS", 2) monkeypatch.setattr(status_module, "REQUEST_TIMEOUT_SECONDS", 0.1) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread( target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True, ) serving.start() try: assert server._request_slots.acquire(blocking=False) assert server._request_slots.acquire(blocking=False) assert not server._request_slots.acquire(blocking=False) server._request_slots.release() server._request_slots.release() with socket.create_connection(server.server_address, timeout=2) as client: client.settimeout(2) client.sendall( b"POST /v1/gitea/push/jens/repo-a HTTP/1.1\r\n" b"Host: localhost\r\n" b"Content-Length: 10\r\n" b"\r\n" b"{" ) chunks: list[bytes] = [] while chunk := client.recv(4096): chunks.append(chunk) response = b"".join(chunks) assert b" 408 " in response assert b"request_timeout" in response finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_status_server_accepts_only_exact_signed_repository_push( status_module, tmp_path, monkeypatch ): secret = b"webhook-" + b"x" * 48 secret_root = tmp_path / "secrets" inbox_root = tmp_path / "inbox" projects_path = tmp_path / "projects.json" _write_webhook_secret(secret_root, "jens__repo-a", secret) project = { **_webhook_project(), "slug": "repo-a", "containers": ["RepoA"], } projects_path.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_SECRET_ROOT", secret_root) monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox_root) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() revision = "a" * 40 raw_body = json.dumps(_push_payload(revision), separators=(",", ":")).encode() signature = hmac.new(secret, raw_body, hashlib.sha256).hexdigest() headers = { "Content-Type": "application/json", "X-Gitea-Delivery": "5f6d9b7a-4d42-4ae0-8ab5-36cf7cb6ea6d", "X-Gitea-Event": "push", "X-Gitea-Event-Type": "push", "X-Gitea-Hook-Installation-Target-Type": "repository", "X-Gitea-Signature": signature, } try: client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/push/jens/repo-a", body=raw_body, headers=headers, ) response = client.getresponse() body = json.loads(response.read()) client.close() assert response.status == 202 assert body["status"] == "accepted" assert body["revision"] == revision request_digest = body["request"] replay_body = json.dumps( { **_push_payload(revision, repository="Jens/repo-a"), "compare_url": "https://example.invalid", }, separators=(",", ":"), ).encode() replay_headers = { **headers, "X-Gitea-Delivery": "145440ab-b7b1-4c74-ae84-996d8d48167e", "X-Gitea-Signature": hmac.new( secret, replay_body, hashlib.sha256 ).hexdigest(), } client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/push/jens/repo-a", body=replay_body, headers=replay_headers, ) response = client.getresponse() replay_response = json.loads(response.read()) client.close() assert response.status == 202 assert replay_response["request"] == request_digest assert len(list(inbox_root.glob("*.json"))) == 1 client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/push/jens/repo-a", body=raw_body + b" ", headers=headers, ) response = client.getresponse() assert response.status == 403 assert b"webhook_signature_rejected" in response.read() client.close() invalid_json = b"{" invalid_headers = {**headers, "X-Gitea-Signature": "0" * 64} client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/push/jens/repo-a", body=invalid_json, headers=invalid_headers, ) response = client.getresponse() assert response.status == 403 assert b"webhook_signature_rejected" in response.read() client.close() valid_invalid_headers = { **headers, "X-Gitea-Signature": hmac.new( secret, invalid_json, hashlib.sha256 ).hexdigest(), } client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/push/jens/repo-a", body=invalid_json, headers=valid_invalid_headers, ) response = client.getresponse() assert response.status == 400 assert b"invalid_json" in response.read() client.close() client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request( "POST", "/v1/gitea/push/Jens/repo-a", body=raw_body, headers=headers, ) response = client.getresponse() assert response.status == 404 client.close() finally: server.shutdown() server.server_close() serving.join(timeout=2) @pytest.mark.parametrize( ("raw_value", "expected"), [ ("1", 1), ("4096", 4096), ("1048576", 1048576), ("0", None), ("1048577", None), ("00001", None), ("-1", None), ("\N{SUPERSCRIPT TWO}", None), ("9" * 5000, None), ], ) def test_status_server_content_length_is_bounded_ascii( status_module, raw_value, expected ): assert status_module._bounded_content_length(raw_value) == expected def test_status_server_closes_completed_json_responses(status_module): handler = status_module.Handler.__new__(status_module.Handler) headers: list[tuple[str, str]] = [] statuses: list[int] = [] handler.close_connection = False handler.wfile = io.BytesIO() handler.send_response = statuses.append handler.send_header = lambda name, value: headers.append((name, value)) handler.end_headers = lambda: None handler._send_json(200, b'{"status":"ok"}\n') assert statuses == [200] assert ("Connection", "close") in headers assert handler.close_connection is True assert handler.wfile.getvalue() == b'{"status":"ok"}\n' def _appops_project() -> dict[str, object]: return { "slug": "repo-a", "compose_project": "repo-a", "compose_files": [".itworx/deployment.yml"], "containers": ["repo-a"], "managed_plan_version": "itworx.deployment/v1", "managed_application_id": "repo-a", "managed_appdata_path": "/srv/example/appdata/repo-a", "managed_image_repository": "repo-a", "managed_image_tag": "1.2.3", "managed_proxy_network": "itworx-npm", "managed_internal_port": 8080, "managed_proxy_hostname": None, "managed_oidc_application_slug": None, "appops_deployment_secret_file": "jens__repo-a", } def _appops_plan(revision: str) -> dict[str, object]: return { "api_version": "itworx.deployment/v1", "application_id": "repo-a", "repository": "jens/repo-a", "revision": revision, "release_version": "1.2.3", "image_repository": "repo-a", "image_tag": "1.2.3", "topology": { "kind": "compose", "compose_file": ".itworx/deployment.yml", "compose_project": "repo-a", "containers": ["repo-a"], "proxy_network": "itworx-npm", "internal_port": 8080, }, "broker_secret_ref": "REPO_A_DEPLOYMENT_BROKER", } def test_appops_plan_root_must_match_the_registered_target_root(status_module): project = _appops_project() plan = _appops_plan("a" * 40) assert status_module._appops_plan({"jens/repo-a": project}, plan) == ( "jens/repo-a", "a" * 40, project, ) project["managed_appdata_root"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data" project["managed_appdata_path"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data/repo-a" alternate_plan = {**plan, "trusted_appdata_root": "/var/lib/docker/volumes/itworx-managed-appdata/_data"} assert status_module._appops_plan({"jens/repo-a": project}, alternate_plan) == ( "jens/repo-a", "a" * 40, project, ) assert status_module._appops_plan({"jens/repo-a": project}, plan) is None drifted = {**project, "managed_appdata_path": "/srv/example/appdata/repo-a"} assert status_module._appops_plan({"jens/repo-a": drifted}, alternate_plan) is None def _write_appops_secret(root: Path, name: str, value: bytes) -> None: root.mkdir(mode=0o700) root.chmod(0o700) path = root / name path.write_bytes(value) path.chmod(0o600) def test_bootstrap_signatures_bind_endpoint_and_are_single_use(status_module, tmp_path, monkeypatch): secret = b"bootstrap-" + b"x" * 48 secret_path = tmp_path / "enrollment-bootstrap" secret_path.write_bytes(secret) secret_path.chmod(0o600) monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", secret_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces") raw = b'{"enrollment":{}}' timestamp = str(int(status_module.time.time())) nonce = "3" * 32 signature = status_module._request_signature( secret, method="POST", path="/v1/appops/enrollments/preflight", timestamp=timestamp, nonce=nonce, raw_body=raw, ) assert not status_module._bootstrap_signature_is_valid( raw, timestamp, nonce, signature, method="POST", path="/v1/appops/enrollments/rollback", ) assert status_module._bootstrap_signature_is_valid( raw, timestamp, nonce, signature, method="POST", path="/v1/appops/enrollments/preflight", ) assert not status_module._bootstrap_signature_is_valid( raw, timestamp, nonce, signature, method="POST", path="/v1/appops/enrollments/preflight", ) def test_repopulse_adoption_capabilities_is_hmac_protected_and_read_only( status_module, tmp_path, monkeypatch ): bootstrap_secret = b"adoption-bootstrap-" + b"x" * 48 bootstrap_path = tmp_path / "bootstrap" bootstrap_path.write_bytes(bootstrap_secret) bootstrap_path.chmod(0o600) nonce_root = tmp_path / "nonces" monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root) monkeypatch.setattr( status_module._repopulse_adoption_store, "capability_report", lambda: { "repository": "jens/repopulse", "executionEnabled": True, "controllerAliasesReady": True, "registryReady": True, "secretRootReady": True, }, ) monkeypatch.setattr(status_module, "_private_tls_configuration", lambda **_kwargs: {}) monkeypatch.setattr(status_module, "_private_tls_context", lambda _configuration: object()) monkeypatch.setattr(status_module, "_repopulse_adoption_alias_mounts_ready", lambda: True) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() body = b"{}" timestamp = str(int(status_module.time.time())) nonce = "5" * 32 path = "/v1/appops/repopulse/adoption/capabilities" signature = status_module._request_signature( bootstrap_secret, method="POST", path=path, timestamp=timestamp, nonce=nonce, raw_body=body, ) headers = { "Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": timestamp, "X-ITWorx-Deployment-Nonce": nonce, "X-ITWorx-Deployment-Signature": signature, } try: client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request("POST", path, body=body, headers=headers) response = client.getresponse() payload = json.loads(response.read()) client.close() assert response.status == 200 assert payload == { "apiVersion": "itworx.repopulse-adoption-capabilities/v1", "status": "ready", "repository": "jens/repopulse", "adoptionStore": {"executionEnabled": True}, "privateTls": {"status": "ready"}, "controller": { "aliases": "ready", "registry": "ready", "secretRoot": "ready", }, } assert bootstrap_secret.decode("ascii") not in json.dumps(payload) assert nonce_root.is_dir() assert len(list(nonce_root.iterdir())) == 1 assert not (tmp_path / "repopulse-adoption").exists() client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request("POST", path, body=body, headers=headers) replay = client.getresponse() replay_payload = json.loads(replay.read()) client.close() assert replay.status == 403 assert replay_payload == {"status": "authentication_rejected"} assert len(list(nonce_root.iterdir())) == 1 for unavailable_path in ( "/v1/appops/repopulse/adoption/execute", "/v1/appops/repopulse/adoption/rollback", ): client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request("POST", unavailable_path, body=body) unsupported = client.getresponse() unsupported.read() client.close() assert unsupported.status == 404 assert len(list(nonce_root.iterdir())) == 1 monkeypatch.setattr( status_module, "_private_tls_configuration", lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("unsafe path detail")), ) tls_nonce = "6" * 32 tls_timestamp = str(int(status_module.time.time())) tls_signature = status_module._request_signature( bootstrap_secret, method="POST", path=path, timestamp=tls_timestamp, nonce=tls_nonce, raw_body=body, ) tls_headers = { **headers, "X-ITWorx-Deployment-Timestamp": tls_timestamp, "X-ITWorx-Deployment-Nonce": tls_nonce, "X-ITWorx-Deployment-Signature": tls_signature, } client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request("POST", path, body=body, headers=tls_headers) tls_unavailable = client.getresponse() tls_payload = json.loads(tls_unavailable.read()) client.close() assert tls_unavailable.status == 503 assert tls_payload["privateTls"] == {"status": "unavailable"} assert tls_payload["controller"]["aliases"] == "ready" assert "unsafe path detail" not in json.dumps(tls_payload) assert len(list(nonce_root.iterdir())) == 2 finally: server.shutdown() server.server_close() serving.join(timeout=2) def _configure_repopulse_adoption_store_for_http_test( status_module, tmp_path, monkeypatch, *, enabled ): store_module = status_module._repopulse_adoption_store controller = tmp_path / "controller" controller.mkdir() controller.chmod(0o755) projects = { "jens/repopulse": store_module._legacy_registry_record(), "jens/other": {"slug": "other"}, } projects_path = controller / "projects.json" projects_path.write_text(json.dumps(projects, sort_keys=True) + "\n", encoding="utf-8") projects_path.chmod(0o600) secrets = controller / "appops-deployment-secrets" secrets.mkdir() secrets.chmod(0o700) state = tmp_path / "state" state.mkdir() state.chmod(0o755) locks = state / "locks" locks.mkdir() locks.chmod(0o755) state_store = state / "repopulse-adoption" project_lock = locks / "repopulse.lock" global_lock = tmp_path / "deploy.lock" for name, value in { "CONTROLLER_READONLY_ROOT": controller, "CONTROLLER_RW_ROOT": controller, "STATE_ROOT": state, "GLOBAL_DEPLOY_LOCK": global_lock, "PROJECT_LOCK": project_lock, "ADOPTION_STATE_ROOT": state_store, "EXECUTION_ENABLED": enabled, "_IS_POSIX_CONTROLLER": True, }.items(): monkeypatch.setattr(store_module, name, value) test_uid = controller.stat().st_uid monkeypatch.setattr(store_module, "_ROOT_UID", test_uid) monkeypatch.setattr(store_module, "_effective_uid", lambda: test_uid) if os.name != "posix": monkeypatch.setattr( store_module, "_mode_bits", lambda path, info: ( 0o700 if path.name in {"appops-deployment-secrets", "repopulse-adoption"} else 0o755 if stat.S_ISDIR(info.st_mode) else 0o600 ), ) monkeypatch.setattr(store_module, "_directory_fsync", lambda _path: None) return store_module, state_store, project_lock, global_lock, controller def _adoption_bootstrap_headers(status_module, secret, *, method, path, body, nonce): timestamp = str(int(status_module.time.time())) signature = status_module._request_signature( secret, method=method, path=path, timestamp=timestamp, nonce=nonce, raw_body=body, ) return { "X-ITWorx-Deployment-Timestamp": timestamp, "X-ITWorx-Deployment-Nonce": nonce, "X-ITWorx-Deployment-Signature": signature, } def _adoption_http_request(server, method, path, *, body=None, headers=None): client = http.client.HTTPConnection(*server.server_address, timeout=2) if body is None: client.request(method, path, headers=headers or {}) else: client.request(method, path, body=body, headers=headers or {}) response = client.getresponse() status = response.status payload = json.loads(response.read()) client.close() return status, payload def _valid_repopulse_reserve_body(status_module, *, seed="reserve-seed"): idempotency_sha256 = hashlib.sha256(seed.encode("ascii")).hexdigest() payload = { "intentId": f"rpa-{idempotency_sha256[:20]}", "idempotencySha256": idempotency_sha256, "preflightSha256": hashlib.sha256(b"fixed-preflight").hexdigest(), } return json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8"), payload def test_repopulse_reserve_is_hmac_protected_strict_and_disabled_before_store_write( status_module, tmp_path, monkeypatch ): store_module, state_store, project_lock, global_lock, _controller = ( _configure_repopulse_adoption_store_for_http_test( status_module, tmp_path, monkeypatch, enabled=False ) ) secret = b"reserve-bootstrap-secret-" + b"x" * 40 bootstrap_path = tmp_path / "bootstrap" bootstrap_path.write_bytes(secret) bootstrap_path.chmod(0o600) nonce_root = tmp_path / "nonces" monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root) transaction_calls = [] def forbidden_transaction(*_args, **_kwargs): transaction_calls.append(True) raise AssertionError("disabled reserve must not enter store transaction") monkeypatch.setattr( store_module.RepoPulseAdoptionStore, "transaction", forbidden_transaction ) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() reserve_path = status_module.REPOPULSE_ADOPTION_RESERVE_PATH raw_body, payload = _valid_repopulse_reserve_body(status_module) try: invalid_payloads = [ {**payload, "path": "/tmp/attacker"}, {key: value for key, value in payload.items() if key != "preflightSha256"}, {**payload, "idempotencySha256": True}, {**payload, "intentId": "rpa-" + "f" * 20}, {**payload, "preflightSha256": "A" * 64}, ] for invalid in invalid_payloads: body = json.dumps(invalid, separators=(",", ":")).encode("utf-8") response_status, response = _adoption_http_request( server, "POST", reserve_path, body=body ) assert response_status == 400 assert response == {"status": "invalid_reserve_request"} # Duplicate keys are rejected before auth and never interpreted as a # different request from the one the caller intended to sign. duplicate_body = ( b'{"intentId":"' + payload["intentId"].encode("ascii") + b'","intentId":"' + payload["intentId"].encode("ascii") + b'","idempotencySha256":"' + payload["idempotencySha256"].encode("ascii") + b'","preflightSha256":"' + payload["preflightSha256"].encode("ascii") + b'"}' ) duplicate_status, duplicate_response = _adoption_http_request( server, "POST", reserve_path, body=duplicate_body ) assert duplicate_status == 400 assert duplicate_response == {"status": "invalid_json"} # Even a valid contract cannot reach the store without the HMAC. unauthenticated_status, unauthenticated = _adoption_http_request( server, "POST", reserve_path, body=raw_body ) assert unauthenticated_status == 403 assert unauthenticated == {"status": "authentication_rejected"} headers = _adoption_bootstrap_headers( status_module, secret, method="POST", path=reserve_path, body=raw_body, nonce="1" * 32, ) disabled_status, disabled = _adoption_http_request( server, "POST", reserve_path, body=raw_body, headers=headers ) assert disabled_status == 503 assert disabled == { "apiVersion": "itworx.repopulse-adoption-reserve/v1", "status": "execution_disabled", } replay_status, replay = _adoption_http_request( server, "POST", reserve_path, body=raw_body, headers=headers ) assert replay_status == 403 assert replay == {"status": "authentication_rejected"} status_path = status_module.REPOPULSE_ADOPTION_STATUS_PATH status_headers = _adoption_bootstrap_headers( status_module, secret, method="GET", path=status_path, body=b"", nonce="2" * 32, ) status_status, status_payload = _adoption_http_request( server, "GET", status_path, headers=status_headers ) assert status_status == 200 assert status_payload == { "apiVersion": "itworx.repopulse-adoption-status/v1", "status": "not_reserved", "phase": None, "hashes": {}, } assert not state_store.exists() assert not project_lock.exists() assert not global_lock.exists() replay_status, replay = _adoption_http_request( server, "GET", status_path, headers=status_headers ) assert replay_status == 403 assert replay == {"status": "authentication_rejected"} state_store.write_text("unsafe status state", encoding="utf-8") unsafe_headers = _adoption_bootstrap_headers( status_module, secret, method="GET", path=status_path, body=b"", nonce="7" * 32, ) unsafe_status, unsafe_payload = _adoption_http_request( server, "GET", status_path, headers=unsafe_headers ) assert unsafe_status == 503 assert unsafe_payload == { "apiVersion": "itworx.repopulse-adoption-status/v1", "status": "unavailable", } assert state_store.read_text(encoding="utf-8") == "unsafe status state" assert "unsafe status state" not in json.dumps(unsafe_payload) assert len(list(nonce_root.iterdir())) == 3 assert transaction_calls == [] assert not project_lock.exists() assert not global_lock.exists() finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_repopulse_reserve_is_idempotent_locked_and_status_redacts_credential( status_module, tmp_path, monkeypatch ): store_module, state_store, project_lock, global_lock, controller = ( _configure_repopulse_adoption_store_for_http_test( status_module, tmp_path, monkeypatch, enabled=True ) ) secret = b"reserve-bootstrap-secret-" + b"y" * 40 bootstrap_path = tmp_path / "bootstrap" bootstrap_path.write_bytes(secret) bootstrap_path.chmod(0o600) nonce_root = tmp_path / "nonces" monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root) credential = b"very-high-entropy-existing-controller-credential" * 2 credential_path = controller / "appops-deployment-secrets" / "jens__repopulse" credential_path.write_bytes(credential) credential_path.chmod(0o600) lock_events = [] original_lock = store_module._exclusive_flock @contextmanager def tracked_lock(path): with original_lock(path): lock_events.append(path) yield monkeypatch.setattr(store_module, "_exclusive_flock", tracked_lock) original_reserve = store_module.RepoPulseAdoptionStore.reserve_intent def checked_reserve(store, **kwargs): assert status_module._enrollment_write_lock.locked() assert lock_events[-2:] == [project_lock, global_lock] return original_reserve(store, **kwargs) monkeypatch.setattr( store_module.RepoPulseAdoptionStore, "reserve_intent", checked_reserve ) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() reserve_path = status_module.REPOPULSE_ADOPTION_RESERVE_PATH raw_body, payload = _valid_repopulse_reserve_body(status_module) try: first_headers = _adoption_bootstrap_headers( status_module, secret, method="POST", path=reserve_path, body=raw_body, nonce="3" * 32, ) first_status, first = _adoption_http_request( server, "POST", reserve_path, body=raw_body, headers=first_headers ) second_headers = _adoption_bootstrap_headers( status_module, secret, method="POST", path=reserve_path, body=raw_body, nonce="4" * 32, ) second_status, second = _adoption_http_request( server, "POST", reserve_path, body=raw_body, headers=second_headers ) assert first_status == second_status == 200 assert first == second assert first["apiVersion"] == "itworx.repopulse-adoption-reserve/v1" assert first["status"] == "reserved" assert first["phase"] == "reserved" assert first["intentId"] == payload["intentId"] assert len(first["intentSha256"]) == 64 assert lock_events == [project_lock, global_lock, project_lock, global_lock] assert not status_module._enrollment_write_lock.locked() assert state_store.is_dir() assert (state_store / "intent.json").is_file() conflict_body, _conflict_payload = _valid_repopulse_reserve_body( status_module, seed="different-reservation" ) conflict_headers = _adoption_bootstrap_headers( status_module, secret, method="POST", path=reserve_path, body=conflict_body, nonce="5" * 32, ) conflict_status, conflict = _adoption_http_request( server, "POST", reserve_path, body=conflict_body, headers=conflict_headers ) assert conflict_status == 409 assert conflict == { "apiVersion": "itworx.repopulse-adoption-reserve/v1", "status": "conflict", } assert lock_events == [ project_lock, global_lock, project_lock, global_lock, project_lock, global_lock, ] status_path = status_module.REPOPULSE_ADOPTION_STATUS_PATH status_headers = _adoption_bootstrap_headers( status_module, secret, method="GET", path=status_path, body=b"", nonce="6" * 32, ) read_status, status_payload = _adoption_http_request( server, "GET", status_path, headers=status_headers ) assert read_status == 200 assert status_payload["status"] == "in_progress" assert status_payload["phase"] == "reserved" assert status_payload["hashes"]["idempotencySha256"] == payload["idempotencySha256"] assert status_payload["hashes"]["preflightSha256"] == payload["preflightSha256"] encoded_status = json.dumps(status_payload) assert credential.decode("ascii") not in encoded_status assert hashlib.sha256(credential).hexdigest() not in encoded_status assert "expectedTargetCredentialSha256" not in encoded_status assert len(list(nonce_root.iterdir())) == 4 assert not status_module._enrollment_write_lock.locked() finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_repopulse_enrollment_rollback_reloads_registry_under_project_global_then_write_lock( status_module, tmp_path, monkeypatch ): store_module, _state_store, project_lock, global_lock, _controller = ( _configure_repopulse_adoption_store_for_http_test( status_module, tmp_path, monkeypatch, enabled=False ) ) secret = b"rollback-bootstrap-secret-" + b"x" * 40 bootstrap_path = tmp_path / "bootstrap" bootstrap_path.write_bytes(secret) bootstrap_path.chmod(0o600) monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", bootstrap_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces") lock_events = [] original_lock = store_module._exclusive_flock project_lock_waiting = threading.Event() allow_project_lock = threading.Event() @contextmanager def tracked_lock(path): if path == project_lock: project_lock_waiting.set() if not allow_project_lock.wait(timeout=5): raise TimeoutError("test did not release the project lock gate") with original_lock(path): lock_events.append(path) yield monkeypatch.setattr(store_module, "_exclusive_flock", tracked_lock) class TrackedEnrollmentLock: def __init__(self): self._lock = threading.Lock() def __enter__(self): lock_events.append("enrollment_lock") self._lock.acquire() return self def __exit__(self, *_args): self._lock.release() def locked(self): return self._lock.locked() monkeypatch.setattr(status_module, "_enrollment_write_lock", TrackedEnrollmentLock()) load_calls = 0 def load_projects(): nonlocal load_calls load_calls += 1 if load_calls == 1: return {"jens/repopulse": {"revision": "stale"}} assert lock_events[-3:] == [project_lock, global_lock, "enrollment_lock"] assert status_module._enrollment_write_lock.locked() assert ( store_module._transaction_owner(store_module._lock_identity()) is None ) return {"jens/repopulse": {"revision": "fresh"}} observed_targets = [] def rollback(projects, target): assert lock_events[-3:] == [project_lock, global_lock, "enrollment_lock"] assert status_module._enrollment_write_lock.locked() assert ( store_module._transaction_owner(store_module._lock_identity()) is None ) assert projects["jens/repopulse"]["revision"] == "fresh" observed_targets.append(target) return "rolled_back", { "enrollmentId": target["enrollment_id"], "applicationId": target["application_id"], } monkeypatch.setattr(status_module, "_load_projects", load_projects) monkeypatch.setattr(status_module, "_rollback_enrollment_target", rollback) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() path = "/v1/appops/enrollments/rollback" target = { "enrollment_id": "enr-" + "c" * 20, "application_id": "repopulse", "repository": "jens/repopulse", } raw_body = json.dumps({"target": target}, separators=(",", ":")).encode("utf-8") headers = { "Content-Type": "application/json", **_adoption_bootstrap_headers( status_module, secret, method="POST", path=path, body=raw_body, nonce="8" * 32, ), } response_values = [] def send_rollback_request(): response_values.append( _adoption_http_request(server, "POST", path, body=raw_body, headers=headers) ) request_thread = threading.Thread(target=send_rollback_request) try: request_thread.start() assert project_lock_waiting.wait(timeout=5) assert not status_module._enrollment_write_lock.locked(), ( "rollback must not hold the enrollment lock while waiting for the project flock" ) allow_project_lock.set() request_thread.join(timeout=5) assert not request_thread.is_alive() response_status, response = response_values[0] assert response_status == 200 assert response == { "status": "rolled_back", "enrollmentId": target["enrollment_id"], "applicationId": target["application_id"], } assert load_calls == 2 assert observed_targets == [target] assert lock_events == [project_lock, global_lock, "enrollment_lock"] assert not status_module._enrollment_write_lock.locked() finally: allow_project_lock.set() server.shutdown() server.server_close() serving.join(timeout=2) def test_repopulse_adoption_capabilities_client_uses_fixed_bootstrap_hmac( monkeypatch, ): from backend.app.services.deployment_broker import DeploymentBrokerClient captured = {} class Response: status_code = 200 @staticmethod def json(): return {"status": "ready", "repository": "jens/repopulse"} class Client: def __init__(self, **kwargs): captured["client"] = kwargs def __enter__(self): return self def __exit__(self, *_args): return None def request(self, method, url, *, content, headers): captured.update( {"method": method, "url": url, "body": content, "headers": headers} ) return Response() monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client) secret = "bootstrap-client-secret-value" * 2 client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True) assert client.repopulse_adoption_capabilities(secret=secret) == { "status": "ready", "repository": "jens/repopulse", } assert captured["method"] == "POST" assert captured["client"]["verify"] is True assert captured["url"] == ( "https://controller.example/v1/appops/repopulse/adoption/capabilities" ) assert captured["body"] == b"{}" headers = captured["headers"] assert headers["X-ITWorx-Deployment-Signature"] == DeploymentBrokerClient._signature( secret, "POST", "/v1/appops/repopulse/adoption/capabilities", headers["X-ITWorx-Deployment-Timestamp"], headers["X-ITWorx-Deployment-Nonce"], b"{}", ) def test_repopulse_adoption_client_binds_commands_to_fixed_paths(monkeypatch): from backend.app.services.deployment_broker import DeploymentBrokerClient requests = [] class Response: status_code = 200 @staticmethod def json(): return {"status": "ok"} class Client: def __init__(self, **_kwargs): pass def __enter__(self): return self def __exit__(self, *_args): return None def request(self, method, url, *, content, headers): requests.append((method, url, content, headers)) return Response() monkeypatch.setattr("backend.app.services.deployment_broker.httpx.Client", Client) secret = "bootstrap-client-secret-value" * 2 client = DeploymentBrokerClient(base_url="https://controller.example", verify_tls=True) digest = "a" * 64 assert client.repopulse_adoption_reserve( secret=secret, intent_id=f"rpa-{digest[:20]}", idempotency_sha256=digest, preflight_sha256="b" * 64, ) == {"status": "ok"} assert client.repopulse_adoption_status(secret=secret) == {"status": "ok"} assert client.repopulse_adoption_execute( secret=secret, intent_id=f"rpa-{digest[:20]}", idempotency_sha256=digest, preflight_sha256="b" * 64, preflight_projection={"repository": "jens/repopulse"}, target_credential="target-secret-value" * 4, ) == {"status": "ok"} assert [(method, url.rsplit("/", 1)[-1]) for method, url, _, _ in requests] == [ ("POST", "reserve"), ("GET", "status"), ("POST", "execute"), ] assert json.loads(requests[0][2]) == { "idempotencySha256": digest, "intentId": f"rpa-{digest[:20]}", "preflightSha256": "b" * 64, } assert requests[1][2] == b"" assert json.loads(requests[2][2]) == { "idempotencySha256": digest, "intentId": f"rpa-{digest[:20]}", "preflightProjection": {"repository": "jens/repopulse"}, "preflightSha256": "b" * 64, "targetCredential": "target-secret-value" * 4, } for method, url, body, headers in requests: path = url.removeprefix("https://controller.example") assert headers["X-ITWorx-Deployment-Signature"] == DeploymentBrokerClient._signature( secret, method, path, headers["X-ITWorx-Deployment-Timestamp"], headers["X-ITWorx-Deployment-Nonce"], body, ) assert DeploymentBrokerClient( base_url="http://127.0.0.1:1300", verify_tls=False ).repopulse_adoption_execute( secret=secret, intent_id=f"rpa-{digest[:20]}", idempotency_sha256=digest, preflight_sha256="b" * 64, preflight_projection={"repository": "jens/repopulse"}, target_credential="target-secret-value" * 4, ) == { "status": "missing_config", "reason": "repopulse_adoption_private_tls_required", } assert len(requests) == 3 def test_controller_rejects_a_group_writable_projects_registry(status_module, tmp_path, monkeypatch): if status_module.os.name == "nt": pytest.skip("POSIX ownership and permissions are enforced by the controller") projects = tmp_path / "projects.json" projects.write_text("{}", encoding="utf-8") projects.chmod(0o666) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) assert status_module._projects_registry_is_safe() is False with pytest.raises(OSError, match="unsafe"): status_module._load_projects() def test_repopulse_capability_requires_fixed_readonly_and_rw_mount_aliases(status_module, monkeypatch): # Exercise the installed aliases, rather than the shared source-tree fallback. readonly_root = Path("/controller-config") rw_root = Path("/controller-hub-releases/controller-config") monkeypatch.setattr(status_module._repopulse_adoption_store, "CONTROLLER_READONLY_ROOT", readonly_root) monkeypatch.setattr(status_module._repopulse_adoption_store, "CONTROLLER_RW_ROOT", rw_root) valid_mountinfo = ( f"25 1 0:2 / {readonly_root.as_posix()} ro - ext4 /dev/test ro\n" f"26 1 0:2 / {rw_root.as_posix()} rw - ext4 /dev/test rw\n" ) assert status_module._repopulse_alias_mounts_match_contract(valid_mountinfo) is True readonly_changed_to_rw = valid_mountinfo.replace( f"{readonly_root.as_posix()} ro", f"{readonly_root.as_posix()} rw" ) assert status_module._repopulse_alias_mounts_match_contract(readonly_changed_to_rw) is False rw_changed_to_readonly = valid_mountinfo.replace( f"{rw_root.as_posix()} rw", f"{rw_root.as_posix()} ro" ) assert status_module._repopulse_alias_mounts_match_contract(rw_changed_to_readonly) is False assert status_module._repopulse_alias_mounts_match_contract("") is False def test_controller_release_identity_is_content_bound(status_module, tmp_path, monkeypatch, capsys): renderer_spec = importlib.util.spec_from_file_location( "prepare_controller_artifact", Path(__file__).resolve().parents[2] / "scripts" / "prepare_controller_artifact.py", ) assert renderer_spec is not None and renderer_spec.loader is not None renderer = importlib.util.module_from_spec(renderer_spec) renderer_spec.loader.exec_module(renderer) assert renderer._ARTIFACT_FILES == status_module._CONTROLLER_ARTIFACT_FILES artifact_root = tmp_path / "controller" artifact_root.mkdir() for filename in status_module._CONTROLLER_ARTIFACT_FILES: source = ( Path(__file__).resolve().parents[2] / "scripts" / filename if filename == "run_repopulse_adoption.py" else status_module.ROOT / filename ) (artifact_root / filename).write_bytes(source.read_bytes()) monkeypatch.setattr(status_module, "ROOT", artifact_root) monkeypatch.setattr(status_module, "CONTROLLER_RELEASE_PATH", artifact_root / "controller-release.json") digest = status_module._controller_artifact_digest() assert digest is not None release = { "apiVersion": "itworx.controller-artifact/v1", "controllerVersion": "2.0.0", "sourceRepository": "jens/unraid-appops-gateway", "sourceRevision": "a" * 40, "sourceTree": "b" * 40, "artifactSha256": digest, "builtAt": "2026-09-14T01:00:00.000Z", } status_module.CONTROLLER_RELEASE_PATH.write_text(json.dumps(release), encoding="utf-8") assert status_module._controller_release() == release monkeypatch.setattr(status_module, "_load_projects", lambda: (_ for _ in ()).throw(AssertionError("legacy config must not be read"))) monkeypatch.setattr(status_module.sys, "argv", ["status_server.py", "--check-artifact"]) status_module.main() assert "validated controller artifact 2.0.0" in capsys.readouterr().out (artifact_root / "deploy.py").write_text("tampered", encoding="utf-8") assert status_module._controller_release() is None def test_appops_deployment_contract_is_pre_registered_and_hmac_bound(status_module, tmp_path, monkeypatch): from backend.app.services.deployment_broker import DeploymentBrokerClient secret_root = tmp_path / "appops-secrets" secret = b"appops-" + b"x" * 48 _write_appops_secret(secret_root, "jens__repo-a", secret) project = _appops_project() plan = _appops_plan("a" * 40) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root) assert status_module._appops_plan({"jens/repo-a": project}, plan) == ( "jens/repo-a", "a" * 40, project, ) raw = json.dumps({"plan": plan}, sort_keys=True, separators=(",", ":")).encode() timestamp = str(int(status_module.time.time())) nonce = "1" * 32 nonce_root = tmp_path / "nonces" monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root) signature = status_module._request_signature( secret, method="POST", path="/v1/appops/deployments/apply", timestamp=timestamp, nonce=nonce, raw_body=raw, ) expected = hmac.new( secret, ( "itworx-controller-request/v2\nPOST\n/v1/appops/deployments/apply\n" f"{timestamp}\n{nonce}\n{hashlib.sha256(raw).hexdigest()}" ).encode("ascii"), hashlib.sha256, ).hexdigest() assert expected == signature # A captured apply request cannot be retargeted to rollback before its # nonce is consumed: the endpoint path is part of the authenticated input. assert not status_module._appops_signature_is_valid( raw, timestamp, nonce, signature, "jens/repo-a", project, method="POST", path="/v1/appops/deployments/rollback", ) assert status_module._appops_signature_is_valid( raw, timestamp, nonce, signature, "jens/repo-a", project, method="POST", path="/v1/appops/deployments/apply", ) # The same authenticated request is rejected after its atomic nonce write. assert not status_module._appops_signature_is_valid( raw, timestamp, nonce, signature, "jens/repo-a", project, method="POST", path="/v1/appops/deployments/apply", ) assert not status_module._appops_signature_is_valid( raw, timestamp, "2" * 32, "0" * 64, "jens/repo-a", project, method="POST", path="/v1/appops/deployments/apply", ) legacy_signature = hmac.new( secret, f"{timestamp}\n{'4' * 32}\n{hashlib.sha256(raw).hexdigest()}".encode(), hashlib.sha256, ).hexdigest() assert DeploymentBrokerClient._signature( secret.decode(), "POST", "/v1/appops/deployments/apply", timestamp, nonce, raw ) == signature assert not status_module._appops_signature_is_valid( raw, timestamp, "4" * 32, legacy_signature, "jens/repo-a", project, method="POST", path="/v1/appops/deployments/apply", ) tampered = {**plan, "image_repository": "other"} assert status_module._appops_plan({"jens/repo-a": project}, tampered) is None def test_appops_request_is_idempotent_and_runs_only_fixed_deploy(status_module, tmp_path, monkeypatch): inbox = tmp_path / "appops-inbox" projects = tmp_path / "projects.json" project = _appops_project() projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") plan = _appops_plan("a" * 40) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) status, request_id = status_module._enqueue_appops_request( repository="jens/repo-a", revision="a" * 40, operation="apply", idempotency_key="appops-repo-a-0001", plan=plan, ) assert status == "pending" assert status_module._enqueue_appops_request( repository="jens/repo-a", revision="a" * 40, operation="apply", idempotency_key="appops-repo-a-0001", plan=plan, ) == ("pending", request_id) marker = inbox / f"{request_id}.json" calls = [] class Result: returncode = 0 monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: calls.append(args) or Result()) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True}) status_module._process_appops_record(marker) assert json.loads(marker.read_text())["status"] == "deployed" assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", "jens/repo-a", "a" * 40]] @pytest.mark.parametrize("operation", ["apply", "rollback"]) def test_generic_appops_hub_request_is_rejected_before_inbox_creation( status_module, tmp_path, monkeypatch, operation ): inbox = tmp_path / "appops-inbox" monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) status, request_id = status_module._enqueue_appops_request( repository="Jens/ITWorx_MCP_Hub", revision="a" * 40, operation=operation, idempotency_key=f"appops-hub-{operation}-0001", plan={}, ) assert (status, request_id) == ("invalid", "") assert not inbox.exists() @pytest.mark.parametrize("operation", ["apply", "rollback"]) @pytest.mark.parametrize( ("prior_status", "expected_status"), [("pending", "failed"), ("running", "outcome_unknown")], ) def test_direct_appops_hub_record_is_denied_before_project_or_deploy_access( status_module, tmp_path, monkeypatch, operation, prior_status, expected_status ): inbox = tmp_path / "appops-inbox" inbox.mkdir() request_id = "a" * 64 marker = inbox / f"{request_id}.json" marker.write_text( json.dumps( { "schema": "appops-deployment-request.v1", "request": request_id, "repository": "jens/itworx_mcp_hub", "revision": "b" * 40, "operation": operation, "plan_sha256": "c" * 64, "status": prior_status, "received_at": 1, "updated_at": 1, } ), encoding="utf-8", ) monkeypatch.setattr( status_module, "_load_projects", lambda: pytest.fail("generic Hub records must not inspect deployment projects"), ) monkeypatch.setattr( status_module.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("generic Hub records must not launch deploy.py"), ) status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == expected_status assert record["reason"] == "canonical_hub_push_required" def test_hub_protected_push_waits_for_candidate_before_fixed_deploy_path( status_module, tmp_path, monkeypatch ): repository = "jens/itworx_mcp_hub" inbox = tmp_path / "webhook-inbox" projects = tmp_path / "projects.json" project = { **_webhook_project(repository), "slug": "itworx-mcp-hub", "containers": ["itworx-mcp-hub"], } projects.write_text(json.dumps({repository: project}), encoding="utf-8") monkeypatch.setattr(status_module, "WEBHOOK_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) revision = "d" * 40 raw_body = json.dumps( _push_payload(revision, repository=repository), separators=(",", ":") ).encode() _, request_id = status_module._enqueue_webhook_delivery( raw_body, "145440ab-b7b1-4c74-ae84-996d8d48167e", 77, "refs/heads/main", repository, revision, status_module._validation_policy(repository, project), ) marker = inbox / f"{request_id}.json" calls = [] class Result: returncode = 0 monkeypatch.setattr( status_module.subprocess, "run", lambda arguments, **_kwargs: calls.append(arguments) or Result(), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"ok": True}) monkeypatch.setattr( status_module, "_fetch_gitea_validation", lambda *_args: ( "pass", [{ "context": "Managed validation / full (push)", "status": "success", "updated_at": "2026-09-14T00:00:00Z", }], ), ) monkeypatch.setattr(status_module, "_prepare_hub_candidate_delivery", lambda *_args: False) status_module._process_webhook_record(marker) waiting = json.loads(marker.read_text(encoding="utf-8")) assert waiting["status"] == "validation_passed" assert "deployment_started" not in [entry["stage"] for entry in waiting["audit"]] assert calls == [] monkeypatch.setattr(status_module, "_prepare_hub_candidate_delivery", lambda *_args: True) status_module._process_webhook_record(marker) assert json.loads(marker.read_text(encoding="utf-8"))["status"] == "deployed" assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "deploy", repository, revision]] def test_durable_rollback_snapshot_is_receipt_bound_and_not_caller_selected(deploy_module, tmp_path, monkeypatch): previous_revision = "a" * 40 current_revision = "b" * 40 project = _project(tmp_path) monkeypatch.setattr(deploy_module, "STATE_ROOT", tmp_path / "state") previous_receipt = _receipt(project, previous_revision) current_receipt = _receipt(project, current_revision) snapshot = { "revision": previous_revision, "legacy_provenance": False, "receipt": previous_receipt, "compose_containers": ["Unraid-AppOps-Gateway"], "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": "container-a", "image_id": "sha256:" + "1" * 64, "image_ref": "Unraid-AppOps-Gateway:production", "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:aaaaaaaaaaaa", "was_running": True, }], } state = { "schema_version": 3, "repository": "jens/unraid-appops-gateway", "live_revision": current_revision, "revision": current_revision, "live_status": "healthy", "live_receipt": current_receipt, "rollback_snapshot": snapshot, } state_path = deploy_module.STATE_ROOT / "unraid-appops-gateway.json" state_path.parent.mkdir(parents=True) state_path.write_text(json.dumps(state), encoding="utf-8") monkeypatch.setattr( deploy_module, "inspect_image", lambda image_id: {"Id": image_id}, ) assert deploy_module.durable_rollback_snapshot( project, "jens/unraid-appops-gateway", current_revision ) == snapshot with pytest.raises(deploy_module.DeployError, match="verified live deployment"): deploy_module.durable_rollback_snapshot( project, "jens/unraid-appops-gateway", "c" * 40 ) def test_managed_runtime_contract_rejects_published_ports_and_accepts_owned_hardened_compose(deploy_module, tmp_path, monkeypatch): project = _project(tmp_path) project.update({ "managed_plan_version": "itworx.deployment/v1", "managed_application_id": "unraid-appops-gateway", "managed_appdata_path": "/srv/example/appdata/unraid-appops-gateway", "managed_image_repository": "unraid-appops-gateway", "managed_image_tag": "1.5.0", "managed_proxy_network": "itworx-npm", "managed_internal_port": 1216, "appops_deployment_secret_file": "jens__unraid-appops-gateway", }) rendered = { "services": { "unraid-appops-gateway": { "image": "unraid-appops-gateway:1.5.0", "restart": "unless-stopped", "read_only": True, "security_opt": ["no-new-privileges:true"], "cap_drop": ["ALL"], "networks": ["itworx-npm"], "volumes": [{ "type": "bind", "source": "/srv/example/appdata/unraid-appops-gateway/data", "target": "/data", }], } }, "networks": {"itworx-npm": {"external": True}}, } monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"])) monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {}) monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: json.dumps(rendered)) deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/unraid-appops-gateway") rendered["services"]["unraid-appops-gateway"]["ports"] = [{"published": "1216"}] with pytest.raises(deploy_module.DeployError, match="published ports"): deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/unraid-appops-gateway") def test_managed_runtime_contract_rejects_appdata_path_traversal_and_controller_env_inheritance(deploy_module, tmp_path, monkeypatch): project = _project(tmp_path) project.update({ "managed_plan_version": "itworx.deployment/v1", "managed_application_id": "unraid-appops-gateway", "managed_appdata_path": "/srv/example/appdata/unraid-appops-gateway", "managed_image_repository": "unraid-appops-gateway", "managed_image_tag": "1.5.0", "managed_proxy_network": "itworx-npm", "managed_internal_port": 1216, "appops_deployment_secret_file": "jens__unraid-appops-gateway", "environment": {"PORTAL_ADMIN_KEY": "managed-secret"}, }) monkeypatch.setenv("UNRELATED_CONTROLLER_SECRET", "must-not-reach-compose") environment = deploy_module.deployment_environment(project, "a" * 40, "jens/unraid-appops-gateway") assert environment["PORTAL_ADMIN_KEY"] == "managed-secret" assert "UNRELATED_CONTROLLER_SECRET" not in environment rendered = { "services": { "unraid-appops-gateway": { "image": "unraid-appops-gateway:1.5.0", "restart": "unless-stopped", "read_only": True, "security_opt": ["no-new-privileges:true"], "cap_drop": ["ALL"], "environment": {"PORTAL_ADMIN_KEY": "managed-secret"}, "networks": ["itworx-npm"], "volumes": [{ "type": "bind", "source": "/srv/example/appdata/unraid-appops-gateway/../../other-app", "target": "/data", }], } }, "networks": {"itworx-npm": {"external": True}}, } monkeypatch.setattr(deploy_module, "compose_command", lambda _project: (tmp_path, ["docker", "compose"])) monkeypatch.setattr(deploy_module, "output", lambda *_args, **_kwargs: json.dumps(rendered)) with pytest.raises(deploy_module.DeployError, match="appdata root"): deploy_module.validate_managed_v1_runtime(project, "a" * 40, "jens/unraid-appops-gateway") def test_appops_rollback_runs_only_receipt_bound_controller_mode(status_module, tmp_path, monkeypatch): inbox = tmp_path / "appops-inbox" projects = tmp_path / "projects.json" project = _appops_project() projects.write_text(json.dumps({"jens/repo-a": project}), encoding="utf-8") plan = _appops_plan("a" * 40) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects) status, request_id = status_module._enqueue_appops_request( repository="jens/repo-a", revision="a" * 40, operation="rollback", idempotency_key="rollback-repo-a-0001", plan=plan, ) assert status == "pending" marker = inbox / f"{request_id}.json" calls = [] class Result: returncode = 0 monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: calls.append(args) or Result()) monkeypatch.setattr( status_module, "_deployment_state", lambda _slug: {"rollback_restored_revision": "b" * 40}, ) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"verified": True}) status_module._process_appops_record(marker) assert json.loads(marker.read_text())["status"] == "rolled_back" assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "rollback", "jens/repo-a", "a" * 40]] def test_webhook_broker_has_no_runner_or_job_secret_artifacts(): assert not (BROKER_ROOT / "docker-host").exists() assert not (BROKER_ROOT / "deploy-runner.config.example.yaml").exists() assert not (BROKER_ROOT / "deploy-runner.compose.example.yaml").exists() source = (BROKER_ROOT / "status_server.py").read_text(encoding="utf-8") assert "BROKER_TOKEN_ROOT" not in source assert "GITEA_DEPLOY_BROKER_TOKEN" not in source assert "UNRAID_DEPLOY_BROKER_TOKEN" not in source assert "X-Gitea-Signature" in source assert "gitea-webhook-inbox.v1" in source def test_appops_uses_push_validation_without_a_deploy_runner_workflow(): workflow_root = ROOT / ".gitea" / "workflows" assert not (workflow_root / "unraid-deploy.yml").exists() managed_validation = (workflow_root / "managed-validation.yml").read_text(encoding="utf-8") assert "push:" in managed_validation assert "branches: [main]" in managed_validation assert "runs-on: linux-validation" in managed_validation assert "unraid-deploy" not in managed_validation def test_chimera_example_requires_exact_source_webhook_auth_and_live_proof(): projects = json.loads( (BROKER_ROOT / "projects.example.json").read_text(encoding="utf-8") ) chimera = projects["jens/chimera"] assert chimera["exact_source"] is True assert chimera["source_dir"].startswith( "/mnt/cache/appdata/gitea-deploy-runners/sources/" ) assert chimera["env_source"] == "/srv/example/appdata/chimera/deploy.env" assert chimera["verify_oci_revision"] is True assert chimera["gitea_repository_id"] == 1234 assert chimera["gitea_webhook_secret_file"] == "jens__chimera" assert "broker_token_file" not in chimera assert "legacy_broker_token" not in chimera assert chimera["live_probe"] == { "type": "chimera-health-live-v1", "container": "Chimera", "port": 6967, "path": "/health/live", "host": "chimera:6967", } assert chimera["environment"] == { "CHIMERA_BUILD_ID": "{build_id}", "CHIMERA_SOURCE_REPOSITORY": "{repository}", "CHIMERA_SOURCE_REVISION": "{revision}", "CHIMERA_SOURCE_TREE": "{tree}", "CHIMERA_BUILD_DATE": "{build_date}", } def _controller_enrollment_payload(*, credential: bool = False) -> dict[str, object]: value: dict[str, object] = { "api_version": "itworx.controller-enrollment/v1", "enrollment_id": "enr-" + "a" * 20, "application_id": "wsbc-portal", "repository": "jens/windows-support-bundle-collector", "revision": "b" * 40, "profile": "internal-webapp-oidc", "container_name": "wsbc-portal", "compose_file": ".itworx/deployment.yml", "trusted_appdata_root": "/srv/example/appdata", "appdata_path": "/srv/example/appdata/wsbc-portal", "data_path": "/srv/example/appdata/wsbc-portal/data", "runtime_uid": 1654, "runtime_gid": 1654, "network": "itworx-npm", "internal_port": 8080, "proxy_hostname": "wsbc.example.com", "image_repository": "wsbc-portal", "image_tag": "0.23.0", "health_path": "/health", "runtime_environment_keys": [ "APPOPS_PUBLIC_URL", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_ISSUER", "OIDC_ROLE_CLAIM", ], } if credential: value["target_credential"] = "x" * 64 return value def _repopulse_controller_enrollment_payload(*, credential: bool = False) -> dict[str, object]: value: dict[str, object] = { "api_version": "itworx.controller-enrollment/v1", "enrollment_id": "enr-" + "c" * 20, "application_id": "repopulse", "repository": "jens/repopulse", "revision": "d" * 40, "profile": "repopulse-authenticated", "container_name": "repopulse", "compose_file": ".itworx/deployment.yml", "trusted_appdata_root": "/srv/example/appdata", "appdata_path": "/srv/example/appdata/repopulse", "data_path": "/srv/example/appdata/repopulse/data", "runtime_uid": 10001, "runtime_gid": 10001, "network": "itworx-mcp-hub-npm", "internal_port": 8000, "proxy_hostname": "repopulse.example.com", "image_repository": "repopulse-managed", "image_tag": "production", "health_path": "/health", "runtime_environment_keys": [ "REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN", ], } if credential: value["target_credential"] = "x" * 64 return value def test_controller_repopulse_profile_is_fixed_and_registers_managed_build(status_module): payload = _repopulse_controller_enrollment_payload() enrollment = status_module._controller_enrollment(payload, require_credential=False) assert enrollment is not None project = status_module._enrollment_project(enrollment) assert project["managed_build_context"] == "." assert project["managed_build_dockerfile"] == "deploy/managed/Dockerfile" assert project["managed_repository"] == "jens/repopulse" assert project["managed_runtime_environment_keys"] == [ "REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN", ] for field, value in ( ("application_id", "another-app"), ("repository", "jens/another-app"), ("network", "other-network"), ("runtime_uid", 1000), ("internal_port", 8080), ("image_tag", "1.5.0"), ("proxy_hostname", "another.example.com"), ): unsafe = {**payload, field: value} assert status_module._controller_enrollment(unsafe, require_credential=False) is None def test_controller_repopulse_runtime_seal_fixes_gitea_url_and_hides_values(status_module, tmp_path, monkeypatch): appdata_root = tmp_path / "appdata" app_root = appdata_root / "repopulse" app_root.mkdir(parents=True) legacy_env_path = app_root / "deploy.env" legacy_env_path.write_text("REPOPULSE_OPERATOR_TOKEN=legacy-preserved\n", encoding="utf-8") env_path = app_root / "deploy.managed.env" env_path.write_text("", encoding="utf-8") monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", appdata_root) monkeypatch.setattr(status_module, "LEGACY_APPDATA_ROOT", appdata_root) monkeypatch.setattr(status_module, "EXACT_SOURCE_ROOT", tmp_path / "sources") payload = _repopulse_controller_enrollment_payload(credential=True) payload["trusted_appdata_root"] = appdata_root.as_posix() payload["appdata_path"] = app_root.as_posix() payload["data_path"] = (app_root / "data").as_posix() enrollment = status_module._controller_enrollment(payload, require_credential=True) assert enrollment is not None project = status_module._enrollment_project(enrollment) assert project["env_source"] == env_path.as_posix() projects = {"jens/repopulse": project} runtime = { "enrollment_id": enrollment["enrollment_id"], "application_id": "repopulse", "repository": "jens/repopulse", "values": { "REPOPULSE_GITEA_URL": "https://gitea.example.com", "REPOPULSE_GITEA_TOKEN": "write-only-gitea-token", "REPOPULSE_MAINTENANCE_TOKEN": "write-only-maintenance-token", }, } status, evidence = status_module._configure_enrollment_runtime(projects, runtime) assert status == "configured" assert evidence == {"enrollmentId": enrollment["enrollment_id"], "applicationId": "repopulse"} assert "write-only-gitea-token" in env_path.read_text(encoding="utf-8") assert "write-only-maintenance-token" in env_path.read_text(encoding="utf-8") assert legacy_env_path.read_text(encoding="utf-8") == "REPOPULSE_OPERATOR_TOKEN=legacy-preserved\n" assert "write-only-gitea-token" not in str(evidence) assert "write-only-maintenance-token" not in str(evidence) unsafe = {**runtime, "values": {**runtime["values"], "REPOPULSE_GITEA_URL": "https://attacker.invalid"}} status, evidence = status_module._configure_enrollment_runtime(projects, unsafe) assert status == "runtime_environment_rejected" assert evidence == {} def test_repopulse_preflight_blocks_existing_private_source_tree(status_module, monkeypatch): payload = _repopulse_controller_enrollment_payload() enrollment = status_module._controller_enrollment(payload, require_credential=False) assert enrollment is not None monkeypatch.setattr(status_module, "_repopulse_controller_storage_ready", lambda: (True, "preflight_ok")) monkeypatch.setattr(status_module, "_repopulse_sources_readable", lambda: False) code, detail = status_module._safe_enrollment_preflight({}, enrollment) assert (code, detail) == ( "source_permissions_unavailable", "fixed_local_sources_are_not_readable_by_runtime_gid", ) @pytest.mark.skipif( os.name == "nt" or not hasattr(os, "geteuid") or os.geteuid() != 0, reason="requires Linux root-owned source trees", ) def test_repopulse_preflight_blocks_an_existing_mode_700_live_source_tree( status_module, deploy_module, tmp_path, monkeypatch ): sources_root = tmp_path / "sources" sources_root.mkdir() for slug in ("repopulse", "privatemuse"): source = sources_root / slug source.mkdir(mode=0o700) os.chown(source, 0, 10001) source.chmod(0o700) monkeypatch.setattr(status_module, "EXACT_SOURCE_ROOT", sources_root) monkeypatch.setattr(deploy_module, "EXACT_SOURCE_ROOT", sources_root) monkeypatch.setattr(status_module, "_repopulse_controller_storage_ready", lambda: (True, "preflight_ok")) payload = _repopulse_controller_enrollment_payload() enrollment = status_module._controller_enrollment(payload, require_credential=False) assert enrollment is not None code, detail = status_module._safe_enrollment_preflight({}, enrollment) assert (code, detail) == ( "source_permissions_unavailable", "fixed_local_sources_are_not_readable_by_runtime_gid", ) for slug in ("repopulse", "privatemuse"): source = sources_root / slug staging = sources_root / f".{slug}-aabbccddeeff-stage" source.rename(staging) deploy_module._prepare_exact_source_group_read(staging, slug) staging.rename(source) assert status_module._repopulse_sources_readable() is True def test_repopulse_preflight_blocks_read_only_controller_mount(status_module, monkeypatch): payload = _repopulse_controller_enrollment_payload() enrollment = status_module._controller_enrollment(payload, require_credential=False) assert enrollment is not None monkeypatch.setattr( status_module, "_repopulse_controller_storage_ready", lambda: (False, "controller_storage_read_only"), ) code, detail = status_module._safe_enrollment_preflight({}, enrollment) assert (code, detail) == ( "controller_storage_read_only", "controller_storage_not_writable_for_repopulse_profile", ) def test_controller_mount_read_only_check_uses_the_most_specific_mount(status_module): mountinfo = ( "30 20 0:30 / /opt/gitea-deploy ro,relatime - ext4 /dev/root ro\n" "31 20 0:31 / /mnt/cache/appdata/gitea-deploy-runners/sources rw,relatime - ext4 /dev/cache rw\n" ) assert status_module._mount_is_read_only_for("/opt/gitea-deploy/status_server.py", mountinfo) is True assert status_module._mount_is_read_only_for( "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse", mountinfo ) is False def test_controller_repopulse_rollback_refuses_legacy_env_collision(status_module, tmp_path, monkeypatch): appdata_root = tmp_path / "appdata" app_root = appdata_root / "repopulse" data_path = app_root / "data" data_path.mkdir(parents=True) managed_env = app_root / "deploy.managed.env" managed_env.write_text("REPOPULSE_GITEA_URL=https://gitea.example.com\n", encoding="utf-8") legacy_env = app_root / "deploy.env" legacy_env.write_text("legacy configuration\n", encoding="utf-8") state_root = tmp_path / "state" state_root.mkdir() secret_root = tmp_path / "target-secrets" secret_root.mkdir() secret = secret_root / "jens__repopulse" secret.write_text("x" * 64, encoding="ascii") monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", appdata_root) monkeypatch.setattr(status_module, "STATE_ROOT", state_root) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secret_root) writes = [] monkeypatch.setattr(status_module, "_write_projects", lambda _projects: writes.append("write")) enrollment_id = "enr-" + "c" * 20 project = { "appops_managed_target": True, "managed_enrollment_id": enrollment_id, "managed_application_id": "repopulse", "managed_profile": "repopulse-authenticated", } status, evidence = status_module._rollback_enrollment_target( {"jens/repopulse": project}, {"enrollment_id": enrollment_id, "application_id": "repopulse", "repository": "jens/repopulse"}, ) assert status == "rollback_failed" assert evidence == {} assert writes == [] assert managed_env.is_file() and legacy_env.is_file() and secret.is_file() assert data_path.is_dir() def test_controller_enrollment_contract_rejects_arbitrary_compose_and_paths(status_module): value = _controller_enrollment_payload() assert status_module._controller_enrollment(value, require_credential=False) is not None unsafe = dict(value) unsafe["compose"] = "services: {}" assert status_module._controller_enrollment(unsafe, require_credential=False) is None unsafe = dict(value) unsafe["data_path"] = "/etc" assert status_module._controller_enrollment(unsafe, require_credential=False) is None unsafe = dict(value) unsafe["proxy_hostname"] = "*.example.com" assert status_module._controller_enrollment(unsafe, require_credential=False) is None unsafe = _controller_enrollment_payload(credential=True) unsafe["target_credential"] = "short" assert status_module._controller_enrollment(unsafe, require_credential=True) is None def test_controller_enrollment_derives_paths_from_the_selected_fixed_root(status_module, monkeypatch): managed_root = "/var/lib/docker/volumes/itworx-managed-appdata/_data" monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", Path(managed_root)) monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda root: root.as_posix() == managed_root) alternate = _controller_enrollment_payload() alternate["trusted_appdata_root"] = managed_root alternate["appdata_path"] = managed_root + "/wsbc-portal" alternate["data_path"] = managed_root + "/wsbc-portal/data" enrollment = status_module._controller_enrollment(alternate, require_credential=False) assert enrollment is not None project = status_module._enrollment_project(enrollment) assert project["managed_appdata_root"] == managed_root assert project["managed_appdata_path"] == managed_root + "/wsbc-portal" unsafe_path = {**alternate, "data_path": "/srv/example/appdata/wsbc-portal/data"} assert status_module._controller_enrollment(unsafe_path, require_credential=False) is None unsupported_root = {**alternate, "trusted_appdata_root": "/srv/example/other"} assert status_module._controller_enrollment(unsupported_root, require_credential=False) is None def test_controller_accepts_legacy_requests_after_managed_marker_is_initialized(status_module, monkeypatch): managed_root = "/var/lib/docker/volumes/itworx-managed-appdata/_data" monkeypatch.setattr(status_module, "_refresh_appdata_root_selection", lambda: Path(managed_root)) monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda root: root.as_posix() == managed_root) legacy = _controller_enrollment_payload() assert legacy["trusted_appdata_root"] == "/srv/example/appdata" enrollment = status_module._controller_enrollment(legacy, require_credential=False) assert enrollment is not None assert enrollment["trusted_appdata_root"] == "/srv/example/appdata" assert "managed_appdata_root" not in status_module._enrollment_project(enrollment) def test_controller_accepts_legacy_v1_payload_without_root_field(status_module): legacy = _controller_enrollment_payload() legacy.pop("trusted_appdata_root") enrollment = status_module._controller_enrollment(legacy, require_credential=False) assert enrollment is not None assert enrollment["trusted_appdata_root"] == "/srv/example/appdata" assert "managed_appdata_root" not in status_module._enrollment_project(enrollment) def test_repopulse_enrollment_remains_bound_to_legacy_appdata_root(status_module, monkeypatch): monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", Path("/var/lib/docker/volumes/itworx-managed-appdata/_data")) payload = _repopulse_controller_enrollment_payload() payload["trusted_appdata_root"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data" payload["appdata_path"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data/repopulse" payload["data_path"] = "/var/lib/docker/volumes/itworx-managed-appdata/_data/repopulse/data" assert status_module._controller_enrollment(payload, require_credential=False) is None def test_fixed_repopulse_profile_stays_legacy_with_managed_marker(status_module, monkeypatch): managed_root = "/var/lib/docker/volumes/itworx-managed-appdata/_data" monkeypatch.setattr(status_module, "_refresh_appdata_root_selection", lambda: Path(managed_root)) monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda root: root.as_posix() == managed_root) enrollment = status_module._controller_enrollment( _repopulse_controller_enrollment_payload(), require_credential=False, ) assert enrollment is not None assert enrollment["trusted_appdata_root"] == "/srv/example/appdata" def test_controller_runtime_seal_accepts_only_profile_owned_keys(status_module, tmp_path, monkeypatch): appdata_root = tmp_path / "appdata" app_root = appdata_root / "wsbc-portal" app_root.mkdir(parents=True) env_path = app_root / "deploy.env" env_path.write_text("", encoding="utf-8") monkeypatch.setattr(status_module, "TRUSTED_APPCONFIG_ROOT", appdata_root) monkeypatch.setattr(status_module, "LEGACY_APPDATA_ROOT", appdata_root) monkeypatch.setattr(status_module, "EXACT_SOURCE_ROOT", tmp_path / "sources") payload = _controller_enrollment_payload(credential=True) payload["trusted_appdata_root"] = appdata_root.as_posix() payload["appdata_path"] = app_root.as_posix() payload["data_path"] = (app_root / "data").as_posix() enrollment = status_module._controller_enrollment(payload, require_credential=True) assert enrollment is not None project = status_module._enrollment_project(enrollment) projects = {"jens/windows-support-bundle-collector": project} runtime = { "enrollment_id": enrollment["enrollment_id"], "application_id": "wsbc-portal", "repository": "jens/windows-support-bundle-collector", "values": { "APPOPS_PUBLIC_URL": "https://wsbc.example.com", "OIDC_CLIENT_ID": "wsbc-portal", "OIDC_CLIENT_SECRET": "opaque-write-only-value", "OIDC_ISSUER": "https://auth.example.com/application/o/wsbc-portal", "OIDC_ROLE_CLAIM": "roles", }, } status, evidence = status_module._configure_enrollment_runtime(projects, runtime) assert status == "configured" assert evidence == {"enrollmentId": enrollment["enrollment_id"], "applicationId": "wsbc-portal"} assert "opaque-write-only-value" in env_path.read_text(encoding="utf-8") unsafe = {**runtime, "values": {**runtime["values"], "ARBITRARY_COMMAND": "id"}} status, _evidence = status_module._configure_enrollment_runtime(projects, unsafe) assert status == "runtime_environment_rejected" def test_controller_enrollment_replay_requires_the_existing_target_credential(status_module, tmp_path, monkeypatch): projects_path = tmp_path / "projects.json" secrets_root = tmp_path / "target-secrets" credential_path = secrets_root / "jens__windows-support-bundle-collector" secrets_root.mkdir() credential_path.write_text("x" * 64, encoding="ascii") enrollment = _controller_enrollment_payload(credential=True) project = status_module._enrollment_project(enrollment) projects_path.write_text(json.dumps({"jens/windows-support-bundle-collector": project}), encoding="utf-8") monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secrets_root) monkeypatch.setattr(status_module, "appdata_root_is_safe", lambda _root: True) enrollment["target_credential"] = "y" * 64 status, _evidence = status_module._enroll_controller_target(enrollment) assert status == "target_credential_conflict" def _workmesh_promotion_project(status_module) -> dict[str, object]: return { "slug": "workmesh", "branch": "main", "source_dir": (status_module.EXACT_SOURCE_ROOT / "workmesh").as_posix(), "exact_source": True, "env_source": "/srv/example/appdata/workmesh/.env", "compose_files": ["compose.yaml"], "compose_project": "workmesh", "containers": [ "workmesh-postgres-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1", ], "retired_compose_containers": ["workmesh-mock-ats-1"], "verify_oci_revision": False, "health_timeout": 300, "appops_deployment_secret_file": "jens__workmesh", "validation": { "validation_gate_version": 1, "required": True, "contexts": ["WorkMesh CI (push)"], "timeout_seconds": 600, "poll_interval_seconds": 10, "max_poll_interval_seconds": 30, }, } def test_workmesh_promotion_policy_accepts_only_root_registered_product_topology(status_module): project = _workmesh_promotion_project(status_module) assert status_module._workmesh_promotion_policy("jens/workmesh", project) == ( "workmesh", "jens__workmesh", ) assert status_module._workmesh_promotion_policy("jens/other", project) is None unsafe = {**project, "source_dir": "/tmp/workmesh"} assert status_module._workmesh_promotion_policy("jens/workmesh", unsafe) is None unsafe = {**project, "containers": [*project["containers"], "sidecar"]} assert status_module._workmesh_promotion_policy("jens/workmesh", unsafe) is None unsafe_oci_mode = {**project, "verify_oci_revision": True} assert status_module._workmesh_promotion_policy("jens/workmesh", unsafe_oci_mode) is None def test_workmesh_preflight_binds_exact_main_validation_and_controller_check( status_module, monkeypatch ): project = _workmesh_promotion_project(status_module) revision = "a" * 40 validation = [{"context": "WorkMesh CI (push)", "status": "success"}] calls: list[list[str]] = [] class Result: returncode = 0 stdout = json.dumps({ "status": "preflight-ok", "repository": "jens/workmesh", "revision": "a" * 40, "backupInputsSha256": "c" * 64, }).encode() monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: True) monkeypatch.setattr(status_module, "_validation_policy", lambda *_args: {"contexts": ["WorkMesh CI (push)"]}) monkeypatch.setattr(status_module, "_fetch_gitea_validation", lambda *_args: ("pass", validation)) monkeypatch.setattr(status_module, "_controller_release", lambda: { "controllerVersion": "2.1.4", "sourceRevision": "c" * 40, "sourceTree": "d" * 40, "artifactSha256": "e" * 64, }) monkeypatch.setattr(status_module.subprocess, "run", lambda args, **_kwargs: calls.append(args) or Result()) result = status_module._workmesh_promotion_preflight("jens/workmesh", revision, project) assert result["status"] == "preflight_ok" assert result["repository"] == "jens/workmesh" assert result["revision"] == revision assert len(result["preflightSha256"]) == 64 assert result["backupInputsSha256"] == "c" * 64 assert result["checks"] == { "exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed", } assert calls == [["python3", str(status_module.ROOT / "deploy.py"), "workmesh-preflight", "jens/workmesh", revision]] def test_workmesh_preflight_blocks_branch_drift_before_controller_command( status_module, monkeypatch ): project = _workmesh_promotion_project(status_module) monkeypatch.setattr(status_module, "_is_current_branch_head", lambda *_args: False) monkeypatch.setattr(status_module.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("controller must not run")) result = status_module._workmesh_promotion_preflight("jens/workmesh", "b" * 40, project) assert result["status"] == "preflight_failed" assert result["checks"]["exactMain"] == "blocked" assert result["checks"]["pushValidation"] == "unknown" assert result["checks"]["controllerPreflight"] == "unknown" def test_workmesh_promotion_queue_deploys_only_after_digest_recheck_and_receipt( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "appops-inbox" projects_path = tmp_path / "projects.json" project = _workmesh_promotion_project(status_module) projects_path.write_text(json.dumps({"jens/workmesh": project}), encoding="utf-8") monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: { "status": "preflight_ok", "repository": "jens/workmesh", "revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64, }) status, request_id = status_module._enqueue_appops_request( repository="jens/workmesh", revision="a" * 40, operation="workmesh_promote", idempotency_key="workmesh-approval-001", plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64}, ) assert status == "pending" assert status_module._enqueue_appops_request( repository="jens/workmesh", revision="a" * 40, operation="workmesh_promote", idempotency_key="workmesh-approval-001", plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64}, ) == ("pending", request_id) calls: list[list[str]] = [] monkeypatch.setattr( status_module, "_run_workmesh_promotion_deploy", lambda args: calls.append(args) or status_module.subprocess.CompletedProcess(args, 0), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "healthy"}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"revision": "a" * 40}) monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: "b" * 64) marker = inbox / f"{request_id}.json" status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == "deployed" assert record["revision"] == "a" * 40 assert record["receipt_sha256"] == hashlib.sha256( b'{"revision":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}' ).hexdigest() assert record["backup_receipt_sha256"] == "b" * 64 assert calls == [[ "python3", str(status_module.ROOT / "deploy.py"), "workmesh-promote", "jens/workmesh", "a" * 40, request_id, "f" * 64, "e" * 64, ]] @pytest.mark.parametrize( ("observed_preflight", "observed_backup_inputs"), [("e" * 64, "e" * 64), ("f" * 64, "9" * 64)], ) def test_workmesh_promotion_worker_refuses_changed_preflight_without_deploy( status_module, tmp_path, monkeypatch, observed_preflight, observed_backup_inputs ): inbox = tmp_path / "appops-inbox" inbox.mkdir() marker = inbox / ("a" * 64 + ".json") marker.write_text(json.dumps({ "schema": "appops-deployment-request.v1", "request": "a" * 64, "repository": "jens/workmesh", "revision": "b" * 40, "operation": "workmesh_promote", "preflight_sha256": "f" * 64, "backup_inputs_sha256": "e" * 64, "status": "pending", }), encoding="utf-8") project = _workmesh_promotion_project(status_module) monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project}) monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: { "status": "preflight_ok", "preflightSha256": observed_preflight, "backupInputsSha256": observed_backup_inputs, }) monkeypatch.setattr( status_module, "_run_workmesh_promotion_deploy", lambda *_args: pytest.fail("changed approval must not deploy"), ) status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == "failed" assert record["reason"] == "workmesh_preflight_changed" def test_workmesh_backup_capture_rejects_changed_inputs_before_stopping_services( deploy_module, monkeypatch, tmp_path ): inputs = { "apiVersion": "itworx.workmesh-backup-inputs/v1", "repository": "jens/workmesh", "target": "workmesh", "candidateRevision": "b" * 40, "priorLiveRevision": "a" * 40, "priorReceiptSha256": "c" * 64, "backupToolSha256": "d" * 64, "environmentSha256": "e" * 64, "externalRecoveryReceiptSha256": "f" * 64, "n8nImageId": "sha256:" + "1" * 64, "n8nImageRefSha256": "2" * 64, } calls: list[str] = [] monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr( deploy_module, "workmesh_backup_preflight_inputs", lambda *_args: inputs ) monkeypatch.setattr( deploy_module, "_workmesh_docker", lambda *args, **_kwargs: calls.append(str(args[0])) or pytest.fail("input drift must be detected before any Docker operation"), ) with pytest.raises(deploy_module.WorkMeshPromotionNoMutation, match="preflight changed"): deploy_module.capture_workmesh_data_backup( {"slug": "workmesh"}, tmp_path, "b" * 40, request_id="3" * 64, preflight_sha256="4" * 64, backup_inputs_sha256="9" * 64, ) assert calls == [] @pytest.mark.parametrize("failure_point", ["backup_input_drift", "backup_restore_gate"]) @pytest.mark.parametrize("source_restore_succeeds", [True, False]) def test_workmesh_promotion_restores_prior_source_before_no_mutation_classification( deploy_module, monkeypatch, tmp_path, failure_point, source_restore_succeeds ): live_revision = "a" * 40 candidate_revision = "b" * 40 source_dir = tmp_path / "source" source_dir.mkdir() (source_dir / "source-revision").write_text("prior", encoding="utf-8") request_id = "c" * 64 preflight_sha256 = "d" * 64 backup_inputs = { "apiVersion": "itworx.workmesh-backup-inputs/v1", "repository": "jens/workmesh", "target": "workmesh", "candidateRevision": candidate_revision, "priorLiveRevision": live_revision, "priorReceiptSha256": "e" * 64, "backupToolSha256": "f" * 64, "environmentSha256": "1" * 64, "externalRecoveryReceiptSha256": "2" * 64, "n8nImageId": "sha256:" + "3" * 64, "n8nImageRefSha256": "4" * 64, } approved_backup_inputs = ( deploy_module.workmesh_backup_inputs_sha256(backup_inputs) if failure_point == "backup_restore_gate" else "9" * 64 ) project = { "slug": "workmesh", "branch": "main", "source_dir": str(source_dir), "env_source": "/srv/example/appdata/workmesh/.env", "compose_project": "workmesh", "compose_files": ["compose.yaml"], "containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS), "retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER], "exact_source": True, "health_timeout": 300, "verify_oci_revision": False, } projects = {"jens/workmesh": project} previous_state = { "schema_version": 3, "repository": "jens/workmesh", "live_revision": live_revision, "live_status": "healthy", "live_receipt": {"containers": {}}, } state_writes: list[dict[str, object]] = [] backup_calls: list[str] = [] @contextmanager def no_lock(*_args): yield monkeypatch.setattr(sys, "argv", [ "deploy.py", "workmesh-promote", "jens/workmesh", candidate_revision, request_id, preflight_sha256, approved_backup_inputs, ]) monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "read_state", lambda _project: previous_state) monkeypatch.setattr( deploy_module, "validate_commit", lambda _repo, _branch, value: value ) monkeypatch.setattr( deploy_module, "deployment_metadata", lambda *_args: {"tree": "5" * 40} ) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "_workmesh_require_product_target", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) monkeypatch.setattr(deploy_module, "exclusive_global_deploy_lock", no_lock) monkeypatch.setattr( deploy_module, "_workmesh_current_runtime", lambda *_args: (previous_state, previous_state["live_receipt"], {}), ) def sync_source(*_args, **_kwargs): (source_dir / "source-revision").write_text("candidate", encoding="utf-8") def restore_source(_project, _repository, requested_revision): assert requested_revision == live_revision if source_restore_succeeds: (source_dir / "source-revision").write_text("prior", encoding="utf-8") monkeypatch.setattr(deploy_module, "sync_source", sync_source) monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) monkeypatch.setattr(deploy_module, "restore_live_source", restore_source) monkeypatch.setattr( deploy_module, "exact_source_matches_revision", lambda _repo, revision, target: revision == live_revision and (target / "source-revision").read_text(encoding="utf-8") == "prior", ) monkeypatch.setattr( deploy_module, "compose_command", lambda *_args: (tmp_path, ["docker", "compose"]) ) monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {}) monkeypatch.setattr(deploy_module, "run", lambda *_args, **_kwargs: None) monkeypatch.setattr(deploy_module, "revalidate_deployment_candidate", lambda *_args: None) monkeypatch.setattr( deploy_module, "workmesh_backup_preflight_inputs", lambda *_args: backup_inputs ) monkeypatch.setattr( deploy_module, "verified_current_receipt", lambda *_args: None, ) monkeypatch.setattr(deploy_module, "validate_forward_revision", lambda *_args: None) monkeypatch.setattr( deploy_module, "write_state", lambda *_args, **kwargs: state_writes.append(kwargs), ) monkeypatch.setattr(deploy_module, "validate_managed_v1_runtime", lambda *_args: None) monkeypatch.setattr(deploy_module, "ensure_docker_capacity", lambda: None) def capture_backup(*_args, **_kwargs): backup_calls.append("capture") raise deploy_module.WorkMeshPromotionNoMutation("isolated backup restore rejected") monkeypatch.setattr(deploy_module, "capture_workmesh_data_backup", capture_backup) monkeypatch.setattr( deploy_module, "deploy_project", lambda *_args: pytest.fail("no-mutation path must not invoke deployment"), ) expected_exception = ( deploy_module.WorkMeshPromotionNoMutation if source_restore_succeeds else deploy_module.DeployError ) with pytest.raises(expected_exception): deploy_module.main() if source_restore_succeeds: assert (source_dir / "source-revision").read_text(encoding="utf-8") == "prior" assert state_writes == [] else: assert (source_dir / "source-revision").read_text(encoding="utf-8") == "candidate" assert state_writes assert state_writes[-1]["live_status"] == "unknown" assert backup_calls == ([] if failure_point == "backup_input_drift" else ["capture"]) @pytest.mark.parametrize("source_restore_succeeds", [True, False]) def test_workmesh_preflight_restores_prior_source_before_returning( deploy_module, monkeypatch, tmp_path, capsys, source_restore_succeeds ): live_revision = "a" * 40 candidate_revision = "b" * 40 source_dir = tmp_path / "source" source_dir.mkdir() (source_dir / "source-revision").write_text("prior", encoding="utf-8") project = { "slug": "workmesh", "branch": "main", "source_dir": str(source_dir), "env_source": "/srv/example/appdata/workmesh/.env", "compose_project": "workmesh", "compose_files": ["compose.yaml"], "containers": list(deploy_module.WORKMESH_PRODUCT_CONTAINERS), "retired_compose_containers": [deploy_module.WORKMESH_RETIRED_CONTAINER], "exact_source": True, "health_timeout": 300, "verify_oci_revision": False, } projects = {"jens/workmesh": project} previous_state = { "schema_version": 3, "repository": "jens/workmesh", "live_revision": live_revision, "live_status": "healthy", "live_receipt": {"containers": {}}, } state_writes: list[dict[str, object]] = [] @contextmanager def no_lock(*_args): yield monkeypatch.setattr(sys, "argv", [ "deploy.py", "workmesh-preflight", "jens/workmesh", candidate_revision, ]) monkeypatch.setattr(deploy_module, "require_host_root", lambda: None) monkeypatch.setattr(deploy_module, "load_projects", lambda: projects) monkeypatch.setattr(deploy_module, "repository_path", lambda *_args: tmp_path) monkeypatch.setattr(deploy_module, "read_state", lambda _project: previous_state) monkeypatch.setattr( deploy_module, "validate_commit", lambda _repo, _branch, value: value ) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *_args: {"tree": "5" * 40}) monkeypatch.setattr(deploy_module, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy_module, "_workmesh_require_product_target", lambda *_args: None) monkeypatch.setattr(deploy_module, "exclusive_lock", no_lock) monkeypatch.setattr(deploy_module, "sync_source", lambda *_args, **_kwargs: ( source_dir / "source-revision" ).write_text("candidate", encoding="utf-8")) monkeypatch.setattr(deploy_module, "copy_environment", lambda *_args: None) monkeypatch.setattr( deploy_module, "compose_command", lambda *_args: (tmp_path, ["docker", "compose"]) ) monkeypatch.setattr(deploy_module, "deployment_environment", lambda *_args: {}) monkeypatch.setattr(deploy_module, "run", lambda *_args, **_kwargs: None) monkeypatch.setattr( deploy_module, "workmesh_backup_preflight_inputs", lambda *_args: ( {"observed": "candidate"} if (source_dir / "source-revision").read_text(encoding="utf-8") == "candidate" else pytest.fail("backup preflight must inspect the prepared candidate") ), ) def restore_source(_project, _repository, requested_revision): assert requested_revision == live_revision if source_restore_succeeds: (source_dir / "source-revision").write_text("prior", encoding="utf-8") monkeypatch.setattr(deploy_module, "restore_live_source", restore_source) monkeypatch.setattr( deploy_module, "exact_source_matches_revision", lambda _repo, revision, target: revision == live_revision and (target / "source-revision").read_text(encoding="utf-8") == "prior", ) monkeypatch.setattr( deploy_module, "write_state", lambda *_args, **kwargs: state_writes.append(kwargs), ) if source_restore_succeeds: assert deploy_module.main() == 0 response = json.loads(capsys.readouterr().out) assert response["status"] == "preflight-ok" assert response["revision"] == candidate_revision assert (source_dir / "source-revision").read_text(encoding="utf-8") == "prior" assert state_writes == [] else: with pytest.raises(deploy_module.DeployError, match="source restoration"): deploy_module.main() assert (source_dir / "source-revision").read_text(encoding="utf-8") == "candidate" assert state_writes assert state_writes[-1]["live_status"] == "unknown" def test_workmesh_promotion_worker_keeps_failed_mutation_without_receipt_ambiguous( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "appops-inbox" project = _workmesh_promotion_project(status_module) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project}) monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: { "status": "preflight_ok", "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64, }) monkeypatch.setattr( status_module, "_run_workmesh_promotion_deploy", lambda args: status_module.subprocess.CompletedProcess(args, 1), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "degraded"}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None) _, request_id = status_module._enqueue_appops_request( repository="jens/workmesh", revision="a" * 40, operation="workmesh_promote", idempotency_key="workmesh-mutation-failure-01", plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64}, ) marker = inbox / f"{request_id}.json" status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == "outcome_unknown" assert record["reason"] == "controller_deployment_failed" def test_workmesh_promotion_worker_reconciles_recovered_request_without_redeploy( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "appops-inbox" inbox.mkdir() request_id = "a" * 64 marker = inbox / f"{request_id}.json" marker.write_text(json.dumps({ "schema": "appops-deployment-request.v1", "request": request_id, "repository": "jens/workmesh", "revision": "b" * 40, "operation": "workmesh_promote", "preflight_sha256": "f" * 64, "backup_inputs_sha256": "e" * 64, "status": "running", }), encoding="utf-8") project = _workmesh_promotion_project(status_module) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project}) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "healthy"}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None) monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: None) monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: { "status": "preflight_ok", "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64, }) monkeypatch.setattr(status_module.subprocess, "run", lambda *_args, **_kwargs: pytest.fail("recovered request must not redeploy")) status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == "outcome_unknown" assert record["reason"] == "controller_recovered_without_receipt" receipt = {"repository": "jens/workmesh", "revision": "b" * 40, "state": "deployed"} monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: receipt) monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: "b" * 64) status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == "deployed" assert record["receipt_sha256"] == hashlib.sha256( json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode("utf-8") ).hexdigest() assert record["backup_receipt_sha256"] == "b" * 64 def test_workmesh_promotion_timeout_is_bounded_and_reconciles_without_retry( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "appops-inbox" project = _workmesh_promotion_project(status_module) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project}) monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: { "status": "preflight_ok", "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64, }) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "degraded"}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: None) calls: list[tuple[list[str], dict[str, object]]] = [] wait_timeouts: list[float] = [] group_stops = [] class Process: pid = 45678 def wait(self, *, timeout): wait_timeouts.append(timeout) raise status_module.subprocess.TimeoutExpired("deploy.py", timeout) def timeout_popen(args, **kwargs): calls.append((args, kwargs)) return Process() monkeypatch.setattr(status_module, "_workmesh_process_group_supported", lambda: True) monkeypatch.setattr(status_module, "_stop_workmesh_process_group", lambda process: group_stops.append(process)) monkeypatch.setattr(status_module.subprocess, "Popen", timeout_popen) _, request_id = status_module._enqueue_appops_request( repository="jens/workmesh", revision="a" * 40, operation="workmesh_promote", idempotency_key="workmesh-timeout-contract-01", plan={"revision": "a" * 40, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64}, ) marker = inbox / f"{request_id}.json" status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert calls[0][1].get("start_new_session") is True assert wait_timeouts == [1800] assert len(group_stops) == 1 assert record["status"] == "outcome_unknown" status_module._process_appops_record(marker) assert len(calls) == 1 assert json.loads(marker.read_text(encoding="utf-8"))["status"] == "outcome_unknown" def test_workmesh_timeout_stops_child_group_before_releasing_lock_owner(status_module): leader = 12345 members = [(leader, "T"), (23456, "S"), (34567, "S")] events = [] def read_members(pgid): assert pgid == leader return list(members) def send_group_signal(pgid, signum): events.append(("group_signal", pgid, signum)) assert signum == 19 members[:] = [(pid, "T") for pid, _state in members] def send_signal(pid, signum): events.append(("signal", pid, signum)) if pid != leader and signum == 9: assert all(state == "T" for _member_pid, state in members) members[:] = [item for item in members if item[0] != pid] elif pid == leader and signum == 9: assert not any(member_pid != leader for member_pid, state in members if state not in {"Z", "X"}) class Process: pid = leader def wait(self): events.append(("wait",)) assert events[-2] == ("signal", leader, 9) status_module._stop_workmesh_process_group( Process(), members_reader=read_members, group_signal_sender=send_group_signal, signal_sender=send_signal, stop_signal=19, kill_signal=9, sleep_fn=lambda _seconds: None, cleanup_timeout_seconds=1, ) assert events == [ ("group_signal", leader, 19), ("signal", 23456, 9), ("signal", 34567, 9), ("signal", leader, 9), ("wait",), ] def test_workmesh_promotion_http_contract_binds_exact_sha_and_preflight_digest( status_module, tmp_path, monkeypatch ): secret = b"workmesh-appops-" + b"x" * 48 secrets_root = tmp_path / "appops-secrets" _write_appops_secret(secrets_root, "jens__workmesh", secret) projects_path = tmp_path / "projects.json" project = _workmesh_promotion_project(status_module) projects_path.write_text(json.dumps({"jens/workmesh": project}), encoding="utf-8") inbox = tmp_path / "inbox" monkeypatch.setattr(status_module, "PROJECTS_PATH", projects_path) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_SECRET_ROOT", secrets_root) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces") monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda repository, revision, _project: { "apiVersion": "itworx.workmesh-promotion/v2", "status": "preflight_ok", "repository": repository, "revision": revision, "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64, "checks": {"exactMain": "passed", "pushValidation": "passed", "controllerPreflight": "passed", "dataBackup": "passed"}, }) server = status_module.BoundedThreadingHTTPServer(("127.0.0.1", 0), status_module.Handler) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() def post(path: str, payload: dict[str, object], nonce: str) -> tuple[int, dict[str, object]]: body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") timestamp = str(int(status_module.time.time())) signature = status_module._request_signature( secret, method="POST", path=path, timestamp=timestamp, nonce=nonce, raw_body=body ) client = http.client.HTTPConnection(*server.server_address, timeout=2) client.request("POST", path, body=body, headers={ "Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": timestamp, "X-ITWorx-Deployment-Nonce": nonce, "X-ITWorx-Deployment-Signature": signature, }) response = client.getresponse() decoded = json.loads(response.read()) client.close() return response.status, decoded try: revision = "a" * 40 preflight_path = status_module.WORKMESH_PROMOTION_PREFLIGHT_PATH preflight_status, preflight = post(preflight_path, {"revision": revision}, "1" * 32) assert preflight_status == 200 assert preflight["preflightSha256"] == "f" * 64 apply_path = status_module.WORKMESH_PROMOTION_APPLY_PATH rejected_status, rejected = post(apply_path, { "revision": revision, "preflightSha256": "e" * 64, "idempotencyKey": "workmesh-approval-001", }, "2" * 32) assert rejected_status == 409 assert rejected["status"] == "preflight_changed" assert rejected["request"] == status_module._appops_request_digest( "jens/workmesh", revision, "workmesh_promote", "workmesh-approval-001" ) assert rejected["repository"] == "jens/workmesh" assert rejected["revision"] == revision assert rejected["operation"] == "workmesh_promote" assert rejected["reason"] == "approved_preflight_no_longer_matches" assert not inbox.exists() or not list(inbox.iterdir()) accepted_status, accepted = post(apply_path, { "revision": revision, "preflightSha256": "f" * 64, "idempotencyKey": "workmesh-approval-002", }, "3" * 32) assert accepted_status == 202 assert accepted["status"] == "accepted" assert accepted["repository"] == "jens/workmesh" assert accepted["revision"] == revision queued = json.loads((inbox / f"{accepted['request']}.json").read_text(encoding="utf-8")) assert queued["operation"] == "workmesh_promote" assert queued["preflight_sha256"] == "f" * 64 assert queued["backup_inputs_sha256"] == "e" * 64 finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_workmesh_isolated_n8n_restore_uses_exact_image_and_no_network( deploy_module, monkeypatch, tmp_path ): backup_dir = tmp_path / "backup" backup_dir.mkdir() state_root = tmp_path / "state" state_root.mkdir() calls: list[tuple[str, ...]] = [] secret = "synthetic-n8n-key-" + "x" * 40 n8n_id = "a" * 64 def docker(*args, timeout=None): command = tuple(str(item) for item in args) calls.append(command) if command[:2] == ("run", "--detach"): env_file = Path(command[command.index("--env-file") + 1]) if os.name != "nt": assert env_file.stat().st_mode & 0o777 == 0o600 assert f"N8N_ENCRYPTION_KEY={secret}" in env_file.read_text(encoding="utf-8") assert secret not in " ".join(command) return subprocess.CompletedProcess(command, 0, stdout=(n8n_id + "\n").encode()) if "volume" in command and "create" in command: return subprocess.CompletedProcess(command, 0, stdout=b"workmesh-restore-volume\n") return subprocess.CompletedProcess(command, 0, stdout=b"") monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) monkeypatch.setattr(deploy_module, "_workmesh_docker", docker) monkeypatch.setattr( deploy_module, "_workmesh_inspect_container", lambda _name: {"Id": n8n_id, "State": {"Running": True}}, ) monkeypatch.setattr(deploy_module.time, "sleep", lambda _seconds: None) evidence = deploy_module.verify_workmesh_n8n_backup_restore( backup_dir=backup_dir, request_id="b" * 64, image_id="sha256:" + "c" * 64, n8n_encryption_key=secret, ) assert evidence == { "status": "verified", "imageId": "sha256:" + "c" * 64, "health": "healthy", "network": "none", } run_commands = [command for command in calls if command[:1] == ("run",)] assert len(run_commands) == 3 assert all(command[command.index("--network") + 1] == "none" for command in run_commands) extractor = next(command for command in run_commands if "-xf" in command) assert extractor[extractor.index("--user") + 1] == "0:0" database_probe = next( command for command in run_commands if any("database.sqlite" in argument for argument in command) ) assert database_probe[database_probe.index("--entrypoint") + 1] == "node" assert database_probe[database_probe.index("--user") + 1] == "1000:1000" assert any(command.endswith("target=/home/node/.n8n,readonly") for command in database_probe) n8n_command = next(command for command in run_commands if "start" in command) assert "sha256:" + "c" * 64 in n8n_command assert "N8N_USER_FOLDER=/home/node/.n8n" in n8n_command assert calls[-2][:2] == ("rm", "-f") assert calls[-1][:2] == ("volume", "rm") def test_workmesh_n8n_restore_rejects_missing_nested_sqlite_and_cleans_scratch_volume( deploy_module, monkeypatch, tmp_path ): backup_dir = tmp_path / "backup" backup_dir.mkdir() state_root = tmp_path / "state" state_root.mkdir() secret = "synthetic-n8n-key-" + "y" * 40 calls: list[tuple[str, ...]] = [] def docker(*args, timeout=None): command = tuple(str(item) for item in args) calls.append(command) if "--entrypoint" in command: raise deploy_module.DeployError("isolated SQLite path/header probe failed") return subprocess.CompletedProcess(command, 0, stdout=b"") monkeypatch.setattr(deploy_module, "STATE_ROOT", state_root) monkeypatch.setattr(deploy_module, "_workmesh_docker", docker) with pytest.raises(deploy_module.DeployError, match="SQLite path/header probe"): deploy_module.verify_workmesh_n8n_backup_restore( backup_dir=backup_dir, request_id="b" * 64, image_id="sha256:" + "c" * 64, n8n_encryption_key=secret, ) assert not any(command[:2] == ("run", "--detach") for command in calls) assert calls[-1][:2] == ("volume", "rm") assert not list(state_root.glob(".workmesh-n8n-restore-*")) def test_workmesh_backup_restart_uses_existing_container_ids_without_compose_recreate( deploy_module, monkeypatch ): revision = "a" * 40 names = ["workmesh-postgres-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1"] ids = {name: f"{index:064x}" for index, name in enumerate(names, start=1)} image = "sha256:" + "b" * 64 receipt = { "revision": revision, "containers": { name: {"container_id": ids[name], "image_id": image} for name in names }, } prior_state = { "live_revision": revision, "live_status": "healthy", "live_receipt": receipt, } containers = { name: { "Id": ids[name], "Image": image, "State": { "Running": name != "workmesh-n8n-1", "Health": {"Status": "healthy"}, }, "Config": { "Labels": { "com.docker.compose.project": "workmesh", "com.docker.compose.service": name.removeprefix("workmesh-").removesuffix("-1"), "com.docker.compose.config-hash": "c" * 64, } }, } for name in names } calls: list[tuple[str, ...]] = [] def docker(*args, timeout=None): calls.append(tuple(args)) if args[:2] == ("ps", "--all"): return subprocess.CompletedProcess(args, 0, stdout=("\n".join(names) + "\n").encode()) if args[0] == "start": containers["workmesh-n8n-1"]["State"]["Running"] = True return subprocess.CompletedProcess(args, 0, stdout=b"") return subprocess.CompletedProcess(args, 0, stdout=b"") monkeypatch.setattr(deploy_module, "_workmesh_docker", docker) monkeypatch.setattr(deploy_module, "_workmesh_inspect_container", lambda name: containers.get(name)) monkeypatch.setattr(deploy_module, "read_state", lambda _project: prior_state) deploy_module._workmesh_restart_previous_runtime( {"slug": "workmesh"}, prior_state, receipt, revision ) assert calls[0] == ("start", ids["workmesh-n8n-1"]) assert not any("compose" in call or call[0] in {"run", "up", "create"} for call in calls) assert containers["workmesh-n8n-1"]["Id"] == ids["workmesh-n8n-1"] assert containers["workmesh-n8n-1"]["Config"]["Labels"]["com.docker.compose.config-hash"] == "c" * 64 def test_workmesh_promotion_success_requires_bound_data_backup_receipt( status_module, tmp_path, monkeypatch ): inbox = tmp_path / "appops-inbox" inbox.mkdir() marker = inbox / ("d" * 64 + ".json") marker.write_text(json.dumps({ "schema": "appops-deployment-request.v1", "request": "d" * 64, "repository": "jens/workmesh", "revision": "a" * 40, "operation": "workmesh_promote", "preflight_sha256": "f" * 64, "backup_inputs_sha256": "e" * 64, "status": "pending", }), encoding="utf-8") project = _workmesh_promotion_project(status_module) monkeypatch.setattr(status_module, "APPOPS_DEPLOYMENT_INBOX_ROOT", inbox) monkeypatch.setattr(status_module, "_load_projects", lambda: {"jens/workmesh": project}) monkeypatch.setattr(status_module, "_workmesh_promotion_preflight", lambda *_args: { "status": "preflight_ok", "preflightSha256": "f" * 64, "backupInputsSha256": "e" * 64, }) monkeypatch.setattr( status_module, "_run_workmesh_promotion_deploy", lambda args: status_module.subprocess.CompletedProcess(args, 0), ) monkeypatch.setattr(status_module, "_deployment_state", lambda _slug: {"live_status": "healthy"}) monkeypatch.setattr(status_module, "_verified_receipt", lambda *_args: {"revision": "a" * 40}) monkeypatch.setattr(status_module, "_verified_workmesh_backup_receipt", lambda *_args: None) status_module._process_appops_record(marker) record = json.loads(marker.read_text(encoding="utf-8")) assert record["status"] == "outcome_unknown" assert record["reason"] == "verified_deployment_or_backup_receipt_unavailable" assert "backup_receipt_sha256" not in record def test_workmesh_external_recovery_receipt_binds_environment_and_both_encryption_keys( deploy_module, monkeypatch, tmp_path ): path = tmp_path / "workmesh-external-recovery.json" now = int(deploy_module.time.time()) n8n_key = "synthetic-n8n-key-" + "n" * 40 workmesh_key = "synthetic-workmesh-key-" + "w" * 40 payload = { "schema": "itworx.workmesh-external-recovery/v1", "repository": "jens/workmesh", "environmentSha256": "a" * 64, "keySha256": { "N8N_ENCRYPTION_KEY": hashlib.sha256(n8n_key.encode()).hexdigest(), "WORKMESH_SECRET_ENCRYPTION_KEY": hashlib.sha256(workmesh_key.encode()).hexdigest(), }, "externalBackupSha256": "b" * 64, "restoreVerifiedAtEpoch": now, } path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") if os.name != "nt": path.chmod(0o600) monkeypatch.setattr(deploy_module, "_root_owned", lambda _path: True) receipt_sha = deploy_module.verify_workmesh_external_recovery_receipt( path, environment_sha256="a" * 64, encryption_keys={ "N8N_ENCRYPTION_KEY": n8n_key, "WORKMESH_SECRET_ENCRYPTION_KEY": workmesh_key, }, ) assert receipt_sha == hashlib.sha256(path.read_bytes()).hexdigest() payload["keySha256"]["N8N_ENCRYPTION_KEY"] = "c" * 64 path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") with pytest.raises(deploy_module.DeployError, match="external recovery receipt"): deploy_module.verify_workmesh_external_recovery_receipt( path, environment_sha256="a" * 64, encryption_keys={ "N8N_ENCRYPTION_KEY": n8n_key, "WORKMESH_SECRET_ENCRYPTION_KEY": workmesh_key, }, ) def test_workmesh_backup_manifest_binds_exact_prior_revision_and_file_digests( deploy_module, tmp_path ): backup = tmp_path / "backup" backup.mkdir() postgres = b"private pg dump bytes" n8n = b"private n8n volume archive" (backup / "postgres.dump").write_bytes(postgres) (backup / "n8n.tar").write_bytes(n8n) manifest = { "format": "workmesh-backup/v1", "created_at": "2026-09-27T12:00:00+00:00", "project": "workmesh", "declared_live_revision": "a" * 40, "postgres": { "file": "postgres.dump", "bytes": len(postgres), "sha256": hashlib.sha256(postgres).hexdigest(), }, "n8n": { "file": "n8n.tar", "bytes": len(n8n), "sha256": hashlib.sha256(n8n).hexdigest(), }, "encryption_root": "external; not included", } manifest_path = backup / "manifest.json" manifest_path.write_text(json.dumps(manifest, sort_keys=True), encoding="utf-8") evidence = deploy_module.verify_workmesh_backup_manifest(backup, "a" * 40) assert evidence["manifestSha256"] == hashlib.sha256(manifest_path.read_bytes()).hexdigest() assert evidence["postgresSha256"] == hashlib.sha256(postgres).hexdigest() assert evidence["n8nSha256"] == hashlib.sha256(n8n).hexdigest() with pytest.raises(deploy_module.DeployError, match="revision"): deploy_module.verify_workmesh_backup_manifest(backup, "c" * 40) (backup / "n8n.tar").write_bytes(b"changed") with pytest.raises(deploy_module.DeployError, match="checksum or size"): deploy_module.verify_workmesh_backup_manifest(backup, "a" * 40) def test_workmesh_backup_receipt_requires_bound_manifest_and_verified_restore_bytes( status_module, tmp_path, monkeypatch ): request_id = "d" * 64 candidate = "b" * 40 prior = "a" * 40 preflight = "f" * 64 inputs = "e" * 64 backup_root = tmp_path / "backups" backup_dir = backup_root / request_id backup_dir.mkdir(parents=True) receipt_root = tmp_path / "receipts" receipt_root.mkdir() postgres = b"synthetic PostgreSQL dump" n8n = b"synthetic n8n volume archive" postgres_sha = hashlib.sha256(postgres).hexdigest() n8n_sha = hashlib.sha256(n8n).hexdigest() (backup_dir / "postgres.dump").write_bytes(postgres) (backup_dir / "n8n.tar").write_bytes(n8n) manifest = { "format": "workmesh-backup/v1", "created_at": "2026-09-27T12:00:00+00:00", "project": "workmesh", "declared_live_revision": prior, "postgres": {"file": "postgres.dump", "bytes": len(postgres), "sha256": postgres_sha}, "n8n": {"file": "n8n.tar", "bytes": len(n8n), "sha256": n8n_sha}, "encryption_root": "external; not included", } manifest_bytes = json.dumps(manifest, sort_keys=True, indent=2).encode() (backup_dir / "manifest.json").write_bytes(manifest_bytes) payload = { "schema": "itworx.workmesh-backup-receipt/v1", "request": request_id, "repository": "jens/workmesh", "operation": "workmesh_promote", "candidateRevision": candidate, "priorLiveRevision": prior, "preflightSha256": preflight, "backupInputsSha256": inputs, "backupToolSha256": "1" * 64, "externalRecoveryReceiptSha256": "2" * 64, "environmentSha256": "3" * 64, "manifestSha256": hashlib.sha256(manifest_bytes).hexdigest(), "postgres": { "sha256": postgres_sha, "bytes": len(postgres), "restore": { "status": "verified", "alembicRevision": "0022", "tableCount": 22, "network": "none", }, }, "n8n": { "sha256": n8n_sha, "bytes": len(n8n), "restore": { "status": "verified", "imageId": "sha256:" + "4" * 64, "health": "healthy", "network": "none", }, }, "automaticDataRollback": False, "createdAt": "2026-09-27T12:01:00+00:00", } receipt_sha = hashlib.sha256( json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() (receipt_root / f"{request_id}.json").write_text( json.dumps({"receipt": payload, "receiptSha256": receipt_sha}), encoding="utf-8" ) if os.name != "nt": for directory in (backup_root, backup_dir, receipt_root): directory.chmod(0o700) for path in ( backup_dir / "manifest.json", backup_dir / "postgres.dump", backup_dir / "n8n.tar", receipt_root / f"{request_id}.json", ): path.chmod(0o600) monkeypatch.setattr(status_module, "WORKMESH_BACKUP_ROOT", backup_root) monkeypatch.setattr(status_module, "WORKMESH_BACKUP_RECEIPT_ROOT", receipt_root) assert status_module._verified_workmesh_backup_receipt( request_id, candidate, preflight, inputs ) == receipt_sha assert status_module._verified_workmesh_backup_receipt( request_id, candidate, "9" * 64, inputs ) is None (backup_dir / "n8n.tar").write_bytes(b"changed") assert status_module._verified_workmesh_backup_receipt( request_id, candidate, preflight, inputs ) is None def _managed_storage_http_request( status_module, server, *, body: bytes, nonce: str, secret: bytes | None, signature: str | None = None, signed_path: str | None = None, ): path = status_module.MANAGED_APPDATA_INITIALIZE_PATH timestamp = str(int(status_module.time.time())) headers = {"Content-Type": "application/json", "Content-Length": str(len(body))} if nonce: headers["X-ITWorx-Deployment-Nonce"] = nonce if timestamp: headers["X-ITWorx-Deployment-Timestamp"] = timestamp if signature is None and secret is not None: signature = status_module._request_signature( secret, method="POST", path=signed_path or path, timestamp=timestamp, nonce=nonce, raw_body=body, ) if signature is not None: headers["X-ITWorx-Deployment-Signature"] = signature connection = http.client.HTTPConnection(*server.server_address, timeout=2) connection.request("POST", path, body=body, headers=headers) response = connection.getresponse() status = response.status payload = json.loads(response.read()) connection.close() return status, payload, headers def test_managed_appdata_initializer_http_requires_exact_signed_one_time_request( status_module, tmp_path, monkeypatch ): secret = b"managed-storage-bootstrap-" + b"x" * 48 secret_path = tmp_path / "bootstrap" secret_path.write_bytes(secret) secret_path.chmod(0o600) nonce_root = tmp_path / "nonces" monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", secret_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", nonce_root) monkeypatch.setattr( status_module.Handler, "_managed_storage_initialization_prerequisites", lambda _handler: True, ) initialized = [] monkeypatch.setattr( status_module, "ensure_managed_appdata_root", lambda: initialized.append(True) or "initialized", ) monkeypatch.setattr( status_module, "_refresh_appdata_root_selection", lambda: status_module.MANAGED_APPDATA_ROOT, ) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() body = json.dumps( { "apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION, "target": status_module.MANAGED_APPDATA_INITIALIZE_TARGET, }, separators=(",", ":"), ).encode("utf-8") try: success_status, success_payload, success_headers = _managed_storage_http_request( status_module, server, body=body, nonce="a" * 32, secret=secret, ) assert success_status == 200 assert success_payload == { "status": "initialized", "trustedAppdataRoot": status_module.MANAGED_APPDATA_ROOT.as_posix(), "backupLocation": "dockerRootDir", } assert initialized == [True] assert secret.decode("ascii") not in json.dumps(success_payload) replay = http.client.HTTPConnection(*server.server_address, timeout=2) replay.request( "POST", status_module.MANAGED_APPDATA_INITIALIZE_PATH, body=body, headers=success_headers, ) replay_response = replay.getresponse() replay_payload = json.loads(replay_response.read()) replay.close() assert replay_response.status == 403 assert replay_payload == {"status": "authentication_rejected"} assert initialized == [True] invalid_requests = [ _managed_storage_http_request( status_module, server, body=body, nonce="b" * 32, secret=secret, signature="0" * 64, ), _managed_storage_http_request( status_module, server, body=body, nonce="c" * 32, secret=secret, signed_path="/v1/appops/enrollments/capabilities", ), _managed_storage_http_request( status_module, server, body=body, nonce="d" * 32, secret=None, ), ] assert [status for status, _payload, _headers in invalid_requests] == [403, 403, 403] assert initialized == [True] for index, payload in enumerate( [ { "apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION, "target": "tower/other-controller", }, { "apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION, "target": status_module.MANAGED_APPDATA_INITIALIZE_TARGET, "path": "C:/private/sentinel", }, ], start=1, ): raw_body = json.dumps(payload, separators=(",", ":")).encode("utf-8") rejected_status, rejected_payload, _headers = _managed_storage_http_request( status_module, server, body=raw_body, nonce=f"{index:032x}", secret=secret, ) assert rejected_status == 400 assert rejected_payload == {"status": "invalid_managed_storage_request"} assert "C:/private/sentinel" not in json.dumps(rejected_payload) oversized_status, oversized_payload, _headers = _managed_storage_http_request( status_module, server, body=b"x" * 1025, nonce="e" * 32, secret=None, ) assert oversized_status == 413 assert oversized_payload == {"status": "invalid_body_size"} assert initialized == [True] finally: server.shutdown() server.server_close() serving.join(timeout=2) def test_managed_appdata_initializer_http_prerequisite_and_refusal_are_sanitized( status_module, tmp_path, monkeypatch ): secret = b"managed-storage-bootstrap-" + b"y" * 48 secret_path = tmp_path / "bootstrap" secret_path.write_bytes(secret) secret_path.chmod(0o600) monkeypatch.setattr(status_module, "APPOPS_ENROLLMENT_BOOTSTRAP_SECRET_FILE", secret_path) monkeypatch.setattr(status_module, "APPOPS_NONCE_ROOT", tmp_path / "nonces") prerequisites = {"ready": False} monkeypatch.setattr( status_module.Handler, "_managed_storage_initialization_prerequisites", lambda _handler: prerequisites["ready"], ) calls = [] monkeypatch.setattr( status_module, "ensure_managed_appdata_root", lambda: calls.append(True) or "initialized", ) monkeypatch.setattr( status_module, "_refresh_appdata_root_selection", lambda: status_module.MANAGED_APPDATA_ROOT, ) server = status_module.BoundedThreadingHTTPServer( ("127.0.0.1", 0), status_module.Handler ) serving = threading.Thread(target=server.serve_forever, daemon=True) serving.start() body = json.dumps( { "apiVersion": status_module.MANAGED_APPDATA_INITIALIZE_API_VERSION, "target": status_module.MANAGED_APPDATA_INITIALIZE_TARGET, }, separators=(",", ":"), ).encode("utf-8") try: status, payload, _headers = _managed_storage_http_request( status_module, server, body=body, nonce="f" * 32, secret=secret ) assert status == 503 assert payload == {"status": "managed_storage_prerequisite_unavailable"} assert calls == [] prerequisites["ready"] = True def refuse_with_sensitive_diagnostic(): raise status_module.UnsafeManagedAppdataRoot( "secret/path sentinel C:/private/appdata" ) monkeypatch.setattr( status_module, "ensure_managed_appdata_root", refuse_with_sensitive_diagnostic ) status, payload, _headers = _managed_storage_http_request( status_module, server, body=body, nonce="1" * 32, secret=secret ) assert status == 409 assert payload == {"status": "managed_storage_initialization_refused"} assert "secret/path sentinel" not in json.dumps(payload) assert "C:/private/appdata" not in json.dumps(payload) assert secret.decode("ascii") not in json.dumps(payload) finally: server.shutdown() server.server_close() serving.join(timeout=2)