from __future__ import annotations import contextlib import copy import hashlib import importlib.util import json import sys import types from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).resolve().parents[2] BROKER = ROOT / "ops" / "gitea-deploy-broker" REPOSITORY = "jens/unraid-appops-gateway" REVISION = "c" * 40 PRIOR_REVISION = "d" * 40 TREE = "e" * 40 RECOVERY_ID = "12345678-1234-4234-8234-123456789abc" TRANSACTION_ID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" SECRET = b"transaction-reconciliation-test-key-32-bytes" REGISTRY_BYTES = b'{"repository":"jens/unraid-appops-gateway"}\n' IMAGE_ID = "sha256:" + "7" * 64 @pytest.fixture def deploy(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) if "fcntl" not in sys.modules: fcntl = types.ModuleType("fcntl") fcntl.LOCK_EX = 2 fcntl.LOCK_UN = 8 fcntl.LOCK_NB = 4 fcntl.flock = lambda *_args: None monkeypatch.setitem(sys.modules, "fcntl", fcntl) spec = importlib.util.spec_from_file_location( "appops_effective_transaction_reconciliation_test", BROKER / "deploy.py", ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _digest(raw: bytes) -> str: return hashlib.sha256(raw).hexdigest() def _mac(secret: bytes, domain: str, raw: bytes) -> str: return hashlib.sha256(secret + b"\0" + domain.encode() + b"\0" + raw).hexdigest() def _metadata( revision: str = REVISION, tree: str = TREE, built_at: str = "2026-10-07T12:00:00.000Z", ) -> dict[str, str]: return { "id": revision[:12], "repository": REPOSITORY, "revision": revision, "tree": tree, "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), "builtAt": built_at, } def _pointer(revision: str) -> dict[str, str]: return { "recoveryId": RECOVERY_ID, "revision": revision, "modelSha256": "1" * 64, "modelRecordSha256": "2" * 64, } def _state_bytes(state: dict[str, object]) -> bytes: return json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + b"\n" def _case(deploy, monkeypatch, tmp_path, *, phase: str = "runtime_mutated"): prior_pointer = _pointer(PRIOR_REVISION) candidate_pointer = _pointer(REVISION) backup = {"sha256": "3" * 64, "revision": REVISION} prior_snapshot = { "revision": PRIOR_REVISION, "appops_runtime_model": prior_pointer, "appops_data_backup": backup, "containers": [{"name": "Unraid-AppOps-Gateway", "image_id": "sha256:" + "4" * 64}], } receipt = { "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY, "revision": REVISION, "tree": TREE, "containers": { "Unraid-AppOps-Gateway": { "container_id": "5" * 64, "image_id": IMAGE_ID, }, }, } live_reference = { "recoveryId": RECOVERY_ID, "revision": REVISION, "liveRecordSha256": "6" * 64, } old_receipt = { "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY, "revision": PRIOR_REVISION, "tree": "9" * 40, } state: dict[str, object] = { "repository": REPOSITORY, "live_revision": PRIOR_REVISION, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": old_receipt, "rollback_snapshot": prior_snapshot, "live_artifacts": { "appops_runtime_model": {"recoveryId": RECOVERY_ID, "revision": PRIOR_REVISION}, "containers": {"Unraid-AppOps-Gateway": {"container_id": "8" * 64}}, }, } original_raw = _state_bytes(state) original_state_sha = _digest(original_raw) candidate_model = { "services": { "appops-gateway": { "image": "itworx/appops-gateway:protected", "build": {"context": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree"}, "environment": {"APPOPS_SOURCE_REVISION": REVISION}, }, }, } payload: dict[str, object] = { "schema": "itworx.appops-self-upgrade-model-transaction/v1", "repository": REPOSITORY, "transactionId": TRANSACTION_ID, "recoveryId": RECOVERY_ID, "kind": "followup", "revision": REVISION, "metadata": _metadata(), "originSemanticSha256": "a" * 64, "originRawSha256": "b" * 64, "originJournalSha256": "c" * 64, "expectedStateSha256": original_state_sha, "registryFingerprint": _mac(SECRET, "registry", REGISTRY_BYTES), "preMutationRuntimeSha256": "d" * 64, "preMutationRuntimeFingerprint": "e" * 64, "sourceFingerprint": "f" * 64, "priorModelPointer": prior_pointer, "priorSnapshot": prior_snapshot, "candidateSourceFingerprint": "0" * 64, "candidateModelPointer": candidate_pointer, "candidateImageId": IMAGE_ID, "buildContexts": { "appops-gateway": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree", }, "phase": phase, "createdAt": 1_791_378_000, "updatedAt": 1_791_378_001, } if phase in {"verified", "committed"}: payload["verifiedLiveReference"] = live_reference payload["verifiedReceipt"] = receipt if phase == "committed": payload["committedStateSha256"] = "0" * 64 envelope = {"payload": payload, "sha256": "9" * 64} registry = {"bytes": REGISTRY_BYTES} calls: dict[str, object] = { "active": [], "active_result": None, "docker": [], "git": [], "backup": [], "verify": [], "current": [], "updates": [], "writes": [], "state_sha_override": None, "metadata_override": None, "receipt_override": None, "fail_commit_once": False, } class StatusReader: PROJECTS_PATH = tmp_path / "projects.json" @staticmethod def _self_upgrade_pending_model_transaction(): return copy.deepcopy(envelope) @staticmethod def _self_upgrade_read_fixed_file(_path, **_kwargs): return registry["bytes"] @staticmethod def _self_upgrade_secret(): return SECRET @staticmethod def _self_upgrade_mac(secret, domain, raw): return _mac(secret, domain, raw) @staticmethod def _self_upgrade_load_compose_model(recovery_id, revision): if recovery_id == RECOVERY_ID and revision == REVISION and calls.get("model_valid", True): return {"model": candidate_model, "pointer": candidate_pointer} return None @staticmethod def _self_upgrade_candidate_model_binding(pointer, observed_receipt): if (pointer == candidate_pointer and observed_receipt == receipt and calls.get("model_valid", True)): return {"modelPointer": pointer, "receipt": observed_receipt} return None @staticmethod def _self_upgrade_store_live_model(pointer, binding): if pointer == candidate_pointer and binding.get("receipt") == receipt: return live_reference return None @staticmethod def _self_upgrade_update_model_transaction(transaction_id, expected_sha, next_phase, bindings): calls["updates"].append((next_phase, copy.deepcopy(bindings))) if transaction_id != TRANSACTION_ID or expected_sha != envelope["sha256"]: return None if next_phase == "committed" and calls["fail_commit_once"]: calls["fail_commit_once"] = False return None updated = dict(envelope["payload"]) updated.update(bindings) updated["phase"] = next_phase updated["updatedAt"] = int(updated["updatedAt"]) + 1 if next_phase == "committed": updated["committedStateSha256"] = _digest(_state_bytes(state)) envelope["payload"] = updated envelope["sha256"] = _digest(json.dumps(updated, sort_keys=True).encode()) return copy.deepcopy(envelope) @staticmethod def _self_upgrade_active_compose_model(observed_state, observed_container): calls["active"].append((copy.deepcopy(observed_state), copy.deepcopy(observed_container))) active = calls.get("active_result") return copy.deepcopy(active) if isinstance(active, dict) else None def read_preimage(_project): raw = _state_bytes(state) override = calls["state_sha_override"] return copy.deepcopy(state), raw, override or _digest(raw) def write_state(repository, project, revision, status, _message, **kwargs): calls["writes"].append((repository, revision, status, copy.deepcopy(kwargs))) reference = project.get("_appops_runtime_model_binding") state.update({ "repository": repository, "live_revision": revision, "live_status": "healthy" if status == "success" else status, "live_legacy_provenance": False, "live_receipt": copy.deepcopy(kwargs["live_receipt"]), "rollback_snapshot": copy.deepcopy(kwargs["rollback_snapshot"]), "live_artifacts": { "appops_runtime_model": copy.deepcopy(reference), "containers": {"Unraid-AppOps-Gateway": {"container_id": receipt["containers"]["Unraid-AppOps-Gateway"]["container_id"]}}, }, }) def validate_commit(_repo, branch, candidate_revision): calls["git"].append((branch, candidate_revision)) if branch != "main" or candidate_revision != REVISION: raise deploy.DeployError("pending source is not the requested main commit") return candidate_revision def deployment_metadata(_repo, candidate_revision, repository, *, built_at=None): if repository != REPOSITORY or candidate_revision != REVISION: raise deploy.DeployError("pending target identity mismatch") observed = _metadata(built_at=built_at or "2026-10-07T12:00:00.000Z") override = calls["metadata_override"] return copy.deepcopy(override or observed) def verify_backup(actual_backup, candidate_revision, previous_revision): calls["backup"].append((actual_backup, candidate_revision, previous_revision)) expected_previous = calls.get("backup_previous_revision", PRIOR_REVISION) if actual_backup != backup or candidate_revision != REVISION or previous_revision != expected_previous: raise deploy.DeployError("pending data backup proof changed") def verify_deployment(_project, observed_metadata, *, expected_image_ids): calls["verify"].append((copy.deepcopy(observed_metadata), dict(expected_image_ids))) if expected_image_ids != {"Unraid-AppOps-Gateway": IMAGE_ID}: raise deploy.DeployError("sealed candidate image differs from the live image") if observed_metadata != _metadata(): raise deploy.DeployError("candidate receipt metadata changed") return copy.deepcopy(calls["receipt_override"] or receipt) def verified_current(repository, _project, candidate_revision, tree): calls["current"].append((repository, candidate_revision, tree)) if (repository, candidate_revision, tree) == (REPOSITORY, REVISION, TREE): return copy.deepcopy(receipt) return None project = { "_repository_name": REPOSITORY, "slug": "unraid-appops-gateway", "branch": "main", "containers": ["Unraid-AppOps-Gateway"], } monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader()) monkeypatch.setattr(deploy, "read_appops_state_preimage", read_preimage) monkeypatch.setattr(deploy, "write_state", write_state) monkeypatch.setattr(deploy, "validate_commit", validate_commit) monkeypatch.setattr(deploy, "deployment_metadata", deployment_metadata) monkeypatch.setattr(deploy, "verify_appops_data_backup", verify_backup) monkeypatch.setattr(deploy, "verify_deployment", verify_deployment) monkeypatch.setattr(deploy, "verified_current_receipt", verified_current) monkeypatch.setattr(deploy, "run", lambda *_args, **_kwargs: pytest.fail("reconciliation must not mutate Docker")) monkeypatch.setattr(deploy, "inspect_container", lambda *_args, **_kwargs: pytest.fail("unexpected inspect path")) monkeypatch.setattr(deploy, "restore_compose_snapshot", lambda *_args, **_kwargs: pytest.fail("unexpected rollback")) return SimpleNamespace( deploy=deploy, project=project, repo=tmp_path / "repository.git", payload=payload, envelope=envelope, state=state, state_sha=original_state_sha, registry=registry, metadata=_metadata(), pointer=candidate_pointer, reference=live_reference, receipt=receipt, backup=backup, calls=calls, ) def _reconcile(case): return case.deploy._appops_reconcile_pending_model_transaction( case.project, REPOSITORY, case.repo, ) def test_runtime_mutated_transaction_finishes_state_publication_without_docker_mutation( deploy, monkeypatch, tmp_path, ): case = _case(deploy, monkeypatch, tmp_path) result = _reconcile(case) assert result == case.receipt assert case.envelope["payload"]["phase"] == "committed" assert [phase for phase, _bindings in case.calls["updates"]] == ["verified", "committed"] assert len(case.calls["writes"]) == 1 repo, revision, status, kwargs = case.calls["writes"][0] assert (repo, revision, status) == (REPOSITORY, REVISION, "success") assert kwargs["expected_state_sha256"] == case.state_sha assert kwargs["live_receipt"] == case.receipt assert case.state["live_legacy_provenance"] is False assert case.state["live_artifacts"]["appops_runtime_model"] == case.reference assert case.calls["git"] == [("main", REVISION)] assert len(case.calls["backup"]) == 1 assert case.calls["verify"] == [(case.metadata, {"Unraid-AppOps-Gateway": IMAGE_ID})] assert case.calls["docker"] == [] def test_verified_state_replay_only_reattests_and_marks_transaction_committed( deploy, monkeypatch, tmp_path, ): case = _case(deploy, monkeypatch, tmp_path, phase="verified") case.state.update({ "live_revision": REVISION, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": copy.deepcopy(case.receipt), "live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)}, }) result = _reconcile(case) assert result == case.receipt assert case.envelope["payload"]["phase"] == "committed" assert [phase for phase, _bindings in case.calls["updates"]] == ["committed"] assert case.calls["current"] == [(REPOSITORY, REVISION, TREE)] assert case.calls["writes"] == [] assert case.calls["git"] == [] assert case.calls["verify"] == [] assert case.calls["docker"] == [] def test_verified_state_replay_rejects_changed_registry_before_reattest( deploy, monkeypatch, tmp_path, ): case = _case(deploy, monkeypatch, tmp_path, phase="verified") case.state.update({ "live_revision": REVISION, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": copy.deepcopy(case.receipt), "live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)}, }) case.registry["bytes"] = b'{"repository":"changed"}\n' with pytest.raises(deploy.DeployError, match="state or registry changed"): _reconcile(case) assert case.calls["current"] == [] assert case.calls["updates"] == [] assert case.calls["writes"] == [] assert case.calls["docker"] == [] def test_state_commit_failure_restarts_from_verified_proof_without_second_state_write( deploy, monkeypatch, tmp_path, ): case = _case(deploy, monkeypatch, tmp_path) case.calls["fail_commit_once"] = True with pytest.raises(deploy.DeployError, match="compare-and-swap failed"): _reconcile(case) assert case.envelope["payload"]["phase"] == "verified" assert len(case.calls["writes"]) == 1 assert case.calls["docker"] == [] result = _reconcile(case) assert result == case.receipt assert case.envelope["payload"]["phase"] == "committed" assert [phase for phase, _bindings in case.calls["updates"]] == [ "verified", "committed", "committed", ] assert len(case.calls["writes"]) == 1 assert len(case.calls["current"]) == 1 assert case.calls["docker"] == [] @pytest.mark.parametrize( "mismatch", ["state", "registry", "model", "target_metadata", "backup", "image", "receipt", "unknown_phase"], ) def test_pending_transaction_mismatches_fail_without_state_or_docker_mutation( deploy, monkeypatch, tmp_path, mismatch, ): case = _case(deploy, monkeypatch, tmp_path) if mismatch == "state": case.calls["state_sha_override"] = "0" * 64 elif mismatch == "registry": case.registry["bytes"] = b'{"repository":"different"}\n' elif mismatch == "model": case.calls["model_valid"] = False elif mismatch == "target_metadata": case.calls["metadata_override"] = _metadata(tree="9" * 40) elif mismatch == "backup": case.payload["priorSnapshot"]["appops_data_backup"] = {"sha256": "0" * 64} elif mismatch == "image": case.payload["candidateImageId"] = "sha256:" + "8" * 64 elif mismatch == "receipt": case.calls["receipt_override"] = {**case.receipt, "revision": "b" * 40} elif mismatch == "unknown_phase": case.payload["phase"] = "prepared" with pytest.raises(deploy.DeployError): _reconcile(case) assert case.calls["writes"] == [] assert case.calls["updates"] == [] assert case.calls["docker"] == [] def _prepare_rolling_back_case(deploy, monkeypatch, tmp_path): case = _case(deploy, monkeypatch, tmp_path, phase="rolling_back") prior_revision = deploy.APPOPS_LEGACY_OLD_REVISION prior_pointer = { **_pointer(prior_revision), "schema": "itworx.appops-effective-compose-model.v1", } restored_reference = {**prior_pointer, "liveRecordSha256": "a" * 64} prior_snapshot = { "revision": prior_revision, "legacy_provenance": True, "receipt": {}, "appops_runtime_model": prior_pointer, "appops_data_backup": case.backup, "containers": [{"name": "Unraid-AppOps-Gateway", "image_id": IMAGE_ID}], } case.payload.update({ "phase": "rolling_back", "priorModelPointer": prior_pointer, "priorSnapshot": prior_snapshot, }) case.state.update({ "repository": REPOSITORY, "live_revision": prior_revision, "live_status": "healthy", "live_legacy_provenance": True, "live_receipt": {}, "live_artifacts": { "appops_runtime_model": restored_reference, "containers": {"Unraid-AppOps-Gateway": {"container_id": "b" * 64}}, }, }) case.calls["active_result"] = {"pointer": copy.deepcopy(restored_reference)} case.calls["backup_previous_revision"] = prior_revision container = {"Id": "b" * 64, "Image": IMAGE_ID, "State": {"Running": True}} monkeypatch.setattr(deploy, "inspect_container", lambda _name: copy.deepcopy(container)) return case, prior_pointer, restored_reference, prior_snapshot, container def test_rolling_back_transaction_seals_exact_restored_legacy_binding_without_mutation( deploy, monkeypatch, tmp_path, ): case, prior_pointer, restored_reference, prior_snapshot, container = _prepare_rolling_back_case( deploy, monkeypatch, tmp_path, ) result = _reconcile(case) assert result == {} assert case.envelope["payload"]["phase"] == "rolled_back" assert case.envelope["payload"]["restoredLiveReference"] == restored_reference assert case.calls["updates"] == [("rolled_back", {"restoredLiveReference": restored_reference})] assert case.calls["active"] == [(case.state, container)] assert case.calls["backup"] == [(case.backup, REVISION, deploy.APPOPS_LEGACY_OLD_REVISION)] assert case.calls["writes"] == [] assert case.calls["git"] == [] assert case.calls["docker"] == [] assert case.payload["priorModelPointer"] == prior_pointer assert case.payload["priorSnapshot"] == prior_snapshot @pytest.mark.parametrize("mismatch", ["state", "reference", "runtime", "registry", "backup"]) def test_rolling_back_transaction_rejects_unbound_prior_state_without_mutation( deploy, monkeypatch, tmp_path, mismatch, ): case, _prior_pointer, restored_reference, prior_snapshot, _container = _prepare_rolling_back_case( deploy, monkeypatch, tmp_path, ) if mismatch == "state": case.state["live_revision"] = "9" * 40 elif mismatch == "reference": case.state["live_artifacts"]["appops_runtime_model"]["liveRecordSha256"] = "c" * 64 elif mismatch == "runtime": case.calls["active_result"] = {"pointer": {**restored_reference, "liveRecordSha256": "d" * 64}} elif mismatch == "registry": case.registry["bytes"] = b'{"repository":"different"}\n' elif mismatch == "backup": prior_snapshot["appops_data_backup"] = {"sha256": "0" * 64} with pytest.raises(deploy.DeployError): _reconcile(case) assert case.calls["updates"] == [] assert case.calls["writes"] == [] assert case.calls["git"] == [] assert case.calls["docker"] == [] def test_explicit_rollback_cli_routes_through_pending_model_reconciliation( deploy, monkeypatch, tmp_path, capsys, ): revision = REVISION project = { "slug": "unraid-appops-gateway", "branch": "main", "source_dir": str(tmp_path / "source"), "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, "environment": {}, } projects = {REPOSITORY: project} events = [] real_exists = deploy.Path.exists def path_exists(path): if str(path) == "/controller-hub-releases/appops-self-upgrade-recoveries/transactions": return True return real_exists(path) def reconcile(actual_project, repository, repo): assert actual_project["_repository_name"] == REPOSITORY assert repository == REPOSITORY assert repo == tmp_path / "repository.git" events.append("reconcile") return {"revision": revision, "proof": "signed-prior-runtime"} monkeypatch.setattr(deploy.Path, "exists", path_exists) monkeypatch.setattr(deploy, "load_projects", lambda: projects) monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repository, item: dict(item)) monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repository.git") monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision) monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: _metadata()) monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None) monkeypatch.setattr(deploy, "bind_appops_self_upgrade_recovery", lambda *_args: None) monkeypatch.setattr(deploy, "bind_registered_legacy_request", lambda *_args: None) monkeypatch.setattr(deploy, "exclusive_lock", lambda *_args, **_kwargs: contextlib.nullcontext()) @contextlib.contextmanager def global_lock(): events.append("global-lock") yield monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", global_lock) monkeypatch.setattr(deploy, "_appops_reconcile_pending_model_transaction", reconcile) monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", REPOSITORY, revision]) assert deploy.main() == 0 assert events == ["global-lock", "reconcile"] assert '"status": "already-current"' in capsys.readouterr().out def _recreated_receipt_case(deploy, monkeypatch): pointer = { "schema": "itworx.appops-effective-compose-model.v1", "recoveryId": RECOVERY_ID, "revision": PRIOR_REVISION, "modelSha256": "1" * 64, "modelRecordSha256": "2" * 64, "liveRecordSha256": "3" * 64, } recorded = { "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY, "revision": PRIOR_REVISION, "tree": "4" * 40, "treeDigest": "5" * 64, "build": { "id": PRIOR_REVISION[:12], "repository": REPOSITORY, "revision": PRIOR_REVISION, "tree": "4" * 40, "treeDigest": "5" * 64, "builtAt": "2026-10-07T11:00:00.000Z", }, "containers": { "Unraid-AppOps-Gateway": { "container_id": "a" * 64, "image_id": IMAGE_ID, "image_revision": PRIOR_REVISION, "image_tree": "4" * 40, "health": "healthy", }, }, "live": {"status": "healthy"}, } observed = copy.deepcopy(recorded) observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "b" * 64 snapshot = { "revision": PRIOR_REVISION, "legacy_provenance": False, "receipt": recorded, "appops_runtime_model": pointer, } project = { "_repository_name": REPOSITORY, "_appops_effective_model_pointer": pointer, } calls = {"attestation": [], "attestation_valid": True} def verify_rollback_model(actual_pointer, actual_snapshot): calls["attestation"].append((copy.deepcopy(actual_pointer), copy.deepcopy(actual_snapshot))) return ( calls["attestation_valid"] and actual_pointer == pointer and actual_snapshot == snapshot ) monkeypatch.setattr( deploy, "_appops_status_reader", lambda: SimpleNamespace(_self_upgrade_verify_rollback_model=verify_rollback_model), ) project["_appops_effective_rollback_snapshot"] = snapshot return SimpleNamespace( project=project, pointer=pointer, recorded=recorded, observed=observed, snapshot=snapshot, calls=calls, ) def test_recreated_receipt_matcher_allows_only_new_valid_container_id( deploy, monkeypatch, ): case = _recreated_receipt_case(deploy, monkeypatch) recorded_id = case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"] observed_id = case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] assert len(recorded_id) == len(observed_id) == 64 assert recorded_id != observed_id assert deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded) assert case.calls["attestation"] == [(case.pointer, case.snapshot)] @pytest.mark.parametrize( "mismatch", [ "repository", "legacy_provenance", "snapshot_revision", "snapshot_receipt", "project_model", "rollback_attestation", "observed_revision", "observed_tree", "observed_image", "observed_container_id", "recorded_container_id", "missing_container", "extra_container", ], ) def test_recreated_receipt_matcher_rejects_every_other_changed_fact( deploy, monkeypatch, mismatch, ): case = _recreated_receipt_case(deploy, monkeypatch) if mismatch == "repository": case.project["_repository_name"] = "jens/other" elif mismatch == "legacy_provenance": case.snapshot["legacy_provenance"] = True elif mismatch == "snapshot_revision": case.snapshot["revision"] = "9" * 40 elif mismatch == "snapshot_receipt": case.snapshot["receipt"] = {**case.recorded, "tree": "8" * 40} elif mismatch == "project_model": case.project["_appops_effective_model_pointer"] = {**case.pointer, "modelSha256": "9" * 64} elif mismatch == "rollback_attestation": case.calls["attestation_valid"] = False elif mismatch == "observed_revision": case.observed["revision"] = "9" * 40 elif mismatch == "observed_tree": case.observed["build"]["tree"] = "8" * 40 elif mismatch == "observed_image": case.observed["containers"]["Unraid-AppOps-Gateway"]["image_id"] = "sha256:" + "8" * 64 elif mismatch == "observed_container_id": case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id" elif mismatch == "recorded_container_id": case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id" elif mismatch == "missing_container": case.observed["containers"] = {} elif mismatch == "extra_container": case.observed["containers"]["unexpected"] = {"container_id": "c" * 64} assert not deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded) def test_known_model_restore_runs_real_receipt_comparison_and_rebinds_recreated_container( deploy, monkeypatch, tmp_path, ): case = _recreated_receipt_case(deploy, monkeypatch) container_name = "Unraid-AppOps-Gateway" old_container_id = case.recorded["containers"][container_name]["container_id"] new_container_id = case.observed["containers"][container_name]["container_id"] case.snapshot["containers"] = [{ "name": container_name, "container_id": old_container_id, "image_id": IMAGE_ID, "image_ref": "unraid-appops-gateway:production", "was_running": True, }] case.project.update({ "slug": "unraid-appops-gateway", "source_dir": str(tmp_path / "source"), "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "containers": [container_name], "environment": {}, }) case.project["_appops_effective_rollback_snapshot"] = case.snapshot model = {"services": {"appops-gateway": {"environment": {"SYNTHETIC": "safe"}}}} live_binding = {**case.pointer, "liveRecordSha256": "f" * 64} calls = {"events": [], "recorded_receipt": None} monkeypatch.setattr( deploy, "_appops_load_effective_model", lambda _project, pointer: {"pointer": pointer, "model": model}, ) monkeypatch.setattr(deploy, "workmesh_transition_enabled", lambda _project: False) monkeypatch.setattr( deploy, "run", lambda command, **_kwargs: calls["events"].append(("docker-tag", list(command))), ) monkeypatch.setattr( deploy, "inspect_image", lambda image_ref: {"Id": IMAGE_ID} if image_ref == "unraid-appops-gateway:production" else None, ) monkeypatch.setattr( deploy, "inspect_container", lambda _name: {"Id": new_container_id, "Image": IMAGE_ID}, ) monkeypatch.setattr(deploy, "repository_path_from_project", lambda _project: tmp_path / "repository.git") monkeypatch.setattr( deploy, "sync_source", lambda *_args, **_kwargs: calls["events"].append(("sync-source", None)), ) monkeypatch.setattr( deploy, "copy_environment", lambda *_args, **_kwargs: calls["events"].append(("copy-environment", None)), ) monkeypatch.setattr( deploy, "remove_attempt_only_compose_containers", lambda *_args: calls["events"].append(("remove-attempt-only", None)), ) monkeypatch.setattr( deploy, "deployment_environment", lambda *_args: {"SYNTHETIC": "safe"}, ) monkeypatch.setattr( deploy, "restore_snapshot_runtime", lambda *_args: calls["events"].append(("restore-runtime", None)), ) monkeypatch.setattr( deploy, "validate_recorded_receipt", lambda _project, receipt, repository, revision: ( {"validated": receipt == case.recorded, "repository": repository, "revision": revision} ), ) def verify_deployment(_project, build, *, expected_image_ids): calls["events"].append(("verify-deployment", copy.deepcopy(expected_image_ids))) assert build == {"validated": True, "repository": REPOSITORY, "revision": PRIOR_REVISION} assert expected_image_ids == {container_name: IMAGE_ID} return copy.deepcopy(case.observed) monkeypatch.setattr(deploy, "verify_deployment", verify_deployment) monkeypatch.setattr(deploy, "cleanup_rollback_image_tags", lambda *_args: None) def record_live_model(_project, receipt): calls["recorded_receipt"] = copy.deepcopy(receipt) return live_binding monkeypatch.setattr(deploy, "_appops_record_live_effective_model", record_live_model) deploy.restore_compose_snapshot(case.project, case.snapshot, REPOSITORY, REVISION) assert calls["recorded_receipt"] == case.observed assert case.project["_rollback_receipt"] == case.observed assert case.project["_appops_restored_model_binding"] == live_binding assert case.project["_rollback_restored_revision"] == PRIOR_REVISION assert "_appops_effective_rollback_snapshot" not in case.project assert old_container_id != new_container_id assert [event[0] for event in calls["events"]] == [ "docker-tag", "sync-source", "copy-environment", "remove-attempt-only", "restore-runtime", "verify-deployment", ] assert len(case.calls["attestation"]) == 2 @pytest.mark.parametrize("legacy_provenance", [True, False]) def test_model_rollback_rebinds_the_restored_live_proof( deploy, monkeypatch, tmp_path, legacy_provenance, ): old_revision = deploy.APPOPS_LEGACY_OLD_REVISION if legacy_provenance else PRIOR_REVISION pointer = { "schema": "itworx.appops-effective-compose-model.v1", "recoveryId": RECOVERY_ID, "revision": old_revision, "modelSha256": "a" * 64, "modelRecordSha256": "b" * 64, } if not legacy_provenance: pointer["liveRecordSha256"] = "c" * 64 model = { "services": { "appops-gateway": { "image": "unraid-appops-gateway:production", "environment": {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"}, }, }, } snapshot_receipt = {} if legacy_provenance else { "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY, "revision": old_revision, "containers": {"Unraid-AppOps-Gateway": {"image_id": IMAGE_ID}}, } snapshot = { "revision": old_revision, "legacy_provenance": legacy_provenance, "receipt": snapshot_receipt, "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": "5" * 64, "image_id": IMAGE_ID, "image_ref": "unraid-appops-gateway:production", }], "appops_runtime_model": pointer, } reference = {**pointer, "liveRecordSha256": "d" * 64} calls = {"plain_snapshot": None, "binding": None, "recorded_receipt": None, "health_verified": False} class StatusReader: @staticmethod def _self_upgrade_verify_rollback_model(observed_pointer, observed_snapshot): return observed_pointer == pointer and observed_snapshot == snapshot @staticmethod def _self_upgrade_legacy_model_binding(observed_pointer): calls["binding"] = copy.deepcopy(observed_pointer) return {"modelPointer": observed_pointer, "runtimeFingerprint": "e" * 64} @staticmethod def _self_upgrade_store_live_model(observed_pointer, binding): if binding.get("modelPointer") == observed_pointer: return reference return None project = {"_repository_name": REPOSITORY, "containers": ["Unraid-AppOps-Gateway"]} monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader()) monkeypatch.setattr( deploy, "_appops_load_effective_model", lambda _project, observed_pointer: {"pointer": observed_pointer, "model": model}, ) monkeypatch.setattr(deploy, "_appops_select_effective_model", lambda *_args: None) def health(_project, *, expected_image_ids): assert expected_image_ids == {"Unraid-AppOps-Gateway": deploy.APPOPS_LEGACY_OLD_IMAGE_ID} calls["health_verified"] = True def verify_runtime(*_args, **_kwargs): assert calls["health_verified"] is True monkeypatch.setattr(deploy, "verify_containers", health) monkeypatch.setattr(deploy, "verify_appops_legacy_runtime", verify_runtime) monkeypatch.setattr( deploy, "_appops_captured_environment", lambda _container: {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"}, ) monkeypatch.setattr(deploy, "inspect_container", lambda _name: {"Id": "6" * 64}) monkeypatch.setattr(deploy, "inspect_image", lambda _image: {"Id": IMAGE_ID}) monkeypatch.setattr( deploy, "_appops_record_live_effective_model", lambda _project, receipt: calls.__setitem__("recorded_receipt", receipt) or reference, ) original_restore = deploy.restore_compose_snapshot def restore_with_plain_snapshot(project_arg, observed_snapshot, repository, failed_revision): if "appops_runtime_model" not in observed_snapshot: calls["plain_snapshot"] = copy.deepcopy(observed_snapshot) if not legacy_provenance: project_arg["_rollback_receipt"] = snapshot_receipt return None return original_restore(project_arg, observed_snapshot, repository, failed_revision) monkeypatch.setattr(deploy, "restore_compose_snapshot", restore_with_plain_snapshot) restore_with_plain_snapshot(project, snapshot, REPOSITORY, REVISION) plain_snapshot = calls["plain_snapshot"] assert isinstance(plain_snapshot, dict) assert "appops_runtime_model" not in plain_snapshot if legacy_provenance: assert plain_snapshot["containers"][0]["restore_image_ref"] == deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF assert project["_appops_legacy_restored"] is True assert calls["binding"] == pointer else: assert calls["recorded_receipt"] == snapshot_receipt assert "_appops_legacy_restored" not in project assert project["_appops_restored_model_binding"] == reference