970 lines
38 KiB
Python
970 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
BROKER = ROOT / "ops" / "gitea-deploy-broker"
|
|
REPOSITORY = "jens/unraid-appops-gateway"
|
|
REVISION = "c" * 40
|
|
PRIOR_REVISION = "d" * 40
|
|
TREE = "e" * 40
|
|
RECOVERY_ID = "12345678-1234-4234-8234-123456789abc"
|
|
TRANSACTION_ID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"
|
|
SECRET = b"transaction-reconciliation-test-key-32-bytes"
|
|
REGISTRY_BYTES = b'{"repository":"jens/unraid-appops-gateway"}\n'
|
|
IMAGE_ID = "sha256:" + "7" * 64
|
|
|
|
|
|
@pytest.fixture
|
|
def deploy(monkeypatch):
|
|
monkeypatch.syspath_prepend(str(BROKER))
|
|
if "fcntl" not in sys.modules:
|
|
fcntl = types.ModuleType("fcntl")
|
|
fcntl.LOCK_EX = 2
|
|
fcntl.LOCK_UN = 8
|
|
fcntl.LOCK_NB = 4
|
|
fcntl.flock = lambda *_args: None
|
|
monkeypatch.setitem(sys.modules, "fcntl", fcntl)
|
|
spec = importlib.util.spec_from_file_location(
|
|
"appops_effective_transaction_reconciliation_test",
|
|
BROKER / "deploy.py",
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _digest(raw: bytes) -> str:
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def _mac(secret: bytes, domain: str, raw: bytes) -> str:
|
|
return hashlib.sha256(secret + b"\0" + domain.encode() + b"\0" + raw).hexdigest()
|
|
|
|
|
|
def _metadata(
|
|
revision: str = REVISION,
|
|
tree: str = TREE,
|
|
built_at: str = "2026-10-07T12:00:00.000Z",
|
|
) -> dict[str, str]:
|
|
return {
|
|
"id": revision[:12],
|
|
"repository": REPOSITORY,
|
|
"revision": revision,
|
|
"tree": tree,
|
|
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
|
|
"builtAt": built_at,
|
|
}
|
|
|
|
|
|
def _pointer(revision: str) -> dict[str, str]:
|
|
return {
|
|
"recoveryId": RECOVERY_ID,
|
|
"revision": revision,
|
|
"modelSha256": "1" * 64,
|
|
"modelRecordSha256": "2" * 64,
|
|
}
|
|
|
|
|
|
def _state_bytes(state: dict[str, object]) -> bytes:
|
|
return json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + b"\n"
|
|
|
|
|
|
def _case(deploy, monkeypatch, tmp_path, *, phase: str = "runtime_mutated"):
|
|
prior_pointer = _pointer(PRIOR_REVISION)
|
|
candidate_pointer = _pointer(REVISION)
|
|
backup = {"sha256": "3" * 64, "revision": REVISION}
|
|
prior_snapshot = {
|
|
"revision": PRIOR_REVISION,
|
|
"appops_runtime_model": prior_pointer,
|
|
"appops_data_backup": backup,
|
|
"containers": [{"name": "Unraid-AppOps-Gateway", "image_id": "sha256:" + "4" * 64}],
|
|
}
|
|
receipt = {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": REPOSITORY,
|
|
"revision": REVISION,
|
|
"tree": TREE,
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"container_id": "5" * 64,
|
|
"image_id": IMAGE_ID,
|
|
},
|
|
},
|
|
}
|
|
live_reference = {
|
|
"recoveryId": RECOVERY_ID,
|
|
"revision": REVISION,
|
|
"liveRecordSha256": "6" * 64,
|
|
}
|
|
old_receipt = {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": REPOSITORY,
|
|
"revision": PRIOR_REVISION,
|
|
"tree": "9" * 40,
|
|
}
|
|
state: dict[str, object] = {
|
|
"repository": REPOSITORY,
|
|
"live_revision": PRIOR_REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": False,
|
|
"live_receipt": old_receipt,
|
|
"rollback_snapshot": prior_snapshot,
|
|
"live_artifacts": {
|
|
"appops_runtime_model": {"recoveryId": RECOVERY_ID, "revision": PRIOR_REVISION},
|
|
"containers": {"Unraid-AppOps-Gateway": {"container_id": "8" * 64}},
|
|
},
|
|
}
|
|
original_raw = _state_bytes(state)
|
|
original_state_sha = _digest(original_raw)
|
|
candidate_model = {
|
|
"services": {
|
|
"appops-gateway": {
|
|
"image": "itworx/appops-gateway:protected",
|
|
"build": {"context": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree"},
|
|
"environment": {"APPOPS_SOURCE_REVISION": REVISION},
|
|
},
|
|
},
|
|
}
|
|
payload: dict[str, object] = {
|
|
"schema": "itworx.appops-self-upgrade-model-transaction/v1",
|
|
"repository": REPOSITORY,
|
|
"transactionId": TRANSACTION_ID,
|
|
"recoveryId": RECOVERY_ID,
|
|
"kind": "followup",
|
|
"revision": REVISION,
|
|
"metadata": _metadata(),
|
|
"originSemanticSha256": "a" * 64,
|
|
"originRawSha256": "b" * 64,
|
|
"originJournalSha256": "c" * 64,
|
|
"expectedStateSha256": original_state_sha,
|
|
"registryFingerprint": _mac(SECRET, "registry", REGISTRY_BYTES),
|
|
"preMutationRuntimeSha256": "d" * 64,
|
|
"preMutationRuntimeFingerprint": "e" * 64,
|
|
"sourceFingerprint": "f" * 64,
|
|
"priorModelPointer": prior_pointer,
|
|
"priorSnapshot": prior_snapshot,
|
|
"candidateSourceFingerprint": "0" * 64,
|
|
"candidateModelPointer": candidate_pointer,
|
|
"candidateImageId": IMAGE_ID,
|
|
"buildContexts": {
|
|
"appops-gateway": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree",
|
|
},
|
|
"phase": phase,
|
|
"createdAt": 1_791_378_000,
|
|
"updatedAt": 1_791_378_001,
|
|
}
|
|
if phase in {"verified", "committed"}:
|
|
payload["verifiedLiveReference"] = live_reference
|
|
payload["verifiedReceipt"] = receipt
|
|
if phase == "committed":
|
|
payload["committedStateSha256"] = "0" * 64
|
|
envelope = {"payload": payload, "sha256": "9" * 64}
|
|
registry = {"bytes": REGISTRY_BYTES}
|
|
calls: dict[str, object] = {
|
|
"active": [],
|
|
"active_result": None,
|
|
"docker": [],
|
|
"git": [],
|
|
"backup": [],
|
|
"verify": [],
|
|
"current": [],
|
|
"updates": [],
|
|
"writes": [],
|
|
"state_sha_override": None,
|
|
"metadata_override": None,
|
|
"receipt_override": None,
|
|
"fail_commit_once": False,
|
|
}
|
|
|
|
class StatusReader:
|
|
PROJECTS_PATH = tmp_path / "projects.json"
|
|
|
|
@staticmethod
|
|
def _self_upgrade_pending_model_transaction():
|
|
return copy.deepcopy(envelope)
|
|
|
|
@staticmethod
|
|
def _self_upgrade_read_fixed_file(_path, **_kwargs):
|
|
return registry["bytes"]
|
|
|
|
@staticmethod
|
|
def _self_upgrade_secret():
|
|
return SECRET
|
|
|
|
@staticmethod
|
|
def _self_upgrade_mac(secret, domain, raw):
|
|
return _mac(secret, domain, raw)
|
|
|
|
@staticmethod
|
|
def _self_upgrade_load_compose_model(recovery_id, revision):
|
|
if recovery_id == RECOVERY_ID and revision == REVISION and calls.get("model_valid", True):
|
|
return {"model": candidate_model, "pointer": candidate_pointer}
|
|
return None
|
|
|
|
@staticmethod
|
|
def _self_upgrade_candidate_model_binding(pointer, observed_receipt):
|
|
if (pointer == candidate_pointer and observed_receipt == receipt
|
|
and calls.get("model_valid", True)):
|
|
return {"modelPointer": pointer, "receipt": observed_receipt}
|
|
return None
|
|
|
|
@staticmethod
|
|
def _self_upgrade_store_live_model(pointer, binding):
|
|
if pointer == candidate_pointer and binding.get("receipt") == receipt:
|
|
return live_reference
|
|
return None
|
|
|
|
@staticmethod
|
|
def _self_upgrade_update_model_transaction(transaction_id, expected_sha, next_phase, bindings):
|
|
calls["updates"].append((next_phase, copy.deepcopy(bindings)))
|
|
if transaction_id != TRANSACTION_ID or expected_sha != envelope["sha256"]:
|
|
return None
|
|
if next_phase == "committed" and calls["fail_commit_once"]:
|
|
calls["fail_commit_once"] = False
|
|
return None
|
|
updated = dict(envelope["payload"])
|
|
updated.update(bindings)
|
|
updated["phase"] = next_phase
|
|
updated["updatedAt"] = int(updated["updatedAt"]) + 1
|
|
if next_phase == "committed":
|
|
updated["committedStateSha256"] = _digest(_state_bytes(state))
|
|
envelope["payload"] = updated
|
|
envelope["sha256"] = _digest(json.dumps(updated, sort_keys=True).encode())
|
|
return copy.deepcopy(envelope)
|
|
|
|
@staticmethod
|
|
def _self_upgrade_active_compose_model(observed_state, observed_container):
|
|
calls["active"].append((copy.deepcopy(observed_state), copy.deepcopy(observed_container)))
|
|
active = calls.get("active_result")
|
|
return copy.deepcopy(active) if isinstance(active, dict) else None
|
|
|
|
def read_preimage(_project):
|
|
raw = _state_bytes(state)
|
|
override = calls["state_sha_override"]
|
|
return copy.deepcopy(state), raw, override or _digest(raw)
|
|
|
|
def write_state(repository, project, revision, status, _message, **kwargs):
|
|
calls["writes"].append((repository, revision, status, copy.deepcopy(kwargs)))
|
|
reference = project.get("_appops_runtime_model_binding")
|
|
state.update({
|
|
"repository": repository,
|
|
"live_revision": revision,
|
|
"live_status": "healthy" if status == "success" else status,
|
|
"live_legacy_provenance": False,
|
|
"live_receipt": copy.deepcopy(kwargs["live_receipt"]),
|
|
"rollback_snapshot": copy.deepcopy(kwargs["rollback_snapshot"]),
|
|
"live_artifacts": {
|
|
"appops_runtime_model": copy.deepcopy(reference),
|
|
"containers": {"Unraid-AppOps-Gateway": {"container_id": receipt["containers"]["Unraid-AppOps-Gateway"]["container_id"]}},
|
|
},
|
|
})
|
|
|
|
def validate_commit(_repo, branch, candidate_revision):
|
|
calls["git"].append((branch, candidate_revision))
|
|
if branch != "main" or candidate_revision != REVISION:
|
|
raise deploy.DeployError("pending source is not the requested main commit")
|
|
return candidate_revision
|
|
|
|
def deployment_metadata(_repo, candidate_revision, repository, *, built_at=None):
|
|
if repository != REPOSITORY or candidate_revision != REVISION:
|
|
raise deploy.DeployError("pending target identity mismatch")
|
|
observed = _metadata(built_at=built_at or "2026-10-07T12:00:00.000Z")
|
|
override = calls["metadata_override"]
|
|
return copy.deepcopy(override or observed)
|
|
|
|
def verify_backup(actual_backup, candidate_revision, previous_revision):
|
|
calls["backup"].append((actual_backup, candidate_revision, previous_revision))
|
|
expected_previous = calls.get("backup_previous_revision", PRIOR_REVISION)
|
|
if actual_backup != backup or candidate_revision != REVISION or previous_revision != expected_previous:
|
|
raise deploy.DeployError("pending data backup proof changed")
|
|
|
|
def verify_deployment(_project, observed_metadata, *, expected_image_ids):
|
|
calls["verify"].append((copy.deepcopy(observed_metadata), dict(expected_image_ids)))
|
|
if expected_image_ids != {"Unraid-AppOps-Gateway": IMAGE_ID}:
|
|
raise deploy.DeployError("sealed candidate image differs from the live image")
|
|
if observed_metadata != _metadata():
|
|
raise deploy.DeployError("candidate receipt metadata changed")
|
|
return copy.deepcopy(calls["receipt_override"] or receipt)
|
|
|
|
def verified_current(repository, _project, candidate_revision, tree):
|
|
calls["current"].append((repository, candidate_revision, tree))
|
|
if (repository, candidate_revision, tree) == (REPOSITORY, REVISION, TREE):
|
|
return copy.deepcopy(receipt)
|
|
return None
|
|
|
|
project = {
|
|
"_repository_name": REPOSITORY,
|
|
"slug": "unraid-appops-gateway",
|
|
"branch": "main",
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
}
|
|
monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader())
|
|
monkeypatch.setattr(deploy, "read_appops_state_preimage", read_preimage)
|
|
monkeypatch.setattr(deploy, "write_state", write_state)
|
|
monkeypatch.setattr(deploy, "validate_commit", validate_commit)
|
|
monkeypatch.setattr(deploy, "deployment_metadata", deployment_metadata)
|
|
monkeypatch.setattr(deploy, "verify_appops_data_backup", verify_backup)
|
|
monkeypatch.setattr(deploy, "verify_deployment", verify_deployment)
|
|
monkeypatch.setattr(deploy, "verified_current_receipt", verified_current)
|
|
monkeypatch.setattr(deploy, "run", lambda *_args, **_kwargs: pytest.fail("reconciliation must not mutate Docker"))
|
|
monkeypatch.setattr(deploy, "inspect_container", lambda *_args, **_kwargs: pytest.fail("unexpected inspect path"))
|
|
monkeypatch.setattr(deploy, "restore_compose_snapshot", lambda *_args, **_kwargs: pytest.fail("unexpected rollback"))
|
|
return SimpleNamespace(
|
|
deploy=deploy,
|
|
project=project,
|
|
repo=tmp_path / "repository.git",
|
|
payload=payload,
|
|
envelope=envelope,
|
|
state=state,
|
|
state_sha=original_state_sha,
|
|
registry=registry,
|
|
metadata=_metadata(),
|
|
pointer=candidate_pointer,
|
|
reference=live_reference,
|
|
receipt=receipt,
|
|
backup=backup,
|
|
calls=calls,
|
|
)
|
|
|
|
|
|
def _reconcile(case):
|
|
return case.deploy._appops_reconcile_pending_model_transaction(
|
|
case.project, REPOSITORY, case.repo,
|
|
)
|
|
|
|
|
|
def test_runtime_mutated_transaction_finishes_state_publication_without_docker_mutation(
|
|
deploy, monkeypatch, tmp_path,
|
|
):
|
|
case = _case(deploy, monkeypatch, tmp_path)
|
|
|
|
result = _reconcile(case)
|
|
|
|
assert result == case.receipt
|
|
assert case.envelope["payload"]["phase"] == "committed"
|
|
assert [phase for phase, _bindings in case.calls["updates"]] == ["verified", "committed"]
|
|
assert len(case.calls["writes"]) == 1
|
|
repo, revision, status, kwargs = case.calls["writes"][0]
|
|
assert (repo, revision, status) == (REPOSITORY, REVISION, "success")
|
|
assert kwargs["expected_state_sha256"] == case.state_sha
|
|
assert kwargs["live_receipt"] == case.receipt
|
|
assert case.state["live_legacy_provenance"] is False
|
|
assert case.state["live_artifacts"]["appops_runtime_model"] == case.reference
|
|
assert case.calls["git"] == [("main", REVISION)]
|
|
assert len(case.calls["backup"]) == 1
|
|
assert case.calls["verify"] == [(case.metadata, {"Unraid-AppOps-Gateway": IMAGE_ID})]
|
|
assert case.calls["docker"] == []
|
|
|
|
|
|
def test_verified_state_replay_only_reattests_and_marks_transaction_committed(
|
|
deploy, monkeypatch, tmp_path,
|
|
):
|
|
case = _case(deploy, monkeypatch, tmp_path, phase="verified")
|
|
case.state.update({
|
|
"live_revision": REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": False,
|
|
"live_receipt": copy.deepcopy(case.receipt),
|
|
"live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)},
|
|
})
|
|
|
|
result = _reconcile(case)
|
|
|
|
assert result == case.receipt
|
|
assert case.envelope["payload"]["phase"] == "committed"
|
|
assert [phase for phase, _bindings in case.calls["updates"]] == ["committed"]
|
|
assert case.calls["current"] == [(REPOSITORY, REVISION, TREE)]
|
|
assert case.calls["writes"] == []
|
|
assert case.calls["git"] == []
|
|
assert case.calls["verify"] == []
|
|
assert case.calls["docker"] == []
|
|
|
|
|
|
def test_verified_state_replay_rejects_changed_registry_before_reattest(
|
|
deploy, monkeypatch, tmp_path,
|
|
):
|
|
case = _case(deploy, monkeypatch, tmp_path, phase="verified")
|
|
case.state.update({
|
|
"live_revision": REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": False,
|
|
"live_receipt": copy.deepcopy(case.receipt),
|
|
"live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)},
|
|
})
|
|
case.registry["bytes"] = b'{"repository":"changed"}\n'
|
|
|
|
with pytest.raises(deploy.DeployError, match="state or registry changed"):
|
|
_reconcile(case)
|
|
|
|
assert case.calls["current"] == []
|
|
assert case.calls["updates"] == []
|
|
assert case.calls["writes"] == []
|
|
assert case.calls["docker"] == []
|
|
|
|
|
|
def test_state_commit_failure_restarts_from_verified_proof_without_second_state_write(
|
|
deploy, monkeypatch, tmp_path,
|
|
):
|
|
case = _case(deploy, monkeypatch, tmp_path)
|
|
case.calls["fail_commit_once"] = True
|
|
|
|
with pytest.raises(deploy.DeployError, match="compare-and-swap failed"):
|
|
_reconcile(case)
|
|
|
|
assert case.envelope["payload"]["phase"] == "verified"
|
|
assert len(case.calls["writes"]) == 1
|
|
assert case.calls["docker"] == []
|
|
|
|
result = _reconcile(case)
|
|
|
|
assert result == case.receipt
|
|
assert case.envelope["payload"]["phase"] == "committed"
|
|
assert [phase for phase, _bindings in case.calls["updates"]] == [
|
|
"verified", "committed", "committed",
|
|
]
|
|
assert len(case.calls["writes"]) == 1
|
|
assert len(case.calls["current"]) == 1
|
|
assert case.calls["docker"] == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mismatch",
|
|
["state", "registry", "model", "target_metadata", "backup", "image", "receipt", "unknown_phase"],
|
|
)
|
|
def test_pending_transaction_mismatches_fail_without_state_or_docker_mutation(
|
|
deploy, monkeypatch, tmp_path, mismatch,
|
|
):
|
|
case = _case(deploy, monkeypatch, tmp_path)
|
|
if mismatch == "state":
|
|
case.calls["state_sha_override"] = "0" * 64
|
|
elif mismatch == "registry":
|
|
case.registry["bytes"] = b'{"repository":"different"}\n'
|
|
elif mismatch == "model":
|
|
case.calls["model_valid"] = False
|
|
elif mismatch == "target_metadata":
|
|
case.calls["metadata_override"] = _metadata(tree="9" * 40)
|
|
elif mismatch == "backup":
|
|
case.payload["priorSnapshot"]["appops_data_backup"] = {"sha256": "0" * 64}
|
|
elif mismatch == "image":
|
|
case.payload["candidateImageId"] = "sha256:" + "8" * 64
|
|
elif mismatch == "receipt":
|
|
case.calls["receipt_override"] = {**case.receipt, "revision": "b" * 40}
|
|
elif mismatch == "unknown_phase":
|
|
case.payload["phase"] = "prepared"
|
|
|
|
with pytest.raises(deploy.DeployError):
|
|
_reconcile(case)
|
|
|
|
assert case.calls["writes"] == []
|
|
assert case.calls["updates"] == []
|
|
assert case.calls["docker"] == []
|
|
|
|
|
|
def _prepare_rolling_back_case(deploy, monkeypatch, tmp_path):
|
|
case = _case(deploy, monkeypatch, tmp_path, phase="rolling_back")
|
|
prior_revision = deploy.APPOPS_LEGACY_OLD_REVISION
|
|
prior_pointer = {
|
|
**_pointer(prior_revision),
|
|
"schema": "itworx.appops-effective-compose-model.v1",
|
|
}
|
|
restored_reference = {**prior_pointer, "liveRecordSha256": "a" * 64}
|
|
prior_snapshot = {
|
|
"revision": prior_revision,
|
|
"legacy_provenance": True,
|
|
"receipt": {},
|
|
"appops_runtime_model": prior_pointer,
|
|
"appops_data_backup": case.backup,
|
|
"containers": [{"name": "Unraid-AppOps-Gateway", "image_id": IMAGE_ID}],
|
|
}
|
|
case.payload.update({
|
|
"phase": "rolling_back",
|
|
"priorModelPointer": prior_pointer,
|
|
"priorSnapshot": prior_snapshot,
|
|
})
|
|
case.state.update({
|
|
"repository": REPOSITORY,
|
|
"live_revision": prior_revision,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": True,
|
|
"live_receipt": {},
|
|
"live_artifacts": {
|
|
"appops_runtime_model": restored_reference,
|
|
"containers": {"Unraid-AppOps-Gateway": {"container_id": "b" * 64}},
|
|
},
|
|
})
|
|
case.calls["active_result"] = {"pointer": copy.deepcopy(restored_reference)}
|
|
case.calls["backup_previous_revision"] = prior_revision
|
|
container = {"Id": "b" * 64, "Image": IMAGE_ID, "State": {"Running": True}}
|
|
monkeypatch.setattr(deploy, "inspect_container", lambda _name: copy.deepcopy(container))
|
|
return case, prior_pointer, restored_reference, prior_snapshot, container
|
|
|
|
|
|
def test_rolling_back_transaction_seals_exact_restored_legacy_binding_without_mutation(
|
|
deploy, monkeypatch, tmp_path,
|
|
):
|
|
case, prior_pointer, restored_reference, prior_snapshot, container = _prepare_rolling_back_case(
|
|
deploy, monkeypatch, tmp_path,
|
|
)
|
|
|
|
result = _reconcile(case)
|
|
|
|
assert result == {}
|
|
assert case.envelope["payload"]["phase"] == "rolled_back"
|
|
assert case.envelope["payload"]["restoredLiveReference"] == restored_reference
|
|
assert case.calls["updates"] == [("rolled_back", {"restoredLiveReference": restored_reference})]
|
|
assert case.calls["active"] == [(case.state, container)]
|
|
assert case.calls["backup"] == [(case.backup, REVISION, deploy.APPOPS_LEGACY_OLD_REVISION)]
|
|
assert case.calls["writes"] == []
|
|
assert case.calls["git"] == []
|
|
assert case.calls["docker"] == []
|
|
assert case.payload["priorModelPointer"] == prior_pointer
|
|
assert case.payload["priorSnapshot"] == prior_snapshot
|
|
|
|
|
|
@pytest.mark.parametrize("mismatch", ["state", "reference", "runtime", "registry", "backup"])
|
|
def test_rolling_back_transaction_rejects_unbound_prior_state_without_mutation(
|
|
deploy, monkeypatch, tmp_path, mismatch,
|
|
):
|
|
case, _prior_pointer, restored_reference, prior_snapshot, _container = _prepare_rolling_back_case(
|
|
deploy, monkeypatch, tmp_path,
|
|
)
|
|
if mismatch == "state":
|
|
case.state["live_revision"] = "9" * 40
|
|
elif mismatch == "reference":
|
|
case.state["live_artifacts"]["appops_runtime_model"]["liveRecordSha256"] = "c" * 64
|
|
elif mismatch == "runtime":
|
|
case.calls["active_result"] = {"pointer": {**restored_reference, "liveRecordSha256": "d" * 64}}
|
|
elif mismatch == "registry":
|
|
case.registry["bytes"] = b'{"repository":"different"}\n'
|
|
elif mismatch == "backup":
|
|
prior_snapshot["appops_data_backup"] = {"sha256": "0" * 64}
|
|
|
|
with pytest.raises(deploy.DeployError):
|
|
_reconcile(case)
|
|
|
|
assert case.calls["updates"] == []
|
|
assert case.calls["writes"] == []
|
|
assert case.calls["git"] == []
|
|
assert case.calls["docker"] == []
|
|
|
|
|
|
def test_explicit_rollback_cli_routes_through_pending_model_reconciliation(
|
|
deploy, monkeypatch, tmp_path, capsys,
|
|
):
|
|
revision = REVISION
|
|
project = {
|
|
"slug": "unraid-appops-gateway",
|
|
"branch": "main",
|
|
"source_dir": str(tmp_path / "source"),
|
|
"compose_project": "unraid-appops-gateway",
|
|
"compose_files": ["docker-compose.yml"],
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
"verify_oci_revision": True,
|
|
"environment": {},
|
|
}
|
|
projects = {REPOSITORY: project}
|
|
events = []
|
|
real_exists = deploy.Path.exists
|
|
|
|
def path_exists(path):
|
|
if str(path) == "/controller-hub-releases/appops-self-upgrade-recoveries/transactions":
|
|
return True
|
|
return real_exists(path)
|
|
|
|
def reconcile(actual_project, repository, repo):
|
|
assert actual_project["_repository_name"] == REPOSITORY
|
|
assert repository == REPOSITORY
|
|
assert repo == tmp_path / "repository.git"
|
|
events.append("reconcile")
|
|
return {"revision": revision, "proof": "signed-prior-runtime"}
|
|
|
|
monkeypatch.setattr(deploy.Path, "exists", path_exists)
|
|
monkeypatch.setattr(deploy, "load_projects", lambda: projects)
|
|
monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repository, item: dict(item))
|
|
monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repository.git")
|
|
monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision)
|
|
monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: _metadata())
|
|
monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None)
|
|
monkeypatch.setattr(deploy, "bind_appops_self_upgrade_recovery", lambda *_args: None)
|
|
monkeypatch.setattr(deploy, "bind_registered_legacy_request", lambda *_args: None)
|
|
monkeypatch.setattr(deploy, "exclusive_lock", lambda *_args, **_kwargs: contextlib.nullcontext())
|
|
|
|
@contextlib.contextmanager
|
|
def global_lock():
|
|
events.append("global-lock")
|
|
yield
|
|
|
|
monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", global_lock)
|
|
monkeypatch.setattr(deploy, "_appops_reconcile_pending_model_transaction", reconcile)
|
|
monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", REPOSITORY, revision])
|
|
|
|
assert deploy.main() == 0
|
|
assert events == ["global-lock", "reconcile"]
|
|
assert '"status": "already-current"' in capsys.readouterr().out
|
|
|
|
|
|
def _recreated_receipt_case(deploy, monkeypatch):
|
|
pointer = {
|
|
"schema": "itworx.appops-effective-compose-model.v1",
|
|
"recoveryId": RECOVERY_ID,
|
|
"revision": PRIOR_REVISION,
|
|
"modelSha256": "1" * 64,
|
|
"modelRecordSha256": "2" * 64,
|
|
"liveRecordSha256": "3" * 64,
|
|
}
|
|
recorded = {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": REPOSITORY,
|
|
"revision": PRIOR_REVISION,
|
|
"tree": "4" * 40,
|
|
"treeDigest": "5" * 64,
|
|
"build": {
|
|
"id": PRIOR_REVISION[:12],
|
|
"repository": REPOSITORY,
|
|
"revision": PRIOR_REVISION,
|
|
"tree": "4" * 40,
|
|
"treeDigest": "5" * 64,
|
|
"builtAt": "2026-10-07T11:00:00.000Z",
|
|
},
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"container_id": "a" * 64,
|
|
"image_id": IMAGE_ID,
|
|
"image_revision": PRIOR_REVISION,
|
|
"image_tree": "4" * 40,
|
|
"health": "healthy",
|
|
},
|
|
},
|
|
"live": {"status": "healthy"},
|
|
}
|
|
observed = copy.deepcopy(recorded)
|
|
observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "b" * 64
|
|
snapshot = {
|
|
"revision": PRIOR_REVISION,
|
|
"legacy_provenance": False,
|
|
"receipt": recorded,
|
|
"appops_runtime_model": pointer,
|
|
}
|
|
project = {
|
|
"_repository_name": REPOSITORY,
|
|
"_appops_effective_model_pointer": pointer,
|
|
}
|
|
calls = {"attestation": [], "attestation_valid": True}
|
|
|
|
def verify_rollback_model(actual_pointer, actual_snapshot):
|
|
calls["attestation"].append((copy.deepcopy(actual_pointer), copy.deepcopy(actual_snapshot)))
|
|
return (
|
|
calls["attestation_valid"]
|
|
and actual_pointer == pointer
|
|
and actual_snapshot == snapshot
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"_appops_status_reader",
|
|
lambda: SimpleNamespace(_self_upgrade_verify_rollback_model=verify_rollback_model),
|
|
)
|
|
project["_appops_effective_rollback_snapshot"] = snapshot
|
|
return SimpleNamespace(
|
|
project=project,
|
|
pointer=pointer,
|
|
recorded=recorded,
|
|
observed=observed,
|
|
snapshot=snapshot,
|
|
calls=calls,
|
|
)
|
|
|
|
|
|
def test_recreated_receipt_matcher_allows_only_new_valid_container_id(
|
|
deploy, monkeypatch,
|
|
):
|
|
case = _recreated_receipt_case(deploy, monkeypatch)
|
|
recorded_id = case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"]
|
|
observed_id = case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"]
|
|
|
|
assert len(recorded_id) == len(observed_id) == 64
|
|
assert recorded_id != observed_id
|
|
assert deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded)
|
|
assert case.calls["attestation"] == [(case.pointer, case.snapshot)]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mismatch",
|
|
[
|
|
"repository", "legacy_provenance", "snapshot_revision", "snapshot_receipt",
|
|
"project_model", "rollback_attestation", "observed_revision", "observed_tree",
|
|
"observed_image", "observed_container_id", "recorded_container_id",
|
|
"missing_container", "extra_container",
|
|
],
|
|
)
|
|
def test_recreated_receipt_matcher_rejects_every_other_changed_fact(
|
|
deploy, monkeypatch, mismatch,
|
|
):
|
|
case = _recreated_receipt_case(deploy, monkeypatch)
|
|
if mismatch == "repository":
|
|
case.project["_repository_name"] = "jens/other"
|
|
elif mismatch == "legacy_provenance":
|
|
case.snapshot["legacy_provenance"] = True
|
|
elif mismatch == "snapshot_revision":
|
|
case.snapshot["revision"] = "9" * 40
|
|
elif mismatch == "snapshot_receipt":
|
|
case.snapshot["receipt"] = {**case.recorded, "tree": "8" * 40}
|
|
elif mismatch == "project_model":
|
|
case.project["_appops_effective_model_pointer"] = {**case.pointer, "modelSha256": "9" * 64}
|
|
elif mismatch == "rollback_attestation":
|
|
case.calls["attestation_valid"] = False
|
|
elif mismatch == "observed_revision":
|
|
case.observed["revision"] = "9" * 40
|
|
elif mismatch == "observed_tree":
|
|
case.observed["build"]["tree"] = "8" * 40
|
|
elif mismatch == "observed_image":
|
|
case.observed["containers"]["Unraid-AppOps-Gateway"]["image_id"] = "sha256:" + "8" * 64
|
|
elif mismatch == "observed_container_id":
|
|
case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id"
|
|
elif mismatch == "recorded_container_id":
|
|
case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id"
|
|
elif mismatch == "missing_container":
|
|
case.observed["containers"] = {}
|
|
elif mismatch == "extra_container":
|
|
case.observed["containers"]["unexpected"] = {"container_id": "c" * 64}
|
|
|
|
assert not deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded)
|
|
|
|
|
|
def test_known_model_restore_runs_real_receipt_comparison_and_rebinds_recreated_container(
|
|
deploy, monkeypatch, tmp_path,
|
|
):
|
|
case = _recreated_receipt_case(deploy, monkeypatch)
|
|
container_name = "Unraid-AppOps-Gateway"
|
|
old_container_id = case.recorded["containers"][container_name]["container_id"]
|
|
new_container_id = case.observed["containers"][container_name]["container_id"]
|
|
case.snapshot["containers"] = [{
|
|
"name": container_name,
|
|
"container_id": old_container_id,
|
|
"image_id": IMAGE_ID,
|
|
"image_ref": "unraid-appops-gateway:production",
|
|
"was_running": True,
|
|
}]
|
|
case.project.update({
|
|
"slug": "unraid-appops-gateway",
|
|
"source_dir": str(tmp_path / "source"),
|
|
"compose_project": "unraid-appops-gateway",
|
|
"compose_files": ["docker-compose.yml"],
|
|
"containers": [container_name],
|
|
"environment": {},
|
|
})
|
|
case.project["_appops_effective_rollback_snapshot"] = case.snapshot
|
|
model = {"services": {"appops-gateway": {"environment": {"SYNTHETIC": "safe"}}}}
|
|
live_binding = {**case.pointer, "liveRecordSha256": "f" * 64}
|
|
calls = {"events": [], "recorded_receipt": None}
|
|
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"_appops_load_effective_model",
|
|
lambda _project, pointer: {"pointer": pointer, "model": model},
|
|
)
|
|
monkeypatch.setattr(deploy, "workmesh_transition_enabled", lambda _project: False)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"run",
|
|
lambda command, **_kwargs: calls["events"].append(("docker-tag", list(command))),
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"inspect_image",
|
|
lambda image_ref: {"Id": IMAGE_ID} if image_ref == "unraid-appops-gateway:production" else None,
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"inspect_container",
|
|
lambda _name: {"Id": new_container_id, "Image": IMAGE_ID},
|
|
)
|
|
monkeypatch.setattr(deploy, "repository_path_from_project", lambda _project: tmp_path / "repository.git")
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"sync_source",
|
|
lambda *_args, **_kwargs: calls["events"].append(("sync-source", None)),
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"copy_environment",
|
|
lambda *_args, **_kwargs: calls["events"].append(("copy-environment", None)),
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"remove_attempt_only_compose_containers",
|
|
lambda *_args: calls["events"].append(("remove-attempt-only", None)),
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"deployment_environment",
|
|
lambda *_args: {"SYNTHETIC": "safe"},
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"restore_snapshot_runtime",
|
|
lambda *_args: calls["events"].append(("restore-runtime", None)),
|
|
)
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"validate_recorded_receipt",
|
|
lambda _project, receipt, repository, revision: (
|
|
{"validated": receipt == case.recorded, "repository": repository, "revision": revision}
|
|
),
|
|
)
|
|
|
|
def verify_deployment(_project, build, *, expected_image_ids):
|
|
calls["events"].append(("verify-deployment", copy.deepcopy(expected_image_ids)))
|
|
assert build == {"validated": True, "repository": REPOSITORY, "revision": PRIOR_REVISION}
|
|
assert expected_image_ids == {container_name: IMAGE_ID}
|
|
return copy.deepcopy(case.observed)
|
|
|
|
monkeypatch.setattr(deploy, "verify_deployment", verify_deployment)
|
|
monkeypatch.setattr(deploy, "cleanup_rollback_image_tags", lambda *_args: None)
|
|
|
|
def record_live_model(_project, receipt):
|
|
calls["recorded_receipt"] = copy.deepcopy(receipt)
|
|
return live_binding
|
|
|
|
monkeypatch.setattr(deploy, "_appops_record_live_effective_model", record_live_model)
|
|
|
|
deploy.restore_compose_snapshot(case.project, case.snapshot, REPOSITORY, REVISION)
|
|
|
|
assert calls["recorded_receipt"] == case.observed
|
|
assert case.project["_rollback_receipt"] == case.observed
|
|
assert case.project["_appops_restored_model_binding"] == live_binding
|
|
assert case.project["_rollback_restored_revision"] == PRIOR_REVISION
|
|
assert "_appops_effective_rollback_snapshot" not in case.project
|
|
assert old_container_id != new_container_id
|
|
assert [event[0] for event in calls["events"]] == [
|
|
"docker-tag", "sync-source", "copy-environment", "remove-attempt-only",
|
|
"restore-runtime", "verify-deployment",
|
|
]
|
|
assert len(case.calls["attestation"]) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("legacy_provenance", [True, False])
|
|
def test_model_rollback_rebinds_the_restored_live_proof(
|
|
deploy, monkeypatch, tmp_path, legacy_provenance,
|
|
):
|
|
old_revision = deploy.APPOPS_LEGACY_OLD_REVISION if legacy_provenance else PRIOR_REVISION
|
|
pointer = {
|
|
"schema": "itworx.appops-effective-compose-model.v1",
|
|
"recoveryId": RECOVERY_ID,
|
|
"revision": old_revision,
|
|
"modelSha256": "a" * 64,
|
|
"modelRecordSha256": "b" * 64,
|
|
}
|
|
if not legacy_provenance:
|
|
pointer["liveRecordSha256"] = "c" * 64
|
|
model = {
|
|
"services": {
|
|
"appops-gateway": {
|
|
"image": "unraid-appops-gateway:production",
|
|
"environment": {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"},
|
|
},
|
|
},
|
|
}
|
|
snapshot_receipt = {} if legacy_provenance else {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": REPOSITORY,
|
|
"revision": old_revision,
|
|
"containers": {"Unraid-AppOps-Gateway": {"image_id": IMAGE_ID}},
|
|
}
|
|
snapshot = {
|
|
"revision": old_revision,
|
|
"legacy_provenance": legacy_provenance,
|
|
"receipt": snapshot_receipt,
|
|
"containers": [{
|
|
"name": "Unraid-AppOps-Gateway",
|
|
"container_id": "5" * 64,
|
|
"image_id": IMAGE_ID,
|
|
"image_ref": "unraid-appops-gateway:production",
|
|
}],
|
|
"appops_runtime_model": pointer,
|
|
}
|
|
reference = {**pointer, "liveRecordSha256": "d" * 64}
|
|
calls = {"plain_snapshot": None, "binding": None, "recorded_receipt": None, "health_verified": False}
|
|
|
|
class StatusReader:
|
|
@staticmethod
|
|
def _self_upgrade_verify_rollback_model(observed_pointer, observed_snapshot):
|
|
return observed_pointer == pointer and observed_snapshot == snapshot
|
|
|
|
@staticmethod
|
|
def _self_upgrade_legacy_model_binding(observed_pointer):
|
|
calls["binding"] = copy.deepcopy(observed_pointer)
|
|
return {"modelPointer": observed_pointer, "runtimeFingerprint": "e" * 64}
|
|
|
|
@staticmethod
|
|
def _self_upgrade_store_live_model(observed_pointer, binding):
|
|
if binding.get("modelPointer") == observed_pointer:
|
|
return reference
|
|
return None
|
|
|
|
project = {"_repository_name": REPOSITORY, "containers": ["Unraid-AppOps-Gateway"]}
|
|
monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader())
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"_appops_load_effective_model",
|
|
lambda _project, observed_pointer: {"pointer": observed_pointer, "model": model},
|
|
)
|
|
monkeypatch.setattr(deploy, "_appops_select_effective_model", lambda *_args: None)
|
|
def health(_project, *, expected_image_ids):
|
|
assert expected_image_ids == {"Unraid-AppOps-Gateway": deploy.APPOPS_LEGACY_OLD_IMAGE_ID}
|
|
calls["health_verified"] = True
|
|
def verify_runtime(*_args, **_kwargs):
|
|
assert calls["health_verified"] is True
|
|
monkeypatch.setattr(deploy, "verify_containers", health)
|
|
monkeypatch.setattr(deploy, "verify_appops_legacy_runtime", verify_runtime)
|
|
monkeypatch.setattr(
|
|
deploy, "_appops_captured_environment",
|
|
lambda _container: {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"},
|
|
)
|
|
monkeypatch.setattr(deploy, "inspect_container", lambda _name: {"Id": "6" * 64})
|
|
monkeypatch.setattr(deploy, "inspect_image", lambda _image: {"Id": IMAGE_ID})
|
|
monkeypatch.setattr(
|
|
deploy,
|
|
"_appops_record_live_effective_model",
|
|
lambda _project, receipt: calls.__setitem__("recorded_receipt", receipt) or reference,
|
|
)
|
|
|
|
original_restore = deploy.restore_compose_snapshot
|
|
|
|
def restore_with_plain_snapshot(project_arg, observed_snapshot, repository, failed_revision):
|
|
if "appops_runtime_model" not in observed_snapshot:
|
|
calls["plain_snapshot"] = copy.deepcopy(observed_snapshot)
|
|
if not legacy_provenance:
|
|
project_arg["_rollback_receipt"] = snapshot_receipt
|
|
return None
|
|
return original_restore(project_arg, observed_snapshot, repository, failed_revision)
|
|
|
|
monkeypatch.setattr(deploy, "restore_compose_snapshot", restore_with_plain_snapshot)
|
|
restore_with_plain_snapshot(project, snapshot, REPOSITORY, REVISION)
|
|
|
|
plain_snapshot = calls["plain_snapshot"]
|
|
assert isinstance(plain_snapshot, dict)
|
|
assert "appops_runtime_model" not in plain_snapshot
|
|
if legacy_provenance:
|
|
assert plain_snapshot["containers"][0]["restore_image_ref"] == deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF
|
|
assert project["_appops_legacy_restored"] is True
|
|
assert calls["binding"] == pointer
|
|
else:
|
|
assert calls["recorded_receipt"] == snapshot_receipt
|
|
assert "_appops_legacy_restored" not in project
|
|
assert project["_appops_restored_model_binding"] == reference
|