Publish curated Unraid-AppOps-Gateway source from 2e6cf771e11a9cdc8b2e2453586a16d518adf40b
This commit is contained in:
Public Source Publisher committed
1 parent
5d598db0df
commit
631b846397
10 files changed
+4709
-8
No files matched your search
@@ -0,0 +1,963 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
BROKER = ROOT / "ops" / "gitea-deploy-broker"
|
||||
REPOSITORY = "jens/unraid-appops-gateway"
|
||||
REVISION = "c" * 40
|
||||
PRIOR_REVISION = "d" * 40
|
||||
TREE = "e" * 40
|
||||
RECOVERY_ID = "12345678-1234-4234-8234-123456789abc"
|
||||
TRANSACTION_ID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"
|
||||
SECRET = b"transaction-reconciliation-test-key-32-bytes"
|
||||
REGISTRY_BYTES = b'{"repository":"jens/unraid-appops-gateway"}\n'
|
||||
IMAGE_ID = "sha256:" + "7" * 64
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deploy(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(BROKER))
|
||||
if "fcntl" not in sys.modules:
|
||||
fcntl = types.ModuleType("fcntl")
|
||||
fcntl.LOCK_EX = 2
|
||||
fcntl.LOCK_UN = 8
|
||||
fcntl.LOCK_NB = 4
|
||||
fcntl.flock = lambda *_args: None
|
||||
monkeypatch.setitem(sys.modules, "fcntl", fcntl)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_effective_transaction_reconciliation_test",
|
||||
BROKER / "deploy.py",
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _digest(raw: bytes) -> str:
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def _mac(secret: bytes, domain: str, raw: bytes) -> str:
|
||||
return hashlib.sha256(secret + b"\0" + domain.encode() + b"\0" + raw).hexdigest()
|
||||
|
||||
|
||||
def _metadata(
|
||||
revision: str = REVISION,
|
||||
tree: str = TREE,
|
||||
built_at: str = "2026-10-07T12:00:00.000Z",
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
"id": revision[:12],
|
||||
"repository": REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
|
||||
"builtAt": built_at,
|
||||
}
|
||||
|
||||
|
||||
def _pointer(revision: str) -> dict[str, str]:
|
||||
return {
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"revision": revision,
|
||||
"modelSha256": "1" * 64,
|
||||
"modelRecordSha256": "2" * 64,
|
||||
}
|
||||
|
||||
|
||||
def _state_bytes(state: dict[str, object]) -> bytes:
|
||||
return json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + b"\n"
|
||||
|
||||
|
||||
def _case(deploy, monkeypatch, tmp_path, *, phase: str = "runtime_mutated"):
|
||||
prior_pointer = _pointer(PRIOR_REVISION)
|
||||
candidate_pointer = _pointer(REVISION)
|
||||
backup = {"sha256": "3" * 64, "revision": REVISION}
|
||||
prior_snapshot = {
|
||||
"revision": PRIOR_REVISION,
|
||||
"appops_runtime_model": prior_pointer,
|
||||
"appops_data_backup": backup,
|
||||
"containers": [{"name": "Unraid-AppOps-Gateway", "image_id": "sha256:" + "4" * 64}],
|
||||
}
|
||||
receipt = {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": REPOSITORY,
|
||||
"revision": REVISION,
|
||||
"tree": TREE,
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": "5" * 64,
|
||||
"image_id": IMAGE_ID,
|
||||
},
|
||||
},
|
||||
}
|
||||
live_reference = {
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"revision": REVISION,
|
||||
"liveRecordSha256": "6" * 64,
|
||||
}
|
||||
old_receipt = {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": REPOSITORY,
|
||||
"revision": PRIOR_REVISION,
|
||||
"tree": "9" * 40,
|
||||
}
|
||||
state: dict[str, object] = {
|
||||
"repository": REPOSITORY,
|
||||
"live_revision": PRIOR_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"live_receipt": old_receipt,
|
||||
"rollback_snapshot": prior_snapshot,
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": {"recoveryId": RECOVERY_ID, "revision": PRIOR_REVISION},
|
||||
"containers": {"Unraid-AppOps-Gateway": {"container_id": "8" * 64}},
|
||||
},
|
||||
}
|
||||
original_raw = _state_bytes(state)
|
||||
original_state_sha = _digest(original_raw)
|
||||
candidate_model = {
|
||||
"services": {
|
||||
"appops-gateway": {
|
||||
"image": "itworx/appops-gateway:protected",
|
||||
"build": {"context": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree"},
|
||||
"environment": {"APPOPS_SOURCE_REVISION": REVISION},
|
||||
},
|
||||
},
|
||||
}
|
||||
payload: dict[str, object] = {
|
||||
"schema": "itworx.appops-self-upgrade-model-transaction/v1",
|
||||
"repository": REPOSITORY,
|
||||
"transactionId": TRANSACTION_ID,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"kind": "followup",
|
||||
"revision": REVISION,
|
||||
"metadata": _metadata(),
|
||||
"originSemanticSha256": "a" * 64,
|
||||
"originRawSha256": "b" * 64,
|
||||
"originJournalSha256": "c" * 64,
|
||||
"expectedStateSha256": original_state_sha,
|
||||
"registryFingerprint": _mac(SECRET, "registry", REGISTRY_BYTES),
|
||||
"preMutationRuntimeSha256": "d" * 64,
|
||||
"preMutationRuntimeFingerprint": "e" * 64,
|
||||
"sourceFingerprint": "f" * 64,
|
||||
"priorModelPointer": prior_pointer,
|
||||
"priorSnapshot": prior_snapshot,
|
||||
"candidateSourceFingerprint": "0" * 64,
|
||||
"candidateModelPointer": candidate_pointer,
|
||||
"candidateImageId": IMAGE_ID,
|
||||
"buildContexts": {
|
||||
"appops-gateway": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree",
|
||||
},
|
||||
"phase": phase,
|
||||
"createdAt": 1_791_378_000,
|
||||
"updatedAt": 1_791_378_001,
|
||||
}
|
||||
if phase in {"verified", "committed"}:
|
||||
payload["verifiedLiveReference"] = live_reference
|
||||
payload["verifiedReceipt"] = receipt
|
||||
if phase == "committed":
|
||||
payload["committedStateSha256"] = "0" * 64
|
||||
envelope = {"payload": payload, "sha256": "9" * 64}
|
||||
registry = {"bytes": REGISTRY_BYTES}
|
||||
calls: dict[str, object] = {
|
||||
"active": [],
|
||||
"active_result": None,
|
||||
"docker": [],
|
||||
"git": [],
|
||||
"backup": [],
|
||||
"verify": [],
|
||||
"current": [],
|
||||
"updates": [],
|
||||
"writes": [],
|
||||
"state_sha_override": None,
|
||||
"metadata_override": None,
|
||||
"receipt_override": None,
|
||||
"fail_commit_once": False,
|
||||
}
|
||||
|
||||
class StatusReader:
|
||||
PROJECTS_PATH = tmp_path / "projects.json"
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_pending_model_transaction():
|
||||
return copy.deepcopy(envelope)
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_read_fixed_file(_path, **_kwargs):
|
||||
return registry["bytes"]
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_secret():
|
||||
return SECRET
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_mac(secret, domain, raw):
|
||||
return _mac(secret, domain, raw)
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_load_compose_model(recovery_id, revision):
|
||||
if recovery_id == RECOVERY_ID and revision == REVISION and calls.get("model_valid", True):
|
||||
return {"model": candidate_model, "pointer": candidate_pointer}
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_candidate_model_binding(pointer, observed_receipt):
|
||||
if (pointer == candidate_pointer and observed_receipt == receipt
|
||||
and calls.get("model_valid", True)):
|
||||
return {"modelPointer": pointer, "receipt": observed_receipt}
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_store_live_model(pointer, binding):
|
||||
if pointer == candidate_pointer and binding.get("receipt") == receipt:
|
||||
return live_reference
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_update_model_transaction(transaction_id, expected_sha, next_phase, bindings):
|
||||
calls["updates"].append((next_phase, copy.deepcopy(bindings)))
|
||||
if transaction_id != TRANSACTION_ID or expected_sha != envelope["sha256"]:
|
||||
return None
|
||||
if next_phase == "committed" and calls["fail_commit_once"]:
|
||||
calls["fail_commit_once"] = False
|
||||
return None
|
||||
updated = dict(envelope["payload"])
|
||||
updated.update(bindings)
|
||||
updated["phase"] = next_phase
|
||||
updated["updatedAt"] = int(updated["updatedAt"]) + 1
|
||||
if next_phase == "committed":
|
||||
updated["committedStateSha256"] = _digest(_state_bytes(state))
|
||||
envelope["payload"] = updated
|
||||
envelope["sha256"] = _digest(json.dumps(updated, sort_keys=True).encode())
|
||||
return copy.deepcopy(envelope)
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_active_compose_model(observed_state, observed_container):
|
||||
calls["active"].append((copy.deepcopy(observed_state), copy.deepcopy(observed_container)))
|
||||
active = calls.get("active_result")
|
||||
return copy.deepcopy(active) if isinstance(active, dict) else None
|
||||
|
||||
def read_preimage(_project):
|
||||
raw = _state_bytes(state)
|
||||
override = calls["state_sha_override"]
|
||||
return copy.deepcopy(state), raw, override or _digest(raw)
|
||||
|
||||
def write_state(repository, project, revision, status, _message, **kwargs):
|
||||
calls["writes"].append((repository, revision, status, copy.deepcopy(kwargs)))
|
||||
reference = project.get("_appops_runtime_model_binding")
|
||||
state.update({
|
||||
"repository": repository,
|
||||
"live_revision": revision,
|
||||
"live_status": "healthy" if status == "success" else status,
|
||||
"live_legacy_provenance": False,
|
||||
"live_receipt": copy.deepcopy(kwargs["live_receipt"]),
|
||||
"rollback_snapshot": copy.deepcopy(kwargs["rollback_snapshot"]),
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": copy.deepcopy(reference),
|
||||
"containers": {"Unraid-AppOps-Gateway": {"container_id": receipt["containers"]["Unraid-AppOps-Gateway"]["container_id"]}},
|
||||
},
|
||||
})
|
||||
|
||||
def validate_commit(_repo, branch, candidate_revision):
|
||||
calls["git"].append((branch, candidate_revision))
|
||||
if branch != "main" or candidate_revision != REVISION:
|
||||
raise deploy.DeployError("pending source is not the requested main commit")
|
||||
return candidate_revision
|
||||
|
||||
def deployment_metadata(_repo, candidate_revision, repository, *, built_at=None):
|
||||
if repository != REPOSITORY or candidate_revision != REVISION:
|
||||
raise deploy.DeployError("pending target identity mismatch")
|
||||
observed = _metadata(built_at=built_at or "2026-10-07T12:00:00.000Z")
|
||||
override = calls["metadata_override"]
|
||||
return copy.deepcopy(override or observed)
|
||||
|
||||
def verify_backup(actual_backup, candidate_revision, previous_revision):
|
||||
calls["backup"].append((actual_backup, candidate_revision, previous_revision))
|
||||
expected_previous = calls.get("backup_previous_revision", PRIOR_REVISION)
|
||||
if actual_backup != backup or candidate_revision != REVISION or previous_revision != expected_previous:
|
||||
raise deploy.DeployError("pending data backup proof changed")
|
||||
|
||||
def verify_deployment(_project, observed_metadata, *, expected_image_ids):
|
||||
calls["verify"].append((copy.deepcopy(observed_metadata), dict(expected_image_ids)))
|
||||
if expected_image_ids != {"Unraid-AppOps-Gateway": IMAGE_ID}:
|
||||
raise deploy.DeployError("sealed candidate image differs from the live image")
|
||||
if observed_metadata != _metadata():
|
||||
raise deploy.DeployError("candidate receipt metadata changed")
|
||||
return copy.deepcopy(calls["receipt_override"] or receipt)
|
||||
|
||||
def verified_current(repository, _project, candidate_revision, tree):
|
||||
calls["current"].append((repository, candidate_revision, tree))
|
||||
if (repository, candidate_revision, tree) == (REPOSITORY, REVISION, TREE):
|
||||
return copy.deepcopy(receipt)
|
||||
return None
|
||||
|
||||
project = {
|
||||
"_repository_name": REPOSITORY,
|
||||
"slug": "unraid-appops-gateway",
|
||||
"branch": "main",
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
}
|
||||
monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader())
|
||||
monkeypatch.setattr(deploy, "read_appops_state_preimage", read_preimage)
|
||||
monkeypatch.setattr(deploy, "write_state", write_state)
|
||||
monkeypatch.setattr(deploy, "validate_commit", validate_commit)
|
||||
monkeypatch.setattr(deploy, "deployment_metadata", deployment_metadata)
|
||||
monkeypatch.setattr(deploy, "verify_appops_data_backup", verify_backup)
|
||||
monkeypatch.setattr(deploy, "verify_deployment", verify_deployment)
|
||||
monkeypatch.setattr(deploy, "verified_current_receipt", verified_current)
|
||||
monkeypatch.setattr(deploy, "run", lambda *_args, **_kwargs: pytest.fail("reconciliation must not mutate Docker"))
|
||||
monkeypatch.setattr(deploy, "inspect_container", lambda *_args, **_kwargs: pytest.fail("unexpected inspect path"))
|
||||
monkeypatch.setattr(deploy, "restore_compose_snapshot", lambda *_args, **_kwargs: pytest.fail("unexpected rollback"))
|
||||
return SimpleNamespace(
|
||||
deploy=deploy,
|
||||
project=project,
|
||||
repo=tmp_path / "repository.git",
|
||||
payload=payload,
|
||||
envelope=envelope,
|
||||
state=state,
|
||||
state_sha=original_state_sha,
|
||||
registry=registry,
|
||||
metadata=_metadata(),
|
||||
pointer=candidate_pointer,
|
||||
reference=live_reference,
|
||||
receipt=receipt,
|
||||
backup=backup,
|
||||
calls=calls,
|
||||
)
|
||||
|
||||
|
||||
def _reconcile(case):
|
||||
return case.deploy._appops_reconcile_pending_model_transaction(
|
||||
case.project, REPOSITORY, case.repo,
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_mutated_transaction_finishes_state_publication_without_docker_mutation(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
):
|
||||
case = _case(deploy, monkeypatch, tmp_path)
|
||||
|
||||
result = _reconcile(case)
|
||||
|
||||
assert result == case.receipt
|
||||
assert case.envelope["payload"]["phase"] == "committed"
|
||||
assert [phase for phase, _bindings in case.calls["updates"]] == ["verified", "committed"]
|
||||
assert len(case.calls["writes"]) == 1
|
||||
repo, revision, status, kwargs = case.calls["writes"][0]
|
||||
assert (repo, revision, status) == (REPOSITORY, REVISION, "success")
|
||||
assert kwargs["expected_state_sha256"] == case.state_sha
|
||||
assert kwargs["live_receipt"] == case.receipt
|
||||
assert case.state["live_legacy_provenance"] is False
|
||||
assert case.state["live_artifacts"]["appops_runtime_model"] == case.reference
|
||||
assert case.calls["git"] == [("main", REVISION)]
|
||||
assert len(case.calls["backup"]) == 1
|
||||
assert case.calls["verify"] == [(case.metadata, {"Unraid-AppOps-Gateway": IMAGE_ID})]
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
|
||||
def test_verified_state_replay_only_reattests_and_marks_transaction_committed(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
):
|
||||
case = _case(deploy, monkeypatch, tmp_path, phase="verified")
|
||||
case.state.update({
|
||||
"live_revision": REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"live_receipt": copy.deepcopy(case.receipt),
|
||||
"live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)},
|
||||
})
|
||||
|
||||
result = _reconcile(case)
|
||||
|
||||
assert result == case.receipt
|
||||
assert case.envelope["payload"]["phase"] == "committed"
|
||||
assert [phase for phase, _bindings in case.calls["updates"]] == ["committed"]
|
||||
assert case.calls["current"] == [(REPOSITORY, REVISION, TREE)]
|
||||
assert case.calls["writes"] == []
|
||||
assert case.calls["git"] == []
|
||||
assert case.calls["verify"] == []
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
|
||||
def test_verified_state_replay_rejects_changed_registry_before_reattest(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
):
|
||||
case = _case(deploy, monkeypatch, tmp_path, phase="verified")
|
||||
case.state.update({
|
||||
"live_revision": REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"live_receipt": copy.deepcopy(case.receipt),
|
||||
"live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)},
|
||||
})
|
||||
case.registry["bytes"] = b'{"repository":"changed"}\n'
|
||||
|
||||
with pytest.raises(deploy.DeployError, match="state or registry changed"):
|
||||
_reconcile(case)
|
||||
|
||||
assert case.calls["current"] == []
|
||||
assert case.calls["updates"] == []
|
||||
assert case.calls["writes"] == []
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
|
||||
def test_state_commit_failure_restarts_from_verified_proof_without_second_state_write(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
):
|
||||
case = _case(deploy, monkeypatch, tmp_path)
|
||||
case.calls["fail_commit_once"] = True
|
||||
|
||||
with pytest.raises(deploy.DeployError, match="compare-and-swap failed"):
|
||||
_reconcile(case)
|
||||
|
||||
assert case.envelope["payload"]["phase"] == "verified"
|
||||
assert len(case.calls["writes"]) == 1
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
result = _reconcile(case)
|
||||
|
||||
assert result == case.receipt
|
||||
assert case.envelope["payload"]["phase"] == "committed"
|
||||
assert [phase for phase, _bindings in case.calls["updates"]] == [
|
||||
"verified", "committed", "committed",
|
||||
]
|
||||
assert len(case.calls["writes"]) == 1
|
||||
assert len(case.calls["current"]) == 1
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mismatch",
|
||||
["state", "registry", "model", "target_metadata", "backup", "image", "receipt", "unknown_phase"],
|
||||
)
|
||||
def test_pending_transaction_mismatches_fail_without_state_or_docker_mutation(
|
||||
deploy, monkeypatch, tmp_path, mismatch,
|
||||
):
|
||||
case = _case(deploy, monkeypatch, tmp_path)
|
||||
if mismatch == "state":
|
||||
case.calls["state_sha_override"] = "0" * 64
|
||||
elif mismatch == "registry":
|
||||
case.registry["bytes"] = b'{"repository":"different"}\n'
|
||||
elif mismatch == "model":
|
||||
case.calls["model_valid"] = False
|
||||
elif mismatch == "target_metadata":
|
||||
case.calls["metadata_override"] = _metadata(tree="9" * 40)
|
||||
elif mismatch == "backup":
|
||||
case.payload["priorSnapshot"]["appops_data_backup"] = {"sha256": "0" * 64}
|
||||
elif mismatch == "image":
|
||||
case.payload["candidateImageId"] = "sha256:" + "8" * 64
|
||||
elif mismatch == "receipt":
|
||||
case.calls["receipt_override"] = {**case.receipt, "revision": "b" * 40}
|
||||
elif mismatch == "unknown_phase":
|
||||
case.payload["phase"] = "prepared"
|
||||
|
||||
with pytest.raises(deploy.DeployError):
|
||||
_reconcile(case)
|
||||
|
||||
assert case.calls["writes"] == []
|
||||
assert case.calls["updates"] == []
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
|
||||
def _prepare_rolling_back_case(deploy, monkeypatch, tmp_path):
|
||||
case = _case(deploy, monkeypatch, tmp_path, phase="rolling_back")
|
||||
prior_revision = deploy.APPOPS_LEGACY_OLD_REVISION
|
||||
prior_pointer = {
|
||||
**_pointer(prior_revision),
|
||||
"schema": "itworx.appops-effective-compose-model.v1",
|
||||
}
|
||||
restored_reference = {**prior_pointer, "liveRecordSha256": "a" * 64}
|
||||
prior_snapshot = {
|
||||
"revision": prior_revision,
|
||||
"legacy_provenance": True,
|
||||
"receipt": {},
|
||||
"appops_runtime_model": prior_pointer,
|
||||
"appops_data_backup": case.backup,
|
||||
"containers": [{"name": "Unraid-AppOps-Gateway", "image_id": IMAGE_ID}],
|
||||
}
|
||||
case.payload.update({
|
||||
"phase": "rolling_back",
|
||||
"priorModelPointer": prior_pointer,
|
||||
"priorSnapshot": prior_snapshot,
|
||||
})
|
||||
case.state.update({
|
||||
"repository": REPOSITORY,
|
||||
"live_revision": prior_revision,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": restored_reference,
|
||||
"containers": {"Unraid-AppOps-Gateway": {"container_id": "b" * 64}},
|
||||
},
|
||||
})
|
||||
case.calls["active_result"] = {"pointer": copy.deepcopy(restored_reference)}
|
||||
case.calls["backup_previous_revision"] = prior_revision
|
||||
container = {"Id": "b" * 64, "Image": IMAGE_ID, "State": {"Running": True}}
|
||||
monkeypatch.setattr(deploy, "inspect_container", lambda _name: copy.deepcopy(container))
|
||||
return case, prior_pointer, restored_reference, prior_snapshot, container
|
||||
|
||||
|
||||
def test_rolling_back_transaction_seals_exact_restored_legacy_binding_without_mutation(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
):
|
||||
case, prior_pointer, restored_reference, prior_snapshot, container = _prepare_rolling_back_case(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
)
|
||||
|
||||
result = _reconcile(case)
|
||||
|
||||
assert result == {}
|
||||
assert case.envelope["payload"]["phase"] == "rolled_back"
|
||||
assert case.envelope["payload"]["restoredLiveReference"] == restored_reference
|
||||
assert case.calls["updates"] == [("rolled_back", {"restoredLiveReference": restored_reference})]
|
||||
assert case.calls["active"] == [(case.state, container)]
|
||||
assert case.calls["backup"] == [(case.backup, REVISION, deploy.APPOPS_LEGACY_OLD_REVISION)]
|
||||
assert case.calls["writes"] == []
|
||||
assert case.calls["git"] == []
|
||||
assert case.calls["docker"] == []
|
||||
assert case.payload["priorModelPointer"] == prior_pointer
|
||||
assert case.payload["priorSnapshot"] == prior_snapshot
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mismatch", ["state", "reference", "runtime", "registry", "backup"])
|
||||
def test_rolling_back_transaction_rejects_unbound_prior_state_without_mutation(
|
||||
deploy, monkeypatch, tmp_path, mismatch,
|
||||
):
|
||||
case, _prior_pointer, restored_reference, prior_snapshot, _container = _prepare_rolling_back_case(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
)
|
||||
if mismatch == "state":
|
||||
case.state["live_revision"] = "9" * 40
|
||||
elif mismatch == "reference":
|
||||
case.state["live_artifacts"]["appops_runtime_model"]["liveRecordSha256"] = "c" * 64
|
||||
elif mismatch == "runtime":
|
||||
case.calls["active_result"] = {"pointer": {**restored_reference, "liveRecordSha256": "d" * 64}}
|
||||
elif mismatch == "registry":
|
||||
case.registry["bytes"] = b'{"repository":"different"}\n'
|
||||
elif mismatch == "backup":
|
||||
prior_snapshot["appops_data_backup"] = {"sha256": "0" * 64}
|
||||
|
||||
with pytest.raises(deploy.DeployError):
|
||||
_reconcile(case)
|
||||
|
||||
assert case.calls["updates"] == []
|
||||
assert case.calls["writes"] == []
|
||||
assert case.calls["git"] == []
|
||||
assert case.calls["docker"] == []
|
||||
|
||||
|
||||
def test_explicit_rollback_cli_routes_through_pending_model_reconciliation(
|
||||
deploy, monkeypatch, tmp_path, capsys,
|
||||
):
|
||||
revision = REVISION
|
||||
project = {
|
||||
"slug": "unraid-appops-gateway",
|
||||
"branch": "main",
|
||||
"source_dir": str(tmp_path / "source"),
|
||||
"compose_project": "unraid-appops-gateway",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
"environment": {},
|
||||
}
|
||||
projects = {REPOSITORY: project}
|
||||
events = []
|
||||
real_exists = deploy.Path.exists
|
||||
|
||||
def path_exists(path):
|
||||
if str(path) == "/controller-hub-releases/appops-self-upgrade-recoveries/transactions":
|
||||
return True
|
||||
return real_exists(path)
|
||||
|
||||
def reconcile(actual_project, repository, repo):
|
||||
assert actual_project["_repository_name"] == REPOSITORY
|
||||
assert repository == REPOSITORY
|
||||
assert repo == tmp_path / "repository.git"
|
||||
events.append("reconcile")
|
||||
return {"revision": revision, "proof": "signed-prior-runtime"}
|
||||
|
||||
monkeypatch.setattr(deploy.Path, "exists", path_exists)
|
||||
monkeypatch.setattr(deploy, "load_projects", lambda: projects)
|
||||
monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repository, item: dict(item))
|
||||
monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repository.git")
|
||||
monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision)
|
||||
monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: _metadata())
|
||||
monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None)
|
||||
monkeypatch.setattr(deploy, "bind_appops_self_upgrade_recovery", lambda *_args: None)
|
||||
monkeypatch.setattr(deploy, "bind_registered_legacy_request", lambda *_args: None)
|
||||
monkeypatch.setattr(deploy, "exclusive_lock", lambda *_args, **_kwargs: contextlib.nullcontext())
|
||||
|
||||
@contextlib.contextmanager
|
||||
def global_lock():
|
||||
events.append("global-lock")
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", global_lock)
|
||||
monkeypatch.setattr(deploy, "_appops_reconcile_pending_model_transaction", reconcile)
|
||||
monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", REPOSITORY, revision])
|
||||
|
||||
assert deploy.main() == 0
|
||||
assert events == ["global-lock", "reconcile"]
|
||||
assert '"status": "already-current"' in capsys.readouterr().out
|
||||
|
||||
|
||||
def _recreated_receipt_case(deploy, monkeypatch):
|
||||
pointer = {
|
||||
"schema": "itworx.appops-effective-compose-model.v1",
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"revision": PRIOR_REVISION,
|
||||
"modelSha256": "1" * 64,
|
||||
"modelRecordSha256": "2" * 64,
|
||||
"liveRecordSha256": "3" * 64,
|
||||
}
|
||||
recorded = {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": REPOSITORY,
|
||||
"revision": PRIOR_REVISION,
|
||||
"tree": "4" * 40,
|
||||
"treeDigest": "5" * 64,
|
||||
"build": {
|
||||
"id": PRIOR_REVISION[:12],
|
||||
"repository": REPOSITORY,
|
||||
"revision": PRIOR_REVISION,
|
||||
"tree": "4" * 40,
|
||||
"treeDigest": "5" * 64,
|
||||
"builtAt": "2026-10-07T11:00:00.000Z",
|
||||
},
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": "a" * 64,
|
||||
"image_id": IMAGE_ID,
|
||||
"image_revision": PRIOR_REVISION,
|
||||
"image_tree": "4" * 40,
|
||||
"health": "healthy",
|
||||
},
|
||||
},
|
||||
"live": {"status": "healthy"},
|
||||
}
|
||||
observed = copy.deepcopy(recorded)
|
||||
observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "b" * 64
|
||||
snapshot = {
|
||||
"revision": PRIOR_REVISION,
|
||||
"legacy_provenance": False,
|
||||
"receipt": recorded,
|
||||
"appops_runtime_model": pointer,
|
||||
}
|
||||
project = {
|
||||
"_repository_name": REPOSITORY,
|
||||
"_appops_effective_model_pointer": pointer,
|
||||
}
|
||||
calls = {"attestation": [], "attestation_valid": True}
|
||||
|
||||
def verify_rollback_model(actual_pointer, actual_snapshot):
|
||||
calls["attestation"].append((copy.deepcopy(actual_pointer), copy.deepcopy(actual_snapshot)))
|
||||
return (
|
||||
calls["attestation_valid"]
|
||||
and actual_pointer == pointer
|
||||
and actual_snapshot == snapshot
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"_appops_status_reader",
|
||||
lambda: SimpleNamespace(_self_upgrade_verify_rollback_model=verify_rollback_model),
|
||||
)
|
||||
project["_appops_effective_rollback_snapshot"] = snapshot
|
||||
return SimpleNamespace(
|
||||
project=project,
|
||||
pointer=pointer,
|
||||
recorded=recorded,
|
||||
observed=observed,
|
||||
snapshot=snapshot,
|
||||
calls=calls,
|
||||
)
|
||||
|
||||
|
||||
def test_recreated_receipt_matcher_allows_only_new_valid_container_id(
|
||||
deploy, monkeypatch,
|
||||
):
|
||||
case = _recreated_receipt_case(deploy, monkeypatch)
|
||||
recorded_id = case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"]
|
||||
observed_id = case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"]
|
||||
|
||||
assert len(recorded_id) == len(observed_id) == 64
|
||||
assert recorded_id != observed_id
|
||||
assert deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded)
|
||||
assert case.calls["attestation"] == [(case.pointer, case.snapshot)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mismatch",
|
||||
[
|
||||
"repository", "legacy_provenance", "snapshot_revision", "snapshot_receipt",
|
||||
"project_model", "rollback_attestation", "observed_revision", "observed_tree",
|
||||
"observed_image", "observed_container_id", "recorded_container_id",
|
||||
"missing_container", "extra_container",
|
||||
],
|
||||
)
|
||||
def test_recreated_receipt_matcher_rejects_every_other_changed_fact(
|
||||
deploy, monkeypatch, mismatch,
|
||||
):
|
||||
case = _recreated_receipt_case(deploy, monkeypatch)
|
||||
if mismatch == "repository":
|
||||
case.project["_repository_name"] = "jens/other"
|
||||
elif mismatch == "legacy_provenance":
|
||||
case.snapshot["legacy_provenance"] = True
|
||||
elif mismatch == "snapshot_revision":
|
||||
case.snapshot["revision"] = "9" * 40
|
||||
elif mismatch == "snapshot_receipt":
|
||||
case.snapshot["receipt"] = {**case.recorded, "tree": "8" * 40}
|
||||
elif mismatch == "project_model":
|
||||
case.project["_appops_effective_model_pointer"] = {**case.pointer, "modelSha256": "9" * 64}
|
||||
elif mismatch == "rollback_attestation":
|
||||
case.calls["attestation_valid"] = False
|
||||
elif mismatch == "observed_revision":
|
||||
case.observed["revision"] = "9" * 40
|
||||
elif mismatch == "observed_tree":
|
||||
case.observed["build"]["tree"] = "8" * 40
|
||||
elif mismatch == "observed_image":
|
||||
case.observed["containers"]["Unraid-AppOps-Gateway"]["image_id"] = "sha256:" + "8" * 64
|
||||
elif mismatch == "observed_container_id":
|
||||
case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id"
|
||||
elif mismatch == "recorded_container_id":
|
||||
case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id"
|
||||
elif mismatch == "missing_container":
|
||||
case.observed["containers"] = {}
|
||||
elif mismatch == "extra_container":
|
||||
case.observed["containers"]["unexpected"] = {"container_id": "c" * 64}
|
||||
|
||||
assert not deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded)
|
||||
|
||||
|
||||
def test_known_model_restore_runs_real_receipt_comparison_and_rebinds_recreated_container(
|
||||
deploy, monkeypatch, tmp_path,
|
||||
):
|
||||
case = _recreated_receipt_case(deploy, monkeypatch)
|
||||
container_name = "Unraid-AppOps-Gateway"
|
||||
old_container_id = case.recorded["containers"][container_name]["container_id"]
|
||||
new_container_id = case.observed["containers"][container_name]["container_id"]
|
||||
case.snapshot["containers"] = [{
|
||||
"name": container_name,
|
||||
"container_id": old_container_id,
|
||||
"image_id": IMAGE_ID,
|
||||
"image_ref": "unraid-appops-gateway:production",
|
||||
"was_running": True,
|
||||
}]
|
||||
case.project.update({
|
||||
"slug": "unraid-appops-gateway",
|
||||
"source_dir": str(tmp_path / "source"),
|
||||
"compose_project": "unraid-appops-gateway",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"containers": [container_name],
|
||||
"environment": {},
|
||||
})
|
||||
case.project["_appops_effective_rollback_snapshot"] = case.snapshot
|
||||
model = {"services": {"appops-gateway": {"environment": {"SYNTHETIC": "safe"}}}}
|
||||
live_binding = {**case.pointer, "liveRecordSha256": "f" * 64}
|
||||
calls = {"events": [], "recorded_receipt": None}
|
||||
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"_appops_load_effective_model",
|
||||
lambda _project, pointer: {"pointer": pointer, "model": model},
|
||||
)
|
||||
monkeypatch.setattr(deploy, "workmesh_transition_enabled", lambda _project: False)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"run",
|
||||
lambda command, **_kwargs: calls["events"].append(("docker-tag", list(command))),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"inspect_image",
|
||||
lambda image_ref: {"Id": IMAGE_ID} if image_ref == "unraid-appops-gateway:production" else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"inspect_container",
|
||||
lambda _name: {"Id": new_container_id, "Image": IMAGE_ID},
|
||||
)
|
||||
monkeypatch.setattr(deploy, "repository_path_from_project", lambda _project: tmp_path / "repository.git")
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"sync_source",
|
||||
lambda *_args, **_kwargs: calls["events"].append(("sync-source", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"copy_environment",
|
||||
lambda *_args, **_kwargs: calls["events"].append(("copy-environment", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"remove_attempt_only_compose_containers",
|
||||
lambda *_args: calls["events"].append(("remove-attempt-only", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"deployment_environment",
|
||||
lambda *_args: {"SYNTHETIC": "safe"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"restore_snapshot_runtime",
|
||||
lambda *_args: calls["events"].append(("restore-runtime", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"validate_recorded_receipt",
|
||||
lambda _project, receipt, repository, revision: (
|
||||
{"validated": receipt == case.recorded, "repository": repository, "revision": revision}
|
||||
),
|
||||
)
|
||||
|
||||
def verify_deployment(_project, build, *, expected_image_ids):
|
||||
calls["events"].append(("verify-deployment", copy.deepcopy(expected_image_ids)))
|
||||
assert build == {"validated": True, "repository": REPOSITORY, "revision": PRIOR_REVISION}
|
||||
assert expected_image_ids == {container_name: IMAGE_ID}
|
||||
return copy.deepcopy(case.observed)
|
||||
|
||||
monkeypatch.setattr(deploy, "verify_deployment", verify_deployment)
|
||||
monkeypatch.setattr(deploy, "cleanup_rollback_image_tags", lambda *_args: None)
|
||||
|
||||
def record_live_model(_project, receipt):
|
||||
calls["recorded_receipt"] = copy.deepcopy(receipt)
|
||||
return live_binding
|
||||
|
||||
monkeypatch.setattr(deploy, "_appops_record_live_effective_model", record_live_model)
|
||||
|
||||
deploy.restore_compose_snapshot(case.project, case.snapshot, REPOSITORY, REVISION)
|
||||
|
||||
assert calls["recorded_receipt"] == case.observed
|
||||
assert case.project["_rollback_receipt"] == case.observed
|
||||
assert case.project["_appops_restored_model_binding"] == live_binding
|
||||
assert case.project["_rollback_restored_revision"] == PRIOR_REVISION
|
||||
assert "_appops_effective_rollback_snapshot" not in case.project
|
||||
assert old_container_id != new_container_id
|
||||
assert [event[0] for event in calls["events"]] == [
|
||||
"docker-tag", "sync-source", "copy-environment", "remove-attempt-only",
|
||||
"restore-runtime", "verify-deployment",
|
||||
]
|
||||
assert len(case.calls["attestation"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy_provenance", [True, False])
|
||||
def test_model_rollback_rebinds_the_restored_live_proof(
|
||||
deploy, monkeypatch, tmp_path, legacy_provenance,
|
||||
):
|
||||
old_revision = deploy.APPOPS_LEGACY_OLD_REVISION if legacy_provenance else PRIOR_REVISION
|
||||
pointer = {
|
||||
"schema": "itworx.appops-effective-compose-model.v1",
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"revision": old_revision,
|
||||
"modelSha256": "a" * 64,
|
||||
"modelRecordSha256": "b" * 64,
|
||||
}
|
||||
if not legacy_provenance:
|
||||
pointer["liveRecordSha256"] = "c" * 64
|
||||
model = {
|
||||
"services": {
|
||||
"appops-gateway": {
|
||||
"image": "unraid-appops-gateway:production",
|
||||
"environment": {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"},
|
||||
},
|
||||
},
|
||||
}
|
||||
snapshot_receipt = {} if legacy_provenance else {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": REPOSITORY,
|
||||
"revision": old_revision,
|
||||
"containers": {"Unraid-AppOps-Gateway": {"image_id": IMAGE_ID}},
|
||||
}
|
||||
snapshot = {
|
||||
"revision": old_revision,
|
||||
"legacy_provenance": legacy_provenance,
|
||||
"receipt": snapshot_receipt,
|
||||
"containers": [{
|
||||
"name": "Unraid-AppOps-Gateway",
|
||||
"container_id": "5" * 64,
|
||||
"image_id": IMAGE_ID,
|
||||
"image_ref": "unraid-appops-gateway:production",
|
||||
}],
|
||||
"appops_runtime_model": pointer,
|
||||
}
|
||||
reference = {**pointer, "liveRecordSha256": "d" * 64}
|
||||
calls = {"plain_snapshot": None, "binding": None, "recorded_receipt": None}
|
||||
|
||||
class StatusReader:
|
||||
@staticmethod
|
||||
def _self_upgrade_verify_rollback_model(observed_pointer, observed_snapshot):
|
||||
return observed_pointer == pointer and observed_snapshot == snapshot
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_legacy_model_binding(observed_pointer):
|
||||
calls["binding"] = copy.deepcopy(observed_pointer)
|
||||
return {"modelPointer": observed_pointer, "runtimeFingerprint": "e" * 64}
|
||||
|
||||
@staticmethod
|
||||
def _self_upgrade_store_live_model(observed_pointer, binding):
|
||||
if binding.get("modelPointer") == observed_pointer:
|
||||
return reference
|
||||
return None
|
||||
|
||||
project = {"_repository_name": REPOSITORY, "containers": ["Unraid-AppOps-Gateway"]}
|
||||
monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader())
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"_appops_load_effective_model",
|
||||
lambda _project, observed_pointer: {"pointer": observed_pointer, "model": model},
|
||||
)
|
||||
monkeypatch.setattr(deploy, "_appops_select_effective_model", lambda *_args: None)
|
||||
monkeypatch.setattr(deploy, "verify_appops_legacy_runtime", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr(
|
||||
deploy, "_appops_captured_environment",
|
||||
lambda _container: {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"},
|
||||
)
|
||||
monkeypatch.setattr(deploy, "inspect_container", lambda _name: {"Id": "6" * 64})
|
||||
monkeypatch.setattr(deploy, "inspect_image", lambda _image: {"Id": IMAGE_ID})
|
||||
monkeypatch.setattr(
|
||||
deploy,
|
||||
"_appops_record_live_effective_model",
|
||||
lambda _project, receipt: calls.__setitem__("recorded_receipt", receipt) or reference,
|
||||
)
|
||||
|
||||
original_restore = deploy.restore_compose_snapshot
|
||||
|
||||
def restore_with_plain_snapshot(project_arg, observed_snapshot, repository, failed_revision):
|
||||
if "appops_runtime_model" not in observed_snapshot:
|
||||
calls["plain_snapshot"] = copy.deepcopy(observed_snapshot)
|
||||
if not legacy_provenance:
|
||||
project_arg["_rollback_receipt"] = snapshot_receipt
|
||||
return None
|
||||
return original_restore(project_arg, observed_snapshot, repository, failed_revision)
|
||||
|
||||
monkeypatch.setattr(deploy, "restore_compose_snapshot", restore_with_plain_snapshot)
|
||||
restore_with_plain_snapshot(project, snapshot, REPOSITORY, REVISION)
|
||||
|
||||
plain_snapshot = calls["plain_snapshot"]
|
||||
assert isinstance(plain_snapshot, dict)
|
||||
assert "appops_runtime_model" not in plain_snapshot
|
||||
if legacy_provenance:
|
||||
assert plain_snapshot["containers"][0]["restore_image_ref"] == deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF
|
||||
assert project["_appops_legacy_restored"] is True
|
||||
assert calls["binding"] == pointer
|
||||
else:
|
||||
assert calls["recorded_receipt"] == snapshot_receipt
|
||||
assert "_appops_legacy_restored" not in project
|
||||
assert project["_appops_restored_model_binding"] == reference
|
||||
File diff suppressed because it is too large.
Load diff
@@ -168,6 +168,7 @@ def test_runtime_requires_exact_container_mount_port_and_health(controller, monk
|
||||
"Config": {
|
||||
"Image": "unraid-appops-gateway:platform-5553c04ce497",
|
||||
"Hostname": "unraid-appops-gateway",
|
||||
"Env": ["APPOPS_RUNTIME_FIXTURE=synthetic-value"],
|
||||
"Labels": {
|
||||
"com.docker.compose.project": "unraid-appops-gateway",
|
||||
"com.docker.compose.service": "appops-gateway",
|
||||
|
||||
@@ -0,0 +1,558 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controller(monkeypatch, tmp_path):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_self_upgrade_recovery_boundaries_test",
|
||||
CONTROLLER / "status_server.py",
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox")
|
||||
module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", tmp_path / "recovery")
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"SELF_UPGRADE_RECOVERY_REQUEST_ROOT",
|
||||
tmp_path / "recovery" / "requests",
|
||||
)
|
||||
return module
|
||||
|
||||
|
||||
def _private_recovery_payload(module, recovery_id: str) -> dict[str, object]:
|
||||
revision = "c" * 40
|
||||
return {
|
||||
"schema": module._SELF_UPGRADE_RECOVERY_SCHEMA,
|
||||
"recoveryId": recovery_id,
|
||||
"semanticSha256": module._webhook_semantic_digest(
|
||||
9, "refs/heads/main", revision,
|
||||
),
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision,
|
||||
"originalRawSha256": "1" * 64,
|
||||
"idempotencyKeySha256": "2" * 64,
|
||||
"preflightSha256": "3" * 64,
|
||||
"expectedCurrentRevision": "4" * 40,
|
||||
"currentContainerId": "5" * 64,
|
||||
"currentImageId": "sha256:" + "6" * 64,
|
||||
"currentSourceTree": "7" * 40,
|
||||
"runtimeFingerprint": "8" * 64,
|
||||
"registryFingerprint": "9" * 64,
|
||||
"sourceFingerprint": "a" * 64,
|
||||
"stateSha256": "b" * 64,
|
||||
"runtimeEnvironment": {"API_TOKEN": "recovery-secret-sentinel"},
|
||||
"createdAt": 1,
|
||||
}
|
||||
|
||||
|
||||
def _recovery_delivery(module, *, secret: str) -> dict[str, object]:
|
||||
return {
|
||||
"schema": module._WEBHOOK_SELF_UPGRADE_RECOVERY_SCHEMA,
|
||||
"semantic_sha256": "a" * 64,
|
||||
"recovery_of_sha256": "b" * 64,
|
||||
"recovery_id": str(uuid.UUID("12345678-1234-4234-8234-123456789abc")),
|
||||
"idempotency_key_sha256": "c" * 64,
|
||||
"preflight_sha256": "d" * 64,
|
||||
"body_sha256": "e" * 64,
|
||||
"delivery": "22345678-1234-4234-8234-123456789abc",
|
||||
"gitea_repository_id": 9,
|
||||
"ref": "refs/heads/main",
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": "f" * 40,
|
||||
"source": "typed_self_upgrade_recovery",
|
||||
"status": "awaiting_validation",
|
||||
"validation": {
|
||||
"last_result": "wait",
|
||||
"required_contexts": ["Managed validation / full (push)"],
|
||||
"observed": [],
|
||||
"checked_at": 1,
|
||||
"deadline_at": 2,
|
||||
},
|
||||
"audit": [
|
||||
{"at": 1, "stage": "self_upgrade_recovery_requested"},
|
||||
{"at": 2, "stage": "awaiting_validation"},
|
||||
],
|
||||
"received_at": 1,
|
||||
"updated_at": 2,
|
||||
# These fields must remain private even if a caller or record contains them.
|
||||
"runtimeEnvironment": {"API_TOKEN": secret},
|
||||
"privateCredential": secret,
|
||||
}
|
||||
|
||||
|
||||
def test_ordinary_deploy_strips_ambient_recovery_binding_and_rejects_bad_bindings(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
ambient_secret = "ambient-recovery-secret-sentinel"
|
||||
monkeypatch.setenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", ambient_secret)
|
||||
monkeypatch.setenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", ambient_secret)
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append((command, kwargs))
|
||||
return subprocess.CompletedProcess(command, 0)
|
||||
|
||||
monkeypatch.setattr(controller.subprocess, "run", fake_run)
|
||||
ordinary = controller._run_registered_deploy("jens/quiz-monitor", "c" * 40)
|
||||
assert ordinary.returncode == 0
|
||||
command, kwargs = calls.pop()
|
||||
assert command[-3:] == ["deploy", "jens/quiz-monitor", "c" * 40]
|
||||
assert "APPOPS_SELF_UPGRADE_RECOVERY_ID" not in kwargs["env"]
|
||||
assert "APPOPS_REGISTERED_LEGACY_REQUEST_ID" not in kwargs["env"]
|
||||
assert ambient_secret not in json.dumps(kwargs["env"])
|
||||
|
||||
recovery_id = str(uuid.UUID("12345678-1234-4234-8234-123456789abc"))
|
||||
authorized = controller._run_registered_deploy(
|
||||
controller.SELF_UPGRADE_REPOSITORY,
|
||||
"c" * 40,
|
||||
self_upgrade_recovery_id=recovery_id,
|
||||
)
|
||||
assert authorized.returncode == 0
|
||||
_, kwargs = calls.pop()
|
||||
assert kwargs["env"]["APPOPS_SELF_UPGRADE_RECOVERY_ID"] == recovery_id
|
||||
assert "APPOPS_REGISTERED_LEGACY_REQUEST_ID" not in kwargs["env"]
|
||||
|
||||
invalid_bindings = [
|
||||
(controller.SELF_UPGRADE_REPOSITORY, "not-a-uuid", None),
|
||||
("jens/quiz-monitor", recovery_id, None),
|
||||
(controller.SELF_UPGRADE_REPOSITORY, recovery_id, recovery_id),
|
||||
]
|
||||
for repository, bad_recovery_id, registered_id in invalid_bindings:
|
||||
with pytest.raises(ValueError):
|
||||
controller._run_registered_deploy(
|
||||
repository,
|
||||
"c" * 40,
|
||||
self_upgrade_recovery_id=bad_recovery_id,
|
||||
registered_legacy_request_id=registered_id,
|
||||
)
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_preflight_projects_runtime_identity_without_environment_values(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
project = {
|
||||
"branch": "main",
|
||||
"gitea_repository_id": 9,
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"compose_project": "unraid-appops-gateway",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"environment": {
|
||||
"APPOPS_DATA_PATH": controller.SELF_UPGRADE_DATA_PATH,
|
||||
"APPOPS_PUBLISH_HOST": controller.SELF_UPGRADE_PUBLISH_HOST,
|
||||
},
|
||||
"verify_oci_revision": True,
|
||||
"validation": {
|
||||
"validation_gate_version": 1,
|
||||
"required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
},
|
||||
}
|
||||
private_environment = {"API_TOKEN": "preflight-secret-sentinel"}
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_load_projects",
|
||||
lambda: {controller.SELF_UPGRADE_REPOSITORY: project},
|
||||
)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda _revision: True)
|
||||
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True)
|
||||
monkeypatch.setattr(
|
||||
controller, "_fetch_gitea_validation", lambda *_args: ("pass", []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_self_upgrade_runtime",
|
||||
lambda: {
|
||||
"revision": "4" * 40,
|
||||
"imageId": "sha256:" + "5" * 64,
|
||||
"containerId": "6" * 64,
|
||||
"imageRef": controller.SELF_UPGRADE_PRODUCTION_IMAGE_REF,
|
||||
"sourceTree": "7" * 40,
|
||||
"runtimeFingerprint": "8" * 64,
|
||||
"runtimeEnvironment": private_environment,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller.shutil,
|
||||
"disk_usage",
|
||||
lambda _path: type("Usage", (), {"free": 50 * 1024**3})(),
|
||||
)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda _revision: True)
|
||||
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True)
|
||||
monkeypatch.setattr(
|
||||
controller, "_fetch_gitea_validation", lambda *_args: ("pass", []),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller, "_validation_policy",
|
||||
lambda _repository, _project: {
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_a, **_k: True)
|
||||
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
|
||||
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
serialized = json.dumps(result)
|
||||
assert result["status"] == "preflight_ok"
|
||||
assert "runtimeEnvironment" not in result
|
||||
assert "API_TOKEN" not in serialized
|
||||
assert "preflight-secret-sentinel" not in serialized
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
body = json.dumps({
|
||||
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": "c" * 40,
|
||||
"reason": "preflight boundary test",
|
||||
"expectedCurrentRevision": None,
|
||||
}).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
f"http://127.0.0.1:{server.server_port}{controller.SELF_UPGRADE_PREFLIGHT_PATH}",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
response_body = response.read()
|
||||
assert response.status == 200
|
||||
response_payload = json.loads(response_body)
|
||||
serialized_response = response_body.decode("utf-8")
|
||||
assert response_payload["status"] == "preflight_ok"
|
||||
assert "runtimeEnvironment" not in response_payload
|
||||
assert "API_TOKEN" not in serialized_response
|
||||
assert "preflight-secret-sentinel" not in serialized_response
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def test_public_recovery_status_projects_away_private_environment(controller, monkeypatch):
|
||||
secret = "public-status-secret-sentinel"
|
||||
record = _recovery_delivery(controller, secret=secret)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_delivery_record", lambda _request_id: record)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_a, **_k: True)
|
||||
monkeypatch.setattr(controller, "_read_inbox_record", lambda _path: None)
|
||||
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
request = urllib.request.Request(
|
||||
f"http://127.0.0.1:{server.server_port}"
|
||||
f"{controller.SELF_UPGRADE_STATUS_PREFIX}{'a' * 64}",
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
status = response.status
|
||||
body = response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
status = error.code
|
||||
body = error.read()
|
||||
assert status == 200, body.decode("utf-8", errors="replace")
|
||||
view = json.loads(body)
|
||||
assert view["repository"] == controller.SELF_UPGRADE_REPOSITORY
|
||||
assert view["revision"] == "f" * 40
|
||||
assert view["attemptType"] == "self-upgrade-recovery"
|
||||
assert view["recoveryOf"] == "b" * 64
|
||||
assert not {
|
||||
"recovery_id", "recovery_of_sha256", "idempotency_key_sha256",
|
||||
"preflight_sha256", "body_sha256", "recovery_hmac",
|
||||
} & view.keys()
|
||||
assert "runtimeEnvironment" not in view
|
||||
assert "privateCredential" not in view
|
||||
assert "recovery_hmac" not in view
|
||||
assert "idempotency_key_sha256" not in view
|
||||
assert secret.encode() not in body
|
||||
assert "API_TOKEN" not in body.decode("utf-8")
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def test_private_recovery_journal_is_0600_and_hmac_tampering_fails_closed(
|
||||
controller, monkeypatch, tmp_path,
|
||||
):
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller recovery store requires root-owned POSIX files")
|
||||
recovery_root = tmp_path / "recovery-store"
|
||||
recovery_root.mkdir(mode=0o700)
|
||||
recovery_root.chmod(0o700)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root)
|
||||
secret = b"fixed-self-upgrade-recovery-test-key-0123456789"
|
||||
monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret)
|
||||
recovery_id = str(uuid.UUID("12345678-1234-4234-8234-123456789abc"))
|
||||
payload = _private_recovery_payload(controller, recovery_id)
|
||||
|
||||
controller._self_upgrade_write_private_journal(recovery_id, payload, secret)
|
||||
marker = recovery_root / f"{recovery_id}.json"
|
||||
assert stat.S_IMODE(marker.stat().st_mode) == 0o600
|
||||
assert controller._self_upgrade_recovery_journal(recovery_id) == payload
|
||||
|
||||
envelope = json.loads(marker.read_text(encoding="utf-8"))
|
||||
envelope["payload"]["runtimeEnvironment"]["API_TOKEN"] = "changed-secret"
|
||||
marker.write_text(json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
marker.chmod(0o600)
|
||||
assert controller._self_upgrade_recovery_journal(recovery_id) is None
|
||||
|
||||
envelope = controller._self_upgrade_recovery_envelope(payload, secret)
|
||||
marker.write_text(json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n")
|
||||
marker.chmod(0o644)
|
||||
assert controller._self_upgrade_recovery_journal(recovery_id) is None
|
||||
|
||||
|
||||
def test_recovery_request_is_immutable_and_idempotent_for_one_key(
|
||||
controller, monkeypatch, tmp_path,
|
||||
):
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller recovery store requires root-owned POSIX files")
|
||||
|
||||
recovery_root = tmp_path / "recovery-store"
|
||||
recovery_root.mkdir(mode=0o700)
|
||||
recovery_root.chmod(0o700)
|
||||
request_root = recovery_root / "requests"
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", request_root)
|
||||
|
||||
secret = b"fixed-self-upgrade-recovery-test-key-0123456789"
|
||||
monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret)
|
||||
revision = "c" * 40
|
||||
semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
|
||||
original_path = tmp_path / "original-failed-delivery.json"
|
||||
original_raw = b'{"immutable":"original failed delivery"}\n'
|
||||
original_path.write_bytes(original_raw)
|
||||
original = {"revision": revision}
|
||||
original_info = (original_path, original_raw, original)
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_self_upgrade_original_failed_delivery",
|
||||
lambda *_args: original_info,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller, "_self_upgrade_delivery_revision", lambda _semantic: revision,
|
||||
)
|
||||
|
||||
expected_preflight = "3" * 64
|
||||
expected_current_revision = controller.SELF_UPGRADE_OLD_REVISION
|
||||
expected_container_id = "5" * 64
|
||||
expected_image_id = "sha256:" + "6" * 64
|
||||
expected_source_tree = "7" * 40
|
||||
expected_runtime_fingerprint = "8" * 64
|
||||
private_environment = {"API_TOKEN": "journal-secret-sentinel"}
|
||||
fresh = {
|
||||
"status": "preflight_ok",
|
||||
"preflightSha256": expected_preflight,
|
||||
"currentRevision": expected_current_revision,
|
||||
"currentContainerId": expected_container_id,
|
||||
"currentImageId": expected_image_id,
|
||||
"currentSourceTree": expected_source_tree,
|
||||
"currentRuntimeFingerprint": expected_runtime_fingerprint,
|
||||
}
|
||||
bindings = {
|
||||
"currentRevision": expected_current_revision,
|
||||
"currentContainerId": expected_container_id,
|
||||
"currentImageId": expected_image_id,
|
||||
"currentSourceTree": expected_source_tree,
|
||||
"runtimeFingerprint": expected_runtime_fingerprint,
|
||||
"registryFingerprint": "9" * 64,
|
||||
"sourceFingerprint": "a" * 64,
|
||||
"stateSha256": "b" * 64,
|
||||
"runtimeEnvironment": private_environment,
|
||||
}
|
||||
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: fresh)
|
||||
monkeypatch.setattr(
|
||||
controller, "_self_upgrade_recovery_current_bindings", lambda: bindings,
|
||||
)
|
||||
|
||||
policy = {
|
||||
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"validation_gate_version": 1,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
project = {"slug": "unraid-appops-gateway"}
|
||||
monkeypatch.setattr(
|
||||
controller, "_load_projects",
|
||||
lambda: {controller.SELF_UPGRADE_REPOSITORY: project},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_validation_policy",
|
||||
lambda _repository, _project: policy,
|
||||
)
|
||||
|
||||
original_sha = controller.hashlib.sha256(original_raw).hexdigest()
|
||||
idempotency_key_sha = "2" * 64
|
||||
reason_sha = "4" * 64
|
||||
first = controller._self_upgrade_recovery_request_status(
|
||||
semantic,
|
||||
original_sha,
|
||||
idempotency_key_sha,
|
||||
expected_preflight,
|
||||
expected_current_revision,
|
||||
reason_sha,
|
||||
policy,
|
||||
)
|
||||
assert first == ("awaiting_validation", semantic)
|
||||
assert original_path.read_bytes() == original_raw
|
||||
|
||||
def store_snapshot() -> dict[str, bytes]:
|
||||
return {
|
||||
path.relative_to(recovery_root).as_posix(): path.read_bytes()
|
||||
for path in sorted(recovery_root.rglob("*.json"))
|
||||
}
|
||||
|
||||
accepted_bytes = store_snapshot()
|
||||
assert accepted_bytes
|
||||
assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(request_root.stat().st_mode) == 0o700
|
||||
assert all(
|
||||
stat.S_IMODE(path.stat().st_mode) == 0o600
|
||||
for path in recovery_root.rglob("*.json")
|
||||
)
|
||||
# The secret is intentionally present only in the root-owned HMAC journal.
|
||||
recovery_id = str(uuid.UUID(bytes=bytes.fromhex(idempotency_key_sha)[:16], version=4))
|
||||
journal_bytes = accepted_bytes[f"{recovery_id}.json"]
|
||||
assert b"journal-secret-sentinel" in journal_bytes
|
||||
assert b"journal-secret-sentinel" not in accepted_bytes[
|
||||
f"{semantic}.delivery.json"
|
||||
]
|
||||
assert b"journal-secret-sentinel" not in accepted_bytes[
|
||||
f"requests/{idempotency_key_sha}.json"
|
||||
]
|
||||
|
||||
same_key = controller._self_upgrade_recovery_request_status(
|
||||
semantic,
|
||||
original_sha,
|
||||
idempotency_key_sha,
|
||||
expected_preflight,
|
||||
expected_current_revision,
|
||||
reason_sha,
|
||||
policy,
|
||||
)
|
||||
assert same_key == ("awaiting_validation", semantic)
|
||||
assert store_snapshot() == accepted_bytes
|
||||
assert original_path.read_bytes() == original_raw
|
||||
|
||||
different_key = controller._self_upgrade_recovery_request_status(
|
||||
semantic,
|
||||
original_sha,
|
||||
"d" * 64,
|
||||
expected_preflight,
|
||||
expected_current_revision,
|
||||
reason_sha,
|
||||
policy,
|
||||
)
|
||||
assert different_key == ("conflict", semantic)
|
||||
assert store_snapshot() == accepted_bytes
|
||||
assert original_path.read_bytes() == original_raw
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"malformed",
|
||||
[b"{", b"[]", b'{"duplicate":1,"duplicate":2}'],
|
||||
)
|
||||
def test_legacy_json_parser_rejects_malformed_or_duplicate_records(
|
||||
controller, monkeypatch, malformed,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_self_upgrade_read_legacy_cache_file",
|
||||
lambda *_args, **_kwargs: malformed,
|
||||
)
|
||||
with pytest.raises((OSError, ValueError)):
|
||||
controller._self_upgrade_read_legacy_cache_json(
|
||||
Path("/mnt/cache/appdata/gitea-deploy-runners/state/webhook-inbox")
|
||||
/ ("a" * 64 + ".json"),
|
||||
maximum_bytes=65536,
|
||||
allowed_modes=frozenset({0o600}),
|
||||
)
|
||||
|
||||
|
||||
def test_original_failed_delivery_rejects_modified_legacy_record(controller, monkeypatch):
|
||||
revision = "c" * 40
|
||||
semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
|
||||
policy = {
|
||||
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"validation_gate_version": 1,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800,
|
||||
"poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
}
|
||||
validation = controller._validation_snapshot(policy, 1)
|
||||
validation["last_result"] = "pass"
|
||||
validation["observed"] = [{
|
||||
"context": "Managed validation / full (push)",
|
||||
"status": "success",
|
||||
}]
|
||||
record = {
|
||||
"schema": controller._WEBHOOK_SCHEMA,
|
||||
"semantic_sha256": semantic,
|
||||
"gitea_repository_id": 9,
|
||||
"ref": "refs/heads/main",
|
||||
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision,
|
||||
"status": "failed",
|
||||
"reason": "deployment_failed",
|
||||
"source": "gitea_webhook",
|
||||
"body_sha256": "1" * 64,
|
||||
"validation": validation,
|
||||
"audit": [
|
||||
{"stage": "validation_passed"},
|
||||
{"stage": "deployment_started"},
|
||||
{"stage": "deployment_failed", "reason": "deployment_failed"},
|
||||
],
|
||||
}
|
||||
raw = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_self_upgrade_read_legacy_cache_json",
|
||||
lambda *_args, **_kwargs: (raw, record),
|
||||
)
|
||||
assert controller._self_upgrade_original_failed_delivery(
|
||||
semantic, revision, policy,
|
||||
) == (controller.WEBHOOK_INBOX_ROOT / f"{semantic}.json", raw, record)
|
||||
|
||||
tampered = {**record, "revision": "d" * 40}
|
||||
monkeypatch.setattr(
|
||||
controller,
|
||||
"_self_upgrade_read_legacy_cache_json",
|
||||
lambda *_args, **_kwargs: (raw, tampered),
|
||||
)
|
||||
assert controller._self_upgrade_original_failed_delivery(
|
||||
semantic, revision, policy,
|
||||
) is None
|
||||
@@ -0,0 +1,815 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
||||
RECOVERY_ID = "12345678-1234-4234-8234-123456789abc"
|
||||
REVISION = "c" * 40
|
||||
TREE = "d" * 40
|
||||
IMAGE_ID = "sha256:" + "e" * 64
|
||||
CONTAINER_ID = "f" * 64
|
||||
BUILT_AT = "2026-10-07T00:00:00.000Z"
|
||||
SECRET_SENTINEL = "model-private-secret$literal$$value"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controller(monkeypatch, tmp_path):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_self_upgrade_runtime_model_test",
|
||||
CONTROLLER / "status_server.py",
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
recovery_root = tmp_path / "recovery"
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root)
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", recovery_root / "requests")
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_MODEL_ROOT", recovery_root / "models")
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_TRANSACTION_ROOT", recovery_root / "transactions")
|
||||
secret = b"runtime-model-test-secret-key-0123456789"
|
||||
monkeypatch.setattr(module, "_self_upgrade_secret", lambda: secret)
|
||||
return module, recovery_root, secret
|
||||
|
||||
|
||||
def _environment(revision: str = REVISION, tree: str = TREE) -> dict[str, str]:
|
||||
return {
|
||||
"APPOPS_SOURCE_REPOSITORY": "jens/unraid-appops-gateway",
|
||||
"APPOPS_SOURCE_REVISION": revision,
|
||||
"APPOPS_SOURCE_TREE": tree,
|
||||
"APPOPS_BUILD_ID": revision[:12],
|
||||
"APPOPS_BUILD_DATE": BUILT_AT,
|
||||
"KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna",
|
||||
"OPERATOR_TOKEN": SECRET_SENTINEL,
|
||||
}
|
||||
|
||||
|
||||
def _model(revision: str = REVISION, tree: str = TREE) -> dict[str, object]:
|
||||
return {
|
||||
"name": "unraid-appops-gateway",
|
||||
"services": {
|
||||
"appops-gateway": {
|
||||
"image": "unraid-appops-gateway:production",
|
||||
"container_name": "Unraid-AppOps-Gateway",
|
||||
"environment": _environment(revision, tree),
|
||||
"ports": [{"target": 1216, "published": "1216", "host_ip": "198.18.10.150"}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _legacy_environment() -> dict[str, str]:
|
||||
return {
|
||||
**{f"LEGACY_SETTING_{index:02d}": f"fixture-value-{index:02d}" for index in range(43)},
|
||||
"KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna",
|
||||
"APPOPS_FRONTEND_ORIGIN": "https://apps.example.test",
|
||||
"OPERATOR_TOKEN": SECRET_SENTINEL,
|
||||
}
|
||||
|
||||
|
||||
def _legacy_model(module, image_ref: str | None = None) -> dict[str, object]:
|
||||
return {
|
||||
"name": "unraid-appops-gateway",
|
||||
"services": {
|
||||
"appops-gateway": {
|
||||
"image": image_ref or module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"container_name": "Unraid-AppOps-Gateway",
|
||||
"environment": _legacy_environment(),
|
||||
"ports": [{
|
||||
"target": 1216, "published": "1216", "host_ip": "198.18.10.150",
|
||||
}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _mock_legacy_docker(monkeypatch, module):
|
||||
environment = _legacy_environment()
|
||||
raw_env = [f"{key}={value}" for key, value in environment.items()]
|
||||
raw_env.append(f"APPOPS_FRONTEND_ORIGIN={environment['APPOPS_FRONTEND_ORIGIN']}")
|
||||
image_labels = {
|
||||
"org.opencontainers.image.source": module.SELF_UPGRADE_REPOSITORY,
|
||||
"org.opencontainers.image.revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"org.opencontainers.image.source-tree": module.SELF_UPGRADE_OLD_TREE,
|
||||
"org.opencontainers.image.version": module.SELF_UPGRADE_OLD_REVISION[:12],
|
||||
"org.opencontainers.image.created": "2026-09-01T00:00:00.000Z",
|
||||
}
|
||||
container = {
|
||||
"Id": CONTAINER_ID,
|
||||
"Image": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Mounts": [
|
||||
{"Type": "bind", "Source": module.SELF_UPGRADE_DATA_PATH,
|
||||
"Destination": "/data", "RW": True},
|
||||
{"Type": "bind", "Source": module.SELF_UPGRADE_MODELS_PATH,
|
||||
"Destination": module.SELF_UPGRADE_MODELS_TARGET, "RW": True},
|
||||
],
|
||||
"Config": {
|
||||
"Image": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"Hostname": "unraid-appops-gateway",
|
||||
"Env": raw_env,
|
||||
"Labels": {
|
||||
**image_labels,
|
||||
"com.docker.compose.project": "unraid-appops-gateway",
|
||||
"com.docker.compose.service": "appops-gateway",
|
||||
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
|
||||
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
|
||||
},
|
||||
},
|
||||
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
|
||||
"NetworkSettings": {
|
||||
"Ports": {"1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}]},
|
||||
"Networks": {
|
||||
"itworx-mcp-hub-npm": {},
|
||||
"unraid-appops-gateway_default": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
image = {"Id": module.SELF_UPGRADE_IMAGE_ID, "Config": {"Labels": image_labels}}
|
||||
|
||||
def fake_run(args, **_kwargs):
|
||||
if args[:3] == ["docker", "ps", "-a"]:
|
||||
return SimpleNamespace(stdout="Unraid-AppOps-Gateway\n")
|
||||
if args[:2] == ["docker", "inspect"]:
|
||||
return SimpleNamespace(stdout=json.dumps([container]))
|
||||
if args[:3] == ["docker", "image", "inspect"]:
|
||||
return SimpleNamespace(stdout=json.dumps([image]))
|
||||
raise AssertionError("unexpected Docker command category")
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {
|
||||
module.SELF_UPGRADE_REPOSITORY: {
|
||||
"containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True,
|
||||
},
|
||||
})
|
||||
monkeypatch.setattr(
|
||||
module, "_self_upgrade_read_legacy_cache_json",
|
||||
lambda *_args, **_kwargs: (b"{}", {"live_artifacts": {"containers": {}}}),
|
||||
)
|
||||
monkeypatch.setattr(module, "_self_upgrade_source_is_valid", lambda _source: True)
|
||||
monkeypatch.setattr(module, "_self_upgrade_source_fingerprint", lambda *_args: "2" * 64)
|
||||
return container
|
||||
|
||||
|
||||
def _receipt(module, revision: str = REVISION, tree: str = TREE) -> dict[str, object]:
|
||||
digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
|
||||
return {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": digest,
|
||||
"build": {
|
||||
"id": revision[:12],
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": digest,
|
||||
"builtAt": BUILT_AT,
|
||||
},
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"image_id": IMAGE_ID,
|
||||
"image_revision": revision,
|
||||
"image_source": module.SELF_UPGRADE_REPOSITORY,
|
||||
"image_tree": tree,
|
||||
"image_version": revision[:12],
|
||||
"image_created": BUILT_AT,
|
||||
},
|
||||
},
|
||||
"live": None,
|
||||
}
|
||||
|
||||
|
||||
def _binding(module, receipt, *, container_id: str = CONTAINER_ID) -> dict[str, object]:
|
||||
return {
|
||||
"containerId": container_id,
|
||||
"imageId": IMAGE_ID,
|
||||
"runtimeFingerprint": "1" * 64,
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json(receipt)).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def test_plain_compose_file_preserves_literal_dollars_under_private_attestation(
|
||||
controller,
|
||||
):
|
||||
module, recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
|
||||
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
||||
assert isinstance(pointer, dict)
|
||||
loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION)
|
||||
assert loaded is not None
|
||||
assert loaded["model"] == _model()
|
||||
assert loaded["pointer"] == pointer
|
||||
assert Path(loaded["path"]).name == f"{RECOVERY_ID}-{REVISION}.compose.json"
|
||||
|
||||
compose_path = Path(loaded["path"])
|
||||
attestation_path = compose_path.with_name(
|
||||
f"{RECOVERY_ID}-{REVISION}.model.json"
|
||||
)
|
||||
compose_raw = compose_path.read_bytes()
|
||||
attestation_raw = attestation_path.read_bytes()
|
||||
assert b"model-private-secret$$literal$$$$value" in compose_raw
|
||||
assert SECRET_SENTINEL.encode() not in compose_raw
|
||||
assert SECRET_SENTINEL.encode() not in attestation_raw
|
||||
assert b"OPERATOR_TOKEN" not in attestation_raw
|
||||
assert stat.S_IMODE(compose_path.stat().st_mode) == 0o600
|
||||
assert stat.S_IMODE(attestation_path.stat().st_mode) == 0o600
|
||||
assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE((recovery_root / "models").stat().st_mode) == 0o700
|
||||
assert module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) == pointer
|
||||
assert module._self_upgrade_store_compose_model(
|
||||
RECOVERY_ID, REVISION, _model(tree="a" * 40),
|
||||
) is None
|
||||
|
||||
compose_path.write_text(compose_path.read_text(encoding="utf-8").replace("gpt-6-luna", "gpt-5.5"))
|
||||
compose_path.chmod(0o600)
|
||||
assert module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) is None
|
||||
|
||||
|
||||
def test_live_model_records_are_content_addressed_and_activate_only_from_state(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
||||
assert isinstance(pointer, dict)
|
||||
receipt = _receipt(module)
|
||||
live_pointer = module._self_upgrade_store_live_model(pointer, _binding(module, receipt))
|
||||
assert isinstance(live_pointer, dict)
|
||||
first_live = module._self_upgrade_read_live_model_reference(live_pointer)
|
||||
assert first_live is not None
|
||||
assert first_live["model"] == _model()
|
||||
|
||||
fresh_container_id = "a" * 64
|
||||
restored_pointer = module._self_upgrade_store_live_model(
|
||||
pointer, _binding(module, receipt, container_id=fresh_container_id),
|
||||
)
|
||||
assert isinstance(restored_pointer, dict)
|
||||
assert restored_pointer["liveRecordSha256"] != live_pointer["liveRecordSha256"]
|
||||
assert module._self_upgrade_read_live_model_reference(live_pointer) == first_live
|
||||
restored = module._self_upgrade_read_live_model_reference(restored_pointer)
|
||||
assert restored is not None
|
||||
assert restored["live"]["containerId"] == fresh_container_id
|
||||
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "failed",
|
||||
"live_revision": REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"live_receipt": receipt,
|
||||
"live_artifacts": {
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": fresh_container_id,
|
||||
"image_id": IMAGE_ID,
|
||||
},
|
||||
},
|
||||
"appops_runtime_model": restored_pointer,
|
||||
},
|
||||
}
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
active = module._self_upgrade_active_compose_model(state, project=project)
|
||||
assert active is not None
|
||||
assert active["model"] == _model()
|
||||
runtime = {
|
||||
"revision": REVISION,
|
||||
"containerId": fresh_container_id,
|
||||
"imageId": IMAGE_ID,
|
||||
"runtimeFingerprint": "1" * 64,
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"runtimeEnvironment": _environment(),
|
||||
}
|
||||
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None
|
||||
runtime["runtimeEnvironment"] = {**_environment(), "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-5.5"}
|
||||
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is None
|
||||
state["live_status"] = "unhealthy"
|
||||
assert module._self_upgrade_active_compose_model(state, project=project) is None
|
||||
|
||||
|
||||
def test_candidate_binding_rechecks_exact_model_receipt_and_runtime(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
||||
assert isinstance(pointer, dict)
|
||||
receipt = _receipt(module)
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project})
|
||||
monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: {
|
||||
"revision": REVISION,
|
||||
"sourceRepository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"sourceTree": TREE,
|
||||
"imageId": IMAGE_ID,
|
||||
"containerId": CONTAINER_ID,
|
||||
"runtimeFingerprint": "1" * 64,
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"runtimeEnvironment": _environment(),
|
||||
})
|
||||
binding = module._self_upgrade_candidate_model_binding(pointer, receipt)
|
||||
assert binding == _binding(module, receipt)
|
||||
wrong_receipt = _receipt(module, tree="b" * 40)
|
||||
assert module._self_upgrade_candidate_model_binding(pointer, wrong_receipt) is None
|
||||
bad_pointer = {**pointer, "modelSha256": "0" * 64}
|
||||
assert module._self_upgrade_candidate_model_binding(bad_pointer, receipt) is None
|
||||
|
||||
|
||||
def test_fixed_555_model_preserves_exact_environment_and_duplicate_origin(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
journal = {
|
||||
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
}
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal)
|
||||
pointer = module._self_upgrade_store_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module),
|
||||
)
|
||||
assert isinstance(pointer, dict)
|
||||
loaded = module._self_upgrade_load_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION,
|
||||
)
|
||||
assert loaded is not None
|
||||
assert loaded["model"] == _legacy_model(module)
|
||||
changed = _legacy_model(module)
|
||||
changed["services"]["appops-gateway"]["environment"]["APPOPS_BUILD_ID"] = "untrusted"
|
||||
assert not module._self_upgrade_compose_model_valid(
|
||||
changed, module.SELF_UPGRADE_OLD_REVISION,
|
||||
)
|
||||
|
||||
container = _mock_legacy_docker(monkeypatch, module)
|
||||
initial_runtime = module._self_upgrade_runtime()
|
||||
assert initial_runtime is not None
|
||||
assert initial_runtime["runtimeEnvironment"] == _legacy_environment()
|
||||
assert len(container["Config"]["Env"]) == 47
|
||||
assert len(initial_runtime["runtimeEnvironment"]) == 46
|
||||
runtime = module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
)
|
||||
assert runtime is not None
|
||||
assert runtime["runtimeEnvironment"] == _legacy_environment()
|
||||
assert len(runtime["runtimeEnvironment"]) == 46
|
||||
assert len(container["Config"]["Env"]) == 47
|
||||
assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID
|
||||
assert runtime["containerId"] == container["Id"]
|
||||
binding = module._self_upgrade_legacy_model_binding(pointer)
|
||||
assert binding == {
|
||||
"containerId": CONTAINER_ID,
|
||||
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeFingerprint": runtime["runtimeFingerprint"],
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(),
|
||||
}
|
||||
|
||||
live_pointer = module._self_upgrade_store_live_model(pointer, binding)
|
||||
assert isinstance(live_pointer, dict)
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "rolled-back",
|
||||
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
},
|
||||
},
|
||||
"appops_runtime_model": live_pointer,
|
||||
},
|
||||
}
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
active = module._self_upgrade_active_compose_model(state, project=project)
|
||||
assert active is not None
|
||||
assert active["legacy"] is True
|
||||
assert active["model"] == _legacy_model(module)
|
||||
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None
|
||||
|
||||
container["State"]["Health"]["Status"] = "unhealthy"
|
||||
assert module._self_upgrade_runtime() is None
|
||||
assert module._self_upgrade_model_runtime_identity(pointer, module.SELF_UPGRADE_IMAGE_ID) is None
|
||||
recovery_identity = module._self_upgrade_model_runtime_identity(
|
||||
pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy=False,
|
||||
)
|
||||
assert isinstance(recovery_identity, dict)
|
||||
assert set(recovery_identity) == {
|
||||
"containerId", "imageId", "revision", "sourceRepository", "sourceTree",
|
||||
"runtimeFingerprint", "sourceFingerprint", "runtimeCasSha256",
|
||||
}
|
||||
assert module._self_upgrade_model_runtime_identity(
|
||||
pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy="false",
|
||||
) is None
|
||||
container["State"]["Health"]["Status"] = "healthy"
|
||||
|
||||
# The identical duplicate is accepted only for the exact immutable old
|
||||
# image; conflicting duplicates remain fail-closed.
|
||||
raw_env = container["Config"]["Env"]
|
||||
raw_env[-1] = "APPOPS_FRONTEND_ORIGIN=https://attacker.invalid"
|
||||
assert module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
) is None
|
||||
|
||||
|
||||
def test_fixed_555_production_reference_is_valid_only_under_signed_legacy_model(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
journal = {
|
||||
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
}
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal)
|
||||
model = _legacy_model(module, module.SELF_UPGRADE_PRODUCTION_IMAGE_REF)
|
||||
pointer = module._self_upgrade_store_compose_model(
|
||||
"42345678-1234-4234-8234-123456789abc",
|
||||
module.SELF_UPGRADE_OLD_REVISION,
|
||||
model,
|
||||
)
|
||||
assert isinstance(pointer, dict)
|
||||
container = _mock_legacy_docker(monkeypatch, module)
|
||||
# The pre-capture container can retain the old platform tag while the
|
||||
# signed effective rollback model uses the closed production tag.
|
||||
container["Config"]["Image"] = module.SELF_UPGRADE_OLD_IMAGE_REF
|
||||
initial_runtime = module._self_upgrade_runtime()
|
||||
assert initial_runtime is not None
|
||||
assert initial_runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF
|
||||
|
||||
runtime = module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
)
|
||||
assert isinstance(runtime, dict)
|
||||
assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID
|
||||
assert runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF
|
||||
binding = module._self_upgrade_legacy_model_binding(pointer)
|
||||
assert isinstance(binding, dict)
|
||||
reference = module._self_upgrade_store_live_model(pointer, binding)
|
||||
assert isinstance(reference, dict)
|
||||
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {
|
||||
module.SELF_UPGRADE_REPOSITORY: project,
|
||||
})
|
||||
snapshot = {
|
||||
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"legacy_provenance": True,
|
||||
"receipt": {},
|
||||
"appops_runtime_model": reference,
|
||||
"containers": [{
|
||||
"name": "Unraid-AppOps-Gateway",
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
|
||||
"restore_image_ref": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF,
|
||||
"was_running": True,
|
||||
}],
|
||||
}
|
||||
active_state = {
|
||||
"schema_version": 3,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "rolled-back",
|
||||
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": reference,
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_self_upgrade_read_legacy_cache_json",
|
||||
lambda *_args, **_kwargs: (b"{}", active_state),
|
||||
)
|
||||
assert module._self_upgrade_runtime() is not None
|
||||
verified = module._self_upgrade_verify_rollback_model(reference, snapshot)
|
||||
assert verified is not None
|
||||
assert verified["model"] == model
|
||||
assert verified["legacy"] is True
|
||||
|
||||
container["Config"]["Image"] = module.SELF_UPGRADE_PRODUCTION_IMAGE_REF
|
||||
production_model = module._self_upgrade_load_compose_model(
|
||||
pointer["recoveryId"], pointer["revision"],
|
||||
)
|
||||
assert production_model is not None
|
||||
container["Config"]["Labels"]["com.docker.compose.project.config_files"] = str(
|
||||
production_model["path"],
|
||||
)
|
||||
restored_runtime = module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
)
|
||||
assert isinstance(restored_runtime, dict)
|
||||
assert restored_runtime["imageRef"] == module.SELF_UPGRADE_PRODUCTION_IMAGE_REF
|
||||
|
||||
|
||||
def test_recovery_transaction_binds_authorized_555_preimage_and_holds_pending_state(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, recovery_root, secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller transaction store requires root-owned POSIX files")
|
||||
recovery_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
semantic = module._webhook_semantic_digest(9, "refs/heads/main", REVISION)
|
||||
original_record = {
|
||||
"schema": module._WEBHOOK_SCHEMA,
|
||||
"semantic_sha256": semantic,
|
||||
"revision": REVISION,
|
||||
"status": "failed",
|
||||
"reason": "deployment_failed",
|
||||
}
|
||||
original_raw = module._self_upgrade_canonical_json(original_record) + b"\n"
|
||||
original_sha = hashlib.sha256(original_raw).hexdigest()
|
||||
journal = {
|
||||
"schema": module._SELF_UPGRADE_RECOVERY_SCHEMA,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"semanticSha256": semantic,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": REVISION,
|
||||
"originalRawSha256": original_sha,
|
||||
"idempotencyKeySha256": "3" * 64,
|
||||
"preflightSha256": "7" * 64,
|
||||
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"runtimeFingerprint": "5" * 64,
|
||||
"registryFingerprint": "b" * 64,
|
||||
"currentContainerId": CONTAINER_ID,
|
||||
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"stateSha256": "a" * 64,
|
||||
"currentSourceTree": module.SELF_UPGRADE_OLD_TREE,
|
||||
"createdAt": 1,
|
||||
}
|
||||
monkeypatch.setattr(module, "_self_upgrade_model_transaction_origin_valid", lambda _payload: True)
|
||||
module._self_upgrade_write_private_journal(RECOVERY_ID, journal, secret)
|
||||
request_root = recovery_root / "requests"
|
||||
request_root.mkdir(mode=0o700)
|
||||
request_payload = {
|
||||
"schema": module._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA,
|
||||
"revision": REVISION,
|
||||
"preflightSha256": "7" * 64,
|
||||
"operationId": semantic,
|
||||
"reasonSha256": "8" * 64,
|
||||
"idempotencyKeySha256": "3" * 64,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"recoveryOfSha256": original_sha,
|
||||
"createdAt": 1,
|
||||
}
|
||||
request_record = {
|
||||
**request_payload,
|
||||
"hmac": module._self_upgrade_mac(
|
||||
secret, "recovery-request", module._self_upgrade_canonical_json(request_payload),
|
||||
),
|
||||
}
|
||||
module._self_upgrade_write_private_record(
|
||||
request_root / f"{'3' * 64}.json",
|
||||
module._self_upgrade_canonical_json(request_record) + b"\n",
|
||||
)
|
||||
inbox_root = recovery_root / "inbox"
|
||||
inbox_root.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", inbox_root)
|
||||
original_path = inbox_root / f"{semantic}.json"
|
||||
original_path.write_bytes(original_raw)
|
||||
original_path.chmod(0o600)
|
||||
recovery_delivery = {
|
||||
"status": "deploying",
|
||||
"recovery_id": RECOVERY_ID,
|
||||
"validation": {"last_result": "pass"},
|
||||
"audit": [
|
||||
{"stage": "validation_passed"},
|
||||
{"stage": "deployment_started"},
|
||||
],
|
||||
}
|
||||
module._self_upgrade_write_private_record(
|
||||
recovery_root / f"{semantic}.delivery.json",
|
||||
module._self_upgrade_canonical_json(recovery_delivery) + b"\n",
|
||||
)
|
||||
project = {"branch": "main", "gitea_repository_id": 9}
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project})
|
||||
monkeypatch.setattr(module, "_validation_policy", lambda *_args: {"repository": module.SELF_UPGRADE_REPOSITORY})
|
||||
monkeypatch.setattr(module, "_self_upgrade_original_failed_delivery", lambda *_args: (
|
||||
original_path, original_raw, original_record,
|
||||
))
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True)
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_preimage_matches", lambda _record: True)
|
||||
pointer = module._self_upgrade_store_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module),
|
||||
)
|
||||
assert isinstance(pointer, dict)
|
||||
monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: {
|
||||
"runtimeCasSha256": "6" * 64,
|
||||
"runtimeFingerprint": "5" * 64,
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"containerId": CONTAINER_ID,
|
||||
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
})
|
||||
|
||||
metadata = {
|
||||
"id": REVISION[:12], "repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": REVISION, "tree": TREE,
|
||||
"treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(),
|
||||
"builtAt": BUILT_AT,
|
||||
}
|
||||
snapshot = {
|
||||
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"legacy_provenance": True,
|
||||
"receipt": {},
|
||||
"compose_containers": ["Unraid-AppOps-Gateway"],
|
||||
"containers": [{
|
||||
"name": "Unraid-AppOps-Gateway", "container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
|
||||
"was_running": True,
|
||||
}],
|
||||
"captured_at_epoch": 1,
|
||||
"appops_runtime_model": pointer,
|
||||
}
|
||||
now = 100
|
||||
transaction_id = "22345678-1234-4234-8234-123456789abc"
|
||||
request = {
|
||||
"schema": module._SELF_UPGRADE_MODEL_TRANSACTION_SCHEMA,
|
||||
"transactionId": transaction_id,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"kind": "recovery",
|
||||
"revision": REVISION,
|
||||
"metadata": metadata,
|
||||
"expectedStateSha256": "a" * 64,
|
||||
"registryFingerprint": "b" * 64,
|
||||
"preMutationRuntimeSha256": "6" * 64,
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"priorModelPointer": pointer,
|
||||
"priorSnapshot": snapshot,
|
||||
"candidateModelPointer": None,
|
||||
"buildContexts": {},
|
||||
"candidateImageId": None,
|
||||
"verifiedLiveReference": None,
|
||||
"verifiedReceipt": None,
|
||||
"phase": "prepared",
|
||||
"createdAt": now,
|
||||
"updatedAt": now,
|
||||
}
|
||||
assert module._self_upgrade_recovery_journal(RECOVERY_ID) == journal
|
||||
assert module._self_upgrade_recovery_request_payload(request_record) == request_payload
|
||||
assert module._self_upgrade_original_failed_delivery(semantic, REVISION, {"repository": module.SELF_UPGRADE_REPOSITORY}) == (
|
||||
original_path, original_raw, original_record,
|
||||
)
|
||||
assert module._self_upgrade_recovery_preimage_matches(recovery_delivery) is True
|
||||
read_legacy = module._self_upgrade_read_legacy_cache_json
|
||||
state_root = recovery_root / "state"
|
||||
state_root.mkdir(mode=0o700)
|
||||
state_path = state_root / "unraid-appops-gateway.json"
|
||||
restored_state_raw: bytes | None = None
|
||||
restored_state: dict[str, object] | None = None
|
||||
|
||||
def read_test_inbox(path, **kwargs):
|
||||
if path == original_path:
|
||||
assert path.read_bytes() == original_raw
|
||||
return original_raw, original_record
|
||||
if path == state_path and restored_state_raw is not None and restored_state is not None:
|
||||
assert path.read_bytes() == restored_state_raw
|
||||
return restored_state_raw, restored_state
|
||||
return read_legacy(path, **kwargs)
|
||||
|
||||
# The production reader pins the real legacy cache directory profile. For
|
||||
# this temp-root test, stub only that fixed-path adapter while preserving
|
||||
# the actual HMAC request, journal, authorization, and transaction checks.
|
||||
monkeypatch.setattr(module, "_self_upgrade_read_legacy_cache_json", read_test_inbox)
|
||||
actual_authorization = module._self_upgrade_recovery_authorization(RECOVERY_ID)
|
||||
assert actual_authorization is not None
|
||||
preimage = module._self_upgrade_transaction_live_preimage(RECOVERY_ID, "recovery")
|
||||
assert preimage is not None
|
||||
assert preimage["currentRevision"] == module.SELF_UPGRADE_OLD_REVISION
|
||||
assert preimage["runtimeEnvironment"] == _legacy_environment()
|
||||
assert module._self_upgrade_load_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION,
|
||||
)["model"]["services"]["appops-gateway"]["environment"] == journal["runtimeEnvironment"]
|
||||
created = module._self_upgrade_begin_model_transaction(request)
|
||||
assert isinstance(created, dict)
|
||||
assert created["payload"]["priorSnapshot"]["appops_runtime_model"] == pointer
|
||||
assert stat.S_IMODE((recovery_root / "transactions" / f"{transaction_id}.json").stat().st_mode) == 0o600
|
||||
assert module._self_upgrade_load_model_transaction(transaction_id) == created
|
||||
|
||||
assert module._self_upgrade_update_model_transaction(
|
||||
transaction_id, "0" * 64, "recovery_required",
|
||||
) is None
|
||||
held = module._self_upgrade_update_model_transaction(
|
||||
transaction_id, created["sha256"], "recovery_required",
|
||||
)
|
||||
assert isinstance(held, dict)
|
||||
assert held["payload"]["phase"] == "recovery_required"
|
||||
assert module._self_upgrade_pending_model_transaction() == held
|
||||
|
||||
rollback_binding = {
|
||||
"containerId": CONTAINER_ID,
|
||||
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeFingerprint": "5" * 64,
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(),
|
||||
}
|
||||
restored_reference = module._self_upgrade_store_live_model(pointer, rollback_binding)
|
||||
assert isinstance(restored_reference, dict)
|
||||
restored_state = {
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "failed",
|
||||
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": restored_reference,
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
restored_state_raw = module._self_upgrade_canonical_json(restored_state) + b"\n"
|
||||
state_path.write_bytes(restored_state_raw)
|
||||
state_path.chmod(0o600)
|
||||
monkeypatch.setattr(module, "STATE_ROOT", state_root)
|
||||
|
||||
rollback_transaction_id = "32345678-1234-4234-8234-123456789abc"
|
||||
rollback_payload = dict(created["payload"])
|
||||
rollback_payload.update({
|
||||
"transactionId": rollback_transaction_id,
|
||||
"phase": "rolling_back",
|
||||
"failureCode": "rollback_started",
|
||||
"updatedAt": now + 1,
|
||||
})
|
||||
transaction_path = module._self_upgrade_transaction_path(rollback_transaction_id)
|
||||
module._self_upgrade_write_private_record(
|
||||
transaction_path,
|
||||
module._self_upgrade_canonical_json(
|
||||
module._self_upgrade_model_transaction_envelope(rollback_payload, secret),
|
||||
) + b"\n",
|
||||
)
|
||||
rollback_before = module._self_upgrade_load_model_transaction(rollback_transaction_id)
|
||||
assert isinstance(rollback_before, dict)
|
||||
rolled_back = module._self_upgrade_update_model_transaction(
|
||||
rollback_transaction_id,
|
||||
rollback_before["sha256"],
|
||||
"rolled_back",
|
||||
{"restoredLiveReference": restored_reference},
|
||||
)
|
||||
assert isinstance(rolled_back, dict)
|
||||
assert rolled_back["payload"]["phase"] == "rolled_back"
|
||||
assert rolled_back["payload"]["restoredLiveReference"] == restored_reference
|
||||
assert rolled_back["payload"]["restoredStateSha256"] == hashlib.sha256(restored_state_raw).hexdigest()
|
||||
assert module._self_upgrade_pending_model_transaction() == held
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
@@ -110,6 +111,235 @@ def run(docker, git):
|
||||
)
|
||||
|
||||
|
||||
_MOUNT_ORDERS = {
|
||||
("name", 1): (3, 0, 4, 1, 2),
|
||||
("name", 2): (4, 2, 0, 3, 1),
|
||||
("name", 3): (1, 3, 0, 4, 2),
|
||||
("id", 1): (2, 4, 1, 0, 3),
|
||||
("id", 2): (4, 1, 3, 2, 0),
|
||||
}
|
||||
|
||||
|
||||
def configure_reinspection(
|
||||
docker,
|
||||
*,
|
||||
drift=None,
|
||||
drift_reference="id",
|
||||
drift_pair=1,
|
||||
identifier=FAILED_ID,
|
||||
):
|
||||
"""Model Docker returning the same inspect with independent mount ordering."""
|
||||
original_inspect = docker.inspect_container
|
||||
counts = {"name": 0, "id": 0}
|
||||
reads = []
|
||||
|
||||
def inspect(reference):
|
||||
if reference == bootstrap.MODERN_HELPER_NAME:
|
||||
scope = "name"
|
||||
elif reference == identifier:
|
||||
scope = "id"
|
||||
else:
|
||||
return original_inspect(reference)
|
||||
|
||||
counts[scope] += 1
|
||||
reads.append((scope, counts[scope]))
|
||||
if docker.existing_child is None:
|
||||
return None
|
||||
|
||||
snapshot = copy.deepcopy(docker.existing_child)
|
||||
order = _MOUNT_ORDERS.get((scope, counts[scope]))
|
||||
if order is not None:
|
||||
snapshot["Mounts"] = [snapshot["Mounts"][index] for index in order]
|
||||
|
||||
pair_number = (
|
||||
1
|
||||
if (scope == "name" and counts[scope] == 2)
|
||||
or (scope == "id" and counts[scope] == 1)
|
||||
else 2
|
||||
if (scope == "name" and counts[scope] == 3)
|
||||
or (scope == "id" and counts[scope] == 2)
|
||||
else None
|
||||
)
|
||||
is_drift_read = pair_number == drift_pair and (
|
||||
drift_reference == "both" or scope == drift_reference
|
||||
)
|
||||
if drift is not None and is_drift_read:
|
||||
mount = snapshot["Mounts"][0]
|
||||
if drift == "mount_type":
|
||||
mount["Type"] = "volume"
|
||||
elif drift == "mount_source":
|
||||
mount["Source"] = "/tmp/changed-source"
|
||||
elif drift == "mount_destination":
|
||||
mount["Destination"] = "/tmp/changed-destination"
|
||||
elif drift == "mount_rw":
|
||||
mount["RW"] = not mount["RW"]
|
||||
elif drift == "mount_propagation":
|
||||
mount["Propagation"] = "shared"
|
||||
elif drift == "mount_name":
|
||||
mount["Name"] = "unexpected-volume"
|
||||
elif drift == "missing_mount":
|
||||
snapshot["Mounts"].pop()
|
||||
elif drift == "extra_mount":
|
||||
extra = copy.deepcopy(mount)
|
||||
extra["Destination"] = "/unexpected"
|
||||
snapshot["Mounts"].append(extra)
|
||||
elif drift == "duplicate_destination":
|
||||
snapshot["Mounts"].append(copy.deepcopy(mount))
|
||||
elif drift == "metadata_field":
|
||||
cache_mount = next(
|
||||
item
|
||||
for item in snapshot["Mounts"]
|
||||
if item["Destination"] == "/mnt/cache"
|
||||
)
|
||||
cache_mount["Mode"] = "ro"
|
||||
elif drift == "environment_order":
|
||||
snapshot["Config"]["Env"] = list(reversed(snapshot["Config"]["Env"]))
|
||||
elif drift == "binds_order":
|
||||
snapshot["HostConfig"]["Binds"] = list(
|
||||
reversed(snapshot["HostConfig"]["Binds"])
|
||||
)
|
||||
elif drift == "state_drift":
|
||||
snapshot["State"]["ExitCode"] = 2
|
||||
else:
|
||||
raise AssertionError(f"unhandled test drift: {drift}")
|
||||
return snapshot
|
||||
|
||||
docker.inspect_container = inspect
|
||||
return counts, reads
|
||||
|
||||
|
||||
def historical_inputs():
|
||||
helper_source = (
|
||||
original.ROOT / "backend/app/services/controller_artifact_helper_spec.py"
|
||||
).read_text(encoding="utf-8")
|
||||
child = original._retired_failed_child(helper_source)
|
||||
docker = original.FakeDocker(
|
||||
original._legacy_helper_spec(), existing_child=child
|
||||
)
|
||||
docker.containers["gitea-deploy-control"] = original._modern_controller_source_inspect(
|
||||
bootstrap.MODERN_CONTROLLER_REVISION
|
||||
)
|
||||
git = original.FakeProtectedGit(helper_source)
|
||||
return docker, git
|
||||
|
||||
|
||||
def run_historical(docker, git):
|
||||
return original._run_bootstrap(
|
||||
docker,
|
||||
git,
|
||||
health_sequence=[
|
||||
original._modern_baseline_health(),
|
||||
original._modern_baseline_health(),
|
||||
original._modern_baseline_health(),
|
||||
original._modern_baseline_health(),
|
||||
original._health(),
|
||||
original._health(),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_failed_child_inspect_ignores_mount_order_across_name_and_id_reads():
|
||||
docker, git, _child = inputs()
|
||||
counts, reads = configure_reinspection(docker)
|
||||
|
||||
result = run(docker, git)
|
||||
|
||||
assert result["status"] == "promoted"
|
||||
assert counts == {"name": 4, "id": 2}
|
||||
assert reads[:5] == [
|
||||
("name", 1),
|
||||
("name", 2),
|
||||
("id", 1),
|
||||
("name", 3),
|
||||
("id", 2),
|
||||
]
|
||||
assert docker.removed == [FAILED_ID]
|
||||
assert len(docker.created) == len(docker.started) == 1
|
||||
|
||||
|
||||
def test_historical_failed_child_mount_order_is_ignored_across_name_and_id_reads():
|
||||
docker, git = historical_inputs()
|
||||
counts, reads = configure_reinspection(
|
||||
docker, identifier=original.RETIRED_HELPER_ID
|
||||
)
|
||||
|
||||
result = run_historical(docker, git)
|
||||
|
||||
assert result["status"] == "promoted"
|
||||
assert counts == {"name": 4, "id": 2}
|
||||
assert reads[:5] == [
|
||||
("name", 1),
|
||||
("name", 2),
|
||||
("id", 1),
|
||||
("name", 3),
|
||||
("id", 2),
|
||||
]
|
||||
assert docker.removed == [original.RETIRED_HELPER_ID]
|
||||
assert len(docker.created) == len(docker.started) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drift_pair", [1, 2])
|
||||
def test_historical_failed_child_rejects_mount_metadata_change_from_initial_snapshot(
|
||||
drift_pair,
|
||||
):
|
||||
docker, git = historical_inputs()
|
||||
configure_reinspection(
|
||||
docker,
|
||||
drift="metadata_field",
|
||||
drift_reference="both",
|
||||
drift_pair=drift_pair,
|
||||
identifier=original.RETIRED_HELPER_ID,
|
||||
)
|
||||
|
||||
with pytest.raises(bootstrap.BootstrapError):
|
||||
run_historical(docker, git)
|
||||
|
||||
assert docker.removed == docker.created == docker.started == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"drift",
|
||||
[
|
||||
"mount_type",
|
||||
"mount_source",
|
||||
"mount_destination",
|
||||
"mount_rw",
|
||||
"mount_propagation",
|
||||
"mount_name",
|
||||
"missing_mount",
|
||||
"extra_mount",
|
||||
"duplicate_destination",
|
||||
"metadata_field",
|
||||
"environment_order",
|
||||
"binds_order",
|
||||
"state_drift",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("drift_reference", ["name", "id"])
|
||||
def test_failed_child_reinspection_rejects_any_non_mount_order_drift(
|
||||
drift, drift_reference
|
||||
):
|
||||
docker, git, _child = inputs()
|
||||
# Make the environment order check meaningful while keeping the initial
|
||||
# image and child Config.Env identical.
|
||||
docker.existing_child["Config"]["Env"] = ["PATH=/usr/bin", "LANG=C"]
|
||||
docker.images[bootstrap.LEGACY_CONTROLLER_IMAGE_ID]["Config"]["Env"] = [
|
||||
"PATH=/usr/bin",
|
||||
"LANG=C",
|
||||
]
|
||||
_counts, _reads = configure_reinspection(
|
||||
docker, drift=drift, drift_reference=drift_reference
|
||||
)
|
||||
|
||||
if drift == "binds_order":
|
||||
assert len(docker.existing_child["HostConfig"]["Binds"]) > 1
|
||||
|
||||
with pytest.raises(bootstrap.BootstrapError):
|
||||
run(docker, git)
|
||||
|
||||
assert docker.removed == docker.created == docker.started == []
|
||||
|
||||
|
||||
def test_noncurrent_protected_ancestor_failure_releases_only_its_verified_name():
|
||||
docker, git, child = inputs()
|
||||
result = run(docker, git)
|
||||
@@ -274,13 +504,20 @@ def test_delete_adapter_rejects_noncanonical_full_container_identity(
|
||||
def test_git_ancestor_proof_reads_original_protected_objects(
|
||||
tmp_path, monkeypatch, scenario
|
||||
):
|
||||
def git(*args):
|
||||
def git(*args, allow_replace_objects=False):
|
||||
environment = None
|
||||
if allow_replace_objects:
|
||||
# Only the attack demonstration opts out of the inherited guard.
|
||||
# The production proof still reads original protected Git objects.
|
||||
environment = dict(os.environ)
|
||||
environment.pop("GIT_NO_REPLACE_OBJECTS", None)
|
||||
return (
|
||||
subprocess.run(
|
||||
["git", "-C", str(tmp_path), *args],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=environment,
|
||||
)
|
||||
.stdout.decode()
|
||||
.strip()
|
||||
@@ -302,7 +539,7 @@ def test_git_ancestor_proof_reads_original_protected_objects(
|
||||
proof = bootstrap.ProtectedGitRepository()
|
||||
if scenario == "replace_ref":
|
||||
git("replace", old, current)
|
||||
assert git("rev-parse", f"{old}^{{tree}}") != tree
|
||||
assert git("rev-parse", f"{old}^{{tree}}", allow_replace_objects=True) != tree
|
||||
if scenario == "unrelated":
|
||||
git("checkout", "-qb", "side", old)
|
||||
path.write_text("unrelated child")
|
||||
|
||||
@@ -0,0 +1,633 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import AuditEvent, Secret
|
||||
from backend.app.services import workmesh_secret_import as importer
|
||||
from backend.app.services.secret_store import get_secret, set_secret
|
||||
|
||||
REVISION = "a" * 40
|
||||
CREDENTIAL = "W" * 48
|
||||
PREFLIGHT_SHA = "b" * 64
|
||||
BACKUP_SHA = "c" * 64
|
||||
SECRET_REF = "WORKMESH_DEPLOYMENT_BROKER"
|
||||
TARGET = "jens/workmesh"
|
||||
CHECKS = {
|
||||
"exactMain": "passed",
|
||||
"pushValidation": "passed",
|
||||
"controllerPreflight": "passed",
|
||||
"dataBackup": "passed",
|
||||
}
|
||||
|
||||
|
||||
def _valid_preflight(revision: str = REVISION) -> dict[str, object]:
|
||||
return {
|
||||
"apiVersion": "itworx.workmesh-promotion/v2",
|
||||
"status": "preflight_ok",
|
||||
"repository": TARGET,
|
||||
"revision": revision,
|
||||
"preflightSha256": PREFLIGHT_SHA,
|
||||
"backupInputsSha256": BACKUP_SHA,
|
||||
"checks": dict(CHECKS),
|
||||
"diagnostic": CREDENTIAL,
|
||||
}
|
||||
|
||||
|
||||
class FakeBroker:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
response: object | None = None,
|
||||
error: Exception | None = None,
|
||||
base_url: str = "https://controller.example",
|
||||
verify_tls: bool = True,
|
||||
config_error: str | None = None,
|
||||
before_response=None,
|
||||
):
|
||||
self.response = _valid_preflight() if response is None else response
|
||||
self.error = error
|
||||
self.base_url = base_url
|
||||
self.verify_tls = verify_tls
|
||||
self._config_error = config_error
|
||||
self.before_response = before_response
|
||||
self.preflight_calls: list[tuple[str, str]] = []
|
||||
self.apply_calls = 0
|
||||
|
||||
def configuration_error(self):
|
||||
return self._config_error
|
||||
|
||||
def workmesh_promotion_preflight(self, revision: str, *, secret: str):
|
||||
self.preflight_calls.append((revision, secret))
|
||||
if self.before_response is not None:
|
||||
self.before_response()
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return self.response
|
||||
|
||||
def workmesh_promotion_apply(self, *args, **kwargs):
|
||||
self.apply_calls += 1
|
||||
raise AssertionError("the credential importer must not invoke promotion apply")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database(tmp_path, monkeypatch):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'workmesh-secret-import.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
monkeypatch.setattr(
|
||||
importer, "secret_storage_status", lambda: {"productionReady": True}
|
||||
)
|
||||
with Session(engine) as session:
|
||||
yield engine, session
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _rows(session: Session) -> tuple[Secret | None, list[AuditEvent]]:
|
||||
return (
|
||||
session.get(Secret, SECRET_REF),
|
||||
list(session.exec(select(AuditEvent)).all()),
|
||||
)
|
||||
|
||||
|
||||
def _database_dump(session: Session) -> str:
|
||||
secret, audits = _rows(session)
|
||||
return repr(
|
||||
{
|
||||
"secret": secret.model_dump() if secret is not None else None,
|
||||
"audits": [event.model_dump() for event in audits],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _import(
|
||||
session: Session,
|
||||
*,
|
||||
credential: str = CREDENTIAL,
|
||||
revision: str = REVISION,
|
||||
broker: FakeBroker | None = None,
|
||||
):
|
||||
return importer.import_workmesh_deployment_secret(
|
||||
session,
|
||||
credential,
|
||||
revision=revision,
|
||||
broker=broker or FakeBroker(),
|
||||
)
|
||||
|
||||
|
||||
def test_import_creates_encrypted_secret_after_verified_fixed_preflight_and_audits(
|
||||
database,
|
||||
):
|
||||
_, session = database
|
||||
broker = FakeBroker()
|
||||
|
||||
result = _import(session, broker=broker)
|
||||
|
||||
row = session.get(Secret, SECRET_REF)
|
||||
assert result["status"] == "imported"
|
||||
assert result["secretRef"] == SECRET_REF
|
||||
assert result["repository"] == TARGET
|
||||
assert result["revision"] == REVISION
|
||||
assert result["preflightSha256"] == PREFLIGHT_SHA
|
||||
assert result["readback"] == "verified"
|
||||
assert row is not None
|
||||
assert row.encrypted_value != CREDENTIAL
|
||||
assert get_secret(session, SECRET_REF, touch=False) == CREDENTIAL
|
||||
assert broker.preflight_calls == [(REVISION, CREDENTIAL)]
|
||||
assert broker.apply_calls == 0
|
||||
audits = session.exec(select(AuditEvent)).all()
|
||||
assert len(audits) == 1
|
||||
audit = audits[0]
|
||||
assert audit.target == TARGET
|
||||
assert audit.result == "success"
|
||||
assert audit.redacted_metadata["storeReference"] == SECRET_REF
|
||||
assert audit.redacted_metadata["revision"] == REVISION
|
||||
assert audit.redacted_metadata["preflightSha256"] == PREFLIGHT_SHA
|
||||
assert CREDENTIAL not in _database_dump(session)
|
||||
|
||||
|
||||
def test_identical_import_verifies_and_audits_without_mutating_secret(database):
|
||||
_, session = database
|
||||
assert _import(session)["status"] == "imported"
|
||||
row = session.get(Secret, SECRET_REF)
|
||||
assert row is not None
|
||||
before = {
|
||||
"encrypted_value": row.encrypted_value,
|
||||
"created_at": row.created_at,
|
||||
"updated_at": row.updated_at,
|
||||
"last_used_at": row.last_used_at,
|
||||
"used_by_json": list(row.used_by_json),
|
||||
"metadata_json": dict(row.metadata_json),
|
||||
}
|
||||
|
||||
result = _import(session)
|
||||
|
||||
session.refresh(row)
|
||||
assert result["status"] == "already_present"
|
||||
assert {
|
||||
"encrypted_value": row.encrypted_value,
|
||||
"created_at": row.created_at,
|
||||
"updated_at": row.updated_at,
|
||||
"last_used_at": row.last_used_at,
|
||||
"used_by_json": list(row.used_by_json),
|
||||
"metadata_json": dict(row.metadata_json),
|
||||
} == before
|
||||
assert len(session.exec(select(AuditEvent)).all()) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("credential", ["W" * 32, "W" * 256])
|
||||
def test_credential_length_contract_accepts_exact_boundaries(database, credential):
|
||||
_, session = database
|
||||
|
||||
result = _import(session, credential=credential)
|
||||
|
||||
assert result["status"] == "imported"
|
||||
assert get_secret(session, SECRET_REF, touch=False) == credential
|
||||
|
||||
|
||||
def test_different_existing_value_is_rejected_before_controller_preflight(database):
|
||||
_, session = database
|
||||
set_secret(session, SECRET_REF, "D" * 48, metadata={"owner": "keep"})
|
||||
original = session.get(Secret, SECRET_REF)
|
||||
assert original is not None
|
||||
before = original.model_dump()
|
||||
broker = FakeBroker()
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
session.rollback()
|
||||
current = session.get(Secret, SECRET_REF)
|
||||
assert current is not None and current.model_dump() == before
|
||||
assert broker.preflight_calls == []
|
||||
assert session.exec(select(AuditEvent)).all() == []
|
||||
|
||||
|
||||
def test_undecryptable_existing_value_is_rejected_without_replacement(database):
|
||||
_, session = database
|
||||
set_secret(session, SECRET_REF, "D" * 48)
|
||||
row = session.get(Secret, SECRET_REF)
|
||||
assert row is not None
|
||||
row.encrypted_value = "unreadable-ciphertext"
|
||||
session.add(row)
|
||||
session.commit()
|
||||
broker = FakeBroker()
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
assert session.get(Secret, SECRET_REF).encrypted_value == "unreadable-ciphertext"
|
||||
assert broker.preflight_calls == []
|
||||
|
||||
|
||||
def test_concurrent_identical_insert_winner_is_retained_and_audited(database):
|
||||
engine, session = database
|
||||
winner_timestamp = datetime(2026, 10, 4, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
def concurrent_winner():
|
||||
with Session(engine) as winner_session:
|
||||
set_secret(
|
||||
winner_session,
|
||||
SECRET_REF,
|
||||
CREDENTIAL,
|
||||
used_by=["concurrent-owner"],
|
||||
metadata={"owner": "concurrent-winner"},
|
||||
)
|
||||
winner = winner_session.get(Secret, SECRET_REF)
|
||||
assert winner is not None
|
||||
winner.created_at = winner_timestamp
|
||||
winner.updated_at = winner_timestamp
|
||||
winner_session.add(winner)
|
||||
winner_session.commit()
|
||||
|
||||
broker = FakeBroker(before_response=concurrent_winner)
|
||||
result = _import(session, broker=broker)
|
||||
|
||||
winner = session.get(Secret, SECRET_REF)
|
||||
assert result["status"] == "already_present"
|
||||
assert winner is not None
|
||||
assert winner.created_at.replace(tzinfo=timezone.utc) == winner_timestamp
|
||||
assert winner.updated_at.replace(tzinfo=timezone.utc) == winner_timestamp
|
||||
assert winner.used_by_json == ["concurrent-owner"]
|
||||
assert winner.metadata_json == {"owner": "concurrent-winner"}
|
||||
assert get_secret(session, SECRET_REF, touch=False) == CREDENTIAL
|
||||
assert len(session.exec(select(AuditEvent)).all()) == 1
|
||||
|
||||
|
||||
def test_concurrent_different_insert_winner_is_never_overwritten(database):
|
||||
engine, session = database
|
||||
|
||||
def concurrent_winner():
|
||||
with Session(engine) as winner_session:
|
||||
set_secret(
|
||||
winner_session,
|
||||
SECRET_REF,
|
||||
"D" * 48,
|
||||
metadata={"owner": "concurrent-winner"},
|
||||
)
|
||||
|
||||
broker = FakeBroker(before_response=concurrent_winner)
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
session.rollback()
|
||||
assert get_secret(session, SECRET_REF, touch=False) == "D" * 48
|
||||
assert session.get(Secret, SECRET_REF).metadata_json == {
|
||||
"owner": "concurrent-winner"
|
||||
}
|
||||
assert session.exec(select(AuditEvent)).all() == []
|
||||
|
||||
|
||||
def test_production_secret_storage_must_be_ready(database, monkeypatch):
|
||||
_, session = database
|
||||
monkeypatch.setattr(
|
||||
importer, "secret_storage_status", lambda: {"productionReady": False}
|
||||
)
|
||||
broker = FakeBroker()
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
assert broker.preflight_calls == []
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"credential",
|
||||
[
|
||||
"W" * 31,
|
||||
"W" * 257,
|
||||
"W" * 31 + " ",
|
||||
"W" * 31 + "\n",
|
||||
"W" * 31 + "\u00e9",
|
||||
"W" * 31 + "\x00",
|
||||
],
|
||||
)
|
||||
def test_credential_must_be_bounded_printable_ascii_without_whitespace(
|
||||
database, credential
|
||||
):
|
||||
_, session = database
|
||||
broker = FakeBroker()
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, credential=credential, broker=broker)
|
||||
|
||||
assert broker.preflight_calls == []
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("revision", ["main", "A" * 40, "a" * 39, "a" * 41])
|
||||
def test_revision_must_be_an_exact_full_lowercase_commit(database, revision):
|
||||
_, session = database
|
||||
broker = FakeBroker()
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, revision=revision, broker=broker)
|
||||
|
||||
assert broker.preflight_calls == []
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_url", "verify_tls", "config_error"),
|
||||
[
|
||||
("http://controller.example", True, None),
|
||||
("https://controller.example", False, None),
|
||||
("https://controller.example", True, "deployment_broker_url_invalid"),
|
||||
],
|
||||
)
|
||||
def test_preflight_requires_verified_https_configuration(
|
||||
database, base_url, verify_tls, config_error
|
||||
):
|
||||
_, session = database
|
||||
broker = FakeBroker(
|
||||
base_url=base_url,
|
||||
verify_tls=verify_tls,
|
||||
config_error=config_error,
|
||||
)
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
assert broker.preflight_calls == []
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda value: value.update(status="preflight_failed"),
|
||||
lambda value: value.update(apiVersion="wrong/v1"),
|
||||
lambda value: value.update(repository="jens/other"),
|
||||
lambda value: value.update(revision="f" * 40),
|
||||
lambda value: value.update(preflightSha256="invalid"),
|
||||
lambda value: value.update(backupInputsSha256="invalid"),
|
||||
lambda value: value.update(checks={"exactMain": "passed"}),
|
||||
lambda value: value["checks"].update(pushValidation="blocked"),
|
||||
],
|
||||
)
|
||||
def test_invalid_fixed_preflight_never_writes_secret_or_audit(database, mutate):
|
||||
_, session = database
|
||||
preflight = _valid_preflight()
|
||||
mutate(preflight)
|
||||
broker = FakeBroker(response=preflight)
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
session.rollback()
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
assert session.exec(select(AuditEvent)).all() == []
|
||||
assert broker.apply_calls == 0
|
||||
|
||||
|
||||
def test_non_mapping_preflight_is_rejected_without_storage_write(database):
|
||||
_, session = database
|
||||
broker = FakeBroker(response=["not", "a", "mapping"])
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError):
|
||||
_import(session, broker=broker)
|
||||
|
||||
session.rollback()
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
|
||||
|
||||
def test_broker_exception_is_sanitized_and_never_persisted(database):
|
||||
_, session = database
|
||||
broker = FakeBroker(error=RuntimeError(f"upstream leaked {CREDENTIAL}"))
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError) as error:
|
||||
_import(session, broker=broker)
|
||||
|
||||
assert CREDENTIAL not in str(error.value)
|
||||
assert broker.preflight_calls == [(REVISION, CREDENTIAL)]
|
||||
session.rollback()
|
||||
assert session.get(Secret, SECRET_REF) is None
|
||||
assert session.exec(select(AuditEvent)).all() == []
|
||||
|
||||
|
||||
def test_audit_insert_failure_rolls_back_secret_and_audit_together(database):
|
||||
engine, session = database
|
||||
|
||||
def fail_audit_insert(conn, cursor, statement, parameters, context, executemany):
|
||||
if statement.lstrip().lower().startswith("insert into auditevent"):
|
||||
raise RuntimeError(f"database detail contained {CREDENTIAL}")
|
||||
|
||||
from sqlalchemy import event
|
||||
|
||||
event.listen(engine, "before_cursor_execute", fail_audit_insert)
|
||||
try:
|
||||
with pytest.raises(importer.WorkMeshSecretImportError) as error:
|
||||
_import(session)
|
||||
finally:
|
||||
event.remove(engine, "before_cursor_execute", fail_audit_insert)
|
||||
|
||||
assert CREDENTIAL not in str(error.value)
|
||||
session.rollback()
|
||||
with Session(engine) as verify:
|
||||
assert verify.get(Secret, SECRET_REF) is None
|
||||
assert verify.exec(select(AuditEvent)).all() == []
|
||||
|
||||
|
||||
def test_uncertain_readback_returns_blocked_without_replacing_committed_secret(
|
||||
database, monkeypatch
|
||||
):
|
||||
engine, session = database
|
||||
actual_get_secret = importer.get_secret
|
||||
calls = 0
|
||||
|
||||
def fail_readback_once(db_session, name, *, touch=False):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 2:
|
||||
return None
|
||||
return actual_get_secret(db_session, name, touch=touch)
|
||||
|
||||
monkeypatch.setattr(importer, "get_secret", fail_readback_once)
|
||||
broker = FakeBroker()
|
||||
|
||||
with pytest.raises(importer.WorkMeshSecretImportError) as error:
|
||||
_import(session, broker=broker)
|
||||
|
||||
assert CREDENTIAL not in str(error.value)
|
||||
monkeypatch.setattr(importer, "get_secret", actual_get_secret)
|
||||
session.rollback()
|
||||
with Session(engine) as verify:
|
||||
row = verify.get(Secret, SECRET_REF)
|
||||
assert row is not None
|
||||
before = row.model_dump()
|
||||
assert get_secret(verify, SECRET_REF, touch=False) == CREDENTIAL
|
||||
assert len(verify.exec(select(AuditEvent)).all()) == 1
|
||||
|
||||
outcome = _import(session, broker=FakeBroker())
|
||||
assert outcome["status"] == "already_present"
|
||||
session.expire_all()
|
||||
after = session.get(Secret, SECRET_REF)
|
||||
assert after is not None and after.model_dump() == before
|
||||
|
||||
|
||||
def test_uncertain_commit_returns_blocked_and_same_value_replay_does_not_replace(
|
||||
database, monkeypatch
|
||||
):
|
||||
engine, session = database
|
||||
actual_commit = session.commit
|
||||
|
||||
def commit_then_lose_acknowledgement():
|
||||
if any(isinstance(item, AuditEvent) for item in session.new):
|
||||
actual_commit()
|
||||
raise RuntimeError(f"commit acknowledgement lost for {CREDENTIAL}")
|
||||
actual_commit()
|
||||
|
||||
monkeypatch.setattr(session, "commit", commit_then_lose_acknowledgement)
|
||||
with pytest.raises(importer.WorkMeshSecretImportError) as error:
|
||||
_import(session)
|
||||
assert CREDENTIAL not in str(error.value)
|
||||
|
||||
monkeypatch.setattr(session, "commit", actual_commit)
|
||||
with Session(engine) as verify:
|
||||
row = verify.get(Secret, SECRET_REF)
|
||||
assert row is not None
|
||||
before = row.model_dump()
|
||||
assert get_secret(verify, SECRET_REF, touch=False) == CREDENTIAL
|
||||
assert len(verify.exec(select(AuditEvent)).all()) == 1
|
||||
|
||||
result = _import(session, broker=FakeBroker())
|
||||
assert result["status"] == "already_present"
|
||||
session.expire_all()
|
||||
after = session.get(Secret, SECRET_REF)
|
||||
assert after is not None and after.model_dump() == before
|
||||
assert len(session.exec(select(AuditEvent)).all()) == 2
|
||||
|
||||
|
||||
def test_cli_imports_stdin_only_and_emits_sanitized_verification(database, monkeypatch):
|
||||
engine, _ = database
|
||||
broker = FakeBroker()
|
||||
monkeypatch.setattr(importer, "engine", engine)
|
||||
monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker)
|
||||
output = io.StringIO()
|
||||
|
||||
exit_code = importer.main(
|
||||
[REVISION],
|
||||
stdin=io.BytesIO(CREDENTIAL.encode("ascii")),
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
result = json.loads(output.getvalue())
|
||||
assert exit_code == 0
|
||||
assert result["status"] == "imported"
|
||||
assert result["secretRef"] == SECRET_REF
|
||||
assert result["revision"] == REVISION
|
||||
assert CREDENTIAL not in output.getvalue()
|
||||
assert broker.preflight_calls == [(REVISION, CREDENTIAL)]
|
||||
|
||||
|
||||
def test_cli_rejects_extra_arguments_before_reading_secret(database, monkeypatch):
|
||||
engine, _ = database
|
||||
broker = FakeBroker()
|
||||
monkeypatch.setattr(importer, "engine", engine)
|
||||
monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker)
|
||||
output = io.StringIO()
|
||||
|
||||
class UnreadableInput:
|
||||
def read(self, _size):
|
||||
pytest.fail("invalid command line must be rejected before reading stdin")
|
||||
|
||||
exit_code = importer.main(
|
||||
[REVISION, CREDENTIAL],
|
||||
stdin=UnreadableInput(),
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
assert exit_code != 0
|
||||
assert json.loads(output.getvalue()) == {
|
||||
"reason": "invalid_arguments",
|
||||
"status": "blocked",
|
||||
}
|
||||
assert broker.preflight_calls == []
|
||||
|
||||
|
||||
def test_cli_rejects_non_ascii_stdin_without_calling_controller(database, monkeypatch):
|
||||
engine, _ = database
|
||||
broker = FakeBroker()
|
||||
monkeypatch.setattr(importer, "engine", engine)
|
||||
monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker)
|
||||
output = io.StringIO()
|
||||
|
||||
exit_code = importer.main(
|
||||
[REVISION],
|
||||
stdin=io.BytesIO(b"W" * 47 + b"\xff"),
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
assert exit_code != 0
|
||||
assert json.loads(output.getvalue()) == {
|
||||
"reason": "invalid_input",
|
||||
"status": "blocked",
|
||||
}
|
||||
assert broker.preflight_calls == []
|
||||
|
||||
|
||||
def test_cli_rejects_oversized_stdin_without_calling_controller(database, monkeypatch):
|
||||
engine, _ = database
|
||||
broker = FakeBroker()
|
||||
monkeypatch.setattr(importer, "engine", engine)
|
||||
monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker)
|
||||
output = io.StringIO()
|
||||
|
||||
exit_code = importer.main(
|
||||
[REVISION],
|
||||
stdin=io.BytesIO(b"W" * 4097),
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
assert exit_code != 0
|
||||
assert json.loads(output.getvalue())["status"] == "blocked"
|
||||
assert "W" * 48 not in output.getvalue()
|
||||
assert broker.preflight_calls == []
|
||||
|
||||
|
||||
def test_cli_unexpected_error_never_echoes_secret(database, monkeypatch):
|
||||
engine, _ = database
|
||||
broker = FakeBroker(error=RuntimeError(f"unexpected {CREDENTIAL}"))
|
||||
monkeypatch.setattr(importer, "engine", engine)
|
||||
monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker)
|
||||
output = io.StringIO()
|
||||
|
||||
exit_code = importer.main(
|
||||
[REVISION],
|
||||
stdin=io.BytesIO(CREDENTIAL.encode("ascii")),
|
||||
stdout=output,
|
||||
)
|
||||
|
||||
result = json.loads(output.getvalue())
|
||||
assert exit_code != 0
|
||||
assert result["status"] == "blocked"
|
||||
assert CREDENTIAL not in output.getvalue()
|
||||
assert "unexpected" not in output.getvalue()
|
||||
assert "traceback" not in output.getvalue().lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tampered", [False, True])
|
||||
def test_cli_unknown_typed_error_code_is_normalized(database, monkeypatch, tampered):
|
||||
engine, _ = database
|
||||
monkeypatch.setattr(importer, "engine", engine)
|
||||
monkeypatch.setattr(importer, "DeploymentBrokerClient", FakeBroker)
|
||||
|
||||
def fail_import(*args, **kwargs):
|
||||
error = importer.WorkMeshSecretImportError(CREDENTIAL)
|
||||
if tampered:
|
||||
error.code = CREDENTIAL
|
||||
raise error
|
||||
|
||||
monkeypatch.setattr(importer, "import_workmesh_deployment_secret", fail_import)
|
||||
output = io.StringIO()
|
||||
assert importer.main(
|
||||
[REVISION], stdin=io.BytesIO(CREDENTIAL.encode("ascii")), stdout=output
|
||||
) == 2
|
||||
assert json.loads(output.getvalue()) == {
|
||||
"status": "blocked", "reason": "import_failed"
|
||||
}
|
||||
assert CREDENTIAL not in output.getvalue()
|
||||
Reference in new issue
Block a user