263 lines
12 KiB
Python
263 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
REPOSITORY = "jens/unraid-appops-gateway"
|
|
CONTAINER = "Unraid-AppOps-Gateway"
|
|
OBSERVED = "5" * 40
|
|
RECORDED = "6" * 40
|
|
ATTEMPT = "a" * 40
|
|
IMAGE = "sha256:" + "5" * 64
|
|
OLD_IMAGE = "sha256:" + "6" * 64
|
|
|
|
|
|
def receipt(revision=RECORDED, image=OLD_IMAGE):
|
|
tree = "b" * 40
|
|
digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
|
|
built = "2026-10-02T00:00:00.000Z"
|
|
return {
|
|
"schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY,
|
|
"revision": revision, "tree": tree, "treeDigest": digest,
|
|
"build": {"id": revision[:12], "repository": REPOSITORY,
|
|
"revision": revision, "tree": tree, "treeDigest": digest,
|
|
"builtAt": built},
|
|
"containers": {CONTAINER: {"image_id": image,
|
|
"image_revision": revision, "image_source": REPOSITORY,
|
|
"image_tree": tree, "image_version": revision[:12],
|
|
"image_created": built}}, "live": None,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def recovery(monkeypatch, tmp_path):
|
|
if "fcntl" not in sys.modules:
|
|
shim = types.ModuleType("fcntl")
|
|
shim.LOCK_EX, shim.LOCK_UN = 2, 8
|
|
shim.flock = lambda *_args: None
|
|
monkeypatch.setitem(sys.modules, "fcntl", shim)
|
|
spec = importlib.util.spec_from_file_location(
|
|
"appops_recovery_test", ROOT / "ops/gitea-deploy-broker/deploy.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
state_root = tmp_path / "state"
|
|
state_root.mkdir(mode=0o700)
|
|
monkeypatch.setattr(module, "STATE_ROOT", state_root)
|
|
monkeypatch.setattr(module, "require_host_root", lambda: None)
|
|
monkeypatch.setattr(module, "_root_owned", lambda _path: True)
|
|
if os.name == "nt":
|
|
# Windows does not expose POSIX write bits; keep real inode/type checks.
|
|
real_lstat = Path.lstat
|
|
def posix_permissions(path):
|
|
observed = real_lstat(path)
|
|
return types.SimpleNamespace(st_mode=observed.st_mode & ~0o022,
|
|
st_dev=observed.st_dev, st_ino=observed.st_ino)
|
|
monkeypatch.setattr(Path, "lstat", posix_permissions)
|
|
fsyncs = []
|
|
monkeypatch.setattr(module, "_fsync_directory", lambda path: fsyncs.append(path))
|
|
project = {"slug": "unraid-appops-gateway", "branch": "main",
|
|
"gitea_repository_id": 9, "containers": [CONTAINER],
|
|
"compose_project": "unraid-appops-gateway",
|
|
"compose_files": ["docker-compose.yml"], "verify_oci_revision": True,
|
|
"environment": {"APPOPS_DATA_PATH": str(module.APPOPS_DATA_ROOT)},
|
|
"source_dir": str(tmp_path), "_repository_name": REPOSITORY}
|
|
(tmp_path / "docker-compose.yml").write_text("services: {}\n")
|
|
public = {key: value for key, value in project.items() if not key.startswith("_")}
|
|
monkeypatch.setattr(module, "load_projects", lambda: {REPOSITORY: copy.deepcopy(public)})
|
|
def validate(_repo, branch, revision):
|
|
assert branch == "main"
|
|
if revision not in {OBSERVED, ATTEMPT}:
|
|
raise module.DeployError("revision is not a protected ancestor")
|
|
return revision
|
|
monkeypatch.setattr(module, "validate_commit", validate)
|
|
source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway"
|
|
labels = {"org.opencontainers.image.revision": OBSERVED,
|
|
"org.opencontainers.image.source": source,
|
|
"com.docker.compose.project": "unraid-appops-gateway"}
|
|
container = {"Id": "c" * 64, "Image": IMAGE,
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
"Config": {"Image": "appops:observed", "Labels": labels},
|
|
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
|
|
"Mounts": [{"Destination": "/data", "Type": "bind", "RW": True,
|
|
"Source": str(module.APPOPS_DATA_ROOT)}]}
|
|
image = {"Id": IMAGE, "Config": {"Labels": copy.deepcopy(labels)}}
|
|
monkeypatch.setattr(module, "inspect_container", lambda name: copy.deepcopy(container))
|
|
monkeypatch.setattr(module, "inspect_image", lambda name: copy.deepcopy(image))
|
|
monkeypatch.setattr(module, "compose_project_container_names", lambda name: {CONTAINER})
|
|
def output(command, **_kwargs):
|
|
assert command == ["docker", "ps", "-a", "--format", "{{.Names}}"]
|
|
return CONTAINER + "\n"
|
|
monkeypatch.setattr(module, "output", output)
|
|
def forbidden(*_args, **_kwargs):
|
|
pytest.fail("recovery attempted a host mutation")
|
|
monkeypatch.setattr(module, "run", forbidden)
|
|
prior = {"schema_version": 3, "repository": REPOSITORY,
|
|
"slug": project["slug"], "compose_project": project["compose_project"],
|
|
"revision": RECORDED, "live_revision": RECORDED,
|
|
"status": "failed", "live_status": "unknown", "live_legacy_provenance": False,
|
|
"live_artifacts": {"containers": {CONTAINER: {"image_id": OLD_IMAGE}}},
|
|
"live_receipt": receipt(), "rollback_snapshot": {"revision": "7" * 40},
|
|
"last_attempt": {"revision": ATTEMPT, "status": "failed",
|
|
"message": module._APPOPS_RECOVERY_FAILURE}}
|
|
path = state_root / "unraid-appops-gateway.json"
|
|
def save():
|
|
path.write_text(json.dumps(prior), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
save()
|
|
return types.SimpleNamespace(m=module, project=project, repo=tmp_path,
|
|
path=path, prior=prior, container=container, image=image,
|
|
public=public, save=save, fsyncs=fsyncs)
|
|
|
|
|
|
def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery):
|
|
r = recovery
|
|
before = r.path.read_bytes()
|
|
compose = (r.repo / "docker-compose.yml").read_bytes()
|
|
result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
state = json.loads(r.path.read_bytes())
|
|
backup = r.path.parent / result["backupFile"]
|
|
assert backup.read_bytes() == before
|
|
assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest()
|
|
assert state["last_attempt"] == r.prior["last_attempt"]
|
|
assert state["live_revision"] == OBSERVED
|
|
assert state["live_legacy_provenance"] is True
|
|
assert state["live_status"] == "healthy"
|
|
assert state["live_receipt"] == state["rollback_snapshot"] == {}
|
|
assert state["live_artifacts"]["compose_sha256"] == ""
|
|
assert state["live_artifacts"]["containers"][CONTAINER]["image_id"] == IMAGE
|
|
assert state["recovery"]["retired_receipt_revision"] == RECORDED
|
|
assert (r.repo / "docker-compose.yml").read_bytes() == compose
|
|
assert r.fsyncs == [r.path.parent, r.path.parent]
|
|
|
|
|
|
@pytest.mark.parametrize("field,value", [
|
|
("schema_version", 2), ("status", "success"), ("live_status", "healthy"),
|
|
("live_legacy_provenance", None), ("live_legacy_provenance", 0),
|
|
("repository", "jens/other"), ("live_revision", OBSERVED),
|
|
("rollback_snapshot", None),
|
|
("live_receipt", None), ("live_receipt", {}), ("live_receipt", "invalid"),
|
|
("live_artifacts", None), ("live_artifacts", {}),
|
|
])
|
|
def test_rejects_ineligible_recorded_state(recovery, field, value):
|
|
r = recovery
|
|
r.prior[field] = value
|
|
r.save()
|
|
before = r.path.read_bytes()
|
|
with pytest.raises(r.m.DeployError):
|
|
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
assert r.path.read_bytes() == before
|
|
assert not list(r.path.parent.glob("*.bak"))
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["wrong-failure", "non-ancestor", "same-image", "receipt-image", "receipt-revision"])
|
|
def test_rejects_unproven_historical_drift(recovery, failure):
|
|
r = recovery
|
|
if failure == "wrong-failure":
|
|
r.prior["last_attempt"]["message"] = "Deployment failed"
|
|
elif failure == "non-ancestor":
|
|
r.prior["last_attempt"]["revision"] = "d" * 40
|
|
elif failure == "same-image":
|
|
r.prior["live_artifacts"]["containers"][CONTAINER]["image_id"] = IMAGE
|
|
elif failure == "receipt-image":
|
|
r.prior["live_receipt"]["containers"][CONTAINER]["image_id"] = IMAGE
|
|
else:
|
|
r.prior["live_receipt"]["revision"] = OBSERVED
|
|
r.save()
|
|
before = r.path.read_bytes()
|
|
with pytest.raises(r.m.DeployError):
|
|
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
assert r.path.read_bytes() == before
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["unhealthy", "volume", "readonly", "mount-path", "port", "image", "source", "revision", "extra-container", "config", "owner"])
|
|
def test_rejects_wrong_actual_runtime_or_target(recovery, monkeypatch, failure):
|
|
r = recovery
|
|
if failure == "unhealthy":
|
|
r.container["State"]["Health"]["Status"] = "unhealthy"
|
|
elif failure in {"volume", "readonly", "mount-path"}:
|
|
key, value = {"volume": ("Type", "volume"), "readonly": ("RW", False),
|
|
"mount-path": ("Source", "/other")}[failure]
|
|
r.container["Mounts"][0][key] = value
|
|
elif failure == "port":
|
|
r.container["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostPort"] = "9999"
|
|
elif failure == "image":
|
|
r.image["Id"] = OLD_IMAGE
|
|
elif failure in {"source", "revision"}:
|
|
r.image["Config"]["Labels"]["org.opencontainers.image." + failure] = "wrong"
|
|
elif failure == "extra-container":
|
|
monkeypatch.setattr(r.m, "compose_project_container_names", lambda name: {CONTAINER, "other"})
|
|
elif failure == "config":
|
|
r.public["branch"] = "other"
|
|
else:
|
|
monkeypatch.setattr(r.m, "_root_owned", lambda path: False)
|
|
before = r.path.read_bytes()
|
|
with pytest.raises(r.m.DeployError):
|
|
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
assert r.path.read_bytes() == before
|
|
|
|
|
|
@pytest.mark.parametrize("change", ["bytes", "inode", "runtime", "config"])
|
|
def test_rechecks_after_durable_backup_before_replace(recovery, monkeypatch, change):
|
|
r = recovery
|
|
def after_backup(_path):
|
|
if change == "bytes":
|
|
r.path.write_bytes(b'{"external": true}')
|
|
elif change == "inode":
|
|
replacement = r.path.with_suffix(".replacement")
|
|
replacement.write_bytes(r.path.read_bytes())
|
|
replacement.chmod(0o600)
|
|
replacement.replace(r.path)
|
|
elif change == "runtime":
|
|
r.container["Id"] = "d" * 64
|
|
else:
|
|
r.public["branch"] = "changed"
|
|
monkeypatch.setattr(r.m, "_fsync_directory", after_backup)
|
|
with pytest.raises(r.m.DeployError):
|
|
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
state = json.loads(r.path.read_bytes())
|
|
assert "recovery" not in state
|
|
assert len(list(r.path.parent.glob("*.bak"))) == 1
|
|
assert not list(r.path.parent.glob("*.tmp"))
|
|
|
|
|
|
def test_next_normal_delivery_keeps_recovery_audit(recovery):
|
|
r = recovery
|
|
result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
audit = json.loads(r.path.read_bytes())["recovery"]
|
|
r.m.write_state(REPOSITORY, r.project, OBSERVED, "success", "Verified normal deployment",
|
|
live_receipt=receipt(OBSERVED, IMAGE), live_legacy_provenance=False)
|
|
state = json.loads(r.path.read_bytes())
|
|
assert state["recovery"] == audit
|
|
assert (r.path.parent / result["backupFile"]).exists()
|
|
assert state["live_receipt"]["revision"] == OBSERVED
|
|
assert state["live_legacy_provenance"] is False
|
|
assert state["last_attempt"]["message"] == "Verified normal deployment"
|
|
|
|
|
|
def test_post_replace_fsync_failure_reports_applied_unknown_and_retains_backup(recovery, monkeypatch):
|
|
r = recovery
|
|
calls = 0
|
|
def fail_after_replace(_path):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 2:
|
|
raise OSError("synthetic directory fsync failure")
|
|
monkeypatch.setattr(r.m, "_fsync_directory", fail_after_replace)
|
|
before = r.path.read_bytes()
|
|
with pytest.raises(r.m.DeployError, match="state was replaced but durability is unknown"):
|
|
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
|
state = json.loads(r.path.read_bytes())
|
|
assert state["live_revision"] == OBSERVED
|
|
assert (r.path.parent / state["recovery"]["backup_file"]).read_bytes() == before
|
|
with pytest.raises(r.m.DeployError, match="exact failed pre-mutation"):
|
|
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|