from __future__ import annotations import copy import hashlib import importlib.util import json import os import sys import types from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] REPOSITORY = "jens/unraid-appops-gateway" CONTAINER = "Unraid-AppOps-Gateway" OBSERVED = "5" * 40 RECORDED = "6" * 40 ATTEMPT = "a" * 40 IMAGE = "sha256:" + "5" * 64 OLD_IMAGE = "sha256:" + "6" * 64 def receipt(revision=RECORDED, image=OLD_IMAGE): tree = "b" * 40 digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest() built = "2026-10-02T00:00:00.000Z" return { "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY, "revision": revision, "tree": tree, "treeDigest": digest, "build": {"id": revision[:12], "repository": REPOSITORY, "revision": revision, "tree": tree, "treeDigest": digest, "builtAt": built}, "containers": {CONTAINER: {"image_id": image, "image_revision": revision, "image_source": REPOSITORY, "image_tree": tree, "image_version": revision[:12], "image_created": built}}, "live": None, } @pytest.fixture def recovery(monkeypatch, tmp_path): if "fcntl" not in sys.modules: shim = types.ModuleType("fcntl") shim.LOCK_EX, shim.LOCK_UN = 2, 8 shim.flock = lambda *_args: None monkeypatch.setitem(sys.modules, "fcntl", shim) spec = importlib.util.spec_from_file_location( "appops_recovery_test", ROOT / "ops/gitea-deploy-broker/deploy.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) state_root = tmp_path / "state" state_root.mkdir(mode=0o700) monkeypatch.setattr(module, "STATE_ROOT", state_root) monkeypatch.setattr(module, "require_host_root", lambda: None) monkeypatch.setattr(module, "_root_owned", lambda _path: True) if os.name == "nt": # Windows does not expose POSIX write bits; keep real inode/type checks. real_lstat = Path.lstat def posix_permissions(path): observed = real_lstat(path) return types.SimpleNamespace(st_mode=observed.st_mode & ~0o022, st_dev=observed.st_dev, st_ino=observed.st_ino) monkeypatch.setattr(Path, "lstat", posix_permissions) fsyncs = [] monkeypatch.setattr(module, "_fsync_directory", lambda path: fsyncs.append(path)) project = {"slug": "unraid-appops-gateway", "branch": "main", "gitea_repository_id": 9, "containers": [CONTAINER], "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "verify_oci_revision": True, "environment": {"APPOPS_DATA_PATH": str(module.APPOPS_DATA_ROOT)}, "source_dir": str(tmp_path), "_repository_name": REPOSITORY} (tmp_path / "docker-compose.yml").write_text("services: {}\n") public = {key: value for key, value in project.items() if not key.startswith("_")} monkeypatch.setattr(module, "load_projects", lambda: {REPOSITORY: copy.deepcopy(public)}) def validate(_repo, branch, revision): assert branch == "main" if revision not in {OBSERVED, ATTEMPT}: raise module.DeployError("revision is not a protected ancestor") return revision monkeypatch.setattr(module, "validate_commit", validate) source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" labels = {"org.opencontainers.image.revision": OBSERVED, "org.opencontainers.image.source": source, "com.docker.compose.project": "unraid-appops-gateway"} container = {"Id": "c" * 64, "Image": IMAGE, "State": {"Running": True, "Health": {"Status": "healthy"}}, "Config": {"Image": "appops:observed", "Labels": labels}, "NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}}, "Mounts": [{"Destination": "/data", "Type": "bind", "RW": True, "Source": str(module.APPOPS_DATA_ROOT)}]} image = {"Id": IMAGE, "Config": {"Labels": copy.deepcopy(labels)}} monkeypatch.setattr(module, "inspect_container", lambda name: copy.deepcopy(container)) monkeypatch.setattr(module, "inspect_image", lambda name: copy.deepcopy(image)) monkeypatch.setattr(module, "compose_project_container_names", lambda name: {CONTAINER}) def output(command, **_kwargs): assert command == ["docker", "ps", "-a", "--format", "{{.Names}}"] return CONTAINER + "\n" monkeypatch.setattr(module, "output", output) def forbidden(*_args, **_kwargs): pytest.fail("recovery attempted a host mutation") monkeypatch.setattr(module, "run", forbidden) prior = {"schema_version": 3, "repository": REPOSITORY, "slug": project["slug"], "compose_project": project["compose_project"], "revision": RECORDED, "live_revision": RECORDED, "status": "failed", "live_status": "unknown", "live_legacy_provenance": False, "live_artifacts": {"containers": {CONTAINER: {"image_id": OLD_IMAGE}}}, "live_receipt": receipt(), "rollback_snapshot": {"revision": "7" * 40}, "last_attempt": {"revision": ATTEMPT, "status": "failed", "message": module._APPOPS_RECOVERY_FAILURE}} path = state_root / "unraid-appops-gateway.json" def save(): path.write_text(json.dumps(prior), encoding="utf-8") path.chmod(0o600) save() return types.SimpleNamespace(m=module, project=project, repo=tmp_path, path=path, prior=prior, container=container, image=image, public=public, save=save, fsyncs=fsyncs) def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery): r = recovery before = r.path.read_bytes() compose = (r.repo / "docker-compose.yml").read_bytes() result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) state = json.loads(r.path.read_bytes()) backup = r.path.parent / result["backupFile"] assert backup.read_bytes() == before assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest() assert state["last_attempt"] == r.prior["last_attempt"] assert state["live_revision"] == OBSERVED assert state["live_legacy_provenance"] is True assert state["live_status"] == "healthy" assert state["live_receipt"] == state["rollback_snapshot"] == {} assert state["live_artifacts"]["compose_sha256"] == "" assert state["live_artifacts"]["containers"][CONTAINER]["image_id"] == IMAGE assert state["recovery"]["retired_receipt_revision"] == RECORDED assert (r.repo / "docker-compose.yml").read_bytes() == compose assert r.fsyncs == [r.path.parent, r.path.parent] @pytest.mark.parametrize("field,value", [ ("schema_version", 2), ("status", "success"), ("live_status", "healthy"), ("live_legacy_provenance", None), ("live_legacy_provenance", 0), ("repository", "jens/other"), ("live_revision", OBSERVED), ("rollback_snapshot", None), ("live_receipt", None), ("live_receipt", {}), ("live_receipt", "invalid"), ("live_artifacts", None), ("live_artifacts", {}), ]) def test_rejects_ineligible_recorded_state(recovery, field, value): r = recovery r.prior[field] = value r.save() before = r.path.read_bytes() with pytest.raises(r.m.DeployError): r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) assert r.path.read_bytes() == before assert not list(r.path.parent.glob("*.bak")) @pytest.mark.parametrize("failure", ["wrong-failure", "non-ancestor", "same-image", "receipt-image", "receipt-revision"]) def test_rejects_unproven_historical_drift(recovery, failure): r = recovery if failure == "wrong-failure": r.prior["last_attempt"]["message"] = "Deployment failed" elif failure == "non-ancestor": r.prior["last_attempt"]["revision"] = "d" * 40 elif failure == "same-image": r.prior["live_artifacts"]["containers"][CONTAINER]["image_id"] = IMAGE elif failure == "receipt-image": r.prior["live_receipt"]["containers"][CONTAINER]["image_id"] = IMAGE else: r.prior["live_receipt"]["revision"] = OBSERVED r.save() before = r.path.read_bytes() with pytest.raises(r.m.DeployError): r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) assert r.path.read_bytes() == before @pytest.mark.parametrize("failure", ["unhealthy", "volume", "readonly", "mount-path", "port", "image", "source", "revision", "extra-container", "config", "owner"]) def test_rejects_wrong_actual_runtime_or_target(recovery, monkeypatch, failure): r = recovery if failure == "unhealthy": r.container["State"]["Health"]["Status"] = "unhealthy" elif failure in {"volume", "readonly", "mount-path"}: key, value = {"volume": ("Type", "volume"), "readonly": ("RW", False), "mount-path": ("Source", "/other")}[failure] r.container["Mounts"][0][key] = value elif failure == "port": r.container["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostPort"] = "9999" elif failure == "image": r.image["Id"] = OLD_IMAGE elif failure in {"source", "revision"}: r.image["Config"]["Labels"]["org.opencontainers.image." + failure] = "wrong" elif failure == "extra-container": monkeypatch.setattr(r.m, "compose_project_container_names", lambda name: {CONTAINER, "other"}) elif failure == "config": r.public["branch"] = "other" else: monkeypatch.setattr(r.m, "_root_owned", lambda path: False) before = r.path.read_bytes() with pytest.raises(r.m.DeployError): r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) assert r.path.read_bytes() == before @pytest.mark.parametrize("change", ["bytes", "inode", "runtime", "config"]) def test_rechecks_after_durable_backup_before_replace(recovery, monkeypatch, change): r = recovery def after_backup(_path): if change == "bytes": r.path.write_bytes(b'{"external": true}') elif change == "inode": replacement = r.path.with_suffix(".replacement") replacement.write_bytes(r.path.read_bytes()) replacement.chmod(0o600) replacement.replace(r.path) elif change == "runtime": r.container["Id"] = "d" * 64 else: r.public["branch"] = "changed" monkeypatch.setattr(r.m, "_fsync_directory", after_backup) with pytest.raises(r.m.DeployError): r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) state = json.loads(r.path.read_bytes()) assert "recovery" not in state assert len(list(r.path.parent.glob("*.bak"))) == 1 assert not list(r.path.parent.glob("*.tmp")) def test_next_normal_delivery_keeps_recovery_audit(recovery): r = recovery result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) audit = json.loads(r.path.read_bytes())["recovery"] r.m.write_state(REPOSITORY, r.project, OBSERVED, "success", "Verified normal deployment", live_receipt=receipt(OBSERVED, IMAGE), live_legacy_provenance=False) state = json.loads(r.path.read_bytes()) assert state["recovery"] == audit assert (r.path.parent / result["backupFile"]).exists() assert state["live_receipt"]["revision"] == OBSERVED assert state["live_legacy_provenance"] is False assert state["last_attempt"]["message"] == "Verified normal deployment" def test_post_replace_fsync_failure_reports_applied_unknown_and_retains_backup(recovery, monkeypatch): r = recovery calls = 0 def fail_after_replace(_path): nonlocal calls calls += 1 if calls == 2: raise OSError("synthetic directory fsync failure") monkeypatch.setattr(r.m, "_fsync_directory", fail_after_replace) before = r.path.read_bytes() with pytest.raises(r.m.DeployError, match="state was replaced but durability is unknown"): r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED) state = json.loads(r.path.read_bytes()) assert state["live_revision"] == OBSERVED assert (r.path.parent / state["recovery"]["backup_file"]).read_bytes() == before with pytest.raises(r.m.DeployError, match="exact failed pre-mutation"): r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)