Publish curated Unraid-AppOps-Gateway source from 3c5fb0c6d1f4e693e4a0fa50e7151509b4450786
This commit is contained in:
Public Source Publisher committed
1 parent
3c7997bf4b
commit
2de6e5c85e
5 files changed
+786
-6
No files matched your search
@@ -0,0 +1,262 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
REPOSITORY = "jens/unraid-appops-gateway"
|
||||
CONTAINER = "Unraid-AppOps-Gateway"
|
||||
OBSERVED = "5" * 40
|
||||
RECORDED = "6" * 40
|
||||
ATTEMPT = "a" * 40
|
||||
IMAGE = "sha256:" + "5" * 64
|
||||
OLD_IMAGE = "sha256:" + "6" * 64
|
||||
|
||||
|
||||
def receipt(revision=RECORDED, image=OLD_IMAGE):
|
||||
tree = "b" * 40
|
||||
digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
|
||||
built = "2026-10-02T00:00:00.000Z"
|
||||
return {
|
||||
"schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY,
|
||||
"revision": revision, "tree": tree, "treeDigest": digest,
|
||||
"build": {"id": revision[:12], "repository": REPOSITORY,
|
||||
"revision": revision, "tree": tree, "treeDigest": digest,
|
||||
"builtAt": built},
|
||||
"containers": {CONTAINER: {"image_id": image,
|
||||
"image_revision": revision, "image_source": REPOSITORY,
|
||||
"image_tree": tree, "image_version": revision[:12],
|
||||
"image_created": built}}, "live": None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recovery(monkeypatch, tmp_path):
|
||||
if "fcntl" not in sys.modules:
|
||||
shim = types.ModuleType("fcntl")
|
||||
shim.LOCK_EX, shim.LOCK_UN = 2, 8
|
||||
shim.flock = lambda *_args: None
|
||||
monkeypatch.setitem(sys.modules, "fcntl", shim)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_recovery_test", ROOT / "ops/gitea-deploy-broker/deploy.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
state_root = tmp_path / "state"
|
||||
state_root.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(module, "STATE_ROOT", state_root)
|
||||
monkeypatch.setattr(module, "require_host_root", lambda: None)
|
||||
monkeypatch.setattr(module, "_root_owned", lambda _path: True)
|
||||
if os.name == "nt":
|
||||
# Windows does not expose POSIX write bits; keep real inode/type checks.
|
||||
real_lstat = Path.lstat
|
||||
def posix_permissions(path):
|
||||
observed = real_lstat(path)
|
||||
return types.SimpleNamespace(st_mode=observed.st_mode & ~0o022,
|
||||
st_dev=observed.st_dev, st_ino=observed.st_ino)
|
||||
monkeypatch.setattr(Path, "lstat", posix_permissions)
|
||||
fsyncs = []
|
||||
monkeypatch.setattr(module, "_fsync_directory", lambda path: fsyncs.append(path))
|
||||
project = {"slug": "unraid-appops-gateway", "branch": "main",
|
||||
"gitea_repository_id": 9, "containers": [CONTAINER],
|
||||
"compose_project": "unraid-appops-gateway",
|
||||
"compose_files": ["docker-compose.yml"], "verify_oci_revision": True,
|
||||
"environment": {"APPOPS_DATA_PATH": str(module.APPOPS_DATA_ROOT)},
|
||||
"source_dir": str(tmp_path), "_repository_name": REPOSITORY}
|
||||
(tmp_path / "docker-compose.yml").write_text("services: {}\n")
|
||||
public = {key: value for key, value in project.items() if not key.startswith("_")}
|
||||
monkeypatch.setattr(module, "load_projects", lambda: {REPOSITORY: copy.deepcopy(public)})
|
||||
def validate(_repo, branch, revision):
|
||||
assert branch == "main"
|
||||
if revision not in {OBSERVED, ATTEMPT}:
|
||||
raise module.DeployError("revision is not a protected ancestor")
|
||||
return revision
|
||||
monkeypatch.setattr(module, "validate_commit", validate)
|
||||
source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway"
|
||||
labels = {"org.opencontainers.image.revision": OBSERVED,
|
||||
"org.opencontainers.image.source": source,
|
||||
"com.docker.compose.project": "unraid-appops-gateway"}
|
||||
container = {"Id": "c" * 64, "Image": IMAGE,
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Config": {"Image": "appops:observed", "Labels": labels},
|
||||
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
|
||||
"Mounts": [{"Destination": "/data", "Type": "bind", "RW": True,
|
||||
"Source": str(module.APPOPS_DATA_ROOT)}]}
|
||||
image = {"Id": IMAGE, "Config": {"Labels": copy.deepcopy(labels)}}
|
||||
monkeypatch.setattr(module, "inspect_container", lambda name: copy.deepcopy(container))
|
||||
monkeypatch.setattr(module, "inspect_image", lambda name: copy.deepcopy(image))
|
||||
monkeypatch.setattr(module, "compose_project_container_names", lambda name: {CONTAINER})
|
||||
def output(command, **_kwargs):
|
||||
assert command == ["docker", "ps", "-a", "--format", "{{.Names}}"]
|
||||
return CONTAINER + "\n"
|
||||
monkeypatch.setattr(module, "output", output)
|
||||
def forbidden(*_args, **_kwargs):
|
||||
pytest.fail("recovery attempted a host mutation")
|
||||
monkeypatch.setattr(module, "run", forbidden)
|
||||
prior = {"schema_version": 3, "repository": REPOSITORY,
|
||||
"slug": project["slug"], "compose_project": project["compose_project"],
|
||||
"revision": RECORDED, "live_revision": RECORDED,
|
||||
"status": "failed", "live_status": "unknown", "live_legacy_provenance": False,
|
||||
"live_artifacts": {"containers": {CONTAINER: {"image_id": OLD_IMAGE}}},
|
||||
"live_receipt": receipt(), "rollback_snapshot": {"revision": "7" * 40},
|
||||
"last_attempt": {"revision": ATTEMPT, "status": "failed",
|
||||
"message": module._APPOPS_RECOVERY_FAILURE}}
|
||||
path = state_root / "unraid-appops-gateway.json"
|
||||
def save():
|
||||
path.write_text(json.dumps(prior), encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
save()
|
||||
return types.SimpleNamespace(m=module, project=project, repo=tmp_path,
|
||||
path=path, prior=prior, container=container, image=image,
|
||||
public=public, save=save, fsyncs=fsyncs)
|
||||
|
||||
|
||||
def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery):
|
||||
r = recovery
|
||||
before = r.path.read_bytes()
|
||||
compose = (r.repo / "docker-compose.yml").read_bytes()
|
||||
result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
state = json.loads(r.path.read_bytes())
|
||||
backup = r.path.parent / result["backupFile"]
|
||||
assert backup.read_bytes() == before
|
||||
assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest()
|
||||
assert state["last_attempt"] == r.prior["last_attempt"]
|
||||
assert state["live_revision"] == OBSERVED
|
||||
assert state["live_legacy_provenance"] is True
|
||||
assert state["live_status"] == "healthy"
|
||||
assert state["live_receipt"] == state["rollback_snapshot"] == {}
|
||||
assert state["live_artifacts"]["compose_sha256"] == ""
|
||||
assert state["live_artifacts"]["containers"][CONTAINER]["image_id"] == IMAGE
|
||||
assert state["recovery"]["retired_receipt_revision"] == RECORDED
|
||||
assert (r.repo / "docker-compose.yml").read_bytes() == compose
|
||||
assert r.fsyncs == [r.path.parent, r.path.parent]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field,value", [
|
||||
("schema_version", 2), ("status", "success"), ("live_status", "healthy"),
|
||||
("live_legacy_provenance", None), ("live_legacy_provenance", 0),
|
||||
("repository", "jens/other"), ("live_revision", OBSERVED),
|
||||
("rollback_snapshot", None),
|
||||
("live_receipt", None), ("live_receipt", {}), ("live_receipt", "invalid"),
|
||||
("live_artifacts", None), ("live_artifacts", {}),
|
||||
])
|
||||
def test_rejects_ineligible_recorded_state(recovery, field, value):
|
||||
r = recovery
|
||||
r.prior[field] = value
|
||||
r.save()
|
||||
before = r.path.read_bytes()
|
||||
with pytest.raises(r.m.DeployError):
|
||||
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
assert r.path.read_bytes() == before
|
||||
assert not list(r.path.parent.glob("*.bak"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["wrong-failure", "non-ancestor", "same-image", "receipt-image", "receipt-revision"])
|
||||
def test_rejects_unproven_historical_drift(recovery, failure):
|
||||
r = recovery
|
||||
if failure == "wrong-failure":
|
||||
r.prior["last_attempt"]["message"] = "Deployment failed"
|
||||
elif failure == "non-ancestor":
|
||||
r.prior["last_attempt"]["revision"] = "d" * 40
|
||||
elif failure == "same-image":
|
||||
r.prior["live_artifacts"]["containers"][CONTAINER]["image_id"] = IMAGE
|
||||
elif failure == "receipt-image":
|
||||
r.prior["live_receipt"]["containers"][CONTAINER]["image_id"] = IMAGE
|
||||
else:
|
||||
r.prior["live_receipt"]["revision"] = OBSERVED
|
||||
r.save()
|
||||
before = r.path.read_bytes()
|
||||
with pytest.raises(r.m.DeployError):
|
||||
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
assert r.path.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["unhealthy", "volume", "readonly", "mount-path", "port", "image", "source", "revision", "extra-container", "config", "owner"])
|
||||
def test_rejects_wrong_actual_runtime_or_target(recovery, monkeypatch, failure):
|
||||
r = recovery
|
||||
if failure == "unhealthy":
|
||||
r.container["State"]["Health"]["Status"] = "unhealthy"
|
||||
elif failure in {"volume", "readonly", "mount-path"}:
|
||||
key, value = {"volume": ("Type", "volume"), "readonly": ("RW", False),
|
||||
"mount-path": ("Source", "/other")}[failure]
|
||||
r.container["Mounts"][0][key] = value
|
||||
elif failure == "port":
|
||||
r.container["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostPort"] = "9999"
|
||||
elif failure == "image":
|
||||
r.image["Id"] = OLD_IMAGE
|
||||
elif failure in {"source", "revision"}:
|
||||
r.image["Config"]["Labels"]["org.opencontainers.image." + failure] = "wrong"
|
||||
elif failure == "extra-container":
|
||||
monkeypatch.setattr(r.m, "compose_project_container_names", lambda name: {CONTAINER, "other"})
|
||||
elif failure == "config":
|
||||
r.public["branch"] = "other"
|
||||
else:
|
||||
monkeypatch.setattr(r.m, "_root_owned", lambda path: False)
|
||||
before = r.path.read_bytes()
|
||||
with pytest.raises(r.m.DeployError):
|
||||
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
assert r.path.read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["bytes", "inode", "runtime", "config"])
|
||||
def test_rechecks_after_durable_backup_before_replace(recovery, monkeypatch, change):
|
||||
r = recovery
|
||||
def after_backup(_path):
|
||||
if change == "bytes":
|
||||
r.path.write_bytes(b'{"external": true}')
|
||||
elif change == "inode":
|
||||
replacement = r.path.with_suffix(".replacement")
|
||||
replacement.write_bytes(r.path.read_bytes())
|
||||
replacement.chmod(0o600)
|
||||
replacement.replace(r.path)
|
||||
elif change == "runtime":
|
||||
r.container["Id"] = "d" * 64
|
||||
else:
|
||||
r.public["branch"] = "changed"
|
||||
monkeypatch.setattr(r.m, "_fsync_directory", after_backup)
|
||||
with pytest.raises(r.m.DeployError):
|
||||
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
state = json.loads(r.path.read_bytes())
|
||||
assert "recovery" not in state
|
||||
assert len(list(r.path.parent.glob("*.bak"))) == 1
|
||||
assert not list(r.path.parent.glob("*.tmp"))
|
||||
|
||||
|
||||
def test_next_normal_delivery_keeps_recovery_audit(recovery):
|
||||
r = recovery
|
||||
result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
audit = json.loads(r.path.read_bytes())["recovery"]
|
||||
r.m.write_state(REPOSITORY, r.project, OBSERVED, "success", "Verified normal deployment",
|
||||
live_receipt=receipt(OBSERVED, IMAGE), live_legacy_provenance=False)
|
||||
state = json.loads(r.path.read_bytes())
|
||||
assert state["recovery"] == audit
|
||||
assert (r.path.parent / result["backupFile"]).exists()
|
||||
assert state["live_receipt"]["revision"] == OBSERVED
|
||||
assert state["live_legacy_provenance"] is False
|
||||
assert state["last_attempt"]["message"] == "Verified normal deployment"
|
||||
|
||||
|
||||
def test_post_replace_fsync_failure_reports_applied_unknown_and_retains_backup(recovery, monkeypatch):
|
||||
r = recovery
|
||||
calls = 0
|
||||
def fail_after_replace(_path):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 2:
|
||||
raise OSError("synthetic directory fsync failure")
|
||||
monkeypatch.setattr(r.m, "_fsync_directory", fail_after_replace)
|
||||
before = r.path.read_bytes()
|
||||
with pytest.raises(r.m.DeployError, match="state was replaced but durability is unknown"):
|
||||
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
state = json.loads(r.path.read_bytes())
|
||||
assert state["live_revision"] == OBSERVED
|
||||
assert (r.path.parent / state["recovery"]["backup_file"]).read_bytes() == before
|
||||
with pytest.raises(r.m.DeployError, match="exact failed pre-mutation"):
|
||||
r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
|
||||
Reference in new issue
Block a user