Publish curated Unraid-AppOps-Gateway source from 445df2c9552640de621323615ea279b59b731dd8

backend/tests/test_runner_failed_cutover_recovery.py

@@ -0,0 +1,376 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import importlib.util
+import json
+import os
+import stat
+import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+BROKER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
+
+
+def load(name, path):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.fixture
+def recovery(monkeypatch):
+    monkeypatch.syspath_prepend(str(BROKER))
+    return load("failed_cutover_recovery_test", BROKER / "runner_failed_cutover_recovery.py")
+
+
+@pytest.fixture
+def owned(recovery, monkeypatch, tmp_path):
+    fixture = load("recovery_envelope_fixture", Path(__file__).with_name("test_runner_resource_envelope.py"))
+    candidate = fixture._module()
+    probe, image, code = fixture._host_probe_fixture(candidate)
+    code = f"{candidate._HOST_VOLUME_ROOT}/controller-artifacts/{recovery.ORIGIN}"
+    probe["Mounts"][1]["Source"] = code
+    probe["HostConfig"]["Mounts"][1]["Source"] = code
+    probe["Id"] = recovery.PROBE_ID
+    probe["Mounts"][0].update(Source="/sys/fs/cgroup", RW=True)
+    probe["HostConfig"]["Mounts"][0].update(Source="/sys/fs/cgroup", ReadOnly=False)
+    probe["HostConfig"]["CapAdd"] = ["CAP_DAC_OVERRIDE"]
+    probe["State"].update(Status="created", Pid=0, StartedAt="0001-01-01T00:00:00Z", FinishedAt="0001-01-01T00:00:00Z")
+    monkeypatch.setattr(candidate, "_controller_runtime_identity", lambda: (image, code))
+    monkeypatch.setattr(candidate, "_inspect_host_probe_by_name", lambda: copy.deepcopy(probe))
+    monkeypatch.setattr(candidate, "_host_probe_image_labels", lambda _: {})
+    fleet = load("recovery_fleet_fixture", BROKER / "runner_fleet_specialize.py")
+    q = load("recovery_quiescence_fixture", BROKER / "runner_acceptance_quiescence.py")
+    root = tmp_path / "bundle"
+    root.mkdir(mode=0o700)
+    manifest_root = tmp_path / "task-images"
+    manifest_root.mkdir(mode=0o700)
+    manifest_path = manifest_root / "task-image-manifest.json"
+    manifest = {"sourceRevision": recovery.ORIGIN, "sourceTree": recovery.TREE}
+    manifest_raw = json.dumps(manifest).encode()
+    manifest_path.write_bytes(manifest_raw)
+    manifest_path.chmod(0o600)
+    manifest_sha = hashlib.sha256(manifest_raw).hexdigest()
+    marker = {"schemaVersion": 1, "status": "failed", "sourceScriptSha256": recovery.SCRIPT_SHA,
+              "imageSourceRevision": recovery.ORIGIN, "imageSourceTree": recovery.TREE,
+              "imageManifestSha256": manifest_sha, "resourceEnvelopeDigest": "a" * 64,
+              "verifiedRunners": list(fleet._SPECIALIZED_NAMES), "updatedAt": "2026-10-10T00:00:00Z"}
+    originals = [{"id": i, "name": name, "labels": [], "disabled": False}
+                 for i, name in zip((9, 22), fleet._legacy_names("v1"), strict=True)]
+    rows = [{**row, "disabled": True, "busy": False} for row in originals]
+    journal = {"status": "in_progress", "stage": "legacy_removed", "sourceRevision": recovery.ORIGIN,
+               "sourceTree": recovery.TREE, "sourceScriptSha256": recovery.SCRIPT_SHA, "legacyTopology": "v1",
+               "cgroupCreated": False, "cgroupEnsurePending": True, "cgroupFamilyInitiallyPresent": False,
+               "newRegistrationIds": {}, "buildProbe": None, "originalRegistrations": originals,
+               "baseComposeSha256": hashlib.sha256(b"compose").hexdigest(), "imageManifestSha256": manifest_sha}
+    dind = {"Id": "b" * 64, "Image": image, "Created": "fixed", "Name": "/gitea-runner-dind",
+            "Path": "entrypoint", "Args": [], "Config": {}, "HostConfig": {}, "Mounts": [],
+            "NetworkSettings": {"Networks": {}}, "RestartCount": 0,
+            "State": {"Running": True, "Pid": 123, "StartedAt": "fixed", "Health": {"Status": "healthy", "Log": []}}}
+    events = []
+    for name in ("SPECIALIZED_STATE_PATH", "MIGRATION_JOURNAL_PATH", "COMPOSE_PATH", "SPECIALIZED_COMPOSE_PATH",
+                 "RUNNER_CONFIG_ROOT", "_SPECIALIZED_DATA_ROOT", "_SPECIALIZED_WORKSPACE_ROOT"):
+        monkeypatch.setattr(fleet, name, tmp_path / name)
+    monkeypatch.setattr(fleet, "TASK_IMAGE_ROOT", manifest_root)
+    monkeypatch.setattr(fleet, "TASK_IMAGE_MANIFEST_PATH", manifest_path)
+    monkeypatch.setattr(fleet, "_trusted_directory", lambda _: None)
+    monkeypatch.setattr(fleet, "_valid_installed_manifest", lambda _: True)
+    monkeypatch.setattr(fleet, "_read_journal", lambda: copy.deepcopy(journal))
+    monkeypatch.setattr(fleet, "_validate_legacy_registrations", lambda current, _: [{k: v for k, v in row.items() if k != "busy"} for row in current])
+    monkeypatch.setattr(fleet, "_inspect", lambda name: copy.deepcopy(dind) if name == fleet.DIND_NAME else None)
+    monkeypatch.setattr(fleet, "_captured_base_source", lambda _: b"compose")
+    monkeypatch.setattr(fleet, "_validate_base_composition", lambda _: "v1")
+    monkeypatch.setattr(fleet, "_captured_compose_mapping", lambda *args, **kwargs: {})
+    monkeypatch.setattr(fleet, "_captured_compose_model", lambda *args, **kwargs: b"fixed")
+    monkeypatch.setattr(fleet, "_read_source_file", lambda *args, **kwargs: b"compose")
+    monkeypatch.setattr(fleet, "_guard_state_unchanged", lambda _: True)
+    monkeypatch.setattr(fleet, "_guard_script_plan", lambda _: events.append("guard-cas-checked"))
+    monkeypatch.setattr(fleet, "_is_captured_original_dind", lambda *args: True)
+    def read(path, **kwargs):
+        if path == fleet.SPECIALIZED_STATE_PATH:
+            return json.dumps(marker).encode()
+        if path == fleet.MIGRATION_JOURNAL_PATH:
+            return json.dumps(journal).encode()
+        if path == manifest_path:
+            return manifest_path.read_bytes()
+        return b"reviewed"
+    monkeypatch.setattr(recovery, "_read", read)
+    monkeypatch.setattr(recovery, "_installed_proof", lambda: b"reviewed")
+    monkeypatch.setattr(recovery, "BUNDLE", root)
+    monkeypatch.setattr(recovery, "__file__", str(root / "runner_failed_cutover_recovery.py"))
+    monkeypatch.setattr(recovery, "CLAIM", tmp_path / "consumed.json")
+    monkeypatch.setattr(recovery, "sys", SimpleNamespace(flags=SimpleNamespace(isolated=True), dont_write_bytecode=True))
+    monkeypatch.setattr(recovery, "os", SimpleNamespace(name="posix", geteuid=lambda: 0, path=os.path))
+    if os.name == "nt":
+        # Windows chmod cannot materialize POSIX directory bits. The native
+        # Linux qualification exercises the real root/mode checks separately.
+        monkeypatch.setattr(recovery, "stat", SimpleNamespace(
+            S_IMODE=lambda mode: 0o700 if stat.S_ISDIR(mode) else stat.S_IMODE(mode),
+            S_ISDIR=stat.S_ISDIR, S_ISREG=stat.S_ISREG,
+        ))
+    pending = []
+    class Driver:
+        def __init__(self, _projects):
+            self.held = False
+        @contextmanager
+        def maintenance(self):
+            events.append("maintenance-enter")
+            self.held = True
+            yield
+            self.held = False
+            events.append("maintenance-exit")
+        def assert_barrier(self):
+            assert self.held
+        def pending_jobs(self):
+            return pending
+        def registrations(self):
+            return copy.deepcopy(rows)
+        def _run(self, args, **kwargs):
+            assert args[6:] == ["ps", "-a", "--no-trunc", "-q"]
+            return ""
+    @contextmanager
+    def migration(_fleet):
+        events.append("migration-enter")
+        yield lambda: None
+        events.append("migration-exit")
+    monkeypatch.setattr(recovery, "_migration_existing_lock", migration)
+    monkeypatch.setattr(fleet, "_migration_lock", lambda **kwargs: pytest.fail("normal lock would create state"))
+    monkeypatch.setattr(q, "ControllerDriver", Driver)
+    monkeypatch.setattr(recovery, "_load", lambda: (fleet, candidate, q, {}))
+    return fleet, candidate, q, Driver, marker, journal, probe, rows, dind, events, pending
+
+
+def test_readonly_preflight_has_no_claim_or_lifecycle_and_uses_both_locks(recovery, owned, monkeypatch):
+    *_rest, events, _pending = owned
+    monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("readonly claimed"))
+    result = recovery.run()
+    assert result["status"] == "ready" and result["effectsPerformed"] is False
+    assert events == ["migration-enter", "maintenance-enter", "guard-cas-checked", "guard-cas-checked",
+                      "maintenance-exit", "migration-exit"]
+
+
+def test_execute_keeps_existing_wrapper_barrier_and_only_candidate_rollback_hook(recovery, owned, monkeypatch):
+    fleet, candidate, _q, _driver, _marker, _journal, _probe, _rows, _dind, events, _pending = owned
+    def claim(_proof):
+        events.append("consumed")
+    monkeypatch.setattr(recovery, "_claim", claim)
+    def rollback(journal, assert_barrier):
+        assert_barrier()
+        assert events[-1] == "consumed"
+        assert fleet.rollback_host_cgroup_family is candidate.rollback_host_cgroup_family
+        events.append("existing-rollback-body")
+        return True
+    monkeypatch.setattr(fleet, "_rollback_before_activation_locked", rollback)
+    result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
+    assert result["status"] == "rolled_back" and result["invocationConsumed"] is True
+    assert events.index("consumed") < events.index("existing-rollback-body") < events.index("maintenance-exit")
+
+
+@pytest.mark.parametrize("change", ["source", "tree", "script", "active", "stage", "pending", "created", "initial",
+                                    "queue", "busy", "registration", "probe_id", "probe_pid", "probe_started", "cap_extra",
+                                    "probe_mount", "guard_cas", "privatebusy", "manifest", "extra_manifest_file"])
+def test_preflight_rejects_drift_without_claim_or_lifecycle(recovery, owned, monkeypatch, change):
+    fleet, _candidate, _q, driver, marker, journal, probe, rows, _dind, _events, pending = owned
+    if change in {"source", "tree", "script"}:
+        journal[{"source": "sourceRevision", "tree": "sourceTree", "script": "sourceScriptSha256"}[change]] = "foreign"
+    elif change == "active":
+        marker["status"] = "active"
+    elif change == "stage":
+        journal["stage"] = "dind_ready"
+    elif change in {"pending", "created", "initial"}:
+        journal[{"pending": "cgroupEnsurePending", "created": "cgroupCreated", "initial": "cgroupFamilyInitiallyPresent"}[change]] = change != "pending"
+    elif change == "queue":
+        pending.append({"id": 1})
+    elif change == "busy":
+        rows[0]["busy"] = True
+    elif change == "registration":
+        rows[0]["labels"] = ["foreign"]
+    elif change == "probe_id":
+        probe["Id"] = "f" * 64
+    elif change in {"probe_pid", "probe_started"}:
+        probe["State"]["Pid" if change == "probe_pid" else "StartedAt"] = 1 if change == "probe_pid" else "foreign"
+    elif change == "cap_extra":
+        probe["HostConfig"]["CapAdd"].append("SYS_ADMIN")
+    elif change == "probe_mount":
+        probe["Mounts"][0]["Source"] = "/foreign"
+    elif change == "guard_cas":
+        monkeypatch.setattr(fleet, "_guard_script_plan", lambda _: (_ for _ in ()).throw(RuntimeError("private-secret-canary")))
+    elif change == "privatebusy":
+        monkeypatch.setattr(driver, "_run", lambda *args, **kwargs: "f" * 64)
+    elif change == "manifest":
+        fleet.TASK_IMAGE_MANIFEST_PATH.write_text('{"sourceRevision":"foreign"}')
+    else:
+        (fleet.TASK_IMAGE_ROOT / "foreign").touch()
+    monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("refusal claimed"))
+    monkeypatch.setattr(fleet, "_rollback_before_activation", lambda _: pytest.fail("refusal rolled back"))
+    result = recovery.run()
+    assert result["status"] == "blocked" and result["effectsPerformed"] is False
+    assert "private-secret-canary" not in json.dumps(result)
+    assert result["reason"] in recovery.REASONS | {"recovery_admission_rejected"}
+
+
+@pytest.mark.parametrize("field", ["health_log", "pid", "config", "mounts", "started"])
+def test_repeated_admission_preserves_process_identity_except_health_log(recovery, owned, monkeypatch, field):
+    fleet, _candidate, q, driver_type, _marker, _journal, _probe, _rows, dind, _events, _pending = owned
+    driver = driver_type({})
+    with driver.maintenance():
+        _, before = recovery._admit(fleet, _candidate, driver, q)
+        if field == "health_log":
+            dind["State"]["Health"]["Log"] = [{"Start": "later", "End": "later"}]
+        elif field in {"pid", "started"}:
+            dind["State"]["Pid" if field == "pid" else "StartedAt"] = "foreign"
+        elif field == "config":
+            dind["Config"]["Labels"] = {"foreign": "value"}
+        else:
+            dind["Mounts"] = [{"Type": "bind", "Source": "/foreign", "Destination": "/foreign"}]
+        _, after = recovery._admit(fleet, _candidate, driver, q)
+    assert (before == after) is (field == "health_log")
+
+
+@pytest.mark.parametrize("change", ["order", "extra_field", "duplicate", "source", "rw", "propagation"])
+def test_probe_mount_order_is_immaterial_but_complete_mount_identity_is_preserved(recovery, owned, change):
+    fleet, candidate, q, driver_type, _marker, _journal, probe, *_rest = owned
+    driver = driver_type({})
+    with driver.maintenance():
+        _, before = recovery._admit(fleet, candidate, driver, q)
+        if change == "order":
+            probe["Mounts"].reverse()
+        elif change == "extra_field":
+            probe["Mounts"][0]["unexpected_identity"] = "foreign"
+        elif change == "duplicate":
+            probe["Mounts"].append(copy.deepcopy(probe["Mounts"][0]))
+        else:
+            probe["Mounts"][0][{"source": "Source", "rw": "RW", "propagation": "Propagation"}[change]] = "foreign"
+        try:
+            _, after = recovery._admit(fleet, candidate, driver, q)
+        except recovery.RecoveryRejected:
+            assert change != "order"
+            return
+    assert (before == after) is (change == "order")
+
+
+def test_after_claim_failure_is_unknown_never_auto_retried_and_redacted(recovery, owned, monkeypatch):
+    fleet, *_rest = owned
+    calls = []
+    def claim(_proof):
+        recovery.CLAIM.write_text("consumed")
+        calls.append(True)
+    monkeypatch.setattr(recovery, "_claim", claim)
+    monkeypatch.setattr(fleet, "_rollback_before_activation_locked", lambda *args: False)
+    result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
+    assert result["status"] == "unknown" and result["effectsPerformed"] is None and calls == [True]
+    result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
+    assert result["status"] == "blocked" and calls == [True] and recovery.CLAIM.exists()
+
+
+@pytest.mark.parametrize("boundary", ["migration_busy", "maintenance_busy", "second_read", "queue_before_execute"])
+def test_refused_barrier_or_changed_admission_never_claims_or_reaches_rollback_body(recovery, owned, monkeypatch, boundary):
+    fleet, _candidate, _q, driver, _marker, _journal, probe, _rows, _dind, _events, pending = owned
+    @contextmanager
+    def refused(*args, **kwargs):
+        raise RuntimeError("private-canary")
+        yield
+    if boundary == "migration_busy":
+        monkeypatch.setattr(recovery, "_migration_existing_lock", refused)
+    elif boundary == "maintenance_busy":
+        monkeypatch.setattr(driver, "maintenance", refused)
+    elif boundary == "queue_before_execute":
+        pending.append({"id": 1})
+    else:
+        reads = []
+        original = _candidate._inspect_host_probe_by_name
+        def changed():
+            reads.append(True)
+            value = original()
+            if len(reads) == 2:
+                value["Created"] = "changed"
+            return value
+        monkeypatch.setattr(_candidate, "_inspect_host_probe_by_name", changed)
+    monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("refusal claimed"))
+    monkeypatch.setattr(fleet, "_rollback_before_activation_locked", lambda *args: pytest.fail("refusal entered rollback body"))
+    result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION)
+    assert result["status"] == "blocked" and result["effectsPerformed"] is False
+    assert "private-canary" not in json.dumps(result)
+    if boundary == "queue_before_execute":
+        assert result["reason"] == "global_ci_work_present"
+    elif boundary == "second_read":
+        assert result["reason"] == "admission_changed"
+
+
+@pytest.mark.parametrize("confirmation", [None, "foreign"])
+def test_execution_requires_exact_fixed_confirmation(recovery, owned, confirmation):
+    result = recovery.run(execute=True, confirmation=confirmation)
+    assert result["reason"] == "fixed_confirmation_required" and result["effectsPerformed"] is False
+
+
+@pytest.mark.skipif(os.name != "posix", reason="native POSIX root ownership and atomic publication")
+@pytest.mark.parametrize("failure", [None, "before_link", "after_link"])
+def test_atomic_consumed_claim_survives_crash_boundaries_without_replay(recovery, monkeypatch, tmp_path, failure):
+    marker = tmp_path / "once.json"
+    tmp_path.chmod(0o700)
+    monkeypatch.setattr(recovery, "CLAIM", marker)
+    monkeypatch.setattr(recovery, "_parents", lambda _: None)
+    original = recovery.os.fsync
+    calls = []
+    def fsync(fd):
+        calls.append(True)
+        if failure == "before_link" and len(calls) == 1 or failure == "after_link" and len(calls) == 2:
+            raise OSError("private-canary")
+        original(fd)
+    monkeypatch.setattr(recovery.os, "fsync", fsync)
+    if failure:
+        with pytest.raises(OSError):
+            recovery._claim({"bound": True})
+        assert marker.exists() is (failure == "after_link")
+    else:
+        recovery._claim({"bound": True})
+        assert json.loads(marker.read_text())["phase"] == "consumed"
+        assert marker.stat().st_mode & 0o777 == 0o600
+        with pytest.raises(FileExistsError):
+            recovery._claim({"bound": True})
+    assert not list(tmp_path.glob(".ca3a-recovery-*"))
+
+
+@pytest.mark.parametrize("missing", ["directory", "lock"])
+def test_existing_migration_lock_never_creates_missing_operation_state(recovery, monkeypatch, tmp_path, missing):
+    parent = tmp_path / "state"
+    if missing == "lock":
+        parent.mkdir()
+    monkeypatch.setattr(recovery, "_parents", lambda _: None)
+    fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=parent / "specialization-journal.json")
+    before = set(tmp_path.rglob("*"))
+    with pytest.raises(FileNotFoundError):
+        with recovery._migration_existing_lock(fleet):
+            pytest.fail("missing lock acquired")
+    assert set(tmp_path.rglob("*")) == before
+
+
+@pytest.mark.skipif(os.name != "posix", reason="native independent-process flock proof")
+def test_existing_migration_lock_excludes_other_process_and_rejects_leaf_replacement(recovery, monkeypatch, tmp_path):
+    lock = tmp_path / "specialization.lock"
+    lock.write_bytes(b"")
+    lock.chmod(0o600)
+    monkeypatch.setattr(recovery, "_parents", lambda _: None)
+    fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=tmp_path / "specialization-journal.json")
+    probe = ("import fcntl,os,sys; f=os.open(sys.argv[1],os.O_RDWR);\n"
+             "try: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)\n"
+             "except BlockingIOError: sys.exit(0)\n"
+             "sys.exit(1)")
+    with pytest.raises(recovery.RecoveryRejected, match="migration_lock_changed"):
+        with recovery._migration_existing_lock(fleet) as assert_lock:
+            result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False)
+            assert result.returncode == 0 and not result.stdout and not result.stderr
+            assert_lock()
+            replacement = tmp_path / "replacement"
+            replacement.write_bytes(b"")
+            replacement.chmod(0o600)
+            replacement.replace(lock)
+            assert_lock()

backend/tests/test_runner_resource_envelope.py

@@ -896,7 +896,8 @@ def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift():
 
 
 @pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"])
-def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode):
+@pytest.mark.parametrize("capability", ["DAC_OVERRIDE", "CAP_DAC_OVERRIDE"])
+def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode, capability):
     envelope = _module()
     probe, image_id, code_source = _host_probe_fixture(envelope)
     inherited = {"io.buildah.version": "1.35.0", "org.opencontainers.image.title": "Official runner image"}
@@ -905,7 +906,7 @@ def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(mon
     source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup"
     probe["Mounts"][0].update(Source=source, RW=rw)
     probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw)
-    probe["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if rw else []
+    probe["HostConfig"]["CapAdd"] = [capability] if rw else []
     assert envelope._validate_host_probe_inspect(
         probe, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited
     )
@@ -934,6 +935,28 @@ def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(mon
     assert calls == [["docker", "rm", probe["Id"]]]
 
 
+@pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"])
+@pytest.mark.parametrize("cap_add", [None, [], ["DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE"],
+                                    ["DAC_OVERRIDE", "CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "DAC_OVERRIDE"],
+                                    ["CAP_DAC_OVERRIDE", "CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "SYS_ADMIN"],
+                                    ["SYS_ADMIN"], ["dac_override"], ["CAP_CAP_DAC_OVERRIDE"],
+                                    "DAC_OVERRIDE", "", {}, (), False, 0, [None], [{"cap": "DAC_OVERRIDE"}]])
+def test_host_probe_capabilities_allow_only_one_exact_equivalent_spelling(mode, cap_add):
+    envelope = _module()
+    probe, image_id, code_source = _host_probe_fixture(envelope)
+    rw = mode in {"ensure", "rollback"}
+    source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup"
+    probe["Mounts"][0].update(Source=source, RW=rw)
+    probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw)
+    probe["HostConfig"]["CapAdd"] = copy.deepcopy(cap_add)
+    before = copy.deepcopy(probe)
+    expected = cap_add in (["DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE"]) if rw else cap_add is None or cap_add == []
+    assert envelope._validate_host_probe_inspect(
+        probe, mode=mode, image_id=image_id, code_source=code_source
+    ) is expected
+    assert probe == before
+
+
 @pytest.mark.parametrize("drift", ["running", "foreign_source", "race", "missing_state"])
 def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch, drift):
     envelope = _module()