from __future__ import annotations import concurrent.futures import copy import importlib.util import json import os import subprocess import sys import threading from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).resolve().parents[2] ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py" def _module(*, real_lock=False): broker_dir = str(ENVELOPE_PATH.parent) if broker_dir not in sys.path: sys.path.insert(0, broker_dir) spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) if not real_lock: # Existing Docker fixtures exercise identity/cleanup independently of # the protected volume. Native lifecycle tests below retain the lock. @contextmanager def fixture_lock(**_kwargs): yield lambda: None module._host_probe_lifecycle_lock = fixture_lock return module class _ProbeLockStore: """Two independently opened fixture views, with real descriptor/inode checks.""" def __init__(self, logical: Path, host: Path | None = None): self.paths = (logical, host or logical) self.logical = SimpleNamespace(fd=os.open(logical, os.O_RDONLY | os.O_DIRECTORY)) self.host = SimpleNamespace(fd=os.open(host or logical, os.O_RDONLY | os.O_DIRECTORY)) def assert_current(self): identities = [] for view, path in zip((self.logical, self.host), self.paths, strict=True): opened, current = os.fstat(view.fd), path.stat(follow_symlinks=False) assert (opened.st_dev, opened.st_ino) == (current.st_dev, current.st_ino) identities.append((opened.st_dev, opened.st_ino)) if identities[0] != identities[1]: raise RuntimeError("private-canary: mismatched volume views") def __enter__(self): return self def __exit__(self, *_args): os.close(self.host.fd) os.close(self.logical.fd) @pytest.mark.parametrize("mutation", [None, "remote", "options", "labels", "root", "malformed"]) @pytest.mark.parametrize("native_cleanup", [False, True]) def test_probe_lock_store_uses_only_attested_fixed_local_volume(monkeypatch, mutation, native_cleanup): envelope = _module(real_lock=True) import mcp_hub_release_store as store document = [{"Name": store.VOLUME_NAME, "Driver": "local", "Scope": "local", "Mountpoint": store.VOLUME_HOST_ROOT.as_posix(), "Options": {}, "Labels": dict(store.EXPECTED_LABELS)}] root = store.DOCKER_ROOT.as_posix() if mutation == "remote": document[0]["Driver"] = "nfs" elif mutation == "options": document[0]["Options"] = {"device": "private-canary"} elif mutation == "labels": document[0]["Labels"] = {} elif mutation == "root": root = "/srv/example/private-canary" calls = [] def run(args, **kwargs): calls.append(args) assert kwargs == {"timeout": 15} if args == ["docker", "volume", "inspect", store.VOLUME_NAME]: return "private-canary" if mutation == "malformed" else json.dumps(document) assert args == ["docker", "info", "--format", "{{.DockerRootDir}}"] return root sentinel = object() def verify(metadata, docker_root): store.validate_volume_metadata(metadata, docker_root) return sentinel monkeypatch.setattr(envelope, "_run", run) monkeypatch.setattr(envelope, "open_fixed_release_store", verify) class PinnedFixture: @staticmethod def open(path): assert path == store.VOLUME_HOST_ROOT return sentinel monkeypatch.setattr(envelope, "PinnedDirectory", PinnedFixture) if mutation is None: result = envelope._open_host_probe_lock_store(native_cleanup=native_cleanup) if native_cleanup: assert result.host is result.logical is sentinel else: assert result is sentinel assert len(calls) == 2 else: with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_store_untrusted$") as error: envelope._open_host_probe_lock_store(native_cleanup=native_cleanup) assert error.value.__cause__ is None def test_native_cleanup_store_rechecks_only_fixed_physical_volume_and_closes_once(): envelope = _module(real_lock=True) calls = [] directory = SimpleNamespace(assert_current=lambda path: calls.append(path), close=lambda: calls.append("close")) with envelope._NativeProbeLockStore(directory) as store: assert store.host is store.logical is directory store.assert_current() assert calls == [envelope.VOLUME_HOST_ROOT, "close"] @pytest.fixture def native_probe_lock(tmp_path, monkeypatch): if os.name != "posix" or os.geteuid() != 0: pytest.skip("real protected flock contract requires root/POSIX") envelope = _module(real_lock=True) root = tmp_path / "private-lock-root" root.mkdir(mode=0o700) monkeypatch.setattr(envelope, "_open_host_probe_lock_store", lambda **_kwargs: _ProbeLockStore(root)) return envelope, root def test_probe_lock_is_private_stable_and_released_after_failure(native_probe_lock): envelope, root = native_probe_lock with pytest.raises(ValueError, match="fixture failure"), envelope._host_probe_lifecycle_lock() as guard: guard() first = (root / envelope._HOST_PROBE_LOCK_NAME).stat() assert first.st_mode & 0o777 == 0o600 raise ValueError("fixture failure") with envelope._host_probe_lifecycle_lock() as guard: guard() second = (root / envelope._HOST_PROBE_LOCK_NAME).stat() assert (second.st_dev, second.st_ino) == (first.st_dev, first.st_ino) @pytest.mark.parametrize("mutation", ["mode", "owner", "symlink", "hardlink", "nonempty", "directory"]) def test_probe_lock_refuses_untrusted_leaf_without_adopting(native_probe_lock, mutation): envelope, root = native_probe_lock path = root / envelope._HOST_PROBE_LOCK_NAME if mutation == "directory": path.mkdir(mode=0o700) elif mutation == "symlink": target = root / "foreign" target.write_bytes(b"") target.chmod(0o600) path.symlink_to(target) else: path.write_bytes(b"private-canary" if mutation == "nonempty" else b"") path.chmod(0o644 if mutation == "mode" else 0o600) if mutation == "owner": os.chown(path, 1000, 1000) elif mutation == "hardlink": os.link(path, root / "foreign-link") with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_untrusted$"), envelope._host_probe_lifecycle_lock(): pytest.fail("unsafe lock adopted") def test_probe_lock_refuses_replacement_and_different_volume_alias(native_probe_lock, monkeypatch, tmp_path): envelope, root = native_probe_lock with envelope._host_probe_lifecycle_lock() as guard: path = root / envelope._HOST_PROBE_LOCK_NAME path.rename(root / "original-owned-lock") path.touch(mode=0o600) with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_untrusted$"): guard() other = tmp_path / "foreign-volume" other.mkdir(mode=0o700) monkeypatch.setattr(envelope, "_open_host_probe_lock_store", lambda **_kwargs: _ProbeLockStore(root, other)) with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_store_untrusted$"), envelope._host_probe_lifecycle_lock(): pytest.fail("unrelated aliases accepted") @pytest.mark.parametrize("wait", [0.05, 3]) def test_probe_lock_excludes_independent_process_through_independent_view(native_probe_lock, wait): envelope, root = native_probe_lock # Independent interpreter and opens: no inherited Python lock or lock FD. child_source = ( "import importlib.util,json,os,sys\n" "from pathlib import Path\n" "from types import SimpleNamespace\n" "sys.path.insert(0,sys.argv[1])\n" "spec=importlib.util.spec_from_file_location('probe_lock_child',Path(sys.argv[1])/'runner_resource_envelope.py')\n" "m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)\n" "class Store:\n" " def __enter__(self):\n" " self.logical=SimpleNamespace(fd=os.open(sys.argv[2],os.O_RDONLY|os.O_DIRECTORY))\n" " self.host=SimpleNamespace(fd=os.open(sys.argv[2],os.O_RDONLY|os.O_DIRECTORY));return self\n" " def assert_current(self):\n" " assert os.fstat(self.logical.fd).st_ino==os.fstat(self.host.fd).st_ino\n" " def __exit__(self,*args):os.close(self.logical.fd);os.close(self.host.fd)\n" "m._open_host_probe_lock_store=lambda **kw:Store();m._HOST_PROBE_LOCK_WAIT_SECONDS=float(sys.argv[3])\n" "print('started',flush=True)\n" "try:\n" " with m._host_probe_lifecycle_lock(native_cleanup=True) as guard:guard();print('acquired',flush=True)\n" "except RuntimeError as error:print(str(error),flush=True)\n" ) with envelope._host_probe_lifecycle_lock(): child = subprocess.Popen([sys.executable, "-c", child_source, str(ENVELOPE_PATH.parent), str(root), str(wait)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: assert child.stdout.readline().strip() == "started" if wait < 1: output, error = child.communicate(timeout=3) assert output.strip() == "resource_cgroup_probe_lock_busy" assert error == "" and child.returncode == 0 else: assert child.poll() is None finally: if wait < 1 and child.poll() is None: child.kill() child.communicate() if wait >= 1: output, error = child.communicate(timeout=5) assert output.strip() == "acquired" assert error == "" and child.returncode == 0 def test_concurrent_probe_lifecycles_both_complete_without_busy_or_stolen_cleanup(native_probe_lock, monkeypatch): envelope, _root = native_probe_lock probe, image_id, code_source = _host_probe_fixture(envelope) started, release = threading.Event(), threading.Event() state = {"exists": False, "starts": 0, "removals": 0} monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source)) monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {}) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: copy.deepcopy(probe) if state["exists"] else None) def run(args, **_kwargs): if args[:2] == ["docker", "create"]: assert not state["exists"] state["exists"] = True return probe["Id"] if args[:2] == ["docker", "inspect"]: assert state["exists"] return json.dumps([probe]) assert args == ["docker", "rm", probe["Id"]] assert state["exists"] and probe["State"]["Running"] is False state["exists"] = False state["removals"] += 1 return "" def start(args, **_kwargs): assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]] state["starts"] += 1 probe["State"]["Running"] = True if state["starts"] == 1: started.set() assert release.wait(3) probe["State"]["Running"] = False return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}') monkeypatch.setattr(envelope, "_run", run) monkeypatch.setattr(envelope.subprocess, "run", start) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: first = pool.submit(envelope._run_host_probe, "snapshot") assert started.wait(3) second = pool.submit(envelope._run_host_probe, "snapshot") try: assert state["starts"] == 1 and not second.done() finally: release.set() assert first.result(timeout=5) == second.result(timeout=5) == {"verified": True} assert state["removals"] == 2 and state["exists"] is False def test_normal_probe_and_promotion_cleanup_share_same_complete_lock(monkeypatch): envelope = _module(real_lock=True) stages = [] @contextmanager def lock(**kwargs): assert kwargs in ({}, {"native_cleanup": True}) stages.append("acquire") yield lambda: stages.append("guard") stages.append("release") monkeypatch.setattr(envelope, "_host_probe_lifecycle_lock", lock) def normal(mode, payload, guard): assert mode == "family" and payload is None guard() stages.append("normal") return {"verified": True} def cleanup(**kwargs): kwargs["assert_lock"]() stages.append("cleanup") return True monkeypatch.setattr(envelope, "_run_host_probe_locked", normal) monkeypatch.setattr(envelope, "_cleanup_stopped_host_probe_locked", cleanup) assert envelope.inspect_host_cgroup_family() == {"verified": True} assert envelope.cleanup_stopped_host_probe(image_id="sha256:" + "a" * 64, code_source=f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'b' * 40}") assert stages == ["acquire", "guard", "normal", "release", "acquire", "guard", "cleanup", "release"] def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]: return { "path": path, "cpuMax": cpu_max, "memoryMax": str(memory_max), "cpuUsageUsec": "1200", "memoryCurrentBytes": "4096", } def _snapshot(): names = ( "itworx-ci-checks", "itworx-ci-backend", "itworx-ci-frontend", "itworx-ci-security", "itworx-ci-browser", "itworx-ci-build", "gitea-runner-dind", ) containers = [] groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)] dind_path = "" for index, name in enumerate(names, start=1): container_id = f"{index:064x}" path = f"/itworx-ci/{container_id}" if name == "gitea-runner-dind": dind_path = path cpu_max = "600000 100000" memory_max = 12 * 1024**3 running = True else: cpu_max = "50000 100000" memory_max = 512 * 1024**2 # Only two exact outer runner processes may be live. Stopped # configured containers remain part of the identity inventory. running = name in {"itworx-ci-checks", "itworx-ci-backend"} containers.append( { "name": name, "id": container_id, "path": path if running else None, "running": running, **( {} if name == "gitea-runner-dind" else { "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2, } ), } ) if running: groups.append(_group(path, cpu_max, memory_max)) groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3)) return { "version": 2, "family": _group("/itworx-ci", "800000 100000", 16 * 1024**3), "dind": { "path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3), "configuredCpuMilli": 6000, "configuredMemoryBytes": 12 * 1024**3, }, "containers": containers, "groups": groups, } def _fixture_outer_namespace(envelope, monkeypatch): # These pre-existing fixtures isolate resource/config collection. The real # namespace/Engine boundary is exercised by the dedicated tests below. def proof(name): item = envelope._inspect_container(name) return envelope._engine_outer_binding(item), [(item["State"]["Pid"], 1)], { "target": {"name": name, "id": item["Id"]}, "processCount": 1, "threadCount": 1, } monkeypatch.setattr(envelope, "_attest_outer_namespace", proof) def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config): snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) workload_specs = [ ( "8" * 64, {"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0}, 1_000, 1024**3, "100000 100000", ), ( "9" * 64, builder_cpu_config, 3_000, 5 * 1024**3, "300000 100000", ), ] workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs] other_outer_ids = { item["id"] for item in snapshot["containers"] if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"} } old_workload_path = f"{dind_path}/docker/job-1" snapshot["groups"] = [ group for group in snapshot["groups"] if group["path"] != old_workload_path and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids} ] for row in snapshot["containers"]: if row["name"] == envelope.DIND_NAME: continue running = row["name"] == "itworx-ci-build" row["running"] = running row["path"] = f"/itworx-ci/{row['id']}" if running else None if running: snapshot["groups"].append( _group(row["path"], "50000 100000", 512 * 1024**2) ) nested_items = [] for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs: config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config} nested_items.append( { "Id": container_id, "State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71}, "HostConfig": config, } ) snapshot["groups"].append( _group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes) ) snapshot["workloads"] = [ { "id": container_id, "path": path, "nestedPid": 70 if container_id.startswith("8") else 71, "cpuMilli": cpu_milli, "memoryBytes": memory_bytes, } for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path in zip(workload_specs, workload_paths, strict=True) ] profile_names = { profile.container_name: profile.id for profile in envelope.RUNNER_PROFILES.values() } ids = {row["name"]: row["id"] for row in snapshot["containers"]} pids = {name: 2000 + index for index, name in enumerate(ids)} outer_items = {} for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: if name == envelope.DIND_NAME: labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": "runner-dind", } host = { "CgroupParent": "itworx-ci", "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, } state = {"Running": True, "Pid": pids[name]} else: profile_id = profile_names[name] labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, } is_running = name == "itworx-ci-build" host = { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * 1024**2, } state = {"Running": is_running, "Pid": pids[name] if is_running else 0} outer_items[name] = { "Name": f"/{name}", "Id": ids[name], "Config": {"Labels": labels}, "State": state, "HostConfig": host, } def command(args, **_kwargs): if "info" in args: return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" if args[-3:] == ["ps", "--no-trunc", "-q"]: return "\n".join(item["Id"] for item in nested_items) if "inspect" in args: return json.dumps(nested_items) raise AssertionError(f"unexpected fixed Docker command: {args}") monkeypatch.setattr(envelope, "_run", command) monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) probe_calls = [] def fake_probe(operation, payload=None): probe_calls.append((operation, payload)) return snapshot monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) _fixture_outer_namespace(envelope, monkeypatch) return snapshot, nested_items, probe_calls def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( ["build", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["reason"] is None assert result["effectiveCpuMilli"] == 8000 assert result["effectiveMemoryMiB"] == 16384 assert result["activeClaims"] == 2 assert result["heavyClaims"] == 1 assert result["runningRunnerContainers"] == 2 assert result["cgroups"]["dindCpuMilli"] == 6000 assert result["cgroups"]["dindMemoryMiB"] == 12288 assert result["familyCpuUsageUsec"] == 1200 assert result["familyMemoryCurrentBytes"] == 4096 assert len(result["digest"]) == 64 def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( ["backend", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["activeClaims"] == 2 assert result["cgroups"]["reservedCpuMilli"] == 7000 assert result["cgroups"]["reservedMemoryMiB"] == 12288 denied = envelope._evaluate_cgroup_snapshot( ["build", "browser"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert denied["ready"] is False assert denied["reason"] == "resource_claim_heavy_limit_exceeded" def test_attestor_rejects_an_unbounded_workload_descendant(): envelope = _module() snapshot = _snapshot() snapshot["groups"].append( _group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3) ) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unbounded_descendant" def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max")) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["cgroups"]["dindCpuMilli"] == 6000 def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) original_workload = next( item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1" ) original_workload["cpuMax"] = "50000 100000" original_workload["memoryMax"] = str(1024**3) snapshot["groups"].extend( [ _group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3), _group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3), ] ) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["cgroups"]["reservedCpuMilli"] == 4000 assert result["cgroups"]["reservedMemoryMiB"] == 7168 def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].extend( [ _group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3), _group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3), ] ) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved" def test_attestor_rejects_workload_without_an_active_claim(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( [], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_workload_without_claim" def test_attestor_rejects_unknown_outer_container_and_parent_escape(): envelope = _module() snapshot = _snapshot() snapshot["containers"].append( {"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"} ) snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2)) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unknown_outer_container" escaped = _snapshot() escaped["containers"][0]["path"] = "/docker/outside" result = envelope._evaluate_cgroup_snapshot( ["checks"], escaped, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_container_outside_family" def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract(): envelope = _module() snapshot = _snapshot() del snapshot["groups"][-1]["memoryCurrentBytes"] result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_usage_unavailable" result = envelope._evaluate_cgroup_snapshot( ["checks"], _snapshot(), expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_budget_contract_mismatch" def test_claim_count_and_unknown_profiles_fail_closed(): envelope = _module() too_many = envelope._evaluate_cgroup_snapshot( ["checks", "security", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert too_many["ready"] is False assert too_many["reason"] == "resource_claim_slot_limit_exceeded" unknown = envelope._evaluate_cgroup_snapshot( ["general"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert unknown["ready"] is False assert unknown["reason"] == "resource_claim_profile_unknown" def test_attestor_reports_and_rejects_more_than_two_running_outer_runners(): envelope = _module() snapshot = _snapshot() name = "itworx-ci-frontend" item = next(container for container in snapshot["containers"] if container["name"] == name) item["running"] = True item["path"] = f"/itworx-ci/{item['id']}" snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2)) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded" assert result["runningRunnerContainers"] == 3 def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" args = envelope._host_probe_create_args("snapshot", image_id, code_source) assert args[0:3] == ["docker", "create", "--interactive"] assert "--pid=host" not in args assert "--cgroupns=private" in args assert "--network=none" in args assert "--read-only" in args assert "--cap-drop=ALL" in args assert "--cpus=0.1" in args assert "--memory=64m" in args assert "--pids-limit=64" in args assert "--mount" in args mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] assert len(mounts) == 2 mount = mounts[0] assert "source=/sys/fs/cgroup" in mount assert "target=/host-cgroup" in mount assert "readonly" in mount assert mounts[1] == ( f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave" ) assert not any("/proc" in item or "docker.sock" in item for item in args) assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"] def _host_probe_fixture(envelope): image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" base = { "Id": "b" * 64, "Name": "/itworx-runner-cgroup-probe", "Image": image_id, "Config": { "Image": image_id, "Entrypoint": ["python3"], "Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"], "Labels": envelope._host_probe_labels(), "User": "0:0", }, "State": {"Running": False, "ExitCode": 0}, "HostConfig": { "NetworkMode": "none", "PidMode": "", "CgroupnsMode": "private", "ReadonlyRootfs": True, "Privileged": False, "NanoCpus": 100_000_000, "Memory": 64 * 1024**2, "PidsLimit": 64, "CapAdd": [], "CapDrop": ["ALL"], "SecurityOpt": ["no-new-privileges:true"], "RestartPolicy": {"Name": "no"}, "Binds": None, "Volumes": None, "Mounts": [ { "Type": "bind", "Source": "/sys/fs/cgroup/itworx-ci", "Target": "/host-cgroup", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, }, { "Type": "bind", "Source": code_source, "Target": "/opt/gitea-deploy", "ReadOnly": True, "BindOptions": {"Propagation": "rslave"}, }, ], }, "Mounts": [ { "Type": "bind", "Source": "/sys/fs/cgroup/itworx-ci", "Destination": "/host-cgroup", "Mode": "", "RW": False, "Propagation": "rprivate", }, { "Type": "bind", "Source": code_source, "Destination": "/opt/gitea-deploy", "Mode": "", "RW": False, "Propagation": "rslave", }, ], } return base, image_id, code_source def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): envelope = _module() base, image_id, code_source = _host_probe_fixture(envelope) assert envelope._validate_host_probe_inspect( base, mode="snapshot", image_id=image_id, code_source=code_source ) for mode in ("family", "ensure", "rollback"): candidate = copy.deepcopy(base) expected_source = "/sys/fs/cgroup" expected_rw = mode in {"ensure", "rollback"} candidate["Mounts"][0].update( Source=expected_source, Mode="", RW=expected_rw, ) candidate["HostConfig"]["Mounts"][0].update( Source=expected_source, ReadOnly=not expected_rw, ) candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else [] assert envelope._validate_host_probe_inspect( candidate, mode=mode, image_id=image_id, code_source=code_source ) for mutate in ( lambda item: item["HostConfig"].update(PidMode="host"), lambda item: item["HostConfig"].update(CgroupnsMode="host"), lambda item: item["HostConfig"].update(NetworkMode="bridge"), lambda item: item["HostConfig"].update(ReadonlyRootfs=False), lambda item: item["HostConfig"].update(Privileged=True), lambda item: item["HostConfig"].update(Memory=0), lambda item: item["Mounts"].append( {"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False} ), lambda item: item["Mounts"][0].update(RW=True), lambda item: item["Mounts"][1].update(RW=True), lambda item: item["Mounts"][0].update(Propagation="rslave"), lambda item: item["Mounts"][1].update(Propagation="rprivate"), lambda item: item["Mounts"][1].update(Propagation="rshared"), lambda item: item["Mounts"][1].update(Propagation="slave"), lambda item: item["HostConfig"]["Mounts"][0]["BindOptions"].update(Propagation="rslave"), lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rprivate"), lambda item: item["Mounts"][0].update(Mode="ro"), lambda item: item["Mounts"][1].update(Mode="ro"), lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"), lambda item: item["HostConfig"]["Mounts"].append( { "Type": "bind", "Source": "/proc", "Target": "/host-proc", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, } ), lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False), lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"), lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"), ): candidate = copy.deepcopy(base) mutate(candidate) assert not envelope._validate_host_probe_inspect( candidate, mode="snapshot", image_id=image_id, code_source=code_source ) @pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"]) @pytest.mark.parametrize("capability", ["DAC_OVERRIDE", "CAP_DAC_OVERRIDE"]) def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode, capability): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) inherited = {"io.buildah.version": "1.35.0", "org.opencontainers.image.title": "Official runner image"} probe["Config"]["Labels"].update(inherited) rw = mode in {"ensure", "rollback"} source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup" probe["Mounts"][0].update(Source=source, RW=rw) probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw) probe["HostConfig"]["CapAdd"] = [capability] if rw else [] assert envelope._validate_host_probe_inspect( probe, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited ) for mutate in ( lambda item: item["Config"]["Labels"].update(unknown="extra"), lambda item: item["Config"]["Labels"].update({"org.opencontainers.image.title": "changed"}), lambda item: item["Config"]["Labels"].pop("io.buildah.version"), lambda item: item["Config"]["Labels"].update({"tech.itworx.owner": "foreign"}), lambda item: item.update(Image="sha256:" + "f" * 64), ): candidate = copy.deepcopy(probe) mutate(candidate) assert not envelope._validate_host_probe_inspect( candidate, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited ) monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited) inspected = iter([probe, copy.deepcopy(probe), None]) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) calls = [] def remove(args, **kw): calls.append(args) assert kw == {"check": False, "capture_output": True, "timeout": 10} return SimpleNamespace(returncode=0, stderr=b"") monkeypatch.setattr(envelope.subprocess, "run", remove) assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) assert calls == [["docker", "rm", probe["Id"]]] @pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"]) @pytest.mark.parametrize("cap_add", [None, [], ["DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE", "CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "SYS_ADMIN"], ["SYS_ADMIN"], ["dac_override"], ["CAP_CAP_DAC_OVERRIDE"], "DAC_OVERRIDE", "", {}, (), False, 0, [None], [{"cap": "DAC_OVERRIDE"}]]) def test_host_probe_capabilities_allow_only_one_exact_equivalent_spelling(mode, cap_add): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) rw = mode in {"ensure", "rollback"} source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup" probe["Mounts"][0].update(Source=source, RW=rw) probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw) probe["HostConfig"]["CapAdd"] = copy.deepcopy(cap_add) before = copy.deepcopy(probe) expected = cap_add in (["DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE"]) if rw else cap_add is None or cap_add == [] assert envelope._validate_host_probe_inspect( probe, mode=mode, image_id=image_id, code_source=code_source ) is expected assert probe == before @pytest.mark.parametrize("drift", ["running", "foreign_source", "race", "missing_state"]) def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch, drift): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) fresh = copy.deepcopy(probe) if drift == "running": probe["State"]["Running"] = True elif drift == "foreign_source": probe["Mounts"][1]["Source"] = code_source.replace("c" * 40, "d" * 40) elif drift == "race": fresh["Id"] = "f" * 64 else: probe.pop("State") monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {}) inspected = iter([probe, fresh]) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) monkeypatch.setattr(envelope, "_run", lambda *a, **kw: pytest.fail("No removal or probe execution")) monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda *a: pytest.fail("No removal")) with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_"): envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) @pytest.mark.parametrize("drift", [None, "extra_mount", "changed_mount_field", "changed_unknown_mount_field", "changed_top_level_field", "changed_host_mounts", "changed_state"]) def test_probe_cleanup_cas_ignores_only_complete_mount_collection_order(monkeypatch, drift): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) fresh = copy.deepcopy(probe) fresh["Mounts"].reverse() if drift == "extra_mount": fresh["Mounts"].append(copy.deepcopy(fresh["Mounts"][0])) elif drift == "changed_mount_field": fresh["Mounts"][0]["RW"] = True elif drift == "changed_unknown_mount_field": fresh["Mounts"][0]["unexpectedMetadata"] = "changed" elif drift == "changed_top_level_field": fresh["unexpectedMetadata"] = "changed" elif drift == "changed_host_mounts": fresh["HostConfig"]["Mounts"].reverse() elif drift == "changed_state": fresh["State"]["unexpectedMetadata"] = "changed" monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {}) inspected = iter([probe, fresh, None]) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) removed = [] monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda identifier: removed.append(identifier)) if drift is None: assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) assert removed == [probe["Id"]] else: with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_changed"): envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) assert removed == [] assert fresh != probe @pytest.mark.parametrize("stderr,condition", [ (b"private-canary: no space left on device", "remove_storage_full"), (b"private-canary: read-only file system", "remove_read_only"), (b"private-canary: permission denied", "remove_permission_denied"), (b"Cannot connect to the Docker daemon at private-canary", "remove_daemon_unavailable"), (b"private-canary: device or resource busy", "remove_busy"), (b"Removal of container private-canary is already in progress", "remove_busy"), (b"No such container: private-canary", "remove_container_missing"), (b"private-canary: client version is too old", "remove_api_version_unsupported"), (b"removal of container private-canary failed unexpectedly", "remove_failed"), (b"private-canary", "remove_failed"), (b"no space left on device" + b"x" * 4096, "remove_failed"), ]) def test_probe_removal_has_finite_diagnostics_and_never_forces(monkeypatch, stderr, condition): envelope = _module() identifier = "d" * 64 def run(args, **kw): assert args == ["docker", "rm", identifier] assert kw == {"check": False, "capture_output": True, "timeout": 10} return SimpleNamespace(returncode=1, stderr=stderr, stdout=b"private-canary") monkeypatch.setattr(envelope.subprocess, "run", run) with pytest.raises(envelope.HostProbeCleanupError) as error: envelope._remove_cleanup_probe(identifier) assert str(error.value) == "resource_cgroup_probe_cleanup_" + condition assert "private-canary" not in str(error.value) assert error.value.__cause__ is None @pytest.mark.parametrize("identifier", ["probe", "d" * 12, "d" * 64 + " --force", "../private-canary"]) def test_probe_removal_rejects_selectors_before_any_command(monkeypatch, identifier): envelope = _module() monkeypatch.setattr(envelope.subprocess, "run", lambda *a, **kw: pytest.fail("No command")) with pytest.raises(envelope.HostProbeCleanupError, match="cleanup_identity_invalid"): envelope._remove_cleanup_probe(identifier) @pytest.mark.parametrize("stage", ["image_metadata_failed", "inspect_failed", "reinspect_failed", "absence_failed"]) def test_probe_read_failures_drop_untrusted_exception_text(stage): envelope = _module() def fail(): raise RuntimeError("Bearer private-canary") with pytest.raises(envelope.HostProbeCleanupError) as error: envelope._cleanup_probe_read(stage, fail) assert str(error.value) == "resource_cgroup_probe_cleanup_" + stage assert error.value.__cause__ is None @pytest.mark.skipif(os.name == "nt", reason="Docker CLI shim uses POSIX executable semantics") @pytest.mark.parametrize("failure,condition", [("read-only file system private-canary", "remove_read_only"), ("Bearer private-canary", "remove_failed")]) def test_real_probe_cleanup_cli_exposes_only_finite_child_failure(tmp_path, failure, condition): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) fixture = tmp_path / "fixture.json" fixture.write_text(json.dumps({"probe": probe, "image": image_id, "failure": failure})) shim = tmp_path / "docker" shim.write_text("#!" + sys.executable + "\n" + "import json,sys,pathlib\n" + "v=json.loads(pathlib.Path(" + repr(str(fixture)) + ").read_text())\n" + "a=sys.argv[1:]\n" + "if a[:2]==['image','inspect']: print(json.dumps([{'Id':v['image'],'Config':{'Labels':{}}}]))\n" + "elif a[:2]==['ps','-a']: print(v['probe']['Id'])\n" + "elif a[0]=='inspect': print(json.dumps([v['probe']]))\n" + "elif a==['rm',v['probe']['Id']]: print(v['failure'],file=sys.stderr); sys.exit(1)\n" + "else: sys.exit(99)\n") shim.chmod(0o700) # Keep the actual CLI branch/diagnostics, isolating only its volume lock. entry = tmp_path / "cleanup-entry.py" source = ENVELOPE_PATH.read_text(encoding="utf-8") source = source.replace('if __name__ == "__main__":', '@contextmanager\ndef _host_probe_lifecycle_lock(**_kwargs):\n' ' yield lambda: None\n\nif __name__ == "__main__":') entry.write_text(source, encoding="utf-8") result = subprocess.run([sys.executable, str(entry), "--cleanup-stopped-host-probe", image_id, code_source], capture_output=True, timeout=15, check=False, env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", ""), "PYTHONPATH": str(ENVELOPE_PATH.parent)}) assert result.returncode == 1 assert result.stdout == b"" assert result.stderr == ("resource_cgroup_probe_cleanup_" + condition + "\n").encode() @pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.title": "runner"}]) def test_probe_label_metadata_comes_from_exact_image(monkeypatch, labels): envelope = _module() image_id = "sha256:" + "a" * 64 calls = [] def run(args, **kw): calls.append(args) return json.dumps([{"Id": image_id, "Config": {"Labels": labels}}]) monkeypatch.setattr(envelope, "_run", run) assert envelope._host_probe_image_labels(image_id) == (labels or {}) assert calls == [["docker", "image", "inspect", image_id]] @pytest.mark.parametrize("row", [ {"Id": "sha256:" + "b" * 64, "Config": {"Labels": {}}}, {"Id": "sha256:" + "a" * 64, "Config": {"Labels": ["foreign"]}}, {"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"secret-canary": 123}}}, {"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"tech.itworx.owner": "foreign"}}}, ]) def test_probe_image_metadata_rejects_drift_without_echo(monkeypatch, row): envelope = _module() monkeypatch.setattr(envelope, "_run", lambda *a, **kw: json.dumps([row])) with pytest.raises(RuntimeError, match="^resource_cgroup_probe_image_labels_invalid$"): envelope._host_probe_image_labels("sha256:" + "a" * 64) def test_normal_probe_executes_and_removes_its_image_label_bound_container(monkeypatch): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) inherited = {"org.opencontainers.image.title": "Official runner image"} probe["Config"]["Labels"].update(inherited) monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source)) monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None) calls = [] def run(args, **kw): calls.append(args) if args[:2] == ["docker", "create"]: return probe["Id"] if args[:2] == ["docker", "inspect"]: return json.dumps([probe]) assert args == ["docker", "rm", probe["Id"]] return "" def start(args, **kw): assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]] assert json.loads(kw["input"]) == {"op": "snapshot"} return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}') monkeypatch.setattr(envelope, "_run", run) monkeypatch.setattr(envelope.subprocess, "run", start) assert envelope._run_host_probe("snapshot") == {"verified": True} assert calls[-1] == ["docker", "rm", probe["Id"]] def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch): envelope = _module() revision = "d" * 40 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}" image_id = "sha256:" + "e" * 64 release = {"sourceRevision": revision} controller = { "Name": "/gitea-deploy-control", "Image": image_id, "State": {"Running": True}, "Mounts": [ { "Type": "bind", "Source": code_source, "Destination": "/opt/gitea-deploy", "RW": False, } ], } monkeypatch.setitem( sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release), ) monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller])) assert envelope._controller_runtime_identity() == (image_id, code_source) controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}" with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"): envelope._controller_runtime_identity() def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch): envelope = _module() calls = [] def probe(operation, payload=None): calls.append((operation, payload)) return {"created": True, "rootSubtreeBefore": ["pids"]} monkeypatch.setattr(envelope, "_run_host_probe", probe) result = envelope.ensure_host_cgroup_family(allow_existing=False) assert result == {"created": True, "rootSubtreeBefore": ["pids"]} assert calls == [("ensure", {"allowExisting": False})] def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch): envelope = _module() calls = [] monkeypatch.setattr( envelope, "_run_host_probe", lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False}, ) result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"]) assert result == {"removed": True, "familyExists": False} assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})] with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"]) def _owned_family_fixture(monkeypatch, *, exact=False, init=True, docker=True): envelope = _module() cid = "a" * 64 leaf = f"/itworx-ci/{cid}" paths = ["/itworx-ci", leaf] if init: paths.append(f"{leaf}/init") if docker: paths.append(f"{leaf}/docker") rows = { path: {"path": path, "cpuMax": "max 100000", "memoryMax": "max"} for path in paths } rows[leaf].update(cpuMax="600000 100000", memoryMax=str(12 * 1024**3)) if exact: rows["/itworx-ci"].update(cpuMax="800000 100000", memoryMax=str(16 * 1024**3)) processes = {path: "" for path in paths} processes[f"{leaf}/init" if init else leaf] = "1 2" descriptors = {path: index for index, path in enumerate(paths, start=101)} reverse = {value: path for path, value in descriptors.items()} required = {"cpu", "memory", "pids"} controls = {"root": required.copy(), "family": required.copy()} inodes = {path: value for path, value in descriptors.items()} monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, controls["root"])) monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 101) monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [dict(row) for row in rows.values()]) monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, path: descriptors[path]) monkeypatch.setattr(envelope.os, "dup", lambda fd: fd) monkeypatch.setattr(envelope.os, "close", lambda _fd: None) monkeypatch.setattr(envelope.os, "fstat", lambda fd: SimpleNamespace( st_dev=36, st_ino=inodes[reverse[fd]], st_uid=0, st_mode=0o40755, )) monkeypatch.setattr(envelope, "_probe_stat", lambda _fd, _name: SimpleNamespace(st_dev=36, st_ino=inodes["/itworx-ci"])) def read(fd, name): path = reverse[fd] if name == "cgroup.procs": return processes[path] if name == "cgroup.threads": return processes[path] if name in {"pids.current", "cgroup.events"}: count = sum(len(value.split()) for group, value in processes.items() if group == path or group.startswith(path + "/")) return str(count) if name == "pids.current" else f"populated {int(count > 0)}\nfrozen 0" if name == "cgroup.subtree_control": return " ".join(sorted(controls["family"])) raise AssertionError(name) writes = [] def write(fd, name, value): assert fd == 101 assert name in {"cpu.max", "memory.max"} writes.append((name, value)) rows["/itworx-ci"]["cpuMax" if name == "cpu.max" else "memoryMax"] = value monkeypatch.setattr(envelope, "_probe_read", read) monkeypatch.setattr(envelope, "_probe_write", write) claim = {"containerId": cid, "processCount": 2, "threadCount": 2} return envelope, claim, rows, processes, controls, writes, inodes @pytest.mark.parametrize("init,docker,exact", [(False, False, False), (True, False, False), (True, True, False), (True, True, True)]) def test_owned_dind_family_binds_only_existing_exact_private_child(monkeypatch, init, docker, exact): envelope, claim, _rows, _processes, _controls, writes, _inodes = _owned_family_fixture( monkeypatch, init=init, docker=docker, exact=exact, ) result = envelope._probe_ensure_owned_dind_family(1, claim) assert result == {"created": False, "rootSubtreeBefore": ["cpu", "memory", "pids"], "familyDevice": 36, "familyInode": 101, "familyCpuMax": "800000 100000", "familyMemoryMax": str(16 * 1024**3)} assert writes == ([] if exact else [("cpu.max", "800000 100000"), ("memory.max", str(16 * 1024**3))]) @pytest.mark.parametrize("mutation", ["missing", "foreign_group", "foreign_pid", "root_pid", "docker_pid", "zero_pid", "pid_missing", "duplicate_pid", "dind_cpu", "dind_memory", "foreign_budget", "mixed_budget", "root_controls", "family_controls", "unknown_controls"]) def test_owned_dind_family_rejects_foreign_or_invisible_work_before_writes(monkeypatch, mutation): envelope, claim, rows, processes, controls, writes, _inodes = _owned_family_fixture(monkeypatch) leaf = f"/itworx-ci/{claim['containerId']}" if mutation == "missing": monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: None) elif mutation == "foreign_group": rows[f"{leaf}/docker/foreign"] = {"path": f"{leaf}/docker/foreign", "cpuMax": "max", "memoryMax": "max"} elif mutation in {"foreign_pid", "zero_pid", "pid_missing", "duplicate_pid"}: processes[f"{leaf}/init"] = {"foreign_pid": "1 0", "zero_pid": "0", "pid_missing": "2", "duplicate_pid": "1 1"}[mutation] elif mutation == "root_pid": processes["/itworx-ci"] = "2" elif mutation == "docker_pid": processes[f"{leaf}/docker"] = "2" elif mutation == "dind_cpu": rows[leaf]["cpuMax"] = "max 100000" elif mutation == "dind_memory": rows[leaf]["memoryMax"] = "max" elif mutation == "foreign_budget": rows["/itworx-ci"]["cpuMax"] = "700000 100000" elif mutation == "mixed_budget": rows["/itworx-ci"]["cpuMax"] = "800000 100000" elif mutation == "root_controls": controls["root"].remove("memory") elif mutation == "family_controls": controls["family"].remove("pids") elif mutation == "unknown_controls": controls["family"].add("foreign") with pytest.raises(RuntimeError, match="resource_cgroup_"): envelope._probe_ensure_owned_dind_family(1, claim) assert writes == [] @pytest.mark.parametrize("boundary", ["before_cpu", "before_memory", "readback"]) def test_owned_dind_family_rechecks_full_identity_at_each_write_boundary(monkeypatch, boundary): envelope, claim, _rows, _processes, _controls, writes, inodes = _owned_family_fixture(monkeypatch) original = envelope._probe_owned_dind_state calls = 0 def changing(*args): nonlocal calls calls += 1 if calls == {"before_cpu": 2, "before_memory": 3, "readback": 4}[boundary]: inodes[f"/itworx-ci/{claim['containerId']}"] += 1 return original(*args) monkeypatch.setattr(envelope, "_probe_owned_dind_state", changing) with pytest.raises(RuntimeError, match="resource_cgroup_probe_path_untrusted"): envelope._probe_ensure_owned_dind_family(1, claim) assert len(writes) == {"before_cpu": 0, "before_memory": 1, "readback": 2}[boundary] @pytest.mark.parametrize("mutation", ["pid", "controller", "budget"]) def test_owned_dind_family_refuses_drift_after_cpu_intent(monkeypatch, mutation): envelope, claim, rows, processes, controls, writes, _inodes = _owned_family_fixture(monkeypatch) write = envelope._probe_write def changing(*args): write(*args) if mutation == "pid": processes[f"/itworx-ci/{claim['containerId']}/init"] = "1 0" elif mutation == "controller": controls["root"].remove("pids") else: rows["/itworx-ci"]["cpuMax"] = "max 100000" monkeypatch.setattr(envelope, "_probe_write", changing) with pytest.raises(RuntimeError, match="resource_cgroup_"): envelope._probe_ensure_owned_dind_family(1, claim) assert writes == [("cpu.max", "800000 100000")] @pytest.mark.parametrize("mixed", [False, True]) def test_owned_dind_dispatch_keeps_empty_ensure_contract_distinct(monkeypatch, mixed): envelope, claim, _rows, _processes, _controls, writes, _inodes = _owned_family_fixture(monkeypatch) monkeypatch.setattr(envelope, "Path", lambda value: SimpleNamespace(lstat=lambda: SimpleNamespace( st_mode=0o40755, st_uid=0, ))) monkeypatch.setattr(envelope.os, "open", lambda *_args: 1) monkeypatch.setattr(envelope.os, "mkdir", lambda *_args, **_kwargs: pytest.fail("owned ensure must not create a family")) request = {"op": "ensure", "ownedDind": claim} if mixed: request["allowExisting"] = True with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"): envelope._host_probe_dispatch(request) assert writes == [] else: assert envelope._host_probe_dispatch(request)["familyInode"] == 101 @pytest.mark.parametrize("mutate", ["id", "pid_bool", "pid_negative", "process_bool", "duplicate", "missing_pid", "too_many", "extra"]) def test_owned_dind_claim_is_typed_and_finite(monkeypatch, mutate): envelope = _module() claim = {"containerId": "a" * 64, "hostPid": 1234, "processIds": [1234]} if mutate == "id": claim["containerId"] = "private-canary" elif mutate == "pid_bool": claim["hostPid"] = True elif mutate == "pid_negative": claim["hostPid"] = -1 elif mutate == "process_bool": claim["processIds"] = [True] elif mutate == "duplicate": claim["processIds"] = [1234, 1234] elif mutate == "missing_pid": claim["processIds"] = [1235] elif mutate == "too_many": claim["processIds"] = [1234, *range(1, 257)] elif mutate == "extra": claim["path"] = "private-canary" with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"): envelope._owned_dind_claim(claim) def test_owned_dind_public_api_uses_existing_fixed_ensure_only(monkeypatch): envelope = _module() calls = [] result = {"created": False, "rootSubtreeBefore": ["cpu", "memory", "pids"], "familyDevice": 36, "familyInode": 101, "familyCpuMax": "800000 100000", "familyMemoryMax": str(16 * 1024**3)} monkeypatch.setattr(envelope, "_engine_outer_population", lambda name: ({"Id": "a" * 64, "State": {"Pid": 1234}}, [(1234, 1), (1235, 1)])) monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, request: calls.append((mode, request)) or result) assert envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234, expected_process_ids=[1235, 1234]) == result assert calls == [("ensure", {"ownedDind": {"containerId": "a" * 64, "processCount": 2, "threadCount": 2, "allowPartial": False}})] with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"): envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234, expected_process_ids="1234") assert len(calls) == 1 @pytest.mark.parametrize("mutation", ["created", "inode", "device", "cpu", "memory", "controllers", "extra"]) def test_owned_dind_public_api_rejects_invalid_readback(monkeypatch, mutation): envelope = _module() result = {"created": False, "rootSubtreeBefore": ["cpu", "memory", "pids"], "familyDevice": 36, "familyInode": 101, "familyCpuMax": "800000 100000", "familyMemoryMax": str(16 * 1024**3)} key, value = {"created": ("created", True), "inode": ("familyInode", True), "device": ("familyDevice", -1), "cpu": ("familyCpuMax", "max 100000"), "memory": ("familyMemoryMax", "max"), "controllers": ("rootSubtreeBefore", ["cpu", "pids"]), "extra": ("private-canary", "secret-canary")}[mutation] result[key] = value monkeypatch.setattr(envelope, "_engine_outer_population", lambda name: ({"Id": "a" * 64, "State": {"Pid": 1234}}, [(1234, 1)])) monkeypatch.setattr(envelope, "_run_host_probe", lambda *_args: result) with pytest.raises(RuntimeError, match="^resource_cgroup_family_readback_failed$"): envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234, expected_process_ids=[1234]) @pytest.mark.parametrize("pid_location", ["leaf", "init", "missing"]) def test_snapshot_dind_pid_allows_only_exact_owned_init_or_leaf(monkeypatch, pid_location): envelope, claim, rows, processes, _controls, _writes, _inodes = _owned_family_fixture(monkeypatch) leaf = f"/itworx-ci/{claim['containerId']}" processes[f"{leaf}/init"] = "" if pid_location == "leaf": processes[leaf] = "1 2" elif pid_location == "init": processes[f"{leaf}/init"] = "1 2" outer = [ {"name": name, "id": claim["containerId"] if name == envelope.DIND_NAME else f"{index:064x}", "running": name == envelope.DIND_NAME, "pid": 1234 if name == envelope.DIND_NAME else None, "processCount": 2 if name == envelope.DIND_NAME else None, "threadCount": 2 if name == envelope.DIND_NAME else None, "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2} for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1) ] request = {"op": "snapshot", "outer": outer, "workloads": []} if pid_location in {"leaf", "init"}: result = envelope._probe_snapshot(request, 101) assert result["dind"]["path"] == leaf else: with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"): envelope._probe_snapshot(request, 101) @pytest.mark.parametrize("suffix", ["init", "init/foreign", "foreign"]) def test_attestor_allows_only_single_known_dind_init_group(suffix): envelope = _module() snapshot = _snapshot() snapshot["groups"].append(_group(f"{snapshot['dind']['path']}/{suffix}", "max 100000", "max")) result = envelope._evaluate_cgroup_snapshot(["build", "backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}) assert result["ready"] is (suffix == "init") if suffix != "init": assert result["reason"] == "resource_cgroup_unmapped_descendant" @pytest.mark.parametrize("cpu,memory,partial", [("800000 100000", "max", False), ("800000 100000", "max", True), ("max 100000", str(16 * 1024**3), False), ("max 100000", str(16 * 1024**3), True)]) def test_owned_partial_budget_requires_explicit_bound_resume(monkeypatch, cpu, memory, partial): envelope, claim, rows, _processes, _controls, writes, _inodes = _owned_family_fixture(monkeypatch) rows["/itworx-ci"].update(cpuMax=cpu, memoryMax=memory) claim["allowPartial"] = partial if not partial or memory != "max": with pytest.raises(RuntimeError, match="^resource_cgroup_family_budget_conflict$"): envelope._probe_ensure_owned_dind_family(1, claim) assert writes == [] else: assert envelope._probe_ensure_owned_dind_family(1, claim)["familyInode"] == 101 assert writes == ([("memory.max", str(16 * 1024**3))] if memory == "max" else [("cpu.max", "800000 100000")]) @pytest.mark.parametrize("partial", [1, "true", None]) def test_owned_partial_flag_is_not_coerced(monkeypatch, partial): envelope, claim, _rows, _processes, _controls, writes, _inodes = _owned_family_fixture(monkeypatch) claim["allowPartial"] = partial with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"): envelope._probe_ensure_owned_dind_family(1, claim) with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"): envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234, expected_process_ids=[1234], allow_owned_partial=partial) assert writes == [] @pytest.mark.parametrize("foreign", [False, True]) def test_namespace_probe_rejects_foreign_zero_even_when_top_counts_match(monkeypatch, foreign): envelope, claim, _rows, processes, _controls, _writes, _inodes = _owned_family_fixture(monkeypatch) if foreign: processes[f"/itworx-ci/{claim['containerId']}/init"] = "1 0" request = {"op": "population", "target": {"name": envelope.DIND_NAME, "id": claim["containerId"]}, "processCount": 2, "threadCount": 2} if foreign: with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"): envelope._probe_namespace_population(101, request) else: assert envelope._probe_namespace_population(101, request) == {"namespaceOwned": True, "processCount": 2, "threadCount": 2} def test_private_zero_is_only_an_opaque_population_entry(monkeypatch): envelope, claim, _rows, processes, _controls, _writes, _inodes = _owned_family_fixture(monkeypatch) processes[f"/itworx-ci/{claim['containerId']}/init"] = "0 0" population = envelope._probe_population(103) assert population == {"processCount": 2, "threadCount": 2, "hierarchicalThreads": 2, "populated": True} assert envelope._probe_empty(103) is False with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"): envelope._probe_population(103, owned_namespace=True) @pytest.mark.parametrize("mode", ["ensure", "population"]) @pytest.mark.parametrize("drift", [None, "host", "private", "other_cid", "label", "mount"]) def test_namespace_join_is_exact_bound_container_only(mode, drift): envelope = _module() base, image_id, source = _host_probe_fixture(envelope) target = {"name": envelope.DIND_NAME, "id": "a" * 64} args = envelope._host_probe_create_args(mode, image_id, source, namespace_target=target) assert args[args.index("--pid") + 1] == "container:" + target["id"] assert not any("/proc" in value or "--pid=host" in value or "docker.sock" in value for value in args) namespace_labels = {"tech.itworx.probe.namespace-name": target["name"], "tech.itworx.probe.namespace-id": target["id"]} base["Config"]["Labels"].update(namespace_labels) base["HostConfig"]["PidMode"] = "container:" + target["id"] expected_source = "/sys/fs/cgroup" if mode == "ensure" else "/sys/fs/cgroup/itworx-ci/" + target["id"] base["Mounts"][0].update(Source=expected_source, RW=mode == "ensure") base["HostConfig"]["Mounts"][0].update(Source=expected_source, ReadOnly=mode != "ensure") base["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if mode == "ensure" else [] if drift in {"host", "private", "other_cid"}: base["HostConfig"]["PidMode"] = {"host": "host", "private": "", "other_cid": "container:" + "b" * 64}[drift] elif drift == "label": base["Config"]["Labels"]["tech.itworx.probe.namespace-id"] = "b" * 64 elif drift == "mount": base["HostConfig"]["Mounts"][0]["Source"] = "/proc" assert envelope._validate_host_probe_inspect(base, mode=mode, image_id=image_id, code_source=source, namespace_target=target) is (drift is None) assert not envelope._validate_host_probe_inspect(base, mode=mode, image_id=image_id, code_source=source) if drift is None: assert envelope._stopped_probe_namespace(base) == target @pytest.mark.parametrize("mode,name", [("snapshot", "gitea-runner-dind"), ("rollback", "gitea-runner-dind"), ("family", "gitea-runner-dind"), ("ensure", "itworx-ci-build"), ("population", "foreign")]) def test_namespace_join_never_expands_other_modes_or_targets(mode, name): envelope = _module() _base, image_id, source = _host_probe_fixture(envelope) with pytest.raises(RuntimeError, match="^resource_cgroup_probe_namespace_invalid$"): envelope._host_probe_create_args(mode, image_id, source, namespace_target={"name": name, "id": "a" * 64}) @pytest.mark.parametrize("drift", [None, "pid", "start", "image", "threads", "foreign_namespace"]) def test_namespace_attestation_rechecks_engine_without_counts_only_adoption(monkeypatch, drift): envelope = _module() before = {"Id": "a" * 64, "Image": "sha256:" + "c" * 64, "State": {"Pid": 1234, "StartedAt": "fixed-start"}} pairs = [(1234, 2), (1235, 1)] after, after_pairs = copy.deepcopy(before), list(pairs) if drift == "pid": after["State"]["Pid"] = 1236 elif drift == "start": after["State"]["StartedAt"] = "another-start" elif drift == "image": after["Image"] = "sha256:" + "d" * 64 elif drift == "threads": after_pairs[0] = (1234, 3) calls = [] def engine(name): assert name == envelope.DIND_NAME calls.append(name) return (before, pairs) if len(calls) == 1 else (after, after_pairs) monkeypatch.setattr(envelope, "_engine_outer_population", engine) def probe(mode, claim): assert mode == "population" and claim["target"] == {"name": envelope.DIND_NAME, "id": "a" * 64} assert claim["processCount"] == 2 and claim["threadCount"] == 3 if drift == "foreign_namespace": raise RuntimeError("resource_cgroup_process_mapping_invalid") return {"namespaceOwned": True, "processCount": 2, "threadCount": 3} monkeypatch.setattr(envelope, "_run_host_probe", probe) if drift is None: assert envelope._attest_outer_namespace(envelope.DIND_NAME)[0] == before else: with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"): envelope._attest_outer_namespace(envelope.DIND_NAME) @pytest.mark.parametrize("output", ["PID NLWP\n1234 2\n1235 1", "PID NLWP\n1234 0", "PID NLWP\n1234 2\n1234 1", "PID NLWP\n1235 2", "PID COMMAND\n1234 private-canary", "PID NLWP\n0 1", "PID NLWP\n1234 16385", "PID NLWP\n1234 1 extra"]) def test_engine_population_reads_only_fixed_cid_numeric_pid_thread_columns(monkeypatch, output): envelope = _module() item = {"Name": "/" + envelope.DIND_NAME, "Id": "a" * 64, "Image": "sha256:" + "b" * 64, "Created": "fixed-created", "State": {"Running": True, "Pid": 1234, "StartedAt": "fixed-start"}, "HostConfig": {"CgroupParent": "itworx-ci"}} monkeypatch.setattr(envelope, "_inspect_container", lambda name: item) commands = [] def run(args, **kwargs): commands.append((args, kwargs)) return output monkeypatch.setattr(envelope, "_run", run) if output == "PID NLWP\n1234 2\n1235 1": binding, pairs = envelope._engine_outer_population(envelope.DIND_NAME) assert binding["Id"] == "a" * 64 and pairs == [(1234, 2), (1235, 1)] else: with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"): envelope._engine_outer_population(envelope.DIND_NAME) assert commands == [(["docker", "top", "a" * 64, "-eo", "pid,nlwp"], {"timeout": 15})] with pytest.raises(RuntimeError, match="^resource_cgroup_container_identity_invalid$"): envelope._engine_outer_population("private-canary") assert len(commands) == 1 def test_engine_binding_preserves_all_mount_fields_but_not_incidental_order(): envelope = _module() item = {"Mounts": [{"Destination": "/a", "Source": "/fixed/a", "RW": True}, {"Destination": "/b", "Source": "/fixed/b", "RW": False}], "State": {"Running": True, "Pid": 1234, "Health": {"Log": ["volatile"]}}} reordered = copy.deepcopy(item) reordered["Mounts"].reverse() reordered["State"]["Health"]["Log"] = ["changed"] assert envelope._engine_outer_binding(item) == envelope._engine_outer_binding(reordered) reordered["Mounts"][0]["RW"] = True assert envelope._engine_outer_binding(item) != envelope._engine_outer_binding(reordered) @pytest.mark.parametrize("drift", [False, True]) def test_namespace_probe_lifecycle_rechecks_target_before_start(monkeypatch, drift): envelope = _module() base, image_id, source = _host_probe_fixture(envelope) target = {"name": envelope.DIND_NAME, "id": "a" * 64} base["Config"]["Labels"].update({"tech.itworx.probe.namespace-name": target["name"], "tech.itworx.probe.namespace-id": target["id"]}) base["HostConfig"]["PidMode"] = "container:" + target["id"] base["HostConfig"]["Mounts"][0]["Source"] += "/" + target["id"] base["Mounts"][0]["Source"] += "/" + target["id"] monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, source)) monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _image: {}) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None) engine_calls = [] def engine(name): assert name == envelope.DIND_NAME engine_calls.append(name) return {"Id": target["id"], "State": {"Pid": 1234, "StartedAt": "changed" if drift and len(engine_calls) > 1 else "fixed"}}, [(1234, 1)] monkeypatch.setattr(envelope, "_engine_outer_population", engine) commands = [] def run(args, **_kwargs): commands.append(args) if args[1] == "create": assert args[args.index("--pid") + 1] == "container:" + target["id"] return base["Id"] if args[1] == "inspect": return json.dumps([base]) assert args == ["docker", "rm", base["Id"]] return "" monkeypatch.setattr(envelope, "_run", run) starts = [] def start(args, **kwargs): starts.append(args) assert kwargs["timeout"] == 60 and b'"population"' in kwargs["input"] return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"namespaceOwned":true,"processCount":1,"threadCount":1}}') monkeypatch.setattr(envelope.subprocess, "run", start) payload = {"target": target, "processCount": 1, "threadCount": 1} if drift: with pytest.raises(RuntimeError, match="^resource_cgroup_probe_namespace_invalid$"): envelope._run_host_probe("population", payload) assert starts == [] else: assert envelope._run_host_probe("population", payload)["namespaceOwned"] is True assert starts == [["docker", "start", "--attach", "--interactive", base["Id"]]] assert commands[-1] == ["docker", "rm", base["Id"]] @pytest.mark.parametrize("hidden", ["threads", "pids", "events", "opaque"]) def test_empty_proof_never_uses_private_pid_visibility_as_absence(monkeypatch, hidden): envelope = _module() reads = {"cgroup.procs": "", "cgroup.threads": "", "pids.current": "0", "cgroup.events": "populated 0"} key, value = {"threads": ("cgroup.threads", "0"), "pids": ("pids.current", "1"), "events": ("cgroup.events", "populated 1"), "opaque": ("cgroup.procs", "0")}[hidden] reads[key] = value monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name]) assert envelope._probe_empty(1) is False def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch): envelope = _module() required = {"cpu", "memory", "pids"} monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required)) monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123) monkeypatch.setattr(envelope.os, "close", lambda _fd: None) groups = [ { "path": envelope.FAMILY_CGROUP, "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3), } ] monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups)) reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": "", "cgroup.threads": "", "pids.current": "0", "cgroup.events": "populated 0\nfrozen 0"} monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name]) ready = envelope._probe_family_info(1) assert ready["safeToStartDind"] is True assert ready["safeToStartDindReason"] is None groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"}) blocked = envelope._probe_family_info(1) assert blocked["safeToStartDind"] is False assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty" groups.pop() reads["cgroup.procs"] = "4321" occupied = envelope._probe_family_info(1) assert occupied["safeToStartDind"] is False assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty" def test_rollback_preserves_unraid_existing_hugetlb_controller(monkeypatch): envelope = _module() initial = ["cpuset", "cpu", "io", "memory", "hugetlb", "pids"] calls = [] monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, payload: calls.append((mode, payload)) or {"removed": True, "familyExists": False}) envelope.rollback_host_cgroup_family(created=True, initial_controllers=initial) assert calls == [("rollback", {"created": True, "initialControllers": initial})] monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123) monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [{ "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3), }]) monkeypatch.setattr(envelope, "_probe_read", lambda fd, name: "" if name in {"cgroup.procs", "cgroup.threads"} else "0" if name == "pids.current" else "populated 0\nfrozen 0" if name == "cgroup.events" else "cpu memory pids" if fd == 123 else " ".join(initial)) monkeypatch.setattr(envelope.os, "close", lambda _fd: None) monkeypatch.setattr(envelope.os, "rmdir", lambda *args, **kwargs: None) writes = [] monkeypatch.setattr(envelope, "_probe_write", lambda *args: writes.append(args)) result = envelope._probe_rollback_family(1, {"op": "rollback", "created": True, "initialControllers": initial}) assert result == {"removed": True, "familyExists": False} assert writes == [(123, "cgroup.subtree_control", "-cpu -memory -pids")] for controllers in (["unknown"], initial + ["unknown"]): with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): envelope.rollback_host_cgroup_family(created=True, initial_controllers=controllers) with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): envelope._probe_rollback_family(1, {"op": "rollback", "created": True, "initialControllers": controllers}) def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source) ensure = envelope._host_probe_create_args("ensure", image_id, code_source) rollback = envelope._host_probe_create_args("rollback", image_id, code_source) family = envelope._host_probe_create_args("family", image_id, code_source) def mounts(args): return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] snapshot_mount, snapshot_code_mount = mounts(snapshot) ensure_mount, ensure_code_mount = mounts(ensure) rollback_mount, rollback_code_mount = mounts(rollback) family_mount, family_code_mount = mounts(family) assert snapshot_mount == ( "type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate" ) assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup") assert "readonly" not in ensure_mount assert "readonly" not in rollback_mount assert family_mount == ( "type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate" ) assert all( code_mount == f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave" for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount) ) for args in (snapshot, ensure, rollback, family): assert "--pid=host" not in args assert "--network=none" in args assert "--cap-drop=ALL" in args assert "--cpus=0.1" in args assert "--memory=64m" in args assert "--pids-limit=64" in args assert not any("/proc" in item or "docker.sock" in item for item in args) assert "--cap-add=DAC_OVERRIDE" not in snapshot assert "--cap-add=DAC_OVERRIDE" in ensure assert "--cap-add=DAC_OVERRIDE" in rollback with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"): envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code") def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path): envelope = _module() family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) dind_id = "7" * 64 workload_id = "8" * 64 dind_path = f"/itworx-ci/{dind_id}" workload_path = f"{dind_path}/docker/{workload_id}" monkeypatch.setattr( envelope, "_probe_walk", lambda _fd: [ {"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)}, {"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)}, {"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)}, ], ) monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd)) # The host cgroup contains global PID 4321; nested Docker reports PID 7 # inside DinD's private namespace. Membership is proven by ID path and a # populated cgroup, never by comparing those unrelated PID numbers. monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw: "0" if name in {"cgroup.procs", "cgroup.threads"} else "2" if name == "pids.current" else "populated 1\nfrozen 0") monkeypatch.setattr(envelope, "_probe_population", lambda fd, **_kwargs: { "processCount": 0 if fd == family_fd else 1, "threadCount": 0 if fd == family_fd else 1, "hierarchicalThreads": 2, "populated": True, }) outer = [] for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1): if name == envelope.DIND_NAME: outer.append({"name": name, "id": dind_id, "running": True, "processCount": 2, "threadCount": 2}) else: outer.append( { "name": name, "id": f"{index:064x}", "running": False, "pid": None, "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2, } ) try: snapshot = envelope._probe_snapshot( { "op": "snapshot", "outer": outer, "workloads": [ { "id": workload_id, "cgroupParent": "docker", "nestedPid": 7, "cpuMilli": 100, "memoryBytes": 1024**3, } ], }, family_fd, ) finally: os.close(family_fd) assert snapshot["workloads"][0]["path"] == workload_path assert snapshot["workloads"][0]["nestedPid"] == 7 assert snapshot["containers"][-1]["path"] == dind_path assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"): try: envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent) except RuntimeError as exc: assert str(exc) == "resource_cgroup_workload_identity_invalid" else: raise AssertionError("unmapped nested Docker cgroup parent was accepted") def test_attestor_rejects_unowned_descendants_below_dind(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].append( _group(f"{dind_path}/unknown-child", "100000 100000", 1024**3) ) snapshot["workloads"] = [ {"path": f"{dind_path}/docker/job-1"} ] result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unmapped_descendant" def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch): envelope = _module() snapshot = _snapshot() family_path = str(snapshot["family"]["path"]) dind_path = str(snapshot["dind"]["path"]) probe_id = "f" * 64 snapshot["containers"] = [ { **item, "running": item["name"] == envelope.DIND_NAME, "path": item["path"] if item["name"] == envelope.DIND_NAME else None, } for item in snapshot["containers"] ] snapshot["groups"] = [ group for group in snapshot["groups"] if group["path"] == family_path or group["path"] == dind_path ] workload_path = f"{dind_path}/docker/{probe_id}" snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3)) snapshot["workloads"] = [{ "id": probe_id, "path": workload_path, "nestedPid": 77, "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, }] monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot) result = envelope.attest_build_probe_workload(probe_id, 77) assert result["ready"] is True, result assert result["runningRunnerContainers"] == 0 assert result["probeWorkload"] == { "id": probe_id, "path": workload_path, "nestedPid": 77, "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, "descendantGroupCount": 0, } assert "snapshot" not in result assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch): envelope = _module() snapshot = _snapshot() profile_names = { profile.container_name: profile.id for profile in envelope.RUNNER_PROFILES.values() } ids = {row["name"]: row["id"] for row in snapshot["containers"]} pids = {name: 2000 + index for index, name in enumerate(ids)} outer_items = {} for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: if name == envelope.DIND_NAME: labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": "runner-dind", } host = { "CgroupParent": "itworx-ci", "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, } state = {"Running": True, "Pid": pids[name]} else: profile_id = profile_names[name] labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, } is_running = name in {"itworx-ci-checks", "itworx-ci-backend"} host = { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * 1024**2, } state = {"Running": is_running, "Pid": pids[name] if is_running else 0} outer_items[name] = { "Name": f"/{name}", "Id": ids[name], "Config": {"Labels": labels}, "State": state, "HostConfig": host, } docker_commands = [] def command(args, **_kwargs): docker_commands.append(args) if "info" in args: return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" if "docker" in args and "ps" in args: return "" raise AssertionError(f"unexpected fixed Docker command: {args}") monkeypatch.setattr(envelope, "_run", command) monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) probe_calls = [] def fake_probe(operation, payload=None): probe_calls.append((operation, payload)) return snapshot monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) _fixture_outer_namespace(envelope, monkeypatch) real_read_text = Path.read_text def reject_local_proc_or_sys(path, *args, **kwargs): if str(path).startswith(("/proc/", "/sys/fs/cgroup")): raise AssertionError("the private controller must use the host helper") return real_read_text(path, *args, **kwargs) monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys) result = envelope.attest_resource_envelope( ["backend"], expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert probe_calls and probe_calls[0][0] == "snapshot" assert result["ready"] is True assert result["familyCpuUsageUsec"] == 1200 assert result["familyMemoryCurrentBytes"] == 4096 dind_calls = [args for args in docker_commands if "exec" in args] assert len(dind_calls) == 2 assert all( f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args for args in dind_calls ) @pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"]) def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support): envelope = _module() _install_classic_builder_collection( envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000} ) original = envelope._run def command(args, **kwargs): if "exec" in args and "info" in args: return support return original(args, **kwargs) monkeypatch.setattr(envelope, "_run", command) with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"): envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch): envelope = _module() snapshot, nested_items, probe_calls = _install_classic_builder_collection( envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}, ) result = envelope.attest_resource_envelope( ["build"], expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True, result assert probe_calls[0][0] == "snapshot" assert probe_calls[0][1]["workloads"] == [ { "id": nested_items[0]["Id"], "cgroupParent": "docker", "nestedPid": 70, "cpuMilli": 1000, "memoryBytes": 1024**3, }, { "id": nested_items[1]["Id"], "cgroupParent": "docker", "nestedPid": 71, "cpuMilli": 3000, "memoryBytes": 5 * 1024**3, }, ] assert snapshot["workloads"] == [ { "id": nested_items[0]["Id"], "path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}", "nestedPid": 70, "cpuMilli": 1000, "memoryBytes": 1024**3, }, { "id": nested_items[1]["Id"], "path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}", "nestedPid": 71, "cpuMilli": 3000, "memoryBytes": 5 * 1024**3, }, ] builder_group = next( group for group in snapshot["groups"] if group["path"].endswith("/docker/" + nested_items[1]["Id"]) ) assert builder_group["cpuMax"] == "300000 100000" assert builder_group["memoryMax"] == str(5 * 1024**3) @pytest.mark.parametrize( "builder_cpu_config", [ pytest.param( {"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000}, id="zero-quota", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0}, id="zero-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000}, id="missing-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000}, id="boolean-quota", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True}, id="boolean-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001}, id="period-out-of-range", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000}, id="fractional-millicpu", ), pytest.param( {"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000}, id="conflicting-encodings", ), pytest.param( {"CpuQuota": 300_000, "CpuPeriod": 100_000}, id="missing-nanocpus-field", ), ], ) def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations( monkeypatch, builder_cpu_config ): envelope = _module() _snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection( envelope, monkeypatch, builder_cpu_config ) with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"): envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) assert probe_calls == []