from __future__ import annotations import copy import importlib.util import json import os import subprocess import sys from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).resolve().parents[2] ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py" def _module(): broker_dir = str(ENVELOPE_PATH.parent) if broker_dir not in sys.path: sys.path.insert(0, broker_dir) spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]: return { "path": path, "cpuMax": cpu_max, "memoryMax": str(memory_max), "cpuUsageUsec": "1200", "memoryCurrentBytes": "4096", } def _snapshot(): names = ( "itworx-ci-checks", "itworx-ci-backend", "itworx-ci-frontend", "itworx-ci-security", "itworx-ci-browser", "itworx-ci-build", "gitea-runner-dind", ) containers = [] groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)] dind_path = "" for index, name in enumerate(names, start=1): container_id = f"{index:064x}" path = f"/itworx-ci/{container_id}" if name == "gitea-runner-dind": dind_path = path cpu_max = "600000 100000" memory_max = 12 * 1024**3 running = True else: cpu_max = "50000 100000" memory_max = 512 * 1024**2 # Only two exact outer runner processes may be live. Stopped # configured containers remain part of the identity inventory. running = name in {"itworx-ci-checks", "itworx-ci-backend"} containers.append( { "name": name, "id": container_id, "path": path if running else None, "running": running, **( {} if name == "gitea-runner-dind" else { "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2, } ), } ) if running: groups.append(_group(path, cpu_max, memory_max)) groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3)) return { "version": 2, "family": _group("/itworx-ci", "800000 100000", 16 * 1024**3), "dind": { "path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3), "configuredCpuMilli": 6000, "configuredMemoryBytes": 12 * 1024**3, }, "containers": containers, "groups": groups, } def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config): snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) workload_specs = [ ( "8" * 64, {"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0}, 1_000, 1024**3, "100000 100000", ), ( "9" * 64, builder_cpu_config, 3_000, 5 * 1024**3, "300000 100000", ), ] workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs] other_outer_ids = { item["id"] for item in snapshot["containers"] if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"} } old_workload_path = f"{dind_path}/docker/job-1" snapshot["groups"] = [ group for group in snapshot["groups"] if group["path"] != old_workload_path and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids} ] for row in snapshot["containers"]: if row["name"] == envelope.DIND_NAME: continue running = row["name"] == "itworx-ci-build" row["running"] = running row["path"] = f"/itworx-ci/{row['id']}" if running else None if running: snapshot["groups"].append( _group(row["path"], "50000 100000", 512 * 1024**2) ) nested_items = [] for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs: config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config} nested_items.append( { "Id": container_id, "State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71}, "HostConfig": config, } ) snapshot["groups"].append( _group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes) ) snapshot["workloads"] = [ { "id": container_id, "path": path, "nestedPid": 70 if container_id.startswith("8") else 71, "cpuMilli": cpu_milli, "memoryBytes": memory_bytes, } for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path in zip(workload_specs, workload_paths, strict=True) ] profile_names = { profile.container_name: profile.id for profile in envelope.RUNNER_PROFILES.values() } ids = {row["name"]: row["id"] for row in snapshot["containers"]} pids = {name: 2000 + index for index, name in enumerate(ids)} outer_items = {} for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: if name == envelope.DIND_NAME: labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": "runner-dind", } host = { "CgroupParent": "itworx-ci", "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, } state = {"Running": True, "Pid": pids[name]} else: profile_id = profile_names[name] labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, } is_running = name == "itworx-ci-build" host = { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * 1024**2, } state = {"Running": is_running, "Pid": pids[name] if is_running else 0} outer_items[name] = { "Name": f"/{name}", "Id": ids[name], "Config": {"Labels": labels}, "State": state, "HostConfig": host, } def command(args, **_kwargs): if "info" in args: return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" if args[-3:] == ["ps", "--no-trunc", "-q"]: return "\n".join(item["Id"] for item in nested_items) if "inspect" in args: return json.dumps(nested_items) raise AssertionError(f"unexpected fixed Docker command: {args}") monkeypatch.setattr(envelope, "_run", command) monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) probe_calls = [] def fake_probe(operation, payload=None): probe_calls.append((operation, payload)) return snapshot monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) return snapshot, nested_items, probe_calls def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( ["build", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["reason"] is None assert result["effectiveCpuMilli"] == 8000 assert result["effectiveMemoryMiB"] == 16384 assert result["activeClaims"] == 2 assert result["heavyClaims"] == 1 assert result["runningRunnerContainers"] == 2 assert result["cgroups"]["dindCpuMilli"] == 6000 assert result["cgroups"]["dindMemoryMiB"] == 12288 assert result["familyCpuUsageUsec"] == 1200 assert result["familyMemoryCurrentBytes"] == 4096 assert len(result["digest"]) == 64 def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( ["backend", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["activeClaims"] == 2 assert result["cgroups"]["reservedCpuMilli"] == 7000 assert result["cgroups"]["reservedMemoryMiB"] == 12288 denied = envelope._evaluate_cgroup_snapshot( ["build", "browser"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert denied["ready"] is False assert denied["reason"] == "resource_claim_heavy_limit_exceeded" def test_attestor_rejects_an_unbounded_workload_descendant(): envelope = _module() snapshot = _snapshot() snapshot["groups"].append( _group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3) ) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unbounded_descendant" def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max")) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["cgroups"]["dindCpuMilli"] == 6000 def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) original_workload = next( item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1" ) original_workload["cpuMax"] = "50000 100000" original_workload["memoryMax"] = str(1024**3) snapshot["groups"].extend( [ _group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3), _group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3), ] ) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["cgroups"]["reservedCpuMilli"] == 4000 assert result["cgroups"]["reservedMemoryMiB"] == 7168 def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].extend( [ _group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3), _group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3), ] ) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved" def test_attestor_rejects_workload_without_an_active_claim(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( [], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_workload_without_claim" def test_attestor_rejects_unknown_outer_container_and_parent_escape(): envelope = _module() snapshot = _snapshot() snapshot["containers"].append( {"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"} ) snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2)) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unknown_outer_container" escaped = _snapshot() escaped["containers"][0]["path"] = "/docker/outside" result = envelope._evaluate_cgroup_snapshot( ["checks"], escaped, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_container_outside_family" def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract(): envelope = _module() snapshot = _snapshot() del snapshot["groups"][-1]["memoryCurrentBytes"] result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_usage_unavailable" result = envelope._evaluate_cgroup_snapshot( ["checks"], _snapshot(), expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_budget_contract_mismatch" def test_claim_count_and_unknown_profiles_fail_closed(): envelope = _module() too_many = envelope._evaluate_cgroup_snapshot( ["checks", "security", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert too_many["ready"] is False assert too_many["reason"] == "resource_claim_slot_limit_exceeded" unknown = envelope._evaluate_cgroup_snapshot( ["general"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert unknown["ready"] is False assert unknown["reason"] == "resource_claim_profile_unknown" def test_attestor_reports_and_rejects_more_than_two_running_outer_runners(): envelope = _module() snapshot = _snapshot() name = "itworx-ci-frontend" item = next(container for container in snapshot["containers"] if container["name"] == name) item["running"] = True item["path"] = f"/itworx-ci/{item['id']}" snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2)) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded" assert result["runningRunnerContainers"] == 3 def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" args = envelope._host_probe_create_args("snapshot", image_id, code_source) assert args[0:3] == ["docker", "create", "--interactive"] assert "--pid=host" not in args assert "--cgroupns=private" in args assert "--network=none" in args assert "--read-only" in args assert "--cap-drop=ALL" in args assert "--cpus=0.1" in args assert "--memory=64m" in args assert "--pids-limit=64" in args assert "--mount" in args mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] assert len(mounts) == 2 mount = mounts[0] assert "source=/sys/fs/cgroup" in mount assert "target=/host-cgroup" in mount assert "readonly" in mount assert mounts[1] == ( f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave" ) assert not any("/proc" in item or "docker.sock" in item for item in args) assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"] def _host_probe_fixture(envelope): image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" base = { "Id": "b" * 64, "Name": "/itworx-runner-cgroup-probe", "Image": image_id, "Config": { "Image": image_id, "Entrypoint": ["python3"], "Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"], "Labels": envelope._host_probe_labels(), "User": "0:0", }, "State": {"Running": False, "ExitCode": 0}, "HostConfig": { "NetworkMode": "none", "PidMode": "", "CgroupnsMode": "private", "ReadonlyRootfs": True, "Privileged": False, "NanoCpus": 100_000_000, "Memory": 64 * 1024**2, "PidsLimit": 64, "CapAdd": [], "CapDrop": ["ALL"], "SecurityOpt": ["no-new-privileges:true"], "RestartPolicy": {"Name": "no"}, "Binds": None, "Volumes": None, "Mounts": [ { "Type": "bind", "Source": "/sys/fs/cgroup/itworx-ci", "Target": "/host-cgroup", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, }, { "Type": "bind", "Source": code_source, "Target": "/opt/gitea-deploy", "ReadOnly": True, "BindOptions": {"Propagation": "rslave"}, }, ], }, "Mounts": [ { "Type": "bind", "Source": "/sys/fs/cgroup/itworx-ci", "Destination": "/host-cgroup", "Mode": "", "RW": False, "Propagation": "rprivate", }, { "Type": "bind", "Source": code_source, "Destination": "/opt/gitea-deploy", "Mode": "", "RW": False, "Propagation": "rslave", }, ], } return base, image_id, code_source def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): envelope = _module() base, image_id, code_source = _host_probe_fixture(envelope) assert envelope._validate_host_probe_inspect( base, mode="snapshot", image_id=image_id, code_source=code_source ) for mode in ("family", "ensure", "rollback"): candidate = copy.deepcopy(base) expected_source = "/sys/fs/cgroup" expected_rw = mode in {"ensure", "rollback"} candidate["Mounts"][0].update( Source=expected_source, Mode="", RW=expected_rw, ) candidate["HostConfig"]["Mounts"][0].update( Source=expected_source, ReadOnly=not expected_rw, ) candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else [] assert envelope._validate_host_probe_inspect( candidate, mode=mode, image_id=image_id, code_source=code_source ) for mutate in ( lambda item: item["HostConfig"].update(PidMode="host"), lambda item: item["HostConfig"].update(CgroupnsMode="host"), lambda item: item["HostConfig"].update(NetworkMode="bridge"), lambda item: item["HostConfig"].update(ReadonlyRootfs=False), lambda item: item["HostConfig"].update(Privileged=True), lambda item: item["HostConfig"].update(Memory=0), lambda item: item["Mounts"].append( {"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False} ), lambda item: item["Mounts"][0].update(RW=True), lambda item: item["Mounts"][1].update(RW=True), lambda item: item["Mounts"][0].update(Propagation="rslave"), lambda item: item["Mounts"][1].update(Propagation="rprivate"), lambda item: item["Mounts"][1].update(Propagation="rshared"), lambda item: item["Mounts"][1].update(Propagation="slave"), lambda item: item["HostConfig"]["Mounts"][0]["BindOptions"].update(Propagation="rslave"), lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rprivate"), lambda item: item["Mounts"][0].update(Mode="ro"), lambda item: item["Mounts"][1].update(Mode="ro"), lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"), lambda item: item["HostConfig"]["Mounts"].append( { "Type": "bind", "Source": "/proc", "Target": "/host-proc", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, } ), lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False), lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"), lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"), ): candidate = copy.deepcopy(base) mutate(candidate) assert not envelope._validate_host_probe_inspect( candidate, mode="snapshot", image_id=image_id, code_source=code_source ) @pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"]) def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) inherited = {"io.buildah.version": "1.35.0", "org.opencontainers.image.title": "Official runner image"} probe["Config"]["Labels"].update(inherited) rw = mode in {"ensure", "rollback"} source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup" probe["Mounts"][0].update(Source=source, RW=rw) probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw) probe["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if rw else [] assert envelope._validate_host_probe_inspect( probe, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited ) for mutate in ( lambda item: item["Config"]["Labels"].update(unknown="extra"), lambda item: item["Config"]["Labels"].update({"org.opencontainers.image.title": "changed"}), lambda item: item["Config"]["Labels"].pop("io.buildah.version"), lambda item: item["Config"]["Labels"].update({"tech.itworx.owner": "foreign"}), lambda item: item.update(Image="sha256:" + "f" * 64), ): candidate = copy.deepcopy(probe) mutate(candidate) assert not envelope._validate_host_probe_inspect( candidate, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited ) monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited) inspected = iter([probe, copy.deepcopy(probe), None]) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) calls = [] def remove(args, **kw): calls.append(args) assert kw == {"check": False, "capture_output": True, "timeout": 10} return SimpleNamespace(returncode=0, stderr=b"") monkeypatch.setattr(envelope.subprocess, "run", remove) assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) assert calls == [["docker", "rm", probe["Id"]]] @pytest.mark.parametrize("drift", ["running", "foreign_source", "race", "missing_state"]) def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch, drift): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) fresh = copy.deepcopy(probe) if drift == "running": probe["State"]["Running"] = True elif drift == "foreign_source": probe["Mounts"][1]["Source"] = code_source.replace("c" * 40, "d" * 40) elif drift == "race": fresh["Id"] = "f" * 64 else: probe.pop("State") monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {}) inspected = iter([probe, fresh]) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) monkeypatch.setattr(envelope, "_run", lambda *a, **kw: pytest.fail("No removal or probe execution")) monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda *a: pytest.fail("No removal")) with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_"): envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) @pytest.mark.parametrize("stderr,condition", [ (b"private-canary: no space left on device", "remove_storage_full"), (b"private-canary: read-only file system", "remove_read_only"), (b"private-canary: permission denied", "remove_permission_denied"), (b"Cannot connect to the Docker daemon at private-canary", "remove_daemon_unavailable"), (b"private-canary: device or resource busy", "remove_busy"), (b"Removal of container private-canary is already in progress", "remove_busy"), (b"No such container: private-canary", "remove_container_missing"), (b"private-canary: client version is too old", "remove_api_version_unsupported"), (b"removal of container private-canary failed unexpectedly", "remove_failed"), (b"private-canary", "remove_failed"), (b"no space left on device" + b"x" * 4096, "remove_failed"), ]) def test_probe_removal_has_finite_diagnostics_and_never_forces(monkeypatch, stderr, condition): envelope = _module() identifier = "d" * 64 def run(args, **kw): assert args == ["docker", "rm", identifier] assert kw == {"check": False, "capture_output": True, "timeout": 10} return SimpleNamespace(returncode=1, stderr=stderr, stdout=b"private-canary") monkeypatch.setattr(envelope.subprocess, "run", run) with pytest.raises(envelope.HostProbeCleanupError) as error: envelope._remove_cleanup_probe(identifier) assert str(error.value) == "resource_cgroup_probe_cleanup_" + condition assert "private-canary" not in str(error.value) assert error.value.__cause__ is None @pytest.mark.parametrize("identifier", ["probe", "d" * 12, "d" * 64 + " --force", "../private-canary"]) def test_probe_removal_rejects_selectors_before_any_command(monkeypatch, identifier): envelope = _module() monkeypatch.setattr(envelope.subprocess, "run", lambda *a, **kw: pytest.fail("No command")) with pytest.raises(envelope.HostProbeCleanupError, match="cleanup_identity_invalid"): envelope._remove_cleanup_probe(identifier) @pytest.mark.parametrize("stage", ["image_metadata_failed", "inspect_failed", "reinspect_failed", "absence_failed"]) def test_probe_read_failures_drop_untrusted_exception_text(stage): envelope = _module() def fail(): raise RuntimeError("Bearer private-canary") with pytest.raises(envelope.HostProbeCleanupError) as error: envelope._cleanup_probe_read(stage, fail) assert str(error.value) == "resource_cgroup_probe_cleanup_" + stage assert error.value.__cause__ is None @pytest.mark.skipif(os.name == "nt", reason="Docker CLI shim uses POSIX executable semantics") @pytest.mark.parametrize("failure,condition", [("read-only file system private-canary", "remove_read_only"), ("Bearer private-canary", "remove_failed")]) def test_real_probe_cleanup_cli_exposes_only_finite_child_failure(tmp_path, failure, condition): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) fixture = tmp_path / "fixture.json" fixture.write_text(json.dumps({"probe": probe, "image": image_id, "failure": failure})) shim = tmp_path / "docker" shim.write_text("#!" + sys.executable + "\n" + "import json,sys,pathlib\n" + "v=json.loads(pathlib.Path(" + repr(str(fixture)) + ").read_text())\n" + "a=sys.argv[1:]\n" + "if a[:2]==['image','inspect']: print(json.dumps([{'Id':v['image'],'Config':{'Labels':{}}}]))\n" + "elif a[:2]==['ps','-a']: print(v['probe']['Id'])\n" + "elif a[0]=='inspect': print(json.dumps([v['probe']]))\n" + "elif a==['rm',v['probe']['Id']]: print(v['failure'],file=sys.stderr); sys.exit(1)\n" + "else: sys.exit(99)\n") shim.chmod(0o700) result = subprocess.run([sys.executable, str(ENVELOPE_PATH), "--cleanup-stopped-host-probe", image_id, code_source], capture_output=True, timeout=15, env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", "")}) assert result.returncode == 1 assert result.stdout == b"" assert result.stderr == ("resource_cgroup_probe_cleanup_" + condition + "\n").encode() @pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.title": "runner"}]) def test_probe_label_metadata_comes_from_exact_image(monkeypatch, labels): envelope = _module() image_id = "sha256:" + "a" * 64 calls = [] def run(args, **kw): calls.append(args) return json.dumps([{"Id": image_id, "Config": {"Labels": labels}}]) monkeypatch.setattr(envelope, "_run", run) assert envelope._host_probe_image_labels(image_id) == (labels or {}) assert calls == [["docker", "image", "inspect", image_id]] @pytest.mark.parametrize("row", [ {"Id": "sha256:" + "b" * 64, "Config": {"Labels": {}}}, {"Id": "sha256:" + "a" * 64, "Config": {"Labels": ["foreign"]}}, {"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"secret-canary": 123}}}, {"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"tech.itworx.owner": "foreign"}}}, ]) def test_probe_image_metadata_rejects_drift_without_echo(monkeypatch, row): envelope = _module() monkeypatch.setattr(envelope, "_run", lambda *a, **kw: json.dumps([row])) with pytest.raises(RuntimeError, match="^resource_cgroup_probe_image_labels_invalid$"): envelope._host_probe_image_labels("sha256:" + "a" * 64) def test_normal_probe_executes_and_removes_its_image_label_bound_container(monkeypatch): envelope = _module() probe, image_id, code_source = _host_probe_fixture(envelope) inherited = {"org.opencontainers.image.title": "Official runner image"} probe["Config"]["Labels"].update(inherited) monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source)) monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited) monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None) calls = [] def run(args, **kw): calls.append(args) if args[:2] == ["docker", "create"]: return probe["Id"] if args[:2] == ["docker", "inspect"]: return json.dumps([probe]) assert args == ["docker", "rm", probe["Id"]] return "" def start(args, **kw): assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]] assert json.loads(kw["input"]) == {"op": "snapshot"} return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}') monkeypatch.setattr(envelope, "_run", run) monkeypatch.setattr(envelope.subprocess, "run", start) assert envelope._run_host_probe("snapshot") == {"verified": True} assert calls[-1] == ["docker", "rm", probe["Id"]] def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch): envelope = _module() revision = "d" * 40 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}" image_id = "sha256:" + "e" * 64 release = {"sourceRevision": revision} controller = { "Name": "/gitea-deploy-control", "Image": image_id, "State": {"Running": True}, "Mounts": [ { "Type": "bind", "Source": code_source, "Destination": "/opt/gitea-deploy", "RW": False, } ], } monkeypatch.setitem( sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release), ) monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller])) assert envelope._controller_runtime_identity() == (image_id, code_source) controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}" with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"): envelope._controller_runtime_identity() def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch): envelope = _module() calls = [] def probe(operation, payload=None): calls.append((operation, payload)) return {"created": True, "rootSubtreeBefore": ["pids"]} monkeypatch.setattr(envelope, "_run_host_probe", probe) result = envelope.ensure_host_cgroup_family(allow_existing=False) assert result == {"created": True, "rootSubtreeBefore": ["pids"]} assert calls == [("ensure", {"allowExisting": False})] def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch): envelope = _module() calls = [] monkeypatch.setattr( envelope, "_run_host_probe", lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False}, ) result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"]) assert result == {"removed": True, "familyExists": False} assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})] with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"]) def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch): envelope = _module() required = {"cpu", "memory", "pids"} monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required)) monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123) monkeypatch.setattr(envelope.os, "close", lambda _fd: None) groups = [ { "path": envelope.FAMILY_CGROUP, "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3), } ] monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups)) reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": ""} monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name]) ready = envelope._probe_family_info(1) assert ready["safeToStartDind"] is True assert ready["safeToStartDindReason"] is None groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"}) blocked = envelope._probe_family_info(1) assert blocked["safeToStartDind"] is False assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty" groups.pop() reads["cgroup.procs"] = "4321" occupied = envelope._probe_family_info(1) assert occupied["safeToStartDind"] is False assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty" def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source) ensure = envelope._host_probe_create_args("ensure", image_id, code_source) rollback = envelope._host_probe_create_args("rollback", image_id, code_source) family = envelope._host_probe_create_args("family", image_id, code_source) def mounts(args): return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] snapshot_mount, snapshot_code_mount = mounts(snapshot) ensure_mount, ensure_code_mount = mounts(ensure) rollback_mount, rollback_code_mount = mounts(rollback) family_mount, family_code_mount = mounts(family) assert snapshot_mount == ( "type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate" ) assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup") assert "readonly" not in ensure_mount assert "readonly" not in rollback_mount assert family_mount == ( "type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate" ) assert all( code_mount == f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave" for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount) ) for args in (snapshot, ensure, rollback, family): assert "--pid=host" not in args assert "--network=none" in args assert "--cap-drop=ALL" in args assert "--cpus=0.1" in args assert "--memory=64m" in args assert "--pids-limit=64" in args assert not any("/proc" in item or "docker.sock" in item for item in args) assert "--cap-add=DAC_OVERRIDE" not in snapshot assert "--cap-add=DAC_OVERRIDE" in ensure assert "--cap-add=DAC_OVERRIDE" in rollback with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"): envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code") def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path): envelope = _module() family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) dind_id = "7" * 64 workload_id = "8" * 64 dind_path = f"/itworx-ci/{dind_id}" workload_path = f"{dind_path}/docker/{workload_id}" monkeypatch.setattr( envelope, "_probe_walk", lambda _fd: [ {"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)}, {"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)}, {"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)}, ], ) monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd)) # The host cgroup contains global PID 4321; nested Docker reports PID 7 # inside DinD's private namespace. Membership is proven by ID path and a # populated cgroup, never by comparing those unrelated PID numbers. monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw: "4321" if name == "cgroup.procs" else "") outer = [] for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1): if name == envelope.DIND_NAME: outer.append({"name": name, "id": dind_id, "running": True, "pid": 4321}) else: outer.append( { "name": name, "id": f"{index:064x}", "running": False, "pid": None, "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2, } ) try: snapshot = envelope._probe_snapshot( { "op": "snapshot", "outer": outer, "workloads": [ { "id": workload_id, "cgroupParent": "docker", "nestedPid": 7, "cpuMilli": 100, "memoryBytes": 1024**3, } ], }, family_fd, ) finally: os.close(family_fd) assert snapshot["workloads"][0]["path"] == workload_path assert snapshot["workloads"][0]["nestedPid"] == 7 assert snapshot["containers"][-1]["path"] == dind_path assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"): try: envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent) except RuntimeError as exc: assert str(exc) == "resource_cgroup_workload_identity_invalid" else: raise AssertionError("unmapped nested Docker cgroup parent was accepted") def test_attestor_rejects_unowned_descendants_below_dind(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].append( _group(f"{dind_path}/unknown-child", "100000 100000", 1024**3) ) snapshot["workloads"] = [ {"path": f"{dind_path}/docker/job-1"} ] result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unmapped_descendant" def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch): envelope = _module() snapshot = _snapshot() family_path = str(snapshot["family"]["path"]) dind_path = str(snapshot["dind"]["path"]) probe_id = "f" * 64 snapshot["containers"] = [ { **item, "running": item["name"] == envelope.DIND_NAME, "path": item["path"] if item["name"] == envelope.DIND_NAME else None, } for item in snapshot["containers"] ] snapshot["groups"] = [ group for group in snapshot["groups"] if group["path"] == family_path or group["path"] == dind_path ] workload_path = f"{dind_path}/docker/{probe_id}" snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3)) snapshot["workloads"] = [{ "id": probe_id, "path": workload_path, "nestedPid": 77, "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, }] monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot) result = envelope.attest_build_probe_workload(probe_id, 77) assert result["ready"] is True, result assert result["runningRunnerContainers"] == 0 assert result["probeWorkload"] == { "id": probe_id, "path": workload_path, "nestedPid": 77, "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, "descendantGroupCount": 0, } assert "snapshot" not in result assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch): envelope = _module() snapshot = _snapshot() profile_names = { profile.container_name: profile.id for profile in envelope.RUNNER_PROFILES.values() } ids = {row["name"]: row["id"] for row in snapshot["containers"]} pids = {name: 2000 + index for index, name in enumerate(ids)} outer_items = {} for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: if name == envelope.DIND_NAME: labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": "runner-dind", } host = { "CgroupParent": "itworx-ci", "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, } state = {"Running": True, "Pid": pids[name]} else: profile_id = profile_names[name] labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, } is_running = name in {"itworx-ci-checks", "itworx-ci-backend"} host = { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * 1024**2, } state = {"Running": is_running, "Pid": pids[name] if is_running else 0} outer_items[name] = { "Name": f"/{name}", "Id": ids[name], "Config": {"Labels": labels}, "State": state, "HostConfig": host, } docker_commands = [] def command(args, **_kwargs): docker_commands.append(args) if "info" in args: return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" if "docker" in args and "ps" in args: return "" raise AssertionError(f"unexpected fixed Docker command: {args}") monkeypatch.setattr(envelope, "_run", command) monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) probe_calls = [] def fake_probe(operation, payload=None): probe_calls.append((operation, payload)) return snapshot monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) real_read_text = Path.read_text def reject_local_proc_or_sys(path, *args, **kwargs): if str(path).startswith(("/proc/", "/sys/fs/cgroup")): raise AssertionError("the private controller must use the host helper") return real_read_text(path, *args, **kwargs) monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys) result = envelope.attest_resource_envelope( ["backend"], expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert probe_calls and probe_calls[0][0] == "snapshot" assert result["ready"] is True assert result["familyCpuUsageUsec"] == 1200 assert result["familyMemoryCurrentBytes"] == 4096 dind_calls = [args for args in docker_commands if "exec" in args] assert len(dind_calls) == 2 assert all( f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args for args in dind_calls ) @pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"]) def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support): envelope = _module() _install_classic_builder_collection( envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000} ) original = envelope._run def command(args, **kwargs): if "exec" in args and "info" in args: return support return original(args, **kwargs) monkeypatch.setattr(envelope, "_run", command) with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"): envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch): envelope = _module() snapshot, nested_items, probe_calls = _install_classic_builder_collection( envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}, ) result = envelope.attest_resource_envelope( ["build"], expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True, result assert probe_calls[0][0] == "snapshot" assert probe_calls[0][1]["workloads"] == [ { "id": nested_items[0]["Id"], "cgroupParent": "docker", "nestedPid": 70, "cpuMilli": 1000, "memoryBytes": 1024**3, }, { "id": nested_items[1]["Id"], "cgroupParent": "docker", "nestedPid": 71, "cpuMilli": 3000, "memoryBytes": 5 * 1024**3, }, ] assert snapshot["workloads"] == [ { "id": nested_items[0]["Id"], "path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}", "nestedPid": 70, "cpuMilli": 1000, "memoryBytes": 1024**3, }, { "id": nested_items[1]["Id"], "path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}", "nestedPid": 71, "cpuMilli": 3000, "memoryBytes": 5 * 1024**3, }, ] builder_group = next( group for group in snapshot["groups"] if group["path"].endswith("/docker/" + nested_items[1]["Id"]) ) assert builder_group["cpuMax"] == "300000 100000" assert builder_group["memoryMax"] == str(5 * 1024**3) @pytest.mark.parametrize( "builder_cpu_config", [ pytest.param( {"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000}, id="zero-quota", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0}, id="zero-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000}, id="missing-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000}, id="boolean-quota", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True}, id="boolean-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001}, id="period-out-of-range", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000}, id="fractional-millicpu", ), pytest.param( {"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000}, id="conflicting-encodings", ), pytest.param( {"CpuQuota": 300_000, "CpuPeriod": 100_000}, id="missing-nanocpus-field", ), ], ) def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations( monkeypatch, builder_cpu_config ): envelope = _module() _snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection( envelope, monkeypatch, builder_cpu_config ) with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"): envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) assert probe_calls == []