from __future__ import annotations import copy import importlib.util import json import os import sys from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).resolve().parents[2] ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py" def _module(): broker_dir = str(ENVELOPE_PATH.parent) if broker_dir not in sys.path: sys.path.insert(0, broker_dir) spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]: return { "path": path, "cpuMax": cpu_max, "memoryMax": str(memory_max), "cpuUsageUsec": "1200", "memoryCurrentBytes": "4096", } def _snapshot(): names = ( "itworx-ci-checks", "itworx-ci-backend", "itworx-ci-frontend", "itworx-ci-security", "itworx-ci-browser", "itworx-ci-build", "gitea-runner-dind", ) containers = [] groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)] dind_path = "" for index, name in enumerate(names, start=1): container_id = f"{index:064x}" path = f"/itworx-ci/{container_id}" if name == "gitea-runner-dind": dind_path = path cpu_max = "600000 100000" memory_max = 12 * 1024**3 running = True else: cpu_max = "50000 100000" memory_max = 512 * 1024**2 # Only two exact outer runner processes may be live. Stopped # configured containers remain part of the identity inventory. running = name in {"itworx-ci-checks", "itworx-ci-backend"} containers.append( { "name": name, "id": container_id, "path": path if running else None, "running": running, **( {} if name == "gitea-runner-dind" else { "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2, } ), } ) if running: groups.append(_group(path, cpu_max, memory_max)) groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3)) return { "version": 2, "family": _group("/itworx-ci", "800000 100000", 16 * 1024**3), "dind": { "path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3), "configuredCpuMilli": 6000, "configuredMemoryBytes": 12 * 1024**3, }, "containers": containers, "groups": groups, } def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config): snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) workload_specs = [ ( "8" * 64, {"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0}, 1_000, 1024**3, "100000 100000", ), ( "9" * 64, builder_cpu_config, 3_000, 5 * 1024**3, "300000 100000", ), ] workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs] other_outer_ids = { item["id"] for item in snapshot["containers"] if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"} } old_workload_path = f"{dind_path}/docker/job-1" snapshot["groups"] = [ group for group in snapshot["groups"] if group["path"] != old_workload_path and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids} ] for row in snapshot["containers"]: if row["name"] == envelope.DIND_NAME: continue running = row["name"] == "itworx-ci-build" row["running"] = running row["path"] = f"/itworx-ci/{row['id']}" if running else None if running: snapshot["groups"].append( _group(row["path"], "50000 100000", 512 * 1024**2) ) nested_items = [] for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs: config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config} nested_items.append( { "Id": container_id, "State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71}, "HostConfig": config, } ) snapshot["groups"].append( _group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes) ) snapshot["workloads"] = [ { "id": container_id, "path": path, "nestedPid": 70 if container_id.startswith("8") else 71, "cpuMilli": cpu_milli, "memoryBytes": memory_bytes, } for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path in zip(workload_specs, workload_paths, strict=True) ] profile_names = { profile.container_name: profile.id for profile in envelope.RUNNER_PROFILES.values() } ids = {row["name"]: row["id"] for row in snapshot["containers"]} pids = {name: 2000 + index for index, name in enumerate(ids)} outer_items = {} for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: if name == envelope.DIND_NAME: labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": "runner-dind", } host = { "CgroupParent": "itworx-ci", "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, } state = {"Running": True, "Pid": pids[name]} else: profile_id = profile_names[name] labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, } is_running = name == "itworx-ci-build" host = { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * 1024**2, } state = {"Running": is_running, "Pid": pids[name] if is_running else 0} outer_items[name] = { "Name": f"/{name}", "Id": ids[name], "Config": {"Labels": labels}, "State": state, "HostConfig": host, } def command(args, **_kwargs): if "info" in args: return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" if args[-3:] == ["ps", "--no-trunc", "-q"]: return "\n".join(item["Id"] for item in nested_items) if "inspect" in args: return json.dumps(nested_items) raise AssertionError(f"unexpected fixed Docker command: {args}") monkeypatch.setattr(envelope, "_run", command) monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) probe_calls = [] def fake_probe(operation, payload=None): probe_calls.append((operation, payload)) return snapshot monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) return snapshot, nested_items, probe_calls def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( ["build", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["reason"] is None assert result["effectiveCpuMilli"] == 8000 assert result["effectiveMemoryMiB"] == 16384 assert result["activeClaims"] == 2 assert result["heavyClaims"] == 1 assert result["runningRunnerContainers"] == 2 assert result["cgroups"]["dindCpuMilli"] == 6000 assert result["cgroups"]["dindMemoryMiB"] == 12288 assert result["familyCpuUsageUsec"] == 1200 assert result["familyMemoryCurrentBytes"] == 4096 assert len(result["digest"]) == 64 def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( ["backend", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["activeClaims"] == 2 assert result["cgroups"]["reservedCpuMilli"] == 7000 assert result["cgroups"]["reservedMemoryMiB"] == 12288 denied = envelope._evaluate_cgroup_snapshot( ["build", "browser"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert denied["ready"] is False assert denied["reason"] == "resource_claim_heavy_limit_exceeded" def test_attestor_rejects_an_unbounded_workload_descendant(): envelope = _module() snapshot = _snapshot() snapshot["groups"].append( _group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3) ) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unbounded_descendant" def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max")) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["cgroups"]["dindCpuMilli"] == 6000 def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) original_workload = next( item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1" ) original_workload["cpuMax"] = "50000 100000" original_workload["memoryMax"] = str(1024**3) snapshot["groups"].extend( [ _group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3), _group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3), ] ) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True assert result["cgroups"]["reservedCpuMilli"] == 4000 assert result["cgroups"]["reservedMemoryMiB"] == 7168 def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].extend( [ _group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3), _group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3), ] ) result = envelope._evaluate_cgroup_snapshot( ["backend"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved" def test_attestor_rejects_workload_without_an_active_claim(): envelope = _module() result = envelope._evaluate_cgroup_snapshot( [], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_workload_without_claim" def test_attestor_rejects_unknown_outer_container_and_parent_escape(): envelope = _module() snapshot = _snapshot() snapshot["containers"].append( {"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"} ) snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2)) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unknown_outer_container" escaped = _snapshot() escaped["containers"][0]["path"] = "/docker/outside" result = envelope._evaluate_cgroup_snapshot( ["checks"], escaped, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_container_outside_family" def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract(): envelope = _module() snapshot = _snapshot() del snapshot["groups"][-1]["memoryCurrentBytes"] result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_usage_unavailable" result = envelope._evaluate_cgroup_snapshot( ["checks"], _snapshot(), expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_budget_contract_mismatch" def test_claim_count_and_unknown_profiles_fail_closed(): envelope = _module() too_many = envelope._evaluate_cgroup_snapshot( ["checks", "security", "backend"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert too_many["ready"] is False assert too_many["reason"] == "resource_claim_slot_limit_exceeded" unknown = envelope._evaluate_cgroup_snapshot( ["general"], _snapshot(), expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert unknown["ready"] is False assert unknown["reason"] == "resource_claim_profile_unknown" def test_attestor_reports_and_rejects_more_than_two_running_outer_runners(): envelope = _module() snapshot = _snapshot() name = "itworx-ci-frontend" item = next(container for container in snapshot["containers"] if container["name"] == name) item["running"] = True item["path"] = f"/itworx-ci/{item['id']}" snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2)) result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded" assert result["runningRunnerContainers"] == 3 def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" args = envelope._host_probe_create_args("snapshot", image_id, code_source) assert args[0:3] == ["docker", "create", "--interactive"] assert "--pid=host" not in args assert "--cgroupns=private" in args assert "--network=none" in args assert "--read-only" in args assert "--cap-drop=ALL" in args assert "--cpus=0.1" in args assert "--memory=64m" in args assert "--pids-limit=64" in args assert "--mount" in args mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] assert len(mounts) == 2 mount = mounts[0] assert "source=/sys/fs/cgroup" in mount assert "target=/host-cgroup" in mount assert "readonly" in mount assert mounts[1] == ( f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rprivate" ) assert not any("/proc" in item or "docker.sock" in item for item in args) assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"] def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" base = { "Id": "b" * 64, "Name": "/itworx-runner-cgroup-probe", "Image": image_id, "Config": { "Image": image_id, "Entrypoint": ["python3"], "Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"], "Labels": envelope._host_probe_labels(), "User": "0:0", }, "State": {"Running": False, "ExitCode": 0}, "HostConfig": { "NetworkMode": "none", "PidMode": "", "CgroupnsMode": "private", "ReadonlyRootfs": True, "Privileged": False, "NanoCpus": 100_000_000, "Memory": 64 * 1024**2, "PidsLimit": 64, "CapAdd": [], "CapDrop": ["ALL"], "SecurityOpt": ["no-new-privileges:true"], "RestartPolicy": {"Name": "no"}, "Binds": None, "Volumes": None, "Mounts": [ { "Type": "bind", "Source": "/sys/fs/cgroup/itworx-ci", "Target": "/host-cgroup", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, }, { "Type": "bind", "Source": code_source, "Target": "/opt/gitea-deploy", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, }, ], }, "Mounts": [ { "Type": "bind", "Source": "/sys/fs/cgroup/itworx-ci", "Destination": "/host-cgroup", "Mode": "", "RW": False, "Propagation": "rprivate", }, { "Type": "bind", "Source": code_source, "Destination": "/opt/gitea-deploy", "Mode": "", "RW": False, "Propagation": "rprivate", }, ], } assert envelope._validate_host_probe_inspect( base, mode="snapshot", image_id=image_id, code_source=code_source ) for mode in ("family", "ensure", "rollback"): candidate = copy.deepcopy(base) expected_source = "/sys/fs/cgroup" expected_rw = mode in {"ensure", "rollback"} candidate["Mounts"][0].update( Source=expected_source, Mode="", RW=expected_rw, ) candidate["HostConfig"]["Mounts"][0].update( Source=expected_source, ReadOnly=not expected_rw, ) candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else [] assert envelope._validate_host_probe_inspect( candidate, mode=mode, image_id=image_id, code_source=code_source ) for mutate in ( lambda item: item["HostConfig"].update(PidMode="host"), lambda item: item["HostConfig"].update(CgroupnsMode="host"), lambda item: item["HostConfig"].update(NetworkMode="bridge"), lambda item: item["HostConfig"].update(ReadonlyRootfs=False), lambda item: item["HostConfig"].update(Privileged=True), lambda item: item["HostConfig"].update(Memory=0), lambda item: item["Mounts"].append( {"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False} ), lambda item: item["Mounts"][0].update(RW=True), lambda item: item["Mounts"][1].update(RW=True), lambda item: item["Mounts"][0].update(Mode="ro"), lambda item: item["Mounts"][1].update(Mode="ro"), lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"), lambda item: item["HostConfig"]["Mounts"].append( { "Type": "bind", "Source": "/proc", "Target": "/host-proc", "ReadOnly": True, "BindOptions": {"Propagation": "rprivate"}, } ), lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False), lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"), lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"), ): candidate = copy.deepcopy(base) mutate(candidate) assert not envelope._validate_host_probe_inspect( candidate, mode="snapshot", image_id=image_id, code_source=code_source ) def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch): envelope = _module() revision = "d" * 40 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}" image_id = "sha256:" + "e" * 64 release = {"sourceRevision": revision} controller = { "Name": "/gitea-deploy-control", "Image": image_id, "State": {"Running": True}, "Mounts": [ { "Type": "bind", "Source": code_source, "Destination": "/opt/gitea-deploy", "RW": False, } ], } monkeypatch.setitem( sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release), ) monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller])) assert envelope._controller_runtime_identity() == (image_id, code_source) controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}" with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"): envelope._controller_runtime_identity() def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch): envelope = _module() calls = [] def probe(operation, payload=None): calls.append((operation, payload)) return {"created": True, "rootSubtreeBefore": ["pids"]} monkeypatch.setattr(envelope, "_run_host_probe", probe) result = envelope.ensure_host_cgroup_family(allow_existing=False) assert result == {"created": True, "rootSubtreeBefore": ["pids"]} assert calls == [("ensure", {"allowExisting": False})] def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch): envelope = _module() calls = [] monkeypatch.setattr( envelope, "_run_host_probe", lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False}, ) result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"]) assert result == {"removed": True, "familyExists": False} assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})] with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"]) def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch): envelope = _module() required = {"cpu", "memory", "pids"} monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required)) monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123) monkeypatch.setattr(envelope.os, "close", lambda _fd: None) groups = [ { "path": envelope.FAMILY_CGROUP, "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3), } ] monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups)) reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": ""} monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name]) ready = envelope._probe_family_info(1) assert ready["safeToStartDind"] is True assert ready["safeToStartDindReason"] is None groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"}) blocked = envelope._probe_family_info(1) assert blocked["safeToStartDind"] is False assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty" groups.pop() reads["cgroup.procs"] = "4321" occupied = envelope._probe_family_info(1) assert occupied["safeToStartDind"] is False assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty" def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback(): envelope = _module() image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source) ensure = envelope._host_probe_create_args("ensure", image_id, code_source) rollback = envelope._host_probe_create_args("rollback", image_id, code_source) family = envelope._host_probe_create_args("family", image_id, code_source) def mounts(args): return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"] snapshot_mount, snapshot_code_mount = mounts(snapshot) ensure_mount, ensure_code_mount = mounts(ensure) rollback_mount, rollback_code_mount = mounts(rollback) family_mount, family_code_mount = mounts(family) assert snapshot_mount == ( "type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate" ) assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup") assert "readonly" not in ensure_mount assert "readonly" not in rollback_mount assert family_mount == ( "type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate" ) assert all( code_mount == f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rprivate" for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount) ) for args in (snapshot, ensure, rollback, family): assert "--pid=host" not in args assert "--network=none" in args assert "--cap-drop=ALL" in args assert "--cpus=0.1" in args assert "--memory=64m" in args assert "--pids-limit=64" in args assert not any("/proc" in item or "docker.sock" in item for item in args) assert "--cap-add=DAC_OVERRIDE" not in snapshot assert "--cap-add=DAC_OVERRIDE" in ensure assert "--cap-add=DAC_OVERRIDE" in rollback with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"): envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code") def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path): envelope = _module() family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) dind_id = "7" * 64 workload_id = "8" * 64 dind_path = f"/itworx-ci/{dind_id}" workload_path = f"{dind_path}/docker/{workload_id}" monkeypatch.setattr( envelope, "_probe_walk", lambda _fd: [ {"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)}, {"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)}, {"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)}, ], ) monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd)) # The host cgroup contains global PID 4321; nested Docker reports PID 7 # inside DinD's private namespace. Membership is proven by ID path and a # populated cgroup, never by comparing those unrelated PID numbers. monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw: "4321" if name == "cgroup.procs" else "") outer = [] for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1): if name == envelope.DIND_NAME: outer.append({"name": name, "id": dind_id, "running": True, "pid": 4321}) else: outer.append( { "name": name, "id": f"{index:064x}", "running": False, "pid": None, "configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2, } ) try: snapshot = envelope._probe_snapshot( { "op": "snapshot", "outer": outer, "workloads": [ { "id": workload_id, "cgroupParent": "docker", "nestedPid": 7, "cpuMilli": 100, "memoryBytes": 1024**3, } ], }, family_fd, ) finally: os.close(family_fd) assert snapshot["workloads"][0]["path"] == workload_path assert snapshot["workloads"][0]["nestedPid"] == 7 assert snapshot["containers"][-1]["path"] == dind_path assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"): try: envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent) except RuntimeError as exc: assert str(exc) == "resource_cgroup_workload_identity_invalid" else: raise AssertionError("unmapped nested Docker cgroup parent was accepted") def test_attestor_rejects_unowned_descendants_below_dind(): envelope = _module() snapshot = _snapshot() dind_path = str(snapshot["dind"]["path"]) snapshot["groups"].append( _group(f"{dind_path}/unknown-child", "100000 100000", 1024**3) ) snapshot["workloads"] = [ {"path": f"{dind_path}/docker/job-1"} ] result = envelope._evaluate_cgroup_snapshot( ["checks"], snapshot, expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is False assert result["reason"] == "resource_cgroup_unmapped_descendant" def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch): envelope = _module() snapshot = _snapshot() family_path = str(snapshot["family"]["path"]) dind_path = str(snapshot["dind"]["path"]) probe_id = "f" * 64 snapshot["containers"] = [ { **item, "running": item["name"] == envelope.DIND_NAME, "path": item["path"] if item["name"] == envelope.DIND_NAME else None, } for item in snapshot["containers"] ] snapshot["groups"] = [ group for group in snapshot["groups"] if group["path"] == family_path or group["path"] == dind_path ] workload_path = f"{dind_path}/docker/{probe_id}" snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3)) snapshot["workloads"] = [{ "id": probe_id, "path": workload_path, "nestedPid": 77, "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, }] monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot) result = envelope.attest_build_probe_workload(probe_id, 77) assert result["ready"] is True, result assert result["runningRunnerContainers"] == 0 assert result["probeWorkload"] == { "id": probe_id, "path": workload_path, "nestedPid": 77, "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, "descendantGroupCount": 0, } assert "snapshot" not in result assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch): envelope = _module() snapshot = _snapshot() profile_names = { profile.container_name: profile.id for profile in envelope.RUNNER_PROFILES.values() } ids = {row["name"]: row["id"] for row in snapshot["containers"]} pids = {name: 2000 + index for index, name in enumerate(ids)} outer_items = {} for name in envelope.FIXED_RUNNER_AND_DIND_NAMES: if name == envelope.DIND_NAME: labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": "runner-dind", } host = { "CgroupParent": "itworx-ci", "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, } state = {"Running": True, "Pid": pids[name]} else: profile_id = profile_names[name] labels = { "com.docker.compose.project": "gitea-deploy-runners", "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, } is_running = name in {"itworx-ci-checks", "itworx-ci-backend"} host = { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * 1024**2, } state = {"Running": is_running, "Pid": pids[name] if is_running else 0} outer_items[name] = { "Name": f"/{name}", "Id": ids[name], "Config": {"Labels": labels}, "State": state, "HostConfig": host, } docker_commands = [] def command(args, **_kwargs): docker_commands.append(args) if "info" in args: return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs" if "docker" in args and "ps" in args: return "" raise AssertionError(f"unexpected fixed Docker command: {args}") monkeypatch.setattr(envelope, "_run", command) monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name]) probe_calls = [] def fake_probe(operation, payload=None): probe_calls.append((operation, payload)) return snapshot monkeypatch.setattr(envelope, "_run_host_probe", fake_probe) real_read_text = Path.read_text def reject_local_proc_or_sys(path, *args, **kwargs): if str(path).startswith(("/proc/", "/sys/fs/cgroup")): raise AssertionError("the private controller must use the host helper") return real_read_text(path, *args, **kwargs) monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys) result = envelope.attest_resource_envelope( ["backend"], expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert probe_calls and probe_calls[0][0] == "snapshot" assert result["ready"] is True assert result["familyCpuUsageUsec"] == 1200 assert result["familyMemoryCurrentBytes"] == 4096 dind_calls = [args for args in docker_commands if "exec" in args] assert len(dind_calls) == 2 assert all( f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args for args in dind_calls ) @pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"]) def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support): envelope = _module() _install_classic_builder_collection( envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000} ) original = envelope._run def command(args, **kwargs): if "exec" in args and "info" in args: return support return original(args, **kwargs) monkeypatch.setattr(envelope, "_run", command) with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"): envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch): envelope = _module() snapshot, nested_items, probe_calls = _install_classic_builder_collection( envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}, ) result = envelope.attest_resource_envelope( ["build"], expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384}, ) assert result["ready"] is True, result assert probe_calls[0][0] == "snapshot" assert probe_calls[0][1]["workloads"] == [ { "id": nested_items[0]["Id"], "cgroupParent": "docker", "nestedPid": 70, "cpuMilli": 1000, "memoryBytes": 1024**3, }, { "id": nested_items[1]["Id"], "cgroupParent": "docker", "nestedPid": 71, "cpuMilli": 3000, "memoryBytes": 5 * 1024**3, }, ] assert snapshot["workloads"] == [ { "id": nested_items[0]["Id"], "path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}", "nestedPid": 70, "cpuMilli": 1000, "memoryBytes": 1024**3, }, { "id": nested_items[1]["Id"], "path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}", "nestedPid": 71, "cpuMilli": 3000, "memoryBytes": 5 * 1024**3, }, ] builder_group = next( group for group in snapshot["groups"] if group["path"].endswith("/docker/" + nested_items[1]["Id"]) ) assert builder_group["cpuMax"] == "300000 100000" assert builder_group["memoryMax"] == str(5 * 1024**3) @pytest.mark.parametrize( "builder_cpu_config", [ pytest.param( {"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000}, id="zero-quota", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0}, id="zero-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000}, id="missing-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000}, id="boolean-quota", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True}, id="boolean-period", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001}, id="period-out-of-range", ), pytest.param( {"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000}, id="fractional-millicpu", ), pytest.param( {"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000}, id="conflicting-encodings", ), pytest.param( {"CpuQuota": 300_000, "CpuPeriod": 100_000}, id="missing-nanocpus-field", ), ], ) def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations( monkeypatch, builder_cpu_config ): envelope = _module() _snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection( envelope, monkeypatch, builder_cpu_config ) with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"): envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES) assert probe_calls == []