1339 lines
52 KiB
Python
1339 lines
52 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py"
|
|
|
|
|
|
def _module():
|
|
broker_dir = str(ENVELOPE_PATH.parent)
|
|
if broker_dir not in sys.path:
|
|
sys.path.insert(0, broker_dir)
|
|
spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]:
|
|
return {
|
|
"path": path,
|
|
"cpuMax": cpu_max,
|
|
"memoryMax": str(memory_max),
|
|
"cpuUsageUsec": "1200",
|
|
"memoryCurrentBytes": "4096",
|
|
}
|
|
|
|
|
|
def _snapshot():
|
|
names = (
|
|
"itworx-ci-checks",
|
|
"itworx-ci-backend",
|
|
"itworx-ci-frontend",
|
|
"itworx-ci-security",
|
|
"itworx-ci-browser",
|
|
"itworx-ci-build",
|
|
"gitea-runner-dind",
|
|
)
|
|
containers = []
|
|
groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)]
|
|
dind_path = ""
|
|
for index, name in enumerate(names, start=1):
|
|
container_id = f"{index:064x}"
|
|
path = f"/itworx-ci/{container_id}"
|
|
if name == "gitea-runner-dind":
|
|
dind_path = path
|
|
cpu_max = "600000 100000"
|
|
memory_max = 12 * 1024**3
|
|
running = True
|
|
else:
|
|
cpu_max = "50000 100000"
|
|
memory_max = 512 * 1024**2
|
|
# Only two exact outer runner processes may be live. Stopped
|
|
# configured containers remain part of the identity inventory.
|
|
running = name in {"itworx-ci-checks", "itworx-ci-backend"}
|
|
containers.append(
|
|
{
|
|
"name": name,
|
|
"id": container_id,
|
|
"path": path if running else None,
|
|
"running": running,
|
|
**(
|
|
{}
|
|
if name == "gitea-runner-dind"
|
|
else {
|
|
"configuredCpuMilli": 500,
|
|
"configuredMemoryBytes": 512 * 1024**2,
|
|
}
|
|
),
|
|
}
|
|
)
|
|
if running:
|
|
groups.append(_group(path, cpu_max, memory_max))
|
|
groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3))
|
|
return {
|
|
"version": 2,
|
|
"family": _group("/itworx-ci", "800000 100000", 16 * 1024**3),
|
|
"dind": {
|
|
"path": dind_path,
|
|
"cpuMax": "600000 100000",
|
|
"memoryMax": str(12 * 1024**3),
|
|
"configuredCpuMilli": 6000,
|
|
"configuredMemoryBytes": 12 * 1024**3,
|
|
},
|
|
"containers": containers,
|
|
"groups": groups,
|
|
}
|
|
|
|
|
|
def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config):
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
workload_specs = [
|
|
(
|
|
"8" * 64,
|
|
{"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0},
|
|
1_000,
|
|
1024**3,
|
|
"100000 100000",
|
|
),
|
|
(
|
|
"9" * 64,
|
|
builder_cpu_config,
|
|
3_000,
|
|
5 * 1024**3,
|
|
"300000 100000",
|
|
),
|
|
]
|
|
workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs]
|
|
other_outer_ids = {
|
|
item["id"]
|
|
for item in snapshot["containers"]
|
|
if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"}
|
|
}
|
|
old_workload_path = f"{dind_path}/docker/job-1"
|
|
snapshot["groups"] = [
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"] != old_workload_path
|
|
and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids}
|
|
]
|
|
for row in snapshot["containers"]:
|
|
if row["name"] == envelope.DIND_NAME:
|
|
continue
|
|
running = row["name"] == "itworx-ci-build"
|
|
row["running"] = running
|
|
row["path"] = f"/itworx-ci/{row['id']}" if running else None
|
|
if running:
|
|
snapshot["groups"].append(
|
|
_group(row["path"], "50000 100000", 512 * 1024**2)
|
|
)
|
|
nested_items = []
|
|
for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs:
|
|
config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config}
|
|
nested_items.append(
|
|
{
|
|
"Id": container_id,
|
|
"State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71},
|
|
"HostConfig": config,
|
|
}
|
|
)
|
|
snapshot["groups"].append(
|
|
_group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes)
|
|
)
|
|
snapshot["workloads"] = [
|
|
{
|
|
"id": container_id,
|
|
"path": path,
|
|
"nestedPid": 70 if container_id.startswith("8") else 71,
|
|
"cpuMilli": cpu_milli,
|
|
"memoryBytes": memory_bytes,
|
|
}
|
|
for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path
|
|
in zip(workload_specs, workload_paths, strict=True)
|
|
]
|
|
|
|
profile_names = {
|
|
profile.container_name: profile.id
|
|
for profile in envelope.RUNNER_PROFILES.values()
|
|
}
|
|
ids = {row["name"]: row["id"] for row in snapshot["containers"]}
|
|
pids = {name: 2000 + index for index, name in enumerate(ids)}
|
|
outer_items = {}
|
|
for name in envelope.FIXED_RUNNER_AND_DIND_NAMES:
|
|
if name == envelope.DIND_NAME:
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": "runner-dind",
|
|
}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 6_000_000_000,
|
|
"Memory": 12 * 1024**3,
|
|
}
|
|
state = {"Running": True, "Pid": pids[name]}
|
|
else:
|
|
profile_id = profile_names[name]
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": f"runner-{profile_id}",
|
|
"tech.itworx.role": "gitea-actions-runner",
|
|
"tech.itworx.ci.profile": profile_id,
|
|
}
|
|
is_running = name == "itworx-ci-build"
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 500_000_000,
|
|
"Memory": 512 * 1024**2,
|
|
}
|
|
state = {"Running": is_running, "Pid": pids[name] if is_running else 0}
|
|
outer_items[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": ids[name],
|
|
"Config": {"Labels": labels},
|
|
"State": state,
|
|
"HostConfig": host,
|
|
}
|
|
|
|
def command(args, **_kwargs):
|
|
if "info" in args:
|
|
return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs"
|
|
if args[-3:] == ["ps", "--no-trunc", "-q"]:
|
|
return "\n".join(item["Id"] for item in nested_items)
|
|
if "inspect" in args:
|
|
return json.dumps(nested_items)
|
|
raise AssertionError(f"unexpected fixed Docker command: {args}")
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name])
|
|
probe_calls = []
|
|
|
|
def fake_probe(operation, payload=None):
|
|
probe_calls.append((operation, payload))
|
|
return snapshot
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", fake_probe)
|
|
return snapshot, nested_items, probe_calls
|
|
|
|
|
|
def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded():
|
|
envelope = _module()
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["build", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["reason"] is None
|
|
assert result["effectiveCpuMilli"] == 8000
|
|
assert result["effectiveMemoryMiB"] == 16384
|
|
assert result["activeClaims"] == 2
|
|
assert result["heavyClaims"] == 1
|
|
assert result["runningRunnerContainers"] == 2
|
|
assert result["cgroups"]["dindCpuMilli"] == 6000
|
|
assert result["cgroups"]["dindMemoryMiB"] == 12288
|
|
assert result["familyCpuUsageUsec"] == 1200
|
|
assert result["familyMemoryCurrentBytes"] == 4096
|
|
assert len(result["digest"]) == 64
|
|
|
|
|
|
def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook():
|
|
envelope = _module()
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["activeClaims"] == 2
|
|
assert result["cgroups"]["reservedCpuMilli"] == 7000
|
|
assert result["cgroups"]["reservedMemoryMiB"] == 12288
|
|
|
|
denied = envelope._evaluate_cgroup_snapshot(
|
|
["build", "browser"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert denied["ready"] is False
|
|
assert denied["reason"] == "resource_claim_heavy_limit_exceeded"
|
|
|
|
|
|
def test_attestor_rejects_an_unbounded_workload_descendant():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["groups"].append(
|
|
_group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3)
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unbounded_descendant"
|
|
|
|
|
|
def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max"))
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["cgroups"]["dindCpuMilli"] == 6000
|
|
|
|
|
|
def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
original_workload = next(
|
|
item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1"
|
|
)
|
|
original_workload["cpuMax"] = "50000 100000"
|
|
original_workload["memoryMax"] = str(1024**3)
|
|
snapshot["groups"].extend(
|
|
[
|
|
_group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3),
|
|
_group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3),
|
|
]
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["cgroups"]["reservedCpuMilli"] == 4000
|
|
assert result["cgroups"]["reservedMemoryMiB"] == 7168
|
|
|
|
|
|
def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].extend(
|
|
[
|
|
_group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3),
|
|
_group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3),
|
|
]
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved"
|
|
|
|
|
|
def test_attestor_rejects_workload_without_an_active_claim():
|
|
envelope = _module()
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
[],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_workload_without_claim"
|
|
|
|
|
|
def test_attestor_rejects_unknown_outer_container_and_parent_escape():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["containers"].append(
|
|
{"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"}
|
|
)
|
|
snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2))
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unknown_outer_container"
|
|
|
|
escaped = _snapshot()
|
|
escaped["containers"][0]["path"] = "/docker/outside"
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
escaped,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_container_outside_family"
|
|
|
|
|
|
def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
del snapshot["groups"][-1]["memoryCurrentBytes"]
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_usage_unavailable"
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_budget_contract_mismatch"
|
|
|
|
|
|
def test_claim_count_and_unknown_profiles_fail_closed():
|
|
envelope = _module()
|
|
|
|
too_many = envelope._evaluate_cgroup_snapshot(
|
|
["checks", "security", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert too_many["ready"] is False
|
|
assert too_many["reason"] == "resource_claim_slot_limit_exceeded"
|
|
|
|
unknown = envelope._evaluate_cgroup_snapshot(
|
|
["general"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert unknown["ready"] is False
|
|
assert unknown["reason"] == "resource_claim_profile_unknown"
|
|
|
|
|
|
def test_attestor_reports_and_rejects_more_than_two_running_outer_runners():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
name = "itworx-ci-frontend"
|
|
item = next(container for container in snapshot["containers"] if container["name"] == name)
|
|
item["running"] = True
|
|
item["path"] = f"/itworx-ci/{item['id']}"
|
|
snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2))
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded"
|
|
assert result["runningRunnerContainers"] == 3
|
|
|
|
|
|
def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
|
|
args = envelope._host_probe_create_args("snapshot", image_id, code_source)
|
|
|
|
assert args[0:3] == ["docker", "create", "--interactive"]
|
|
assert "--pid=host" not in args
|
|
assert "--cgroupns=private" in args
|
|
assert "--network=none" in args
|
|
assert "--read-only" in args
|
|
assert "--cap-drop=ALL" in args
|
|
assert "--cpus=0.1" in args
|
|
assert "--memory=64m" in args
|
|
assert "--pids-limit=64" in args
|
|
assert "--mount" in args
|
|
mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"]
|
|
assert len(mounts) == 2
|
|
mount = mounts[0]
|
|
assert "source=/sys/fs/cgroup" in mount
|
|
assert "target=/host-cgroup" in mount
|
|
assert "readonly" in mount
|
|
assert mounts[1] == (
|
|
f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave"
|
|
)
|
|
assert not any("/proc" in item or "docker.sock" in item for item in args)
|
|
assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"]
|
|
|
|
|
|
def _host_probe_fixture(envelope):
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
base = {
|
|
"Id": "b" * 64,
|
|
"Name": "/itworx-runner-cgroup-probe",
|
|
"Image": image_id,
|
|
"Config": {
|
|
"Image": image_id,
|
|
"Entrypoint": ["python3"],
|
|
"Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"],
|
|
"Labels": envelope._host_probe_labels(),
|
|
"User": "0:0",
|
|
},
|
|
"State": {"Running": False, "ExitCode": 0},
|
|
"HostConfig": {
|
|
"NetworkMode": "none",
|
|
"PidMode": "",
|
|
"CgroupnsMode": "private",
|
|
"ReadonlyRootfs": True,
|
|
"Privileged": False,
|
|
"NanoCpus": 100_000_000,
|
|
"Memory": 64 * 1024**2,
|
|
"PidsLimit": 64,
|
|
"CapAdd": [],
|
|
"CapDrop": ["ALL"],
|
|
"SecurityOpt": ["no-new-privileges:true"],
|
|
"RestartPolicy": {"Name": "no"},
|
|
"Binds": None,
|
|
"Volumes": None,
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/sys/fs/cgroup/itworx-ci",
|
|
"Target": "/host-cgroup",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Target": "/opt/gitea-deploy",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rslave"},
|
|
},
|
|
],
|
|
},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/sys/fs/cgroup/itworx-ci",
|
|
"Destination": "/host-cgroup",
|
|
"Mode": "",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Destination": "/opt/gitea-deploy",
|
|
"Mode": "",
|
|
"RW": False,
|
|
"Propagation": "rslave",
|
|
},
|
|
],
|
|
}
|
|
|
|
return base, image_id, code_source
|
|
|
|
|
|
def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift():
|
|
envelope = _module()
|
|
base, image_id, code_source = _host_probe_fixture(envelope)
|
|
assert envelope._validate_host_probe_inspect(
|
|
base, mode="snapshot", image_id=image_id, code_source=code_source
|
|
)
|
|
for mode in ("family", "ensure", "rollback"):
|
|
candidate = copy.deepcopy(base)
|
|
expected_source = "/sys/fs/cgroup"
|
|
expected_rw = mode in {"ensure", "rollback"}
|
|
candidate["Mounts"][0].update(
|
|
Source=expected_source,
|
|
Mode="",
|
|
RW=expected_rw,
|
|
)
|
|
candidate["HostConfig"]["Mounts"][0].update(
|
|
Source=expected_source,
|
|
ReadOnly=not expected_rw,
|
|
)
|
|
candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else []
|
|
assert envelope._validate_host_probe_inspect(
|
|
candidate, mode=mode, image_id=image_id, code_source=code_source
|
|
)
|
|
|
|
for mutate in (
|
|
lambda item: item["HostConfig"].update(PidMode="host"),
|
|
lambda item: item["HostConfig"].update(CgroupnsMode="host"),
|
|
lambda item: item["HostConfig"].update(NetworkMode="bridge"),
|
|
lambda item: item["HostConfig"].update(ReadonlyRootfs=False),
|
|
lambda item: item["HostConfig"].update(Privileged=True),
|
|
lambda item: item["HostConfig"].update(Memory=0),
|
|
lambda item: item["Mounts"].append(
|
|
{"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False}
|
|
),
|
|
lambda item: item["Mounts"][0].update(RW=True),
|
|
lambda item: item["Mounts"][1].update(RW=True),
|
|
lambda item: item["Mounts"][0].update(Propagation="rslave"),
|
|
lambda item: item["Mounts"][1].update(Propagation="rprivate"),
|
|
lambda item: item["Mounts"][1].update(Propagation="rshared"),
|
|
lambda item: item["Mounts"][1].update(Propagation="slave"),
|
|
lambda item: item["HostConfig"]["Mounts"][0]["BindOptions"].update(Propagation="rslave"),
|
|
lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rprivate"),
|
|
lambda item: item["Mounts"][0].update(Mode="ro"),
|
|
lambda item: item["Mounts"][1].update(Mode="ro"),
|
|
lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"),
|
|
lambda item: item["HostConfig"]["Mounts"].append(
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/proc",
|
|
"Target": "/host-proc",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
}
|
|
),
|
|
lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False),
|
|
lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"),
|
|
lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"),
|
|
):
|
|
candidate = copy.deepcopy(base)
|
|
mutate(candidate)
|
|
assert not envelope._validate_host_probe_inspect(
|
|
candidate, mode="snapshot", image_id=image_id, code_source=code_source
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"])
|
|
def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
inherited = {"io.buildah.version": "1.35.0", "org.opencontainers.image.title": "Official runner image"}
|
|
probe["Config"]["Labels"].update(inherited)
|
|
rw = mode in {"ensure", "rollback"}
|
|
source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup"
|
|
probe["Mounts"][0].update(Source=source, RW=rw)
|
|
probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw)
|
|
probe["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if rw else []
|
|
assert envelope._validate_host_probe_inspect(
|
|
probe, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited
|
|
)
|
|
for mutate in (
|
|
lambda item: item["Config"]["Labels"].update(unknown="extra"),
|
|
lambda item: item["Config"]["Labels"].update({"org.opencontainers.image.title": "changed"}),
|
|
lambda item: item["Config"]["Labels"].pop("io.buildah.version"),
|
|
lambda item: item["Config"]["Labels"].update({"tech.itworx.owner": "foreign"}),
|
|
lambda item: item.update(Image="sha256:" + "f" * 64),
|
|
):
|
|
candidate = copy.deepcopy(probe)
|
|
mutate(candidate)
|
|
assert not envelope._validate_host_probe_inspect(
|
|
candidate, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited
|
|
)
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited)
|
|
inspected = iter([probe, copy.deepcopy(probe), None])
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
|
calls = []
|
|
def remove(args, **kw):
|
|
calls.append(args)
|
|
assert kw == {"check": False, "capture_output": True, "timeout": 10}
|
|
return SimpleNamespace(returncode=0, stderr=b"")
|
|
monkeypatch.setattr(envelope.subprocess, "run", remove)
|
|
assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
|
assert calls == [["docker", "rm", probe["Id"]]]
|
|
|
|
|
|
@pytest.mark.parametrize("drift", ["running", "foreign_source", "race", "missing_state"])
|
|
def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch, drift):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
fresh = copy.deepcopy(probe)
|
|
if drift == "running":
|
|
probe["State"]["Running"] = True
|
|
elif drift == "foreign_source":
|
|
probe["Mounts"][1]["Source"] = code_source.replace("c" * 40, "d" * 40)
|
|
elif drift == "race":
|
|
fresh["Id"] = "f" * 64
|
|
else:
|
|
probe.pop("State")
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {})
|
|
inspected = iter([probe, fresh])
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
|
monkeypatch.setattr(envelope, "_run", lambda *a, **kw: pytest.fail("No removal or probe execution"))
|
|
monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda *a: pytest.fail("No removal"))
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_"):
|
|
envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
|
|
|
|
|
@pytest.mark.parametrize("stderr,condition", [
|
|
(b"private-canary: no space left on device", "remove_storage_full"),
|
|
(b"private-canary: read-only file system", "remove_read_only"),
|
|
(b"private-canary: permission denied", "remove_permission_denied"),
|
|
(b"Cannot connect to the Docker daemon at private-canary", "remove_daemon_unavailable"),
|
|
(b"private-canary: device or resource busy", "remove_busy"),
|
|
(b"Removal of container private-canary is already in progress", "remove_busy"),
|
|
(b"No such container: private-canary", "remove_container_missing"),
|
|
(b"private-canary: client version is too old", "remove_api_version_unsupported"),
|
|
(b"removal of container private-canary failed unexpectedly", "remove_failed"),
|
|
(b"private-canary", "remove_failed"),
|
|
(b"no space left on device" + b"x" * 4096, "remove_failed"),
|
|
])
|
|
def test_probe_removal_has_finite_diagnostics_and_never_forces(monkeypatch, stderr, condition):
|
|
envelope = _module()
|
|
identifier = "d" * 64
|
|
def run(args, **kw):
|
|
assert args == ["docker", "rm", identifier]
|
|
assert kw == {"check": False, "capture_output": True, "timeout": 10}
|
|
return SimpleNamespace(returncode=1, stderr=stderr, stdout=b"private-canary")
|
|
monkeypatch.setattr(envelope.subprocess, "run", run)
|
|
with pytest.raises(envelope.HostProbeCleanupError) as error:
|
|
envelope._remove_cleanup_probe(identifier)
|
|
assert str(error.value) == "resource_cgroup_probe_cleanup_" + condition
|
|
assert "private-canary" not in str(error.value)
|
|
assert error.value.__cause__ is None
|
|
|
|
|
|
@pytest.mark.parametrize("identifier", ["probe", "d" * 12, "d" * 64 + " --force", "../private-canary"])
|
|
def test_probe_removal_rejects_selectors_before_any_command(monkeypatch, identifier):
|
|
envelope = _module()
|
|
monkeypatch.setattr(envelope.subprocess, "run", lambda *a, **kw: pytest.fail("No command"))
|
|
with pytest.raises(envelope.HostProbeCleanupError, match="cleanup_identity_invalid"):
|
|
envelope._remove_cleanup_probe(identifier)
|
|
|
|
|
|
@pytest.mark.parametrize("stage", ["image_metadata_failed", "inspect_failed", "reinspect_failed", "absence_failed"])
|
|
def test_probe_read_failures_drop_untrusted_exception_text(stage):
|
|
envelope = _module()
|
|
def fail():
|
|
raise RuntimeError("Bearer private-canary")
|
|
with pytest.raises(envelope.HostProbeCleanupError) as error:
|
|
envelope._cleanup_probe_read(stage, fail)
|
|
assert str(error.value) == "resource_cgroup_probe_cleanup_" + stage
|
|
assert error.value.__cause__ is None
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="Docker CLI shim uses POSIX executable semantics")
|
|
@pytest.mark.parametrize("failure,condition", [("read-only file system private-canary", "remove_read_only"),
|
|
("Bearer private-canary", "remove_failed")])
|
|
def test_real_probe_cleanup_cli_exposes_only_finite_child_failure(tmp_path, failure, condition):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
fixture = tmp_path / "fixture.json"
|
|
fixture.write_text(json.dumps({"probe": probe, "image": image_id, "failure": failure}))
|
|
shim = tmp_path / "docker"
|
|
shim.write_text("#!" + sys.executable + "\n" +
|
|
"import json,sys,pathlib\n" +
|
|
"v=json.loads(pathlib.Path(" + repr(str(fixture)) + ").read_text())\n" +
|
|
"a=sys.argv[1:]\n" +
|
|
"if a[:2]==['image','inspect']: print(json.dumps([{'Id':v['image'],'Config':{'Labels':{}}}]))\n" +
|
|
"elif a[:2]==['ps','-a']: print(v['probe']['Id'])\n" +
|
|
"elif a[0]=='inspect': print(json.dumps([v['probe']]))\n" +
|
|
"elif a==['rm',v['probe']['Id']]: print(v['failure'],file=sys.stderr); sys.exit(1)\n" +
|
|
"else: sys.exit(99)\n")
|
|
shim.chmod(0o700)
|
|
result = subprocess.run([sys.executable, str(ENVELOPE_PATH), "--cleanup-stopped-host-probe",
|
|
image_id, code_source], capture_output=True, timeout=15,
|
|
env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", "")})
|
|
assert result.returncode == 1
|
|
assert result.stdout == b""
|
|
assert result.stderr == ("resource_cgroup_probe_cleanup_" + condition + "\n").encode()
|
|
|
|
|
|
@pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.title": "runner"}])
|
|
def test_probe_label_metadata_comes_from_exact_image(monkeypatch, labels):
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
calls = []
|
|
def run(args, **kw):
|
|
calls.append(args)
|
|
return json.dumps([{"Id": image_id, "Config": {"Labels": labels}}])
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
assert envelope._host_probe_image_labels(image_id) == (labels or {})
|
|
assert calls == [["docker", "image", "inspect", image_id]]
|
|
|
|
|
|
@pytest.mark.parametrize("row", [
|
|
{"Id": "sha256:" + "b" * 64, "Config": {"Labels": {}}},
|
|
{"Id": "sha256:" + "a" * 64, "Config": {"Labels": ["foreign"]}},
|
|
{"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"secret-canary": 123}}},
|
|
{"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"tech.itworx.owner": "foreign"}}},
|
|
])
|
|
def test_probe_image_metadata_rejects_drift_without_echo(monkeypatch, row):
|
|
envelope = _module()
|
|
monkeypatch.setattr(envelope, "_run", lambda *a, **kw: json.dumps([row]))
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_image_labels_invalid$"):
|
|
envelope._host_probe_image_labels("sha256:" + "a" * 64)
|
|
|
|
|
|
def test_normal_probe_executes_and_removes_its_image_label_bound_container(monkeypatch):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
inherited = {"org.opencontainers.image.title": "Official runner image"}
|
|
probe["Config"]["Labels"].update(inherited)
|
|
monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source))
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited)
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None)
|
|
calls = []
|
|
def run(args, **kw):
|
|
calls.append(args)
|
|
if args[:2] == ["docker", "create"]:
|
|
return probe["Id"]
|
|
if args[:2] == ["docker", "inspect"]:
|
|
return json.dumps([probe])
|
|
assert args == ["docker", "rm", probe["Id"]]
|
|
return ""
|
|
def start(args, **kw):
|
|
assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]]
|
|
assert json.loads(kw["input"]) == {"op": "snapshot"}
|
|
return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}')
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
monkeypatch.setattr(envelope.subprocess, "run", start)
|
|
assert envelope._run_host_probe("snapshot") == {"verified": True}
|
|
assert calls[-1] == ["docker", "rm", probe["Id"]]
|
|
|
|
|
|
def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch):
|
|
envelope = _module()
|
|
revision = "d" * 40
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}"
|
|
image_id = "sha256:" + "e" * 64
|
|
release = {"sourceRevision": revision}
|
|
controller = {
|
|
"Name": "/gitea-deploy-control",
|
|
"Image": image_id,
|
|
"State": {"Running": True},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Destination": "/opt/gitea-deploy",
|
|
"RW": False,
|
|
}
|
|
],
|
|
}
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"status_server",
|
|
SimpleNamespace(_require_controller_release=lambda: release),
|
|
)
|
|
monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller]))
|
|
|
|
assert envelope._controller_runtime_identity() == (image_id, code_source)
|
|
|
|
controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}"
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"):
|
|
envelope._controller_runtime_identity()
|
|
|
|
|
|
def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
|
|
def probe(operation, payload=None):
|
|
calls.append((operation, payload))
|
|
return {"created": True, "rootSubtreeBefore": ["pids"]}
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", probe)
|
|
result = envelope.ensure_host_cgroup_family(allow_existing=False)
|
|
|
|
assert result == {"created": True, "rootSubtreeBefore": ["pids"]}
|
|
assert calls == [("ensure", {"allowExisting": False})]
|
|
|
|
|
|
def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
envelope,
|
|
"_run_host_probe",
|
|
lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False},
|
|
)
|
|
|
|
result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"])
|
|
|
|
assert result == {"removed": True, "familyExists": False}
|
|
assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})]
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"])
|
|
|
|
|
|
def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch):
|
|
envelope = _module()
|
|
required = {"cpu", "memory", "pids"}
|
|
monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required))
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123)
|
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
|
groups = [
|
|
{
|
|
"path": envelope.FAMILY_CGROUP,
|
|
"cpuMax": "800000 100000",
|
|
"memoryMax": str(16 * 1024**3),
|
|
}
|
|
]
|
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups))
|
|
reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": ""}
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name])
|
|
|
|
ready = envelope._probe_family_info(1)
|
|
assert ready["safeToStartDind"] is True
|
|
assert ready["safeToStartDindReason"] is None
|
|
|
|
groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"})
|
|
blocked = envelope._probe_family_info(1)
|
|
assert blocked["safeToStartDind"] is False
|
|
assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
|
|
|
groups.pop()
|
|
reads["cgroup.procs"] = "4321"
|
|
occupied = envelope._probe_family_info(1)
|
|
assert occupied["safeToStartDind"] is False
|
|
assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
|
|
|
|
|
def test_rollback_preserves_unraid_existing_hugetlb_controller(monkeypatch):
|
|
envelope = _module()
|
|
initial = ["cpuset", "cpu", "io", "memory", "hugetlb", "pids"]
|
|
calls = []
|
|
monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, payload:
|
|
calls.append((mode, payload)) or {"removed": True, "familyExists": False})
|
|
envelope.rollback_host_cgroup_family(created=True, initial_controllers=initial)
|
|
assert calls == [("rollback", {"created": True, "initialControllers": initial})]
|
|
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123)
|
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [{
|
|
"cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3),
|
|
}])
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda fd, name:
|
|
"" if name == "cgroup.procs" else
|
|
"cpu memory pids" if fd == 123 else " ".join(initial))
|
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
|
monkeypatch.setattr(envelope.os, "rmdir", lambda *args, **kwargs: None)
|
|
writes = []
|
|
monkeypatch.setattr(envelope, "_probe_write", lambda *args: writes.append(args))
|
|
result = envelope._probe_rollback_family(1, {"op": "rollback", "created": True,
|
|
"initialControllers": initial})
|
|
assert result == {"removed": True, "familyExists": False}
|
|
assert writes == [(123, "cgroup.subtree_control", "-cpu -memory -pids")]
|
|
for controllers in (["unknown"], initial + ["unknown"]):
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope.rollback_host_cgroup_family(created=True, initial_controllers=controllers)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope._probe_rollback_family(1, {"op": "rollback", "created": True,
|
|
"initialControllers": controllers})
|
|
|
|
|
|
def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
|
|
snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source)
|
|
ensure = envelope._host_probe_create_args("ensure", image_id, code_source)
|
|
rollback = envelope._host_probe_create_args("rollback", image_id, code_source)
|
|
family = envelope._host_probe_create_args("family", image_id, code_source)
|
|
|
|
def mounts(args):
|
|
return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"]
|
|
|
|
snapshot_mount, snapshot_code_mount = mounts(snapshot)
|
|
ensure_mount, ensure_code_mount = mounts(ensure)
|
|
rollback_mount, rollback_code_mount = mounts(rollback)
|
|
family_mount, family_code_mount = mounts(family)
|
|
assert snapshot_mount == (
|
|
"type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup")
|
|
assert "readonly" not in ensure_mount
|
|
assert "readonly" not in rollback_mount
|
|
assert family_mount == (
|
|
"type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert all(
|
|
code_mount
|
|
== f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave"
|
|
for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount)
|
|
)
|
|
for args in (snapshot, ensure, rollback, family):
|
|
assert "--pid=host" not in args
|
|
assert "--network=none" in args
|
|
assert "--cap-drop=ALL" in args
|
|
assert "--cpus=0.1" in args
|
|
assert "--memory=64m" in args
|
|
assert "--pids-limit=64" in args
|
|
assert not any("/proc" in item or "docker.sock" in item for item in args)
|
|
assert "--cap-add=DAC_OVERRIDE" not in snapshot
|
|
assert "--cap-add=DAC_OVERRIDE" in ensure
|
|
assert "--cap-add=DAC_OVERRIDE" in rollback
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"):
|
|
envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code")
|
|
|
|
|
|
def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path):
|
|
envelope = _module()
|
|
family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
dind_id = "7" * 64
|
|
workload_id = "8" * 64
|
|
dind_path = f"/itworx-ci/{dind_id}"
|
|
workload_path = f"{dind_path}/docker/{workload_id}"
|
|
monkeypatch.setattr(
|
|
envelope,
|
|
"_probe_walk",
|
|
lambda _fd: [
|
|
{"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)},
|
|
{"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)},
|
|
{"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)},
|
|
],
|
|
)
|
|
monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd))
|
|
# The host cgroup contains global PID 4321; nested Docker reports PID 7
|
|
# inside DinD's private namespace. Membership is proven by ID path and a
|
|
# populated cgroup, never by comparing those unrelated PID numbers.
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw: "4321" if name == "cgroup.procs" else "")
|
|
outer = []
|
|
for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1):
|
|
if name == envelope.DIND_NAME:
|
|
outer.append({"name": name, "id": dind_id, "running": True, "pid": 4321})
|
|
else:
|
|
outer.append(
|
|
{
|
|
"name": name,
|
|
"id": f"{index:064x}",
|
|
"running": False,
|
|
"pid": None,
|
|
"configuredCpuMilli": 500,
|
|
"configuredMemoryBytes": 512 * 1024**2,
|
|
}
|
|
)
|
|
try:
|
|
snapshot = envelope._probe_snapshot(
|
|
{
|
|
"op": "snapshot",
|
|
"outer": outer,
|
|
"workloads": [
|
|
{
|
|
"id": workload_id,
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 7,
|
|
"cpuMilli": 100,
|
|
"memoryBytes": 1024**3,
|
|
}
|
|
],
|
|
},
|
|
family_fd,
|
|
)
|
|
finally:
|
|
os.close(family_fd)
|
|
|
|
assert snapshot["workloads"][0]["path"] == workload_path
|
|
assert snapshot["workloads"][0]["nestedPid"] == 7
|
|
assert snapshot["containers"][-1]["path"] == dind_path
|
|
assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path
|
|
for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"):
|
|
try:
|
|
envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent)
|
|
except RuntimeError as exc:
|
|
assert str(exc) == "resource_cgroup_workload_identity_invalid"
|
|
else:
|
|
raise AssertionError("unmapped nested Docker cgroup parent was accepted")
|
|
|
|
|
|
def test_attestor_rejects_unowned_descendants_below_dind():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].append(
|
|
_group(f"{dind_path}/unknown-child", "100000 100000", 1024**3)
|
|
)
|
|
snapshot["workloads"] = [
|
|
{"path": f"{dind_path}/docker/job-1"}
|
|
]
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unmapped_descendant"
|
|
|
|
|
|
def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
family_path = str(snapshot["family"]["path"])
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
probe_id = "f" * 64
|
|
snapshot["containers"] = [
|
|
{
|
|
**item,
|
|
"running": item["name"] == envelope.DIND_NAME,
|
|
"path": item["path"] if item["name"] == envelope.DIND_NAME else None,
|
|
}
|
|
for item in snapshot["containers"]
|
|
]
|
|
snapshot["groups"] = [
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"] == family_path or group["path"] == dind_path
|
|
]
|
|
workload_path = f"{dind_path}/docker/{probe_id}"
|
|
snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3))
|
|
snapshot["workloads"] = [{
|
|
"id": probe_id,
|
|
"path": workload_path,
|
|
"nestedPid": 77,
|
|
"cpuMilli": 4000,
|
|
"memoryBytes": 6 * 1024**3,
|
|
}]
|
|
monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot)
|
|
|
|
result = envelope.attest_build_probe_workload(probe_id, 77)
|
|
|
|
assert result["ready"] is True, result
|
|
assert result["runningRunnerContainers"] == 0
|
|
assert result["probeWorkload"] == {
|
|
"id": probe_id,
|
|
"path": workload_path,
|
|
"nestedPid": 77,
|
|
"cpuMilli": 4000,
|
|
"memoryBytes": 6 * 1024**3,
|
|
"descendantGroupCount": 0,
|
|
}
|
|
assert "snapshot" not in result
|
|
assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False
|
|
|
|
|
|
def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
profile_names = {
|
|
profile.container_name: profile.id
|
|
for profile in envelope.RUNNER_PROFILES.values()
|
|
}
|
|
ids = {row["name"]: row["id"] for row in snapshot["containers"]}
|
|
pids = {name: 2000 + index for index, name in enumerate(ids)}
|
|
outer_items = {}
|
|
for name in envelope.FIXED_RUNNER_AND_DIND_NAMES:
|
|
if name == envelope.DIND_NAME:
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": "runner-dind",
|
|
}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 6_000_000_000,
|
|
"Memory": 12 * 1024**3,
|
|
}
|
|
state = {"Running": True, "Pid": pids[name]}
|
|
else:
|
|
profile_id = profile_names[name]
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": f"runner-{profile_id}",
|
|
"tech.itworx.role": "gitea-actions-runner",
|
|
"tech.itworx.ci.profile": profile_id,
|
|
}
|
|
is_running = name in {"itworx-ci-checks", "itworx-ci-backend"}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 500_000_000,
|
|
"Memory": 512 * 1024**2,
|
|
}
|
|
state = {"Running": is_running, "Pid": pids[name] if is_running else 0}
|
|
outer_items[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": ids[name],
|
|
"Config": {"Labels": labels},
|
|
"State": state,
|
|
"HostConfig": host,
|
|
}
|
|
|
|
docker_commands = []
|
|
|
|
def command(args, **_kwargs):
|
|
docker_commands.append(args)
|
|
if "info" in args:
|
|
return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs"
|
|
if "docker" in args and "ps" in args:
|
|
return ""
|
|
raise AssertionError(f"unexpected fixed Docker command: {args}")
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name])
|
|
probe_calls = []
|
|
|
|
def fake_probe(operation, payload=None):
|
|
probe_calls.append((operation, payload))
|
|
return snapshot
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", fake_probe)
|
|
real_read_text = Path.read_text
|
|
|
|
def reject_local_proc_or_sys(path, *args, **kwargs):
|
|
if str(path).startswith(("/proc/", "/sys/fs/cgroup")):
|
|
raise AssertionError("the private controller must use the host helper")
|
|
return real_read_text(path, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys)
|
|
result = envelope.attest_resource_envelope(
|
|
["backend"],
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert probe_calls and probe_calls[0][0] == "snapshot"
|
|
assert result["ready"] is True
|
|
assert result["familyCpuUsageUsec"] == 1200
|
|
assert result["familyMemoryCurrentBytes"] == 4096
|
|
dind_calls = [args for args in docker_commands if "exec" in args]
|
|
assert len(dind_calls) == 2
|
|
assert all(
|
|
f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args
|
|
for args in dind_calls
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"])
|
|
def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support):
|
|
envelope = _module()
|
|
_install_classic_builder_collection(
|
|
envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}
|
|
)
|
|
original = envelope._run
|
|
|
|
def command(args, **kwargs):
|
|
if "exec" in args and "info" in args:
|
|
return support
|
|
return original(args, **kwargs)
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"):
|
|
envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES)
|
|
|
|
|
|
def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch):
|
|
envelope = _module()
|
|
snapshot, nested_items, probe_calls = _install_classic_builder_collection(
|
|
envelope,
|
|
monkeypatch,
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
)
|
|
|
|
result = envelope.attest_resource_envelope(
|
|
["build"],
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True, result
|
|
assert probe_calls[0][0] == "snapshot"
|
|
assert probe_calls[0][1]["workloads"] == [
|
|
{
|
|
"id": nested_items[0]["Id"],
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 70,
|
|
"cpuMilli": 1000,
|
|
"memoryBytes": 1024**3,
|
|
},
|
|
{
|
|
"id": nested_items[1]["Id"],
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 71,
|
|
"cpuMilli": 3000,
|
|
"memoryBytes": 5 * 1024**3,
|
|
},
|
|
]
|
|
assert snapshot["workloads"] == [
|
|
{
|
|
"id": nested_items[0]["Id"],
|
|
"path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}",
|
|
"nestedPid": 70,
|
|
"cpuMilli": 1000,
|
|
"memoryBytes": 1024**3,
|
|
},
|
|
{
|
|
"id": nested_items[1]["Id"],
|
|
"path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}",
|
|
"nestedPid": 71,
|
|
"cpuMilli": 3000,
|
|
"memoryBytes": 5 * 1024**3,
|
|
},
|
|
]
|
|
builder_group = next(
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"].endswith("/docker/" + nested_items[1]["Id"])
|
|
)
|
|
assert builder_group["cpuMax"] == "300000 100000"
|
|
assert builder_group["memoryMax"] == str(5 * 1024**3)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"builder_cpu_config",
|
|
[
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000},
|
|
id="zero-quota",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0},
|
|
id="zero-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000},
|
|
id="missing-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000},
|
|
id="boolean-quota",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True},
|
|
id="boolean-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001},
|
|
id="period-out-of-range",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000},
|
|
id="fractional-millicpu",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
id="conflicting-encodings",
|
|
),
|
|
pytest.param(
|
|
{"CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
id="missing-nanocpus-field",
|
|
),
|
|
],
|
|
)
|
|
def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations(
|
|
monkeypatch, builder_cpu_config
|
|
):
|
|
envelope = _module()
|
|
_snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection(
|
|
envelope, monkeypatch, builder_cpu_config
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"):
|
|
envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES)
|
|
|
|
assert probe_calls == []
|