1107 lines
40 KiB
Python
1107 lines
40 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py"
|
|
|
|
|
|
def _module():
|
|
broker_dir = str(ENVELOPE_PATH.parent)
|
|
if broker_dir not in sys.path:
|
|
sys.path.insert(0, broker_dir)
|
|
spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]:
|
|
return {
|
|
"path": path,
|
|
"cpuMax": cpu_max,
|
|
"memoryMax": str(memory_max),
|
|
"cpuUsageUsec": "1200",
|
|
"memoryCurrentBytes": "4096",
|
|
}
|
|
|
|
|
|
def _snapshot():
|
|
names = (
|
|
"itworx-ci-checks",
|
|
"itworx-ci-backend",
|
|
"itworx-ci-frontend",
|
|
"itworx-ci-security",
|
|
"itworx-ci-browser",
|
|
"itworx-ci-build",
|
|
"gitea-runner-dind",
|
|
)
|
|
containers = []
|
|
groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)]
|
|
dind_path = ""
|
|
for index, name in enumerate(names, start=1):
|
|
container_id = f"{index:064x}"
|
|
path = f"/itworx-ci/{container_id}"
|
|
if name == "gitea-runner-dind":
|
|
dind_path = path
|
|
cpu_max = "600000 100000"
|
|
memory_max = 12 * 1024**3
|
|
running = True
|
|
else:
|
|
cpu_max = "50000 100000"
|
|
memory_max = 512 * 1024**2
|
|
# Only two exact outer runner processes may be live. Stopped
|
|
# configured containers remain part of the identity inventory.
|
|
running = name in {"itworx-ci-checks", "itworx-ci-backend"}
|
|
containers.append(
|
|
{
|
|
"name": name,
|
|
"id": container_id,
|
|
"path": path if running else None,
|
|
"running": running,
|
|
**(
|
|
{}
|
|
if name == "gitea-runner-dind"
|
|
else {
|
|
"configuredCpuMilli": 500,
|
|
"configuredMemoryBytes": 512 * 1024**2,
|
|
}
|
|
),
|
|
}
|
|
)
|
|
if running:
|
|
groups.append(_group(path, cpu_max, memory_max))
|
|
groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3))
|
|
return {
|
|
"version": 2,
|
|
"family": _group("/itworx-ci", "800000 100000", 16 * 1024**3),
|
|
"dind": {
|
|
"path": dind_path,
|
|
"cpuMax": "600000 100000",
|
|
"memoryMax": str(12 * 1024**3),
|
|
"configuredCpuMilli": 6000,
|
|
"configuredMemoryBytes": 12 * 1024**3,
|
|
},
|
|
"containers": containers,
|
|
"groups": groups,
|
|
}
|
|
|
|
|
|
def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config):
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
workload_specs = [
|
|
(
|
|
"8" * 64,
|
|
{"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0},
|
|
1_000,
|
|
1024**3,
|
|
"100000 100000",
|
|
),
|
|
(
|
|
"9" * 64,
|
|
builder_cpu_config,
|
|
3_000,
|
|
5 * 1024**3,
|
|
"300000 100000",
|
|
),
|
|
]
|
|
workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs]
|
|
other_outer_ids = {
|
|
item["id"]
|
|
for item in snapshot["containers"]
|
|
if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"}
|
|
}
|
|
old_workload_path = f"{dind_path}/docker/job-1"
|
|
snapshot["groups"] = [
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"] != old_workload_path
|
|
and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids}
|
|
]
|
|
for row in snapshot["containers"]:
|
|
if row["name"] == envelope.DIND_NAME:
|
|
continue
|
|
running = row["name"] == "itworx-ci-build"
|
|
row["running"] = running
|
|
row["path"] = f"/itworx-ci/{row['id']}" if running else None
|
|
if running:
|
|
snapshot["groups"].append(
|
|
_group(row["path"], "50000 100000", 512 * 1024**2)
|
|
)
|
|
nested_items = []
|
|
for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs:
|
|
config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config}
|
|
nested_items.append(
|
|
{
|
|
"Id": container_id,
|
|
"State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71},
|
|
"HostConfig": config,
|
|
}
|
|
)
|
|
snapshot["groups"].append(
|
|
_group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes)
|
|
)
|
|
snapshot["workloads"] = [
|
|
{
|
|
"id": container_id,
|
|
"path": path,
|
|
"nestedPid": 70 if container_id.startswith("8") else 71,
|
|
"cpuMilli": cpu_milli,
|
|
"memoryBytes": memory_bytes,
|
|
}
|
|
for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path
|
|
in zip(workload_specs, workload_paths, strict=True)
|
|
]
|
|
|
|
profile_names = {
|
|
profile.container_name: profile.id
|
|
for profile in envelope.RUNNER_PROFILES.values()
|
|
}
|
|
ids = {row["name"]: row["id"] for row in snapshot["containers"]}
|
|
pids = {name: 2000 + index for index, name in enumerate(ids)}
|
|
outer_items = {}
|
|
for name in envelope.FIXED_RUNNER_AND_DIND_NAMES:
|
|
if name == envelope.DIND_NAME:
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": "runner-dind",
|
|
}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 6_000_000_000,
|
|
"Memory": 12 * 1024**3,
|
|
}
|
|
state = {"Running": True, "Pid": pids[name]}
|
|
else:
|
|
profile_id = profile_names[name]
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": f"runner-{profile_id}",
|
|
"tech.itworx.role": "gitea-actions-runner",
|
|
"tech.itworx.ci.profile": profile_id,
|
|
}
|
|
is_running = name == "itworx-ci-build"
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 500_000_000,
|
|
"Memory": 512 * 1024**2,
|
|
}
|
|
state = {"Running": is_running, "Pid": pids[name] if is_running else 0}
|
|
outer_items[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": ids[name],
|
|
"Config": {"Labels": labels},
|
|
"State": state,
|
|
"HostConfig": host,
|
|
}
|
|
|
|
def command(args, **_kwargs):
|
|
if "info" in args:
|
|
return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs"
|
|
if args[-3:] == ["ps", "--no-trunc", "-q"]:
|
|
return "\n".join(item["Id"] for item in nested_items)
|
|
if "inspect" in args:
|
|
return json.dumps(nested_items)
|
|
raise AssertionError(f"unexpected fixed Docker command: {args}")
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name])
|
|
probe_calls = []
|
|
|
|
def fake_probe(operation, payload=None):
|
|
probe_calls.append((operation, payload))
|
|
return snapshot
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", fake_probe)
|
|
return snapshot, nested_items, probe_calls
|
|
|
|
|
|
def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded():
|
|
envelope = _module()
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["build", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["reason"] is None
|
|
assert result["effectiveCpuMilli"] == 8000
|
|
assert result["effectiveMemoryMiB"] == 16384
|
|
assert result["activeClaims"] == 2
|
|
assert result["heavyClaims"] == 1
|
|
assert result["runningRunnerContainers"] == 2
|
|
assert result["cgroups"]["dindCpuMilli"] == 6000
|
|
assert result["cgroups"]["dindMemoryMiB"] == 12288
|
|
assert result["familyCpuUsageUsec"] == 1200
|
|
assert result["familyMemoryCurrentBytes"] == 4096
|
|
assert len(result["digest"]) == 64
|
|
|
|
|
|
def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook():
|
|
envelope = _module()
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["activeClaims"] == 2
|
|
assert result["cgroups"]["reservedCpuMilli"] == 7000
|
|
assert result["cgroups"]["reservedMemoryMiB"] == 12288
|
|
|
|
denied = envelope._evaluate_cgroup_snapshot(
|
|
["build", "browser"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert denied["ready"] is False
|
|
assert denied["reason"] == "resource_claim_heavy_limit_exceeded"
|
|
|
|
|
|
def test_attestor_rejects_an_unbounded_workload_descendant():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["groups"].append(
|
|
_group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3)
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unbounded_descendant"
|
|
|
|
|
|
def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max"))
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["cgroups"]["dindCpuMilli"] == 6000
|
|
|
|
|
|
def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
original_workload = next(
|
|
item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1"
|
|
)
|
|
original_workload["cpuMax"] = "50000 100000"
|
|
original_workload["memoryMax"] = str(1024**3)
|
|
snapshot["groups"].extend(
|
|
[
|
|
_group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3),
|
|
_group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3),
|
|
]
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["cgroups"]["reservedCpuMilli"] == 4000
|
|
assert result["cgroups"]["reservedMemoryMiB"] == 7168
|
|
|
|
|
|
def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].extend(
|
|
[
|
|
_group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3),
|
|
_group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3),
|
|
]
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved"
|
|
|
|
|
|
def test_attestor_rejects_workload_without_an_active_claim():
|
|
envelope = _module()
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
[],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_workload_without_claim"
|
|
|
|
|
|
def test_attestor_rejects_unknown_outer_container_and_parent_escape():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["containers"].append(
|
|
{"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"}
|
|
)
|
|
snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2))
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unknown_outer_container"
|
|
|
|
escaped = _snapshot()
|
|
escaped["containers"][0]["path"] = "/docker/outside"
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
escaped,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_container_outside_family"
|
|
|
|
|
|
def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
del snapshot["groups"][-1]["memoryCurrentBytes"]
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_usage_unavailable"
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_budget_contract_mismatch"
|
|
|
|
|
|
def test_claim_count_and_unknown_profiles_fail_closed():
|
|
envelope = _module()
|
|
|
|
too_many = envelope._evaluate_cgroup_snapshot(
|
|
["checks", "security", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert too_many["ready"] is False
|
|
assert too_many["reason"] == "resource_claim_slot_limit_exceeded"
|
|
|
|
unknown = envelope._evaluate_cgroup_snapshot(
|
|
["general"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert unknown["ready"] is False
|
|
assert unknown["reason"] == "resource_claim_profile_unknown"
|
|
|
|
|
|
def test_attestor_reports_and_rejects_more_than_two_running_outer_runners():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
name = "itworx-ci-frontend"
|
|
item = next(container for container in snapshot["containers"] if container["name"] == name)
|
|
item["running"] = True
|
|
item["path"] = f"/itworx-ci/{item['id']}"
|
|
snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2))
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded"
|
|
assert result["runningRunnerContainers"] == 3
|
|
|
|
|
|
def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
|
|
args = envelope._host_probe_create_args("snapshot", image_id, code_source)
|
|
|
|
assert args[0:3] == ["docker", "create", "--interactive"]
|
|
assert "--pid=host" not in args
|
|
assert "--cgroupns=private" in args
|
|
assert "--network=none" in args
|
|
assert "--read-only" in args
|
|
assert "--cap-drop=ALL" in args
|
|
assert "--cpus=0.1" in args
|
|
assert "--memory=64m" in args
|
|
assert "--pids-limit=64" in args
|
|
assert "--mount" in args
|
|
mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"]
|
|
assert len(mounts) == 2
|
|
mount = mounts[0]
|
|
assert "source=/sys/fs/cgroup" in mount
|
|
assert "target=/host-cgroup" in mount
|
|
assert "readonly" in mount
|
|
assert mounts[1] == (
|
|
f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert not any("/proc" in item or "docker.sock" in item for item in args)
|
|
assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"]
|
|
|
|
|
|
def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
base = {
|
|
"Id": "b" * 64,
|
|
"Name": "/itworx-runner-cgroup-probe",
|
|
"Image": image_id,
|
|
"Config": {
|
|
"Image": image_id,
|
|
"Entrypoint": ["python3"],
|
|
"Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"],
|
|
"Labels": envelope._host_probe_labels(),
|
|
"User": "0:0",
|
|
},
|
|
"State": {"Running": False, "ExitCode": 0},
|
|
"HostConfig": {
|
|
"NetworkMode": "none",
|
|
"PidMode": "",
|
|
"CgroupnsMode": "private",
|
|
"ReadonlyRootfs": True,
|
|
"Privileged": False,
|
|
"NanoCpus": 100_000_000,
|
|
"Memory": 64 * 1024**2,
|
|
"PidsLimit": 64,
|
|
"CapAdd": [],
|
|
"CapDrop": ["ALL"],
|
|
"SecurityOpt": ["no-new-privileges:true"],
|
|
"RestartPolicy": {"Name": "no"},
|
|
"Binds": None,
|
|
"Volumes": None,
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/sys/fs/cgroup/itworx-ci",
|
|
"Target": "/host-cgroup",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Target": "/opt/gitea-deploy",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
},
|
|
],
|
|
},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/sys/fs/cgroup/itworx-ci",
|
|
"Destination": "/host-cgroup",
|
|
"Mode": "",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Destination": "/opt/gitea-deploy",
|
|
"Mode": "",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
],
|
|
}
|
|
|
|
assert envelope._validate_host_probe_inspect(
|
|
base, mode="snapshot", image_id=image_id, code_source=code_source
|
|
)
|
|
for mode in ("family", "ensure", "rollback"):
|
|
candidate = copy.deepcopy(base)
|
|
expected_source = "/sys/fs/cgroup"
|
|
expected_rw = mode in {"ensure", "rollback"}
|
|
candidate["Mounts"][0].update(
|
|
Source=expected_source,
|
|
Mode="",
|
|
RW=expected_rw,
|
|
)
|
|
candidate["HostConfig"]["Mounts"][0].update(
|
|
Source=expected_source,
|
|
ReadOnly=not expected_rw,
|
|
)
|
|
candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else []
|
|
assert envelope._validate_host_probe_inspect(
|
|
candidate, mode=mode, image_id=image_id, code_source=code_source
|
|
)
|
|
|
|
for mutate in (
|
|
lambda item: item["HostConfig"].update(PidMode="host"),
|
|
lambda item: item["HostConfig"].update(CgroupnsMode="host"),
|
|
lambda item: item["HostConfig"].update(NetworkMode="bridge"),
|
|
lambda item: item["HostConfig"].update(ReadonlyRootfs=False),
|
|
lambda item: item["HostConfig"].update(Privileged=True),
|
|
lambda item: item["HostConfig"].update(Memory=0),
|
|
lambda item: item["Mounts"].append(
|
|
{"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False}
|
|
),
|
|
lambda item: item["Mounts"][0].update(RW=True),
|
|
lambda item: item["Mounts"][1].update(RW=True),
|
|
lambda item: item["Mounts"][0].update(Mode="ro"),
|
|
lambda item: item["Mounts"][1].update(Mode="ro"),
|
|
lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"),
|
|
lambda item: item["HostConfig"]["Mounts"].append(
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/proc",
|
|
"Target": "/host-proc",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
}
|
|
),
|
|
lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False),
|
|
lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"),
|
|
lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"),
|
|
):
|
|
candidate = copy.deepcopy(base)
|
|
mutate(candidate)
|
|
assert not envelope._validate_host_probe_inspect(
|
|
candidate, mode="snapshot", image_id=image_id, code_source=code_source
|
|
)
|
|
|
|
|
|
def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch):
|
|
envelope = _module()
|
|
revision = "d" * 40
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}"
|
|
image_id = "sha256:" + "e" * 64
|
|
release = {"sourceRevision": revision}
|
|
controller = {
|
|
"Name": "/gitea-deploy-control",
|
|
"Image": image_id,
|
|
"State": {"Running": True},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Destination": "/opt/gitea-deploy",
|
|
"RW": False,
|
|
}
|
|
],
|
|
}
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"status_server",
|
|
SimpleNamespace(_require_controller_release=lambda: release),
|
|
)
|
|
monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller]))
|
|
|
|
assert envelope._controller_runtime_identity() == (image_id, code_source)
|
|
|
|
controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}"
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"):
|
|
envelope._controller_runtime_identity()
|
|
|
|
|
|
def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
|
|
def probe(operation, payload=None):
|
|
calls.append((operation, payload))
|
|
return {"created": True, "rootSubtreeBefore": ["pids"]}
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", probe)
|
|
result = envelope.ensure_host_cgroup_family(allow_existing=False)
|
|
|
|
assert result == {"created": True, "rootSubtreeBefore": ["pids"]}
|
|
assert calls == [("ensure", {"allowExisting": False})]
|
|
|
|
|
|
def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
envelope,
|
|
"_run_host_probe",
|
|
lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False},
|
|
)
|
|
|
|
result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"])
|
|
|
|
assert result == {"removed": True, "familyExists": False}
|
|
assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})]
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"])
|
|
|
|
|
|
def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch):
|
|
envelope = _module()
|
|
required = {"cpu", "memory", "pids"}
|
|
monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required))
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123)
|
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
|
groups = [
|
|
{
|
|
"path": envelope.FAMILY_CGROUP,
|
|
"cpuMax": "800000 100000",
|
|
"memoryMax": str(16 * 1024**3),
|
|
}
|
|
]
|
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups))
|
|
reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": ""}
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name])
|
|
|
|
ready = envelope._probe_family_info(1)
|
|
assert ready["safeToStartDind"] is True
|
|
assert ready["safeToStartDindReason"] is None
|
|
|
|
groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"})
|
|
blocked = envelope._probe_family_info(1)
|
|
assert blocked["safeToStartDind"] is False
|
|
assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
|
|
|
groups.pop()
|
|
reads["cgroup.procs"] = "4321"
|
|
occupied = envelope._probe_family_info(1)
|
|
assert occupied["safeToStartDind"] is False
|
|
assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
|
|
|
|
|
def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
|
|
snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source)
|
|
ensure = envelope._host_probe_create_args("ensure", image_id, code_source)
|
|
rollback = envelope._host_probe_create_args("rollback", image_id, code_source)
|
|
family = envelope._host_probe_create_args("family", image_id, code_source)
|
|
|
|
def mounts(args):
|
|
return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"]
|
|
|
|
snapshot_mount, snapshot_code_mount = mounts(snapshot)
|
|
ensure_mount, ensure_code_mount = mounts(ensure)
|
|
rollback_mount, rollback_code_mount = mounts(rollback)
|
|
family_mount, family_code_mount = mounts(family)
|
|
assert snapshot_mount == (
|
|
"type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup")
|
|
assert "readonly" not in ensure_mount
|
|
assert "readonly" not in rollback_mount
|
|
assert family_mount == (
|
|
"type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert all(
|
|
code_mount
|
|
== f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rprivate"
|
|
for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount)
|
|
)
|
|
for args in (snapshot, ensure, rollback, family):
|
|
assert "--pid=host" not in args
|
|
assert "--network=none" in args
|
|
assert "--cap-drop=ALL" in args
|
|
assert "--cpus=0.1" in args
|
|
assert "--memory=64m" in args
|
|
assert "--pids-limit=64" in args
|
|
assert not any("/proc" in item or "docker.sock" in item for item in args)
|
|
assert "--cap-add=DAC_OVERRIDE" not in snapshot
|
|
assert "--cap-add=DAC_OVERRIDE" in ensure
|
|
assert "--cap-add=DAC_OVERRIDE" in rollback
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"):
|
|
envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code")
|
|
|
|
|
|
def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path):
|
|
envelope = _module()
|
|
family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
dind_id = "7" * 64
|
|
workload_id = "8" * 64
|
|
dind_path = f"/itworx-ci/{dind_id}"
|
|
workload_path = f"{dind_path}/docker/{workload_id}"
|
|
monkeypatch.setattr(
|
|
envelope,
|
|
"_probe_walk",
|
|
lambda _fd: [
|
|
{"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)},
|
|
{"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)},
|
|
{"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)},
|
|
],
|
|
)
|
|
monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd))
|
|
# The host cgroup contains global PID 4321; nested Docker reports PID 7
|
|
# inside DinD's private namespace. Membership is proven by ID path and a
|
|
# populated cgroup, never by comparing those unrelated PID numbers.
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw: "4321" if name == "cgroup.procs" else "")
|
|
outer = []
|
|
for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1):
|
|
if name == envelope.DIND_NAME:
|
|
outer.append({"name": name, "id": dind_id, "running": True, "pid": 4321})
|
|
else:
|
|
outer.append(
|
|
{
|
|
"name": name,
|
|
"id": f"{index:064x}",
|
|
"running": False,
|
|
"pid": None,
|
|
"configuredCpuMilli": 500,
|
|
"configuredMemoryBytes": 512 * 1024**2,
|
|
}
|
|
)
|
|
try:
|
|
snapshot = envelope._probe_snapshot(
|
|
{
|
|
"op": "snapshot",
|
|
"outer": outer,
|
|
"workloads": [
|
|
{
|
|
"id": workload_id,
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 7,
|
|
"cpuMilli": 100,
|
|
"memoryBytes": 1024**3,
|
|
}
|
|
],
|
|
},
|
|
family_fd,
|
|
)
|
|
finally:
|
|
os.close(family_fd)
|
|
|
|
assert snapshot["workloads"][0]["path"] == workload_path
|
|
assert snapshot["workloads"][0]["nestedPid"] == 7
|
|
assert snapshot["containers"][-1]["path"] == dind_path
|
|
assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path
|
|
for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"):
|
|
try:
|
|
envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent)
|
|
except RuntimeError as exc:
|
|
assert str(exc) == "resource_cgroup_workload_identity_invalid"
|
|
else:
|
|
raise AssertionError("unmapped nested Docker cgroup parent was accepted")
|
|
|
|
|
|
def test_attestor_rejects_unowned_descendants_below_dind():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].append(
|
|
_group(f"{dind_path}/unknown-child", "100000 100000", 1024**3)
|
|
)
|
|
snapshot["workloads"] = [
|
|
{"path": f"{dind_path}/docker/job-1"}
|
|
]
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unmapped_descendant"
|
|
|
|
|
|
def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
family_path = str(snapshot["family"]["path"])
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
probe_id = "f" * 64
|
|
snapshot["containers"] = [
|
|
{
|
|
**item,
|
|
"running": item["name"] == envelope.DIND_NAME,
|
|
"path": item["path"] if item["name"] == envelope.DIND_NAME else None,
|
|
}
|
|
for item in snapshot["containers"]
|
|
]
|
|
snapshot["groups"] = [
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"] == family_path or group["path"] == dind_path
|
|
]
|
|
workload_path = f"{dind_path}/docker/{probe_id}"
|
|
snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3))
|
|
snapshot["workloads"] = [{
|
|
"id": probe_id,
|
|
"path": workload_path,
|
|
"nestedPid": 77,
|
|
"cpuMilli": 4000,
|
|
"memoryBytes": 6 * 1024**3,
|
|
}]
|
|
monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot)
|
|
|
|
result = envelope.attest_build_probe_workload(probe_id, 77)
|
|
|
|
assert result["ready"] is True, result
|
|
assert result["runningRunnerContainers"] == 0
|
|
assert result["probeWorkload"] == {
|
|
"id": probe_id,
|
|
"path": workload_path,
|
|
"nestedPid": 77,
|
|
"cpuMilli": 4000,
|
|
"memoryBytes": 6 * 1024**3,
|
|
"descendantGroupCount": 0,
|
|
}
|
|
assert "snapshot" not in result
|
|
assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False
|
|
|
|
|
|
def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
profile_names = {
|
|
profile.container_name: profile.id
|
|
for profile in envelope.RUNNER_PROFILES.values()
|
|
}
|
|
ids = {row["name"]: row["id"] for row in snapshot["containers"]}
|
|
pids = {name: 2000 + index for index, name in enumerate(ids)}
|
|
outer_items = {}
|
|
for name in envelope.FIXED_RUNNER_AND_DIND_NAMES:
|
|
if name == envelope.DIND_NAME:
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": "runner-dind",
|
|
}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 6_000_000_000,
|
|
"Memory": 12 * 1024**3,
|
|
}
|
|
state = {"Running": True, "Pid": pids[name]}
|
|
else:
|
|
profile_id = profile_names[name]
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": f"runner-{profile_id}",
|
|
"tech.itworx.role": "gitea-actions-runner",
|
|
"tech.itworx.ci.profile": profile_id,
|
|
}
|
|
is_running = name in {"itworx-ci-checks", "itworx-ci-backend"}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 500_000_000,
|
|
"Memory": 512 * 1024**2,
|
|
}
|
|
state = {"Running": is_running, "Pid": pids[name] if is_running else 0}
|
|
outer_items[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": ids[name],
|
|
"Config": {"Labels": labels},
|
|
"State": state,
|
|
"HostConfig": host,
|
|
}
|
|
|
|
docker_commands = []
|
|
|
|
def command(args, **_kwargs):
|
|
docker_commands.append(args)
|
|
if "info" in args:
|
|
return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs"
|
|
if "docker" in args and "ps" in args:
|
|
return ""
|
|
raise AssertionError(f"unexpected fixed Docker command: {args}")
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name])
|
|
probe_calls = []
|
|
|
|
def fake_probe(operation, payload=None):
|
|
probe_calls.append((operation, payload))
|
|
return snapshot
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", fake_probe)
|
|
real_read_text = Path.read_text
|
|
|
|
def reject_local_proc_or_sys(path, *args, **kwargs):
|
|
if str(path).startswith(("/proc/", "/sys/fs/cgroup")):
|
|
raise AssertionError("the private controller must use the host helper")
|
|
return real_read_text(path, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys)
|
|
result = envelope.attest_resource_envelope(
|
|
["backend"],
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert probe_calls and probe_calls[0][0] == "snapshot"
|
|
assert result["ready"] is True
|
|
assert result["familyCpuUsageUsec"] == 1200
|
|
assert result["familyMemoryCurrentBytes"] == 4096
|
|
dind_calls = [args for args in docker_commands if "exec" in args]
|
|
assert len(dind_calls) == 2
|
|
assert all(
|
|
f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args
|
|
for args in dind_calls
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"])
|
|
def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support):
|
|
envelope = _module()
|
|
_install_classic_builder_collection(
|
|
envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}
|
|
)
|
|
original = envelope._run
|
|
|
|
def command(args, **kwargs):
|
|
if "exec" in args and "info" in args:
|
|
return support
|
|
return original(args, **kwargs)
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"):
|
|
envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES)
|
|
|
|
|
|
def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch):
|
|
envelope = _module()
|
|
snapshot, nested_items, probe_calls = _install_classic_builder_collection(
|
|
envelope,
|
|
monkeypatch,
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
)
|
|
|
|
result = envelope.attest_resource_envelope(
|
|
["build"],
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True, result
|
|
assert probe_calls[0][0] == "snapshot"
|
|
assert probe_calls[0][1]["workloads"] == [
|
|
{
|
|
"id": nested_items[0]["Id"],
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 70,
|
|
"cpuMilli": 1000,
|
|
"memoryBytes": 1024**3,
|
|
},
|
|
{
|
|
"id": nested_items[1]["Id"],
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 71,
|
|
"cpuMilli": 3000,
|
|
"memoryBytes": 5 * 1024**3,
|
|
},
|
|
]
|
|
assert snapshot["workloads"] == [
|
|
{
|
|
"id": nested_items[0]["Id"],
|
|
"path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}",
|
|
"nestedPid": 70,
|
|
"cpuMilli": 1000,
|
|
"memoryBytes": 1024**3,
|
|
},
|
|
{
|
|
"id": nested_items[1]["Id"],
|
|
"path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}",
|
|
"nestedPid": 71,
|
|
"cpuMilli": 3000,
|
|
"memoryBytes": 5 * 1024**3,
|
|
},
|
|
]
|
|
builder_group = next(
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"].endswith("/docker/" + nested_items[1]["Id"])
|
|
)
|
|
assert builder_group["cpuMax"] == "300000 100000"
|
|
assert builder_group["memoryMax"] == str(5 * 1024**3)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"builder_cpu_config",
|
|
[
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000},
|
|
id="zero-quota",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0},
|
|
id="zero-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000},
|
|
id="missing-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000},
|
|
id="boolean-quota",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True},
|
|
id="boolean-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001},
|
|
id="period-out-of-range",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000},
|
|
id="fractional-millicpu",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
id="conflicting-encodings",
|
|
),
|
|
pytest.param(
|
|
{"CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
id="missing-nanocpus-field",
|
|
),
|
|
],
|
|
)
|
|
def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations(
|
|
monkeypatch, builder_cpu_config
|
|
):
|
|
envelope = _module()
|
|
_snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection(
|
|
envelope, monkeypatch, builder_cpu_config
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"):
|
|
envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES)
|
|
|
|
assert probe_calls == []
|