2187 lines
96 KiB
Python
2187 lines
96 KiB
Python
from __future__ import annotations
|
|
|
|
import concurrent.futures
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
ENVELOPE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_resource_envelope.py"
|
|
|
|
|
|
def _module(*, real_lock=False):
|
|
broker_dir = str(ENVELOPE_PATH.parent)
|
|
if broker_dir not in sys.path:
|
|
sys.path.insert(0, broker_dir)
|
|
spec = importlib.util.spec_from_file_location("appops_runner_resource_envelope_test", ENVELOPE_PATH)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
if not real_lock:
|
|
# Existing Docker fixtures exercise identity/cleanup independently of
|
|
# the protected volume. Native lifecycle tests below retain the lock.
|
|
@contextmanager
|
|
def fixture_lock(**_kwargs):
|
|
yield lambda: None
|
|
module._host_probe_lifecycle_lock = fixture_lock
|
|
return module
|
|
|
|
|
|
class _ProbeLockStore:
|
|
"""Two independently opened fixture views, with real descriptor/inode checks."""
|
|
|
|
def __init__(self, logical: Path, host: Path | None = None):
|
|
self.paths = (logical, host or logical)
|
|
self.logical = SimpleNamespace(fd=os.open(logical, os.O_RDONLY | os.O_DIRECTORY))
|
|
self.host = SimpleNamespace(fd=os.open(host or logical, os.O_RDONLY | os.O_DIRECTORY))
|
|
|
|
def assert_current(self):
|
|
identities = []
|
|
for view, path in zip((self.logical, self.host), self.paths, strict=True):
|
|
opened, current = os.fstat(view.fd), path.stat(follow_symlinks=False)
|
|
assert (opened.st_dev, opened.st_ino) == (current.st_dev, current.st_ino)
|
|
identities.append((opened.st_dev, opened.st_ino))
|
|
if identities[0] != identities[1]:
|
|
raise RuntimeError("private-canary: mismatched volume views")
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
os.close(self.host.fd)
|
|
os.close(self.logical.fd)
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", [None, "remote", "options", "labels", "root", "malformed"])
|
|
@pytest.mark.parametrize("native_cleanup", [False, True])
|
|
def test_probe_lock_store_uses_only_attested_fixed_local_volume(monkeypatch, mutation, native_cleanup):
|
|
envelope = _module(real_lock=True)
|
|
import mcp_hub_release_store as store
|
|
document = [{"Name": store.VOLUME_NAME, "Driver": "local", "Scope": "local",
|
|
"Mountpoint": store.VOLUME_HOST_ROOT.as_posix(), "Options": {},
|
|
"Labels": dict(store.EXPECTED_LABELS)}]
|
|
root = store.DOCKER_ROOT.as_posix()
|
|
if mutation == "remote":
|
|
document[0]["Driver"] = "nfs"
|
|
elif mutation == "options":
|
|
document[0]["Options"] = {"device": "private-canary"}
|
|
elif mutation == "labels":
|
|
document[0]["Labels"] = {}
|
|
elif mutation == "root":
|
|
root = "/srv/example/private-canary"
|
|
calls = []
|
|
def run(args, **kwargs):
|
|
calls.append(args)
|
|
assert kwargs == {"timeout": 15}
|
|
if args == ["docker", "volume", "inspect", store.VOLUME_NAME]:
|
|
return "private-canary" if mutation == "malformed" else json.dumps(document)
|
|
assert args == ["docker", "info", "--format", "{{.DockerRootDir}}"]
|
|
return root
|
|
sentinel = object()
|
|
def verify(metadata, docker_root):
|
|
store.validate_volume_metadata(metadata, docker_root)
|
|
return sentinel
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
monkeypatch.setattr(envelope, "open_fixed_release_store", verify)
|
|
class PinnedFixture:
|
|
@staticmethod
|
|
def open(path):
|
|
assert path == store.VOLUME_HOST_ROOT
|
|
return sentinel
|
|
monkeypatch.setattr(envelope, "PinnedDirectory", PinnedFixture)
|
|
if mutation is None:
|
|
result = envelope._open_host_probe_lock_store(native_cleanup=native_cleanup)
|
|
if native_cleanup:
|
|
assert result.host is result.logical is sentinel
|
|
else:
|
|
assert result is sentinel
|
|
assert len(calls) == 2
|
|
else:
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_store_untrusted$") as error:
|
|
envelope._open_host_probe_lock_store(native_cleanup=native_cleanup)
|
|
assert error.value.__cause__ is None
|
|
|
|
|
|
def test_native_cleanup_store_rechecks_only_fixed_physical_volume_and_closes_once():
|
|
envelope = _module(real_lock=True)
|
|
calls = []
|
|
directory = SimpleNamespace(assert_current=lambda path: calls.append(path), close=lambda: calls.append("close"))
|
|
with envelope._NativeProbeLockStore(directory) as store:
|
|
assert store.host is store.logical is directory
|
|
store.assert_current()
|
|
assert calls == [envelope.VOLUME_HOST_ROOT, "close"]
|
|
|
|
|
|
@pytest.fixture
|
|
def native_probe_lock(tmp_path, monkeypatch):
|
|
if os.name != "posix" or os.geteuid() != 0:
|
|
pytest.skip("real protected flock contract requires root/POSIX")
|
|
envelope = _module(real_lock=True)
|
|
root = tmp_path / "private-lock-root"
|
|
root.mkdir(mode=0o700)
|
|
monkeypatch.setattr(envelope, "_open_host_probe_lock_store", lambda **_kwargs: _ProbeLockStore(root))
|
|
return envelope, root
|
|
|
|
|
|
def test_probe_lock_is_private_stable_and_released_after_failure(native_probe_lock):
|
|
envelope, root = native_probe_lock
|
|
with pytest.raises(ValueError, match="fixture failure"), envelope._host_probe_lifecycle_lock() as guard:
|
|
guard()
|
|
first = (root / envelope._HOST_PROBE_LOCK_NAME).stat()
|
|
assert first.st_mode & 0o777 == 0o600
|
|
raise ValueError("fixture failure")
|
|
with envelope._host_probe_lifecycle_lock() as guard:
|
|
guard()
|
|
second = (root / envelope._HOST_PROBE_LOCK_NAME).stat()
|
|
assert (second.st_dev, second.st_ino) == (first.st_dev, first.st_ino)
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["mode", "owner", "symlink", "hardlink", "nonempty", "directory"])
|
|
def test_probe_lock_refuses_untrusted_leaf_without_adopting(native_probe_lock, mutation):
|
|
envelope, root = native_probe_lock
|
|
path = root / envelope._HOST_PROBE_LOCK_NAME
|
|
if mutation == "directory":
|
|
path.mkdir(mode=0o700)
|
|
elif mutation == "symlink":
|
|
target = root / "foreign"
|
|
target.write_bytes(b"")
|
|
target.chmod(0o600)
|
|
path.symlink_to(target)
|
|
else:
|
|
path.write_bytes(b"private-canary" if mutation == "nonempty" else b"")
|
|
path.chmod(0o644 if mutation == "mode" else 0o600)
|
|
if mutation == "owner":
|
|
os.chown(path, 1000, 1000)
|
|
elif mutation == "hardlink":
|
|
os.link(path, root / "foreign-link")
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_untrusted$"), envelope._host_probe_lifecycle_lock():
|
|
pytest.fail("unsafe lock adopted")
|
|
|
|
|
|
def test_probe_lock_refuses_replacement_and_different_volume_alias(native_probe_lock, monkeypatch, tmp_path):
|
|
envelope, root = native_probe_lock
|
|
with envelope._host_probe_lifecycle_lock() as guard:
|
|
path = root / envelope._HOST_PROBE_LOCK_NAME
|
|
path.rename(root / "original-owned-lock")
|
|
path.touch(mode=0o600)
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_untrusted$"):
|
|
guard()
|
|
other = tmp_path / "foreign-volume"
|
|
other.mkdir(mode=0o700)
|
|
monkeypatch.setattr(envelope, "_open_host_probe_lock_store", lambda **_kwargs: _ProbeLockStore(root, other))
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_lock_store_untrusted$"), envelope._host_probe_lifecycle_lock():
|
|
pytest.fail("unrelated aliases accepted")
|
|
|
|
|
|
@pytest.mark.parametrize("wait", [0.05, 3])
|
|
def test_probe_lock_excludes_independent_process_through_independent_view(native_probe_lock, wait):
|
|
envelope, root = native_probe_lock
|
|
# Independent interpreter and opens: no inherited Python lock or lock FD.
|
|
child_source = (
|
|
"import importlib.util,json,os,sys\n"
|
|
"from pathlib import Path\n"
|
|
"from types import SimpleNamespace\n"
|
|
"sys.path.insert(0,sys.argv[1])\n"
|
|
"spec=importlib.util.spec_from_file_location('probe_lock_child',Path(sys.argv[1])/'runner_resource_envelope.py')\n"
|
|
"m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)\n"
|
|
"class Store:\n"
|
|
" def __enter__(self):\n"
|
|
" self.logical=SimpleNamespace(fd=os.open(sys.argv[2],os.O_RDONLY|os.O_DIRECTORY))\n"
|
|
" self.host=SimpleNamespace(fd=os.open(sys.argv[2],os.O_RDONLY|os.O_DIRECTORY));return self\n"
|
|
" def assert_current(self):\n"
|
|
" assert os.fstat(self.logical.fd).st_ino==os.fstat(self.host.fd).st_ino\n"
|
|
" def __exit__(self,*args):os.close(self.logical.fd);os.close(self.host.fd)\n"
|
|
"m._open_host_probe_lock_store=lambda **kw:Store();m._HOST_PROBE_LOCK_WAIT_SECONDS=float(sys.argv[3])\n"
|
|
"print('started',flush=True)\n"
|
|
"try:\n"
|
|
" with m._host_probe_lifecycle_lock(native_cleanup=True) as guard:guard();print('acquired',flush=True)\n"
|
|
"except RuntimeError as error:print(str(error),flush=True)\n"
|
|
)
|
|
with envelope._host_probe_lifecycle_lock():
|
|
child = subprocess.Popen([sys.executable, "-c", child_source, str(ENVELOPE_PATH.parent), str(root), str(wait)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
try:
|
|
assert child.stdout.readline().strip() == "started"
|
|
if wait < 1:
|
|
output, error = child.communicate(timeout=3)
|
|
assert output.strip() == "resource_cgroup_probe_lock_busy"
|
|
assert error == "" and child.returncode == 0
|
|
else:
|
|
assert child.poll() is None
|
|
finally:
|
|
if wait < 1 and child.poll() is None:
|
|
child.kill()
|
|
child.communicate()
|
|
if wait >= 1:
|
|
output, error = child.communicate(timeout=5)
|
|
assert output.strip() == "acquired"
|
|
assert error == "" and child.returncode == 0
|
|
|
|
|
|
def test_concurrent_probe_lifecycles_both_complete_without_busy_or_stolen_cleanup(native_probe_lock, monkeypatch):
|
|
envelope, _root = native_probe_lock
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
started, release = threading.Event(), threading.Event()
|
|
state = {"exists": False, "starts": 0, "removals": 0}
|
|
monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source))
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {})
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: copy.deepcopy(probe) if state["exists"] else None)
|
|
|
|
def run(args, **_kwargs):
|
|
if args[:2] == ["docker", "create"]:
|
|
assert not state["exists"]
|
|
state["exists"] = True
|
|
return probe["Id"]
|
|
if args[:2] == ["docker", "inspect"]:
|
|
assert state["exists"]
|
|
return json.dumps([probe])
|
|
assert args == ["docker", "rm", probe["Id"]]
|
|
assert state["exists"] and probe["State"]["Running"] is False
|
|
state["exists"] = False
|
|
state["removals"] += 1
|
|
return ""
|
|
|
|
def start(args, **_kwargs):
|
|
assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]]
|
|
state["starts"] += 1
|
|
probe["State"]["Running"] = True
|
|
if state["starts"] == 1:
|
|
started.set()
|
|
assert release.wait(3)
|
|
probe["State"]["Running"] = False
|
|
return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}')
|
|
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
monkeypatch.setattr(envelope.subprocess, "run", start)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
|
first = pool.submit(envelope._run_host_probe, "snapshot")
|
|
assert started.wait(3)
|
|
second = pool.submit(envelope._run_host_probe, "snapshot")
|
|
try:
|
|
assert state["starts"] == 1 and not second.done()
|
|
finally:
|
|
release.set()
|
|
assert first.result(timeout=5) == second.result(timeout=5) == {"verified": True}
|
|
assert state["removals"] == 2 and state["exists"] is False
|
|
|
|
|
|
def test_normal_probe_and_promotion_cleanup_share_same_complete_lock(monkeypatch):
|
|
envelope = _module(real_lock=True)
|
|
stages = []
|
|
@contextmanager
|
|
def lock(**kwargs):
|
|
assert kwargs in ({}, {"native_cleanup": True})
|
|
stages.append("acquire")
|
|
yield lambda: stages.append("guard")
|
|
stages.append("release")
|
|
monkeypatch.setattr(envelope, "_host_probe_lifecycle_lock", lock)
|
|
def normal(mode, payload, guard):
|
|
assert mode == "family" and payload is None
|
|
guard()
|
|
stages.append("normal")
|
|
return {"verified": True}
|
|
def cleanup(**kwargs):
|
|
kwargs["assert_lock"]()
|
|
stages.append("cleanup")
|
|
return True
|
|
monkeypatch.setattr(envelope, "_run_host_probe_locked", normal)
|
|
monkeypatch.setattr(envelope, "_cleanup_stopped_host_probe_locked", cleanup)
|
|
assert envelope.inspect_host_cgroup_family() == {"verified": True}
|
|
assert envelope.cleanup_stopped_host_probe(image_id="sha256:" + "a" * 64,
|
|
code_source=f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'b' * 40}")
|
|
assert stages == ["acquire", "guard", "normal", "release", "acquire", "guard", "cleanup", "release"]
|
|
|
|
|
|
def _group(path: str, cpu_max: str, memory_max: int) -> dict[str, object]:
|
|
return {
|
|
"path": path,
|
|
"cpuMax": cpu_max,
|
|
"memoryMax": str(memory_max),
|
|
"cpuUsageUsec": "1200",
|
|
"memoryCurrentBytes": "4096",
|
|
}
|
|
|
|
|
|
def _snapshot():
|
|
names = (
|
|
"itworx-ci-checks",
|
|
"itworx-ci-backend",
|
|
"itworx-ci-frontend",
|
|
"itworx-ci-security",
|
|
"itworx-ci-browser",
|
|
"itworx-ci-build",
|
|
"gitea-runner-dind",
|
|
)
|
|
containers = []
|
|
groups = [_group("/itworx-ci", "800000 100000", 16 * 1024**3)]
|
|
dind_path = ""
|
|
for index, name in enumerate(names, start=1):
|
|
container_id = f"{index:064x}"
|
|
path = f"/itworx-ci/{container_id}"
|
|
if name == "gitea-runner-dind":
|
|
dind_path = path
|
|
cpu_max = "600000 100000"
|
|
memory_max = 12 * 1024**3
|
|
running = True
|
|
else:
|
|
cpu_max = "50000 100000"
|
|
memory_max = 512 * 1024**2
|
|
# Only two exact outer runner processes may be live. Stopped
|
|
# configured containers remain part of the identity inventory.
|
|
running = name in {"itworx-ci-checks", "itworx-ci-backend"}
|
|
containers.append(
|
|
{
|
|
"name": name,
|
|
"id": container_id,
|
|
"path": path if running else None,
|
|
"running": running,
|
|
**(
|
|
{}
|
|
if name == "gitea-runner-dind"
|
|
else {
|
|
"configuredCpuMilli": 500,
|
|
"configuredMemoryBytes": 512 * 1024**2,
|
|
}
|
|
),
|
|
}
|
|
)
|
|
if running:
|
|
groups.append(_group(path, cpu_max, memory_max))
|
|
groups.append(_group(f"{dind_path}/docker/job-1", "200000 100000", 4 * 1024**3))
|
|
return {
|
|
"version": 2,
|
|
"family": _group("/itworx-ci", "800000 100000", 16 * 1024**3),
|
|
"dind": {
|
|
"path": dind_path,
|
|
"cpuMax": "600000 100000",
|
|
"memoryMax": str(12 * 1024**3),
|
|
"configuredCpuMilli": 6000,
|
|
"configuredMemoryBytes": 12 * 1024**3,
|
|
},
|
|
"containers": containers,
|
|
"groups": groups,
|
|
}
|
|
|
|
|
|
def _fixture_outer_namespace(envelope, monkeypatch):
|
|
# These pre-existing fixtures isolate resource/config collection. The real
|
|
# namespace/Engine boundary is exercised by the dedicated tests below.
|
|
def proof(name):
|
|
item = envelope._inspect_container(name)
|
|
return envelope._engine_outer_binding(item), [(item["State"]["Pid"], 1)], {
|
|
"target": {"name": name, "id": item["Id"]}, "processCount": 1, "threadCount": 1,
|
|
}
|
|
monkeypatch.setattr(envelope, "_attest_outer_namespace", proof)
|
|
|
|
|
|
def _install_classic_builder_collection(envelope, monkeypatch, builder_cpu_config):
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
workload_specs = [
|
|
(
|
|
"8" * 64,
|
|
{"NanoCpus": 1_000_000_000, "CpuQuota": 0, "CpuPeriod": 0},
|
|
1_000,
|
|
1024**3,
|
|
"100000 100000",
|
|
),
|
|
(
|
|
"9" * 64,
|
|
builder_cpu_config,
|
|
3_000,
|
|
5 * 1024**3,
|
|
"300000 100000",
|
|
),
|
|
]
|
|
workload_paths = [f"{dind_path}/docker/{container_id}" for container_id, *_ in workload_specs]
|
|
other_outer_ids = {
|
|
item["id"]
|
|
for item in snapshot["containers"]
|
|
if item["name"] not in {envelope.DIND_NAME, "itworx-ci-build"}
|
|
}
|
|
old_workload_path = f"{dind_path}/docker/job-1"
|
|
snapshot["groups"] = [
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"] != old_workload_path
|
|
and group["path"] not in {f"/itworx-ci/{container_id}" for container_id in other_outer_ids}
|
|
]
|
|
for row in snapshot["containers"]:
|
|
if row["name"] == envelope.DIND_NAME:
|
|
continue
|
|
running = row["name"] == "itworx-ci-build"
|
|
row["running"] = running
|
|
row["path"] = f"/itworx-ci/{row['id']}" if running else None
|
|
if running:
|
|
snapshot["groups"].append(
|
|
_group(row["path"], "50000 100000", 512 * 1024**2)
|
|
)
|
|
nested_items = []
|
|
for container_id, cpu_config, _cpu_milli, memory_bytes, cpu_max in workload_specs:
|
|
config = {"CgroupParent": "docker", "Memory": memory_bytes, **cpu_config}
|
|
nested_items.append(
|
|
{
|
|
"Id": container_id,
|
|
"State": {"Running": True, "Pid": 70 if container_id.startswith("8") else 71},
|
|
"HostConfig": config,
|
|
}
|
|
)
|
|
snapshot["groups"].append(
|
|
_group(workload_paths[len(nested_items) - 1], cpu_max, memory_bytes)
|
|
)
|
|
snapshot["workloads"] = [
|
|
{
|
|
"id": container_id,
|
|
"path": path,
|
|
"nestedPid": 70 if container_id.startswith("8") else 71,
|
|
"cpuMilli": cpu_milli,
|
|
"memoryBytes": memory_bytes,
|
|
}
|
|
for (container_id, _cpu_config, cpu_milli, memory_bytes, _cpu_max), path
|
|
in zip(workload_specs, workload_paths, strict=True)
|
|
]
|
|
|
|
profile_names = {
|
|
profile.container_name: profile.id
|
|
for profile in envelope.RUNNER_PROFILES.values()
|
|
}
|
|
ids = {row["name"]: row["id"] for row in snapshot["containers"]}
|
|
pids = {name: 2000 + index for index, name in enumerate(ids)}
|
|
outer_items = {}
|
|
for name in envelope.FIXED_RUNNER_AND_DIND_NAMES:
|
|
if name == envelope.DIND_NAME:
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": "runner-dind",
|
|
}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 6_000_000_000,
|
|
"Memory": 12 * 1024**3,
|
|
}
|
|
state = {"Running": True, "Pid": pids[name]}
|
|
else:
|
|
profile_id = profile_names[name]
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": f"runner-{profile_id}",
|
|
"tech.itworx.role": "gitea-actions-runner",
|
|
"tech.itworx.ci.profile": profile_id,
|
|
}
|
|
is_running = name == "itworx-ci-build"
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 500_000_000,
|
|
"Memory": 512 * 1024**2,
|
|
}
|
|
state = {"Running": is_running, "Pid": pids[name] if is_running else 0}
|
|
outer_items[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": ids[name],
|
|
"Config": {"Labels": labels},
|
|
"State": state,
|
|
"HostConfig": host,
|
|
}
|
|
|
|
def command(args, **_kwargs):
|
|
if "info" in args:
|
|
return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs"
|
|
if args[-3:] == ["ps", "--no-trunc", "-q"]:
|
|
return "\n".join(item["Id"] for item in nested_items)
|
|
if "inspect" in args:
|
|
return json.dumps(nested_items)
|
|
raise AssertionError(f"unexpected fixed Docker command: {args}")
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name])
|
|
probe_calls = []
|
|
|
|
def fake_probe(operation, payload=None):
|
|
probe_calls.append((operation, payload))
|
|
return snapshot
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", fake_probe)
|
|
_fixture_outer_namespace(envelope, monkeypatch)
|
|
return snapshot, nested_items, probe_calls
|
|
|
|
|
|
def test_attestor_accepts_two_fixed_claims_when_family_and_dind_are_bounded():
|
|
envelope = _module()
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["build", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["reason"] is None
|
|
assert result["effectiveCpuMilli"] == 8000
|
|
assert result["effectiveMemoryMiB"] == 16384
|
|
assert result["activeClaims"] == 2
|
|
assert result["heavyClaims"] == 1
|
|
assert result["runningRunnerContainers"] == 2
|
|
assert result["cgroups"]["dindCpuMilli"] == 6000
|
|
assert result["cgroups"]["dindMemoryMiB"] == 12288
|
|
assert result["familyCpuUsageUsec"] == 1200
|
|
assert result["familyMemoryCurrentBytes"] == 4096
|
|
assert len(result["digest"]) == 64
|
|
|
|
|
|
def test_duplicate_profile_ids_are_separate_slot_claims_and_cannot_overbook():
|
|
envelope = _module()
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["activeClaims"] == 2
|
|
assert result["cgroups"]["reservedCpuMilli"] == 7000
|
|
assert result["cgroups"]["reservedMemoryMiB"] == 12288
|
|
|
|
denied = envelope._evaluate_cgroup_snapshot(
|
|
["build", "browser"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert denied["ready"] is False
|
|
assert denied["reason"] == "resource_claim_heavy_limit_exceeded"
|
|
|
|
|
|
def test_attestor_rejects_an_unbounded_workload_descendant():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["groups"].append(
|
|
_group(f"{snapshot['dind']['path']}/docker/job-unbounded", "max 100000", 4 * 1024**3)
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unbounded_descendant"
|
|
|
|
|
|
def test_attestor_accepts_unbounded_intermediate_when_workload_and_ancestors_are_bounded():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].append(_group(f"{dind_path}/docker", "max 100000", "max"))
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["cgroups"]["dindCpuMilli"] == 6000
|
|
|
|
|
|
def test_attestor_allows_job_and_service_descendants_to_share_one_profile_reservation():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
original_workload = next(
|
|
item for item in snapshot["groups"] if item["path"] == f"{dind_path}/docker/job-1"
|
|
)
|
|
original_workload["cpuMax"] = "50000 100000"
|
|
original_workload["memoryMax"] = str(1024**3)
|
|
snapshot["groups"].extend(
|
|
[
|
|
_group(f"{dind_path}/docker/job-2", "50000 100000", 1024**3),
|
|
_group(f"{dind_path}/docker/service-1", "50000 100000", 1024**3),
|
|
]
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True
|
|
assert result["cgroups"]["reservedCpuMilli"] == 4000
|
|
assert result["cgroups"]["reservedMemoryMiB"] == 7168
|
|
|
|
|
|
def test_attestor_rejects_multiple_workloads_whose_combined_caps_exceed_claim():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].extend(
|
|
[
|
|
_group(f"{dind_path}/docker/job-2", "200000 100000", 3 * 1024**3),
|
|
_group(f"{dind_path}/docker/service-1", "200000 100000", 3 * 1024**3),
|
|
]
|
|
)
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["backend"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_workload_cpu_over_reserved"
|
|
|
|
|
|
def test_attestor_rejects_workload_without_an_active_claim():
|
|
envelope = _module()
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
[],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_workload_without_claim"
|
|
|
|
|
|
def test_attestor_rejects_unknown_outer_container_and_parent_escape():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["containers"].append(
|
|
{"name": "unmapped-runner", "id": "f" * 64, "path": f"/itworx-ci/{'f' * 64}"}
|
|
)
|
|
snapshot["groups"].append(_group(f"/itworx-ci/{'f' * 64}", "50000 100000", 512 * 1024**2))
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unknown_outer_container"
|
|
|
|
escaped = _snapshot()
|
|
escaped["containers"][0]["path"] = "/docker/outside"
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
escaped,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_container_outside_family"
|
|
|
|
|
|
def test_attestor_requires_real_usage_metrics_and_fixed_budget_contract():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
del snapshot["groups"][-1]["memoryCurrentBytes"]
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_usage_unavailable"
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 9000, "memoryMiB": 16384},
|
|
)
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_budget_contract_mismatch"
|
|
|
|
|
|
def test_claim_count_and_unknown_profiles_fail_closed():
|
|
envelope = _module()
|
|
|
|
too_many = envelope._evaluate_cgroup_snapshot(
|
|
["checks", "security", "backend"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert too_many["ready"] is False
|
|
assert too_many["reason"] == "resource_claim_slot_limit_exceeded"
|
|
|
|
unknown = envelope._evaluate_cgroup_snapshot(
|
|
["general"],
|
|
_snapshot(),
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
assert unknown["ready"] is False
|
|
assert unknown["reason"] == "resource_claim_profile_unknown"
|
|
|
|
|
|
def test_attestor_reports_and_rejects_more_than_two_running_outer_runners():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
name = "itworx-ci-frontend"
|
|
item = next(container for container in snapshot["containers"] if container["name"] == name)
|
|
item["running"] = True
|
|
item["path"] = f"/itworx-ci/{item['id']}"
|
|
snapshot["groups"].append(_group(item["path"], "50000 100000", 512 * 1024**2))
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_running_outer_limit_exceeded"
|
|
assert result["runningRunnerContainers"] == 3
|
|
|
|
|
|
def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_socket():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
|
|
args = envelope._host_probe_create_args("snapshot", image_id, code_source)
|
|
|
|
assert args[0:3] == ["docker", "create", "--interactive"]
|
|
assert "--pid=host" not in args
|
|
assert "--cgroupns=private" in args
|
|
assert "--network=none" in args
|
|
assert "--read-only" in args
|
|
assert "--cap-drop=ALL" in args
|
|
assert "--cpus=0.1" in args
|
|
assert "--memory=64m" in args
|
|
assert "--pids-limit=64" in args
|
|
assert "--mount" in args
|
|
mounts = [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"]
|
|
assert len(mounts) == 2
|
|
mount = mounts[0]
|
|
assert "source=/sys/fs/cgroup" in mount
|
|
assert "target=/host-cgroup" in mount
|
|
assert "readonly" in mount
|
|
assert mounts[1] == (
|
|
f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave"
|
|
)
|
|
assert not any("/proc" in item or "docker.sock" in item for item in args)
|
|
assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"]
|
|
|
|
|
|
def _host_probe_fixture(envelope):
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
base = {
|
|
"Id": "b" * 64,
|
|
"Name": "/itworx-runner-cgroup-probe",
|
|
"Image": image_id,
|
|
"Config": {
|
|
"Image": image_id,
|
|
"Entrypoint": ["python3"],
|
|
"Cmd": ["/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"],
|
|
"Labels": envelope._host_probe_labels(),
|
|
"User": "0:0",
|
|
},
|
|
"State": {"Running": False, "ExitCode": 0},
|
|
"HostConfig": {
|
|
"NetworkMode": "none",
|
|
"PidMode": "",
|
|
"CgroupnsMode": "private",
|
|
"ReadonlyRootfs": True,
|
|
"Privileged": False,
|
|
"NanoCpus": 100_000_000,
|
|
"Memory": 64 * 1024**2,
|
|
"PidsLimit": 64,
|
|
"CapAdd": [],
|
|
"CapDrop": ["ALL"],
|
|
"SecurityOpt": ["no-new-privileges:true"],
|
|
"RestartPolicy": {"Name": "no"},
|
|
"Binds": None,
|
|
"Volumes": None,
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/sys/fs/cgroup/itworx-ci",
|
|
"Target": "/host-cgroup",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Target": "/opt/gitea-deploy",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rslave"},
|
|
},
|
|
],
|
|
},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/sys/fs/cgroup/itworx-ci",
|
|
"Destination": "/host-cgroup",
|
|
"Mode": "",
|
|
"RW": False,
|
|
"Propagation": "rprivate",
|
|
},
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Destination": "/opt/gitea-deploy",
|
|
"Mode": "",
|
|
"RW": False,
|
|
"Propagation": "rslave",
|
|
},
|
|
],
|
|
}
|
|
|
|
return base, image_id, code_source
|
|
|
|
|
|
def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift():
|
|
envelope = _module()
|
|
base, image_id, code_source = _host_probe_fixture(envelope)
|
|
assert envelope._validate_host_probe_inspect(
|
|
base, mode="snapshot", image_id=image_id, code_source=code_source
|
|
)
|
|
for mode in ("family", "ensure", "rollback"):
|
|
candidate = copy.deepcopy(base)
|
|
expected_source = "/sys/fs/cgroup"
|
|
expected_rw = mode in {"ensure", "rollback"}
|
|
candidate["Mounts"][0].update(
|
|
Source=expected_source,
|
|
Mode="",
|
|
RW=expected_rw,
|
|
)
|
|
candidate["HostConfig"]["Mounts"][0].update(
|
|
Source=expected_source,
|
|
ReadOnly=not expected_rw,
|
|
)
|
|
candidate["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if expected_rw else []
|
|
assert envelope._validate_host_probe_inspect(
|
|
candidate, mode=mode, image_id=image_id, code_source=code_source
|
|
)
|
|
|
|
for mutate in (
|
|
lambda item: item["HostConfig"].update(PidMode="host"),
|
|
lambda item: item["HostConfig"].update(CgroupnsMode="host"),
|
|
lambda item: item["HostConfig"].update(NetworkMode="bridge"),
|
|
lambda item: item["HostConfig"].update(ReadonlyRootfs=False),
|
|
lambda item: item["HostConfig"].update(Privileged=True),
|
|
lambda item: item["HostConfig"].update(Memory=0),
|
|
lambda item: item["Mounts"].append(
|
|
{"Type": "bind", "Source": "/proc", "Destination": "/host-proc", "RW": False}
|
|
),
|
|
lambda item: item["Mounts"][0].update(RW=True),
|
|
lambda item: item["Mounts"][1].update(RW=True),
|
|
lambda item: item["Mounts"][0].update(Propagation="rslave"),
|
|
lambda item: item["Mounts"][1].update(Propagation="rprivate"),
|
|
lambda item: item["Mounts"][1].update(Propagation="rshared"),
|
|
lambda item: item["Mounts"][1].update(Propagation="slave"),
|
|
lambda item: item["HostConfig"]["Mounts"][0]["BindOptions"].update(Propagation="rslave"),
|
|
lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rprivate"),
|
|
lambda item: item["Mounts"][0].update(Mode="ro"),
|
|
lambda item: item["Mounts"][1].update(Mode="ro"),
|
|
lambda item: item["Mounts"][1].update(Source="/mnt/shared/code"),
|
|
lambda item: item["HostConfig"]["Mounts"].append(
|
|
{
|
|
"Type": "bind",
|
|
"Source": "/proc",
|
|
"Target": "/host-proc",
|
|
"ReadOnly": True,
|
|
"BindOptions": {"Propagation": "rprivate"},
|
|
}
|
|
),
|
|
lambda item: item["HostConfig"]["Mounts"][0].update(ReadOnly=False),
|
|
lambda item: item["HostConfig"]["Mounts"][1]["BindOptions"].update(Propagation="rshared"),
|
|
lambda item: item["HostConfig"]["Mounts"][1].update(Consistency="delegated"),
|
|
):
|
|
candidate = copy.deepcopy(base)
|
|
mutate(candidate)
|
|
assert not envelope._validate_host_probe_inspect(
|
|
candidate, mode="snapshot", image_id=image_id, code_source=code_source
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"])
|
|
@pytest.mark.parametrize("capability", ["DAC_OVERRIDE", "CAP_DAC_OVERRIDE"])
|
|
def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode, capability):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
inherited = {"io.buildah.version": "1.35.0", "org.opencontainers.image.title": "Official runner image"}
|
|
probe["Config"]["Labels"].update(inherited)
|
|
rw = mode in {"ensure", "rollback"}
|
|
source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup"
|
|
probe["Mounts"][0].update(Source=source, RW=rw)
|
|
probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw)
|
|
probe["HostConfig"]["CapAdd"] = [capability] if rw else []
|
|
assert envelope._validate_host_probe_inspect(
|
|
probe, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited
|
|
)
|
|
for mutate in (
|
|
lambda item: item["Config"]["Labels"].update(unknown="extra"),
|
|
lambda item: item["Config"]["Labels"].update({"org.opencontainers.image.title": "changed"}),
|
|
lambda item: item["Config"]["Labels"].pop("io.buildah.version"),
|
|
lambda item: item["Config"]["Labels"].update({"tech.itworx.owner": "foreign"}),
|
|
lambda item: item.update(Image="sha256:" + "f" * 64),
|
|
):
|
|
candidate = copy.deepcopy(probe)
|
|
mutate(candidate)
|
|
assert not envelope._validate_host_probe_inspect(
|
|
candidate, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited
|
|
)
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited)
|
|
inspected = iter([probe, copy.deepcopy(probe), None])
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
|
calls = []
|
|
def remove(args, **kw):
|
|
calls.append(args)
|
|
assert kw == {"check": False, "capture_output": True, "timeout": 10}
|
|
return SimpleNamespace(returncode=0, stderr=b"")
|
|
monkeypatch.setattr(envelope.subprocess, "run", remove)
|
|
assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
|
assert calls == [["docker", "rm", probe["Id"]]]
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"])
|
|
@pytest.mark.parametrize("cap_add", [None, [], ["DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE"],
|
|
["DAC_OVERRIDE", "CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "DAC_OVERRIDE"],
|
|
["CAP_DAC_OVERRIDE", "CAP_DAC_OVERRIDE"], ["DAC_OVERRIDE", "SYS_ADMIN"],
|
|
["SYS_ADMIN"], ["dac_override"], ["CAP_CAP_DAC_OVERRIDE"],
|
|
"DAC_OVERRIDE", "", {}, (), False, 0, [None], [{"cap": "DAC_OVERRIDE"}]])
|
|
def test_host_probe_capabilities_allow_only_one_exact_equivalent_spelling(mode, cap_add):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
rw = mode in {"ensure", "rollback"}
|
|
source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup"
|
|
probe["Mounts"][0].update(Source=source, RW=rw)
|
|
probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw)
|
|
probe["HostConfig"]["CapAdd"] = copy.deepcopy(cap_add)
|
|
before = copy.deepcopy(probe)
|
|
expected = cap_add in (["DAC_OVERRIDE"], ["CAP_DAC_OVERRIDE"]) if rw else cap_add is None or cap_add == []
|
|
assert envelope._validate_host_probe_inspect(
|
|
probe, mode=mode, image_id=image_id, code_source=code_source
|
|
) is expected
|
|
assert probe == before
|
|
|
|
|
|
@pytest.mark.parametrize("drift", ["running", "foreign_source", "race", "missing_state"])
|
|
def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch, drift):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
fresh = copy.deepcopy(probe)
|
|
if drift == "running":
|
|
probe["State"]["Running"] = True
|
|
elif drift == "foreign_source":
|
|
probe["Mounts"][1]["Source"] = code_source.replace("c" * 40, "d" * 40)
|
|
elif drift == "race":
|
|
fresh["Id"] = "f" * 64
|
|
else:
|
|
probe.pop("State")
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {})
|
|
inspected = iter([probe, fresh])
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
|
monkeypatch.setattr(envelope, "_run", lambda *a, **kw: pytest.fail("No removal or probe execution"))
|
|
monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda *a: pytest.fail("No removal"))
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_"):
|
|
envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
|
|
|
|
|
@pytest.mark.parametrize("drift", [None, "extra_mount", "changed_mount_field", "changed_unknown_mount_field",
|
|
"changed_top_level_field", "changed_host_mounts", "changed_state"])
|
|
def test_probe_cleanup_cas_ignores_only_complete_mount_collection_order(monkeypatch, drift):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
fresh = copy.deepcopy(probe)
|
|
fresh["Mounts"].reverse()
|
|
if drift == "extra_mount":
|
|
fresh["Mounts"].append(copy.deepcopy(fresh["Mounts"][0]))
|
|
elif drift == "changed_mount_field":
|
|
fresh["Mounts"][0]["RW"] = True
|
|
elif drift == "changed_unknown_mount_field":
|
|
fresh["Mounts"][0]["unexpectedMetadata"] = "changed"
|
|
elif drift == "changed_top_level_field":
|
|
fresh["unexpectedMetadata"] = "changed"
|
|
elif drift == "changed_host_mounts":
|
|
fresh["HostConfig"]["Mounts"].reverse()
|
|
elif drift == "changed_state":
|
|
fresh["State"]["unexpectedMetadata"] = "changed"
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {})
|
|
inspected = iter([probe, fresh, None])
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected))
|
|
removed = []
|
|
monkeypatch.setattr(envelope, "_remove_cleanup_probe", lambda identifier: removed.append(identifier))
|
|
if drift is None:
|
|
assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
|
assert removed == [probe["Id"]]
|
|
else:
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_changed"):
|
|
envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source)
|
|
assert removed == []
|
|
assert fresh != probe
|
|
|
|
|
|
@pytest.mark.parametrize("stderr,condition", [
|
|
(b"private-canary: no space left on device", "remove_storage_full"),
|
|
(b"private-canary: read-only file system", "remove_read_only"),
|
|
(b"private-canary: permission denied", "remove_permission_denied"),
|
|
(b"Cannot connect to the Docker daemon at private-canary", "remove_daemon_unavailable"),
|
|
(b"private-canary: device or resource busy", "remove_busy"),
|
|
(b"Removal of container private-canary is already in progress", "remove_busy"),
|
|
(b"No such container: private-canary", "remove_container_missing"),
|
|
(b"private-canary: client version is too old", "remove_api_version_unsupported"),
|
|
(b"removal of container private-canary failed unexpectedly", "remove_failed"),
|
|
(b"private-canary", "remove_failed"),
|
|
(b"no space left on device" + b"x" * 4096, "remove_failed"),
|
|
])
|
|
def test_probe_removal_has_finite_diagnostics_and_never_forces(monkeypatch, stderr, condition):
|
|
envelope = _module()
|
|
identifier = "d" * 64
|
|
def run(args, **kw):
|
|
assert args == ["docker", "rm", identifier]
|
|
assert kw == {"check": False, "capture_output": True, "timeout": 10}
|
|
return SimpleNamespace(returncode=1, stderr=stderr, stdout=b"private-canary")
|
|
monkeypatch.setattr(envelope.subprocess, "run", run)
|
|
with pytest.raises(envelope.HostProbeCleanupError) as error:
|
|
envelope._remove_cleanup_probe(identifier)
|
|
assert str(error.value) == "resource_cgroup_probe_cleanup_" + condition
|
|
assert "private-canary" not in str(error.value)
|
|
assert error.value.__cause__ is None
|
|
|
|
|
|
@pytest.mark.parametrize("identifier", ["probe", "d" * 12, "d" * 64 + " --force", "../private-canary"])
|
|
def test_probe_removal_rejects_selectors_before_any_command(monkeypatch, identifier):
|
|
envelope = _module()
|
|
monkeypatch.setattr(envelope.subprocess, "run", lambda *a, **kw: pytest.fail("No command"))
|
|
with pytest.raises(envelope.HostProbeCleanupError, match="cleanup_identity_invalid"):
|
|
envelope._remove_cleanup_probe(identifier)
|
|
|
|
|
|
@pytest.mark.parametrize("stage", ["image_metadata_failed", "inspect_failed", "reinspect_failed", "absence_failed"])
|
|
def test_probe_read_failures_drop_untrusted_exception_text(stage):
|
|
envelope = _module()
|
|
def fail():
|
|
raise RuntimeError("Bearer private-canary")
|
|
with pytest.raises(envelope.HostProbeCleanupError) as error:
|
|
envelope._cleanup_probe_read(stage, fail)
|
|
assert str(error.value) == "resource_cgroup_probe_cleanup_" + stage
|
|
assert error.value.__cause__ is None
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="Docker CLI shim uses POSIX executable semantics")
|
|
@pytest.mark.parametrize("failure,condition", [("read-only file system private-canary", "remove_read_only"),
|
|
("Bearer private-canary", "remove_failed")])
|
|
def test_real_probe_cleanup_cli_exposes_only_finite_child_failure(tmp_path, failure, condition):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
fixture = tmp_path / "fixture.json"
|
|
fixture.write_text(json.dumps({"probe": probe, "image": image_id, "failure": failure}))
|
|
shim = tmp_path / "docker"
|
|
shim.write_text("#!" + sys.executable + "\n" +
|
|
"import json,sys,pathlib\n" +
|
|
"v=json.loads(pathlib.Path(" + repr(str(fixture)) + ").read_text())\n" +
|
|
"a=sys.argv[1:]\n" +
|
|
"if a[:2]==['image','inspect']: print(json.dumps([{'Id':v['image'],'Config':{'Labels':{}}}]))\n" +
|
|
"elif a[:2]==['ps','-a']: print(v['probe']['Id'])\n" +
|
|
"elif a[0]=='inspect': print(json.dumps([v['probe']]))\n" +
|
|
"elif a==['rm',v['probe']['Id']]: print(v['failure'],file=sys.stderr); sys.exit(1)\n" +
|
|
"else: sys.exit(99)\n")
|
|
shim.chmod(0o700)
|
|
# Keep the actual CLI branch/diagnostics, isolating only its volume lock.
|
|
entry = tmp_path / "cleanup-entry.py"
|
|
source = ENVELOPE_PATH.read_text(encoding="utf-8")
|
|
source = source.replace('if __name__ == "__main__":',
|
|
'@contextmanager\ndef _host_probe_lifecycle_lock(**_kwargs):\n'
|
|
' yield lambda: None\n\nif __name__ == "__main__":')
|
|
entry.write_text(source, encoding="utf-8")
|
|
result = subprocess.run([sys.executable, str(entry), "--cleanup-stopped-host-probe",
|
|
image_id, code_source], capture_output=True, timeout=15, check=False,
|
|
env={**os.environ, "PATH": str(tmp_path) + os.pathsep + os.environ.get("PATH", ""),
|
|
"PYTHONPATH": str(ENVELOPE_PATH.parent)})
|
|
assert result.returncode == 1
|
|
assert result.stdout == b""
|
|
assert result.stderr == ("resource_cgroup_probe_cleanup_" + condition + "\n").encode()
|
|
|
|
|
|
@pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.title": "runner"}])
|
|
def test_probe_label_metadata_comes_from_exact_image(monkeypatch, labels):
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
calls = []
|
|
def run(args, **kw):
|
|
calls.append(args)
|
|
return json.dumps([{"Id": image_id, "Config": {"Labels": labels}}])
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
assert envelope._host_probe_image_labels(image_id) == (labels or {})
|
|
assert calls == [["docker", "image", "inspect", image_id]]
|
|
|
|
|
|
@pytest.mark.parametrize("row", [
|
|
{"Id": "sha256:" + "b" * 64, "Config": {"Labels": {}}},
|
|
{"Id": "sha256:" + "a" * 64, "Config": {"Labels": ["foreign"]}},
|
|
{"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"secret-canary": 123}}},
|
|
{"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"tech.itworx.owner": "foreign"}}},
|
|
])
|
|
def test_probe_image_metadata_rejects_drift_without_echo(monkeypatch, row):
|
|
envelope = _module()
|
|
monkeypatch.setattr(envelope, "_run", lambda *a, **kw: json.dumps([row]))
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_image_labels_invalid$"):
|
|
envelope._host_probe_image_labels("sha256:" + "a" * 64)
|
|
|
|
|
|
def test_normal_probe_executes_and_removes_its_image_label_bound_container(monkeypatch):
|
|
envelope = _module()
|
|
probe, image_id, code_source = _host_probe_fixture(envelope)
|
|
inherited = {"org.opencontainers.image.title": "Official runner image"}
|
|
probe["Config"]["Labels"].update(inherited)
|
|
monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source))
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited)
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None)
|
|
calls = []
|
|
def run(args, **kw):
|
|
calls.append(args)
|
|
if args[:2] == ["docker", "create"]:
|
|
return probe["Id"]
|
|
if args[:2] == ["docker", "inspect"]:
|
|
return json.dumps([probe])
|
|
assert args == ["docker", "rm", probe["Id"]]
|
|
return ""
|
|
def start(args, **kw):
|
|
assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]]
|
|
assert json.loads(kw["input"]) == {"op": "snapshot"}
|
|
return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}')
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
monkeypatch.setattr(envelope.subprocess, "run", start)
|
|
assert envelope._run_host_probe("snapshot") == {"verified": True}
|
|
assert calls[-1] == ["docker", "rm", probe["Id"]]
|
|
|
|
|
|
def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch):
|
|
envelope = _module()
|
|
revision = "d" * 40
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{revision}"
|
|
image_id = "sha256:" + "e" * 64
|
|
release = {"sourceRevision": revision}
|
|
controller = {
|
|
"Name": "/gitea-deploy-control",
|
|
"Image": image_id,
|
|
"State": {"Running": True},
|
|
"Mounts": [
|
|
{
|
|
"Type": "bind",
|
|
"Source": code_source,
|
|
"Destination": "/opt/gitea-deploy",
|
|
"RW": False,
|
|
}
|
|
],
|
|
}
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"status_server",
|
|
SimpleNamespace(_require_controller_release=lambda: release),
|
|
)
|
|
monkeypatch.setattr(envelope, "_run", lambda _args, **_kwargs: json.dumps([controller]))
|
|
|
|
assert envelope._controller_runtime_identity() == (image_id, code_source)
|
|
|
|
controller["Mounts"][0]["Source"] = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'f' * 40}"
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_controller_identity_invalid"):
|
|
envelope._controller_runtime_identity()
|
|
|
|
|
|
def test_host_family_initialization_is_fixed_and_uses_the_host_probe(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
|
|
def probe(operation, payload=None):
|
|
calls.append((operation, payload))
|
|
return {"created": True, "rootSubtreeBefore": ["pids"]}
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", probe)
|
|
result = envelope.ensure_host_cgroup_family(allow_existing=False)
|
|
|
|
assert result == {"created": True, "rootSubtreeBefore": ["pids"]}
|
|
assert calls == [("ensure", {"allowExisting": False})]
|
|
|
|
|
|
def test_host_family_rollback_is_fixed_and_rejects_untrusted_arguments(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
envelope,
|
|
"_run_host_probe",
|
|
lambda operation, payload=None: calls.append((operation, payload)) or {"removed": True, "familyExists": False},
|
|
)
|
|
|
|
result = envelope.rollback_host_cgroup_family(created=True, initial_controllers=["pids"])
|
|
|
|
assert result == {"removed": True, "familyExists": False}
|
|
assert calls == [("rollback", {"created": True, "initialControllers": ["pids"]})]
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope.rollback_host_cgroup_family(created="yes", initial_controllers=["pids"])
|
|
|
|
|
|
def _owned_family_fixture(monkeypatch, *, exact=False, init=True, docker=True):
|
|
envelope = _module()
|
|
cid = "a" * 64
|
|
leaf = f"/itworx-ci/{cid}"
|
|
paths = ["/itworx-ci", leaf]
|
|
if init:
|
|
paths.append(f"{leaf}/init")
|
|
if docker:
|
|
paths.append(f"{leaf}/docker")
|
|
rows = {
|
|
path: {"path": path, "cpuMax": "max 100000", "memoryMax": "max"}
|
|
for path in paths
|
|
}
|
|
rows[leaf].update(cpuMax="600000 100000", memoryMax=str(12 * 1024**3))
|
|
if exact:
|
|
rows["/itworx-ci"].update(cpuMax="800000 100000", memoryMax=str(16 * 1024**3))
|
|
processes = {path: "" for path in paths}
|
|
processes[f"{leaf}/init" if init else leaf] = "1 2"
|
|
descriptors = {path: index for index, path in enumerate(paths, start=101)}
|
|
reverse = {value: path for path, value in descriptors.items()}
|
|
required = {"cpu", "memory", "pids"}
|
|
controls = {"root": required.copy(), "family": required.copy()}
|
|
inodes = {path: value for path, value in descriptors.items()}
|
|
monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, controls["root"]))
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 101)
|
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [dict(row) for row in rows.values()])
|
|
monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, path: descriptors[path])
|
|
monkeypatch.setattr(envelope.os, "dup", lambda fd: fd)
|
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
|
monkeypatch.setattr(envelope.os, "fstat", lambda fd: SimpleNamespace(
|
|
st_dev=36, st_ino=inodes[reverse[fd]], st_uid=0, st_mode=0o40755,
|
|
))
|
|
monkeypatch.setattr(envelope, "_probe_stat", lambda _fd, _name: SimpleNamespace(st_dev=36, st_ino=inodes["/itworx-ci"]))
|
|
|
|
def read(fd, name):
|
|
path = reverse[fd]
|
|
if name == "cgroup.procs":
|
|
return processes[path]
|
|
if name == "cgroup.threads":
|
|
return processes[path]
|
|
if name in {"pids.current", "cgroup.events"}:
|
|
count = sum(len(value.split()) for group, value in processes.items() if group == path or group.startswith(path + "/"))
|
|
return str(count) if name == "pids.current" else f"populated {int(count > 0)}\nfrozen 0"
|
|
if name == "cgroup.subtree_control":
|
|
return " ".join(sorted(controls["family"]))
|
|
raise AssertionError(name)
|
|
|
|
writes = []
|
|
|
|
def write(fd, name, value):
|
|
assert fd == 101
|
|
assert name in {"cpu.max", "memory.max"}
|
|
writes.append((name, value))
|
|
rows["/itworx-ci"]["cpuMax" if name == "cpu.max" else "memoryMax"] = value
|
|
|
|
monkeypatch.setattr(envelope, "_probe_read", read)
|
|
monkeypatch.setattr(envelope, "_probe_write", write)
|
|
claim = {"containerId": cid, "processCount": 2, "threadCount": 2}
|
|
return envelope, claim, rows, processes, controls, writes, inodes
|
|
|
|
|
|
@pytest.mark.parametrize("init,docker,exact", [(False, False, False), (True, False, False), (True, True, False), (True, True, True)])
|
|
def test_owned_dind_family_binds_only_existing_exact_private_child(monkeypatch, init, docker, exact):
|
|
envelope, claim, _rows, _processes, _controls, writes, _inodes = _owned_family_fixture(
|
|
monkeypatch, init=init, docker=docker, exact=exact,
|
|
)
|
|
result = envelope._probe_ensure_owned_dind_family(1, claim)
|
|
assert result == {"created": False, "rootSubtreeBefore": ["cpu", "memory", "pids"],
|
|
"familyDevice": 36, "familyInode": 101,
|
|
"familyCpuMax": "800000 100000", "familyMemoryMax": str(16 * 1024**3)}
|
|
assert writes == ([] if exact else [("cpu.max", "800000 100000"), ("memory.max", str(16 * 1024**3))])
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["missing", "foreign_group", "foreign_pid", "root_pid", "docker_pid", "zero_pid",
|
|
"pid_missing", "duplicate_pid", "dind_cpu", "dind_memory", "foreign_budget",
|
|
"mixed_budget", "root_controls", "family_controls", "unknown_controls"])
|
|
def test_owned_dind_family_rejects_foreign_or_invisible_work_before_writes(monkeypatch, mutation):
|
|
envelope, claim, rows, processes, controls, writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
leaf = f"/itworx-ci/{claim['containerId']}"
|
|
if mutation == "missing":
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: None)
|
|
elif mutation == "foreign_group":
|
|
rows[f"{leaf}/docker/foreign"] = {"path": f"{leaf}/docker/foreign", "cpuMax": "max", "memoryMax": "max"}
|
|
elif mutation in {"foreign_pid", "zero_pid", "pid_missing", "duplicate_pid"}:
|
|
processes[f"{leaf}/init"] = {"foreign_pid": "1 0", "zero_pid": "0", "pid_missing": "2",
|
|
"duplicate_pid": "1 1"}[mutation]
|
|
elif mutation == "root_pid":
|
|
processes["/itworx-ci"] = "2"
|
|
elif mutation == "docker_pid":
|
|
processes[f"{leaf}/docker"] = "2"
|
|
elif mutation == "dind_cpu":
|
|
rows[leaf]["cpuMax"] = "max 100000"
|
|
elif mutation == "dind_memory":
|
|
rows[leaf]["memoryMax"] = "max"
|
|
elif mutation == "foreign_budget":
|
|
rows["/itworx-ci"]["cpuMax"] = "700000 100000"
|
|
elif mutation == "mixed_budget":
|
|
rows["/itworx-ci"]["cpuMax"] = "800000 100000"
|
|
elif mutation == "root_controls":
|
|
controls["root"].remove("memory")
|
|
elif mutation == "family_controls":
|
|
controls["family"].remove("pids")
|
|
elif mutation == "unknown_controls":
|
|
controls["family"].add("foreign")
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_"):
|
|
envelope._probe_ensure_owned_dind_family(1, claim)
|
|
assert writes == []
|
|
|
|
|
|
@pytest.mark.parametrize("boundary", ["before_cpu", "before_memory", "readback"])
|
|
def test_owned_dind_family_rechecks_full_identity_at_each_write_boundary(monkeypatch, boundary):
|
|
envelope, claim, _rows, _processes, _controls, writes, inodes = _owned_family_fixture(monkeypatch)
|
|
original = envelope._probe_owned_dind_state
|
|
calls = 0
|
|
|
|
def changing(*args):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == {"before_cpu": 2, "before_memory": 3, "readback": 4}[boundary]:
|
|
inodes[f"/itworx-ci/{claim['containerId']}"] += 1
|
|
return original(*args)
|
|
|
|
monkeypatch.setattr(envelope, "_probe_owned_dind_state", changing)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_path_untrusted"):
|
|
envelope._probe_ensure_owned_dind_family(1, claim)
|
|
assert len(writes) == {"before_cpu": 0, "before_memory": 1, "readback": 2}[boundary]
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["pid", "controller", "budget"])
|
|
def test_owned_dind_family_refuses_drift_after_cpu_intent(monkeypatch, mutation):
|
|
envelope, claim, rows, processes, controls, writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
write = envelope._probe_write
|
|
|
|
def changing(*args):
|
|
write(*args)
|
|
if mutation == "pid":
|
|
processes[f"/itworx-ci/{claim['containerId']}/init"] = "1 0"
|
|
elif mutation == "controller":
|
|
controls["root"].remove("pids")
|
|
else:
|
|
rows["/itworx-ci"]["cpuMax"] = "max 100000"
|
|
|
|
monkeypatch.setattr(envelope, "_probe_write", changing)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_"):
|
|
envelope._probe_ensure_owned_dind_family(1, claim)
|
|
assert writes == [("cpu.max", "800000 100000")]
|
|
|
|
|
|
@pytest.mark.parametrize("mixed", [False, True])
|
|
def test_owned_dind_dispatch_keeps_empty_ensure_contract_distinct(monkeypatch, mixed):
|
|
envelope, claim, _rows, _processes, _controls, writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
monkeypatch.setattr(envelope, "Path", lambda value: SimpleNamespace(lstat=lambda: SimpleNamespace(
|
|
st_mode=0o40755, st_uid=0,
|
|
)))
|
|
monkeypatch.setattr(envelope.os, "open", lambda *_args: 1)
|
|
monkeypatch.setattr(envelope.os, "mkdir", lambda *_args, **_kwargs: pytest.fail("owned ensure must not create a family"))
|
|
request = {"op": "ensure", "ownedDind": claim}
|
|
if mixed:
|
|
request["allowExisting"] = True
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"):
|
|
envelope._host_probe_dispatch(request)
|
|
assert writes == []
|
|
else:
|
|
assert envelope._host_probe_dispatch(request)["familyInode"] == 101
|
|
|
|
|
|
@pytest.mark.parametrize("mutate", ["id", "pid_bool", "pid_negative", "process_bool", "duplicate", "missing_pid", "too_many", "extra"])
|
|
def test_owned_dind_claim_is_typed_and_finite(monkeypatch, mutate):
|
|
envelope = _module()
|
|
claim = {"containerId": "a" * 64, "hostPid": 1234, "processIds": [1234]}
|
|
if mutate == "id":
|
|
claim["containerId"] = "private-canary"
|
|
elif mutate == "pid_bool":
|
|
claim["hostPid"] = True
|
|
elif mutate == "pid_negative":
|
|
claim["hostPid"] = -1
|
|
elif mutate == "process_bool":
|
|
claim["processIds"] = [True]
|
|
elif mutate == "duplicate":
|
|
claim["processIds"] = [1234, 1234]
|
|
elif mutate == "missing_pid":
|
|
claim["processIds"] = [1235]
|
|
elif mutate == "too_many":
|
|
claim["processIds"] = [1234, *range(1, 257)]
|
|
elif mutate == "extra":
|
|
claim["path"] = "private-canary"
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"):
|
|
envelope._owned_dind_claim(claim)
|
|
|
|
|
|
def test_owned_dind_public_api_uses_existing_fixed_ensure_only(monkeypatch):
|
|
envelope = _module()
|
|
calls = []
|
|
result = {"created": False, "rootSubtreeBefore": ["cpu", "memory", "pids"],
|
|
"familyDevice": 36, "familyInode": 101,
|
|
"familyCpuMax": "800000 100000", "familyMemoryMax": str(16 * 1024**3)}
|
|
monkeypatch.setattr(envelope, "_engine_outer_population", lambda name:
|
|
({"Id": "a" * 64, "State": {"Pid": 1234}}, [(1234, 1), (1235, 1)]))
|
|
monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, request: calls.append((mode, request)) or result)
|
|
assert envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234,
|
|
expected_process_ids=[1235, 1234]) == result
|
|
assert calls == [("ensure", {"ownedDind": {"containerId": "a" * 64, "processCount": 2, "threadCount": 2, "allowPartial": False}})]
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"):
|
|
envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234,
|
|
expected_process_ids="1234")
|
|
assert len(calls) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["created", "inode", "device", "cpu", "memory", "controllers", "extra"])
|
|
def test_owned_dind_public_api_rejects_invalid_readback(monkeypatch, mutation):
|
|
envelope = _module()
|
|
result = {"created": False, "rootSubtreeBefore": ["cpu", "memory", "pids"],
|
|
"familyDevice": 36, "familyInode": 101,
|
|
"familyCpuMax": "800000 100000", "familyMemoryMax": str(16 * 1024**3)}
|
|
key, value = {"created": ("created", True), "inode": ("familyInode", True),
|
|
"device": ("familyDevice", -1), "cpu": ("familyCpuMax", "max 100000"),
|
|
"memory": ("familyMemoryMax", "max"), "controllers": ("rootSubtreeBefore", ["cpu", "pids"]),
|
|
"extra": ("private-canary", "secret-canary")}[mutation]
|
|
result[key] = value
|
|
monkeypatch.setattr(envelope, "_engine_outer_population", lambda name:
|
|
({"Id": "a" * 64, "State": {"Pid": 1234}}, [(1234, 1)]))
|
|
monkeypatch.setattr(envelope, "_run_host_probe", lambda *_args: result)
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_readback_failed$"):
|
|
envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234,
|
|
expected_process_ids=[1234])
|
|
|
|
|
|
@pytest.mark.parametrize("pid_location", ["leaf", "init", "missing"])
|
|
def test_snapshot_dind_pid_allows_only_exact_owned_init_or_leaf(monkeypatch, pid_location):
|
|
envelope, claim, rows, processes, _controls, _writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
leaf = f"/itworx-ci/{claim['containerId']}"
|
|
processes[f"{leaf}/init"] = ""
|
|
if pid_location == "leaf":
|
|
processes[leaf] = "1 2"
|
|
elif pid_location == "init":
|
|
processes[f"{leaf}/init"] = "1 2"
|
|
outer = [
|
|
{"name": name, "id": claim["containerId"] if name == envelope.DIND_NAME else f"{index:064x}",
|
|
"running": name == envelope.DIND_NAME, "pid": 1234 if name == envelope.DIND_NAME else None,
|
|
"processCount": 2 if name == envelope.DIND_NAME else None, "threadCount": 2 if name == envelope.DIND_NAME else None,
|
|
"configuredCpuMilli": 500, "configuredMemoryBytes": 512 * 1024**2}
|
|
for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1)
|
|
]
|
|
request = {"op": "snapshot", "outer": outer, "workloads": []}
|
|
if pid_location in {"leaf", "init"}:
|
|
result = envelope._probe_snapshot(request, 101)
|
|
assert result["dind"]["path"] == leaf
|
|
else:
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"):
|
|
envelope._probe_snapshot(request, 101)
|
|
|
|
|
|
@pytest.mark.parametrize("suffix", ["init", "init/foreign", "foreign"])
|
|
def test_attestor_allows_only_single_known_dind_init_group(suffix):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
snapshot["groups"].append(_group(f"{snapshot['dind']['path']}/{suffix}", "max 100000", "max"))
|
|
result = envelope._evaluate_cgroup_snapshot(["build", "backend"], snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384})
|
|
assert result["ready"] is (suffix == "init")
|
|
if suffix != "init":
|
|
assert result["reason"] == "resource_cgroup_unmapped_descendant"
|
|
|
|
|
|
@pytest.mark.parametrize("cpu,memory,partial", [("800000 100000", "max", False), ("800000 100000", "max", True),
|
|
("max 100000", str(16 * 1024**3), False),
|
|
("max 100000", str(16 * 1024**3), True)])
|
|
def test_owned_partial_budget_requires_explicit_bound_resume(monkeypatch, cpu, memory, partial):
|
|
envelope, claim, rows, _processes, _controls, writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
rows["/itworx-ci"].update(cpuMax=cpu, memoryMax=memory)
|
|
claim["allowPartial"] = partial
|
|
if not partial or memory != "max":
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_budget_conflict$"):
|
|
envelope._probe_ensure_owned_dind_family(1, claim)
|
|
assert writes == []
|
|
else:
|
|
assert envelope._probe_ensure_owned_dind_family(1, claim)["familyInode"] == 101
|
|
assert writes == ([("memory.max", str(16 * 1024**3))] if memory == "max" else [("cpu.max", "800000 100000")])
|
|
|
|
|
|
@pytest.mark.parametrize("partial", [1, "true", None])
|
|
def test_owned_partial_flag_is_not_coerced(monkeypatch, partial):
|
|
envelope, claim, _rows, _processes, _controls, writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
claim["allowPartial"] = partial
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"):
|
|
envelope._probe_ensure_owned_dind_family(1, claim)
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_family_request_invalid$"):
|
|
envelope.ensure_owned_dind_cgroup_family(expected_container_id="a" * 64, expected_host_pid=1234,
|
|
expected_process_ids=[1234], allow_owned_partial=partial)
|
|
assert writes == []
|
|
|
|
|
|
@pytest.mark.parametrize("foreign", [False, True])
|
|
def test_namespace_probe_rejects_foreign_zero_even_when_top_counts_match(monkeypatch, foreign):
|
|
envelope, claim, _rows, processes, _controls, _writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
if foreign:
|
|
processes[f"/itworx-ci/{claim['containerId']}/init"] = "1 0"
|
|
request = {"op": "population", "target": {"name": envelope.DIND_NAME, "id": claim["containerId"]},
|
|
"processCount": 2, "threadCount": 2}
|
|
if foreign:
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"):
|
|
envelope._probe_namespace_population(101, request)
|
|
else:
|
|
assert envelope._probe_namespace_population(101, request) == {"namespaceOwned": True, "processCount": 2, "threadCount": 2}
|
|
|
|
|
|
def test_private_zero_is_only_an_opaque_population_entry(monkeypatch):
|
|
envelope, claim, _rows, processes, _controls, _writes, _inodes = _owned_family_fixture(monkeypatch)
|
|
processes[f"/itworx-ci/{claim['containerId']}/init"] = "0 0"
|
|
population = envelope._probe_population(103)
|
|
assert population == {"processCount": 2, "threadCount": 2, "hierarchicalThreads": 2, "populated": True}
|
|
assert envelope._probe_empty(103) is False
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"):
|
|
envelope._probe_population(103, owned_namespace=True)
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["ensure", "population"])
|
|
@pytest.mark.parametrize("drift", [None, "host", "private", "other_cid", "label", "mount"])
|
|
def test_namespace_join_is_exact_bound_container_only(mode, drift):
|
|
envelope = _module()
|
|
base, image_id, source = _host_probe_fixture(envelope)
|
|
target = {"name": envelope.DIND_NAME, "id": "a" * 64}
|
|
args = envelope._host_probe_create_args(mode, image_id, source, namespace_target=target)
|
|
assert args[args.index("--pid") + 1] == "container:" + target["id"]
|
|
assert not any("/proc" in value or "--pid=host" in value or "docker.sock" in value for value in args)
|
|
namespace_labels = {"tech.itworx.probe.namespace-name": target["name"], "tech.itworx.probe.namespace-id": target["id"]}
|
|
base["Config"]["Labels"].update(namespace_labels)
|
|
base["HostConfig"]["PidMode"] = "container:" + target["id"]
|
|
expected_source = "/sys/fs/cgroup" if mode == "ensure" else "/sys/fs/cgroup/itworx-ci/" + target["id"]
|
|
base["Mounts"][0].update(Source=expected_source, RW=mode == "ensure")
|
|
base["HostConfig"]["Mounts"][0].update(Source=expected_source, ReadOnly=mode != "ensure")
|
|
base["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if mode == "ensure" else []
|
|
if drift in {"host", "private", "other_cid"}:
|
|
base["HostConfig"]["PidMode"] = {"host": "host", "private": "", "other_cid": "container:" + "b" * 64}[drift]
|
|
elif drift == "label":
|
|
base["Config"]["Labels"]["tech.itworx.probe.namespace-id"] = "b" * 64
|
|
elif drift == "mount":
|
|
base["HostConfig"]["Mounts"][0]["Source"] = "/proc"
|
|
assert envelope._validate_host_probe_inspect(base, mode=mode, image_id=image_id, code_source=source,
|
|
namespace_target=target) is (drift is None)
|
|
assert not envelope._validate_host_probe_inspect(base, mode=mode, image_id=image_id, code_source=source)
|
|
if drift is None:
|
|
assert envelope._stopped_probe_namespace(base) == target
|
|
|
|
|
|
@pytest.mark.parametrize("mode,name", [("snapshot", "gitea-runner-dind"), ("rollback", "gitea-runner-dind"),
|
|
("family", "gitea-runner-dind"), ("ensure", "itworx-ci-build"),
|
|
("population", "foreign")])
|
|
def test_namespace_join_never_expands_other_modes_or_targets(mode, name):
|
|
envelope = _module()
|
|
_base, image_id, source = _host_probe_fixture(envelope)
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_namespace_invalid$"):
|
|
envelope._host_probe_create_args(mode, image_id, source, namespace_target={"name": name, "id": "a" * 64})
|
|
|
|
|
|
@pytest.mark.parametrize("drift", [None, "pid", "start", "image", "threads", "foreign_namespace"])
|
|
def test_namespace_attestation_rechecks_engine_without_counts_only_adoption(monkeypatch, drift):
|
|
envelope = _module()
|
|
before = {"Id": "a" * 64, "Image": "sha256:" + "c" * 64, "State": {"Pid": 1234, "StartedAt": "fixed-start"}}
|
|
pairs = [(1234, 2), (1235, 1)]
|
|
after, after_pairs = copy.deepcopy(before), list(pairs)
|
|
if drift == "pid":
|
|
after["State"]["Pid"] = 1236
|
|
elif drift == "start":
|
|
after["State"]["StartedAt"] = "another-start"
|
|
elif drift == "image":
|
|
after["Image"] = "sha256:" + "d" * 64
|
|
elif drift == "threads":
|
|
after_pairs[0] = (1234, 3)
|
|
calls = []
|
|
def engine(name):
|
|
assert name == envelope.DIND_NAME
|
|
calls.append(name)
|
|
return (before, pairs) if len(calls) == 1 else (after, after_pairs)
|
|
monkeypatch.setattr(envelope, "_engine_outer_population", engine)
|
|
def probe(mode, claim):
|
|
assert mode == "population" and claim["target"] == {"name": envelope.DIND_NAME, "id": "a" * 64}
|
|
assert claim["processCount"] == 2 and claim["threadCount"] == 3
|
|
if drift == "foreign_namespace":
|
|
raise RuntimeError("resource_cgroup_process_mapping_invalid")
|
|
return {"namespaceOwned": True, "processCount": 2, "threadCount": 3}
|
|
monkeypatch.setattr(envelope, "_run_host_probe", probe)
|
|
if drift is None:
|
|
assert envelope._attest_outer_namespace(envelope.DIND_NAME)[0] == before
|
|
else:
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"):
|
|
envelope._attest_outer_namespace(envelope.DIND_NAME)
|
|
|
|
|
|
@pytest.mark.parametrize("output", ["PID NLWP\n1234 2\n1235 1", "PID NLWP\n1234 0", "PID NLWP\n1234 2\n1234 1",
|
|
"PID NLWP\n1235 2", "PID COMMAND\n1234 private-canary",
|
|
"PID NLWP\n0 1", "PID NLWP\n1234 16385", "PID NLWP\n1234 1 extra"])
|
|
def test_engine_population_reads_only_fixed_cid_numeric_pid_thread_columns(monkeypatch, output):
|
|
envelope = _module()
|
|
item = {"Name": "/" + envelope.DIND_NAME, "Id": "a" * 64, "Image": "sha256:" + "b" * 64,
|
|
"Created": "fixed-created", "State": {"Running": True, "Pid": 1234, "StartedAt": "fixed-start"},
|
|
"HostConfig": {"CgroupParent": "itworx-ci"}}
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: item)
|
|
commands = []
|
|
def run(args, **kwargs):
|
|
commands.append((args, kwargs))
|
|
return output
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
if output == "PID NLWP\n1234 2\n1235 1":
|
|
binding, pairs = envelope._engine_outer_population(envelope.DIND_NAME)
|
|
assert binding["Id"] == "a" * 64 and pairs == [(1234, 2), (1235, 1)]
|
|
else:
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_process_mapping_invalid$"):
|
|
envelope._engine_outer_population(envelope.DIND_NAME)
|
|
assert commands == [(["docker", "top", "a" * 64, "-eo", "pid,nlwp"], {"timeout": 15})]
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_container_identity_invalid$"):
|
|
envelope._engine_outer_population("private-canary")
|
|
assert len(commands) == 1
|
|
|
|
|
|
def test_engine_binding_preserves_all_mount_fields_but_not_incidental_order():
|
|
envelope = _module()
|
|
item = {"Mounts": [{"Destination": "/a", "Source": "/fixed/a", "RW": True},
|
|
{"Destination": "/b", "Source": "/fixed/b", "RW": False}],
|
|
"State": {"Running": True, "Pid": 1234, "Health": {"Log": ["volatile"]}}}
|
|
reordered = copy.deepcopy(item)
|
|
reordered["Mounts"].reverse()
|
|
reordered["State"]["Health"]["Log"] = ["changed"]
|
|
assert envelope._engine_outer_binding(item) == envelope._engine_outer_binding(reordered)
|
|
reordered["Mounts"][0]["RW"] = True
|
|
assert envelope._engine_outer_binding(item) != envelope._engine_outer_binding(reordered)
|
|
|
|
|
|
@pytest.mark.parametrize("drift", [False, True])
|
|
def test_namespace_probe_lifecycle_rechecks_target_before_start(monkeypatch, drift):
|
|
envelope = _module()
|
|
base, image_id, source = _host_probe_fixture(envelope)
|
|
target = {"name": envelope.DIND_NAME, "id": "a" * 64}
|
|
base["Config"]["Labels"].update({"tech.itworx.probe.namespace-name": target["name"],
|
|
"tech.itworx.probe.namespace-id": target["id"]})
|
|
base["HostConfig"]["PidMode"] = "container:" + target["id"]
|
|
base["HostConfig"]["Mounts"][0]["Source"] += "/" + target["id"]
|
|
base["Mounts"][0]["Source"] += "/" + target["id"]
|
|
monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, source))
|
|
monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _image: {})
|
|
monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None)
|
|
engine_calls = []
|
|
def engine(name):
|
|
assert name == envelope.DIND_NAME
|
|
engine_calls.append(name)
|
|
return {"Id": target["id"], "State": {"Pid": 1234, "StartedAt": "changed" if drift and len(engine_calls) > 1 else "fixed"}}, [(1234, 1)]
|
|
monkeypatch.setattr(envelope, "_engine_outer_population", engine)
|
|
commands = []
|
|
def run(args, **_kwargs):
|
|
commands.append(args)
|
|
if args[1] == "create":
|
|
assert args[args.index("--pid") + 1] == "container:" + target["id"]
|
|
return base["Id"]
|
|
if args[1] == "inspect":
|
|
return json.dumps([base])
|
|
assert args == ["docker", "rm", base["Id"]]
|
|
return ""
|
|
monkeypatch.setattr(envelope, "_run", run)
|
|
starts = []
|
|
def start(args, **kwargs):
|
|
starts.append(args)
|
|
assert kwargs["timeout"] == 60 and b'"population"' in kwargs["input"]
|
|
return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"namespaceOwned":true,"processCount":1,"threadCount":1}}')
|
|
monkeypatch.setattr(envelope.subprocess, "run", start)
|
|
payload = {"target": target, "processCount": 1, "threadCount": 1}
|
|
if drift:
|
|
with pytest.raises(RuntimeError, match="^resource_cgroup_probe_namespace_invalid$"):
|
|
envelope._run_host_probe("population", payload)
|
|
assert starts == []
|
|
else:
|
|
assert envelope._run_host_probe("population", payload)["namespaceOwned"] is True
|
|
assert starts == [["docker", "start", "--attach", "--interactive", base["Id"]]]
|
|
assert commands[-1] == ["docker", "rm", base["Id"]]
|
|
|
|
|
|
@pytest.mark.parametrize("hidden", ["threads", "pids", "events", "opaque"])
|
|
def test_empty_proof_never_uses_private_pid_visibility_as_absence(monkeypatch, hidden):
|
|
envelope = _module()
|
|
reads = {"cgroup.procs": "", "cgroup.threads": "", "pids.current": "0", "cgroup.events": "populated 0"}
|
|
key, value = {"threads": ("cgroup.threads", "0"), "pids": ("pids.current", "1"),
|
|
"events": ("cgroup.events", "populated 1"), "opaque": ("cgroup.procs", "0")}[hidden]
|
|
reads[key] = value
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name])
|
|
assert envelope._probe_empty(1) is False
|
|
|
|
|
|
def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(monkeypatch):
|
|
envelope = _module()
|
|
required = {"cpu", "memory", "pids"}
|
|
monkeypatch.setattr(envelope, "_probe_controllers", lambda _fd: (required, required))
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123)
|
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
|
groups = [
|
|
{
|
|
"path": envelope.FAMILY_CGROUP,
|
|
"cpuMax": "800000 100000",
|
|
"memoryMax": str(16 * 1024**3),
|
|
}
|
|
]
|
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: list(groups))
|
|
reads = {"cgroup.subtree_control": "cpu memory pids", "cgroup.procs": "", "cgroup.threads": "",
|
|
"pids.current": "0", "cgroup.events": "populated 0\nfrozen 0"}
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name: reads[name])
|
|
|
|
ready = envelope._probe_family_info(1)
|
|
assert ready["safeToStartDind"] is True
|
|
assert ready["safeToStartDindReason"] is None
|
|
|
|
groups.append({"path": f"{envelope.FAMILY_CGROUP}/unknown", "cpuMax": "max", "memoryMax": "max"})
|
|
blocked = envelope._probe_family_info(1)
|
|
assert blocked["safeToStartDind"] is False
|
|
assert blocked["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
|
|
|
groups.pop()
|
|
reads["cgroup.procs"] = "4321"
|
|
occupied = envelope._probe_family_info(1)
|
|
assert occupied["safeToStartDind"] is False
|
|
assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty"
|
|
|
|
|
|
def test_rollback_preserves_unraid_existing_hugetlb_controller(monkeypatch):
|
|
envelope = _module()
|
|
initial = ["cpuset", "cpu", "io", "memory", "hugetlb", "pids"]
|
|
calls = []
|
|
monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, payload:
|
|
calls.append((mode, payload)) or {"removed": True, "familyExists": False})
|
|
envelope.rollback_host_cgroup_family(created=True, initial_controllers=initial)
|
|
assert calls == [("rollback", {"created": True, "initialControllers": initial})]
|
|
|
|
monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123)
|
|
monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [{
|
|
"cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3),
|
|
}])
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda fd, name:
|
|
"" if name in {"cgroup.procs", "cgroup.threads"} else
|
|
"0" if name == "pids.current" else
|
|
"populated 0\nfrozen 0" if name == "cgroup.events" else
|
|
"cpu memory pids" if fd == 123 else " ".join(initial))
|
|
monkeypatch.setattr(envelope.os, "close", lambda _fd: None)
|
|
monkeypatch.setattr(envelope.os, "rmdir", lambda *args, **kwargs: None)
|
|
writes = []
|
|
monkeypatch.setattr(envelope, "_probe_write", lambda *args: writes.append(args))
|
|
result = envelope._probe_rollback_family(1, {"op": "rollback", "created": True,
|
|
"initialControllers": initial})
|
|
assert result == {"removed": True, "familyExists": False}
|
|
assert writes == [(123, "cgroup.subtree_control", "-cpu -memory -pids")]
|
|
for controllers in (["unknown"], initial + ["unknown"]):
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope.rollback_host_cgroup_family(created=True, initial_controllers=controllers)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"):
|
|
envelope._probe_rollback_family(1, {"op": "rollback", "created": True,
|
|
"initialControllers": controllers})
|
|
|
|
|
|
def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback():
|
|
envelope = _module()
|
|
image_id = "sha256:" + "a" * 64
|
|
code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}"
|
|
|
|
snapshot = envelope._host_probe_create_args("snapshot", image_id, code_source)
|
|
ensure = envelope._host_probe_create_args("ensure", image_id, code_source)
|
|
rollback = envelope._host_probe_create_args("rollback", image_id, code_source)
|
|
family = envelope._host_probe_create_args("family", image_id, code_source)
|
|
|
|
def mounts(args):
|
|
return [args[index + 1] for index, item in enumerate(args[:-1]) if item == "--mount"]
|
|
|
|
snapshot_mount, snapshot_code_mount = mounts(snapshot)
|
|
ensure_mount, ensure_code_mount = mounts(ensure)
|
|
rollback_mount, rollback_code_mount = mounts(rollback)
|
|
family_mount, family_code_mount = mounts(family)
|
|
assert snapshot_mount == (
|
|
"type=bind,source=/sys/fs/cgroup/itworx-ci,target=/host-cgroup,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert ensure_mount.startswith("type=bind,source=/sys/fs/cgroup,target=/host-cgroup")
|
|
assert "readonly" not in ensure_mount
|
|
assert "readonly" not in rollback_mount
|
|
assert family_mount == (
|
|
"type=bind,source=/sys/fs/cgroup,target=/host-cgroup,readonly,bind-propagation=rprivate"
|
|
)
|
|
assert all(
|
|
code_mount
|
|
== f"type=bind,source={code_source},target=/opt/gitea-deploy,readonly,bind-propagation=rslave"
|
|
for code_mount in (snapshot_code_mount, ensure_code_mount, rollback_code_mount, family_code_mount)
|
|
)
|
|
for args in (snapshot, ensure, rollback, family):
|
|
assert "--pid=host" not in args
|
|
assert "--network=none" in args
|
|
assert "--cap-drop=ALL" in args
|
|
assert "--cpus=0.1" in args
|
|
assert "--memory=64m" in args
|
|
assert "--pids-limit=64" in args
|
|
assert not any("/proc" in item or "docker.sock" in item for item in args)
|
|
assert "--cap-add=DAC_OVERRIDE" not in snapshot
|
|
assert "--cap-add=DAC_OVERRIDE" in ensure
|
|
assert "--cap-add=DAC_OVERRIDE" in rollback
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_probe_source_invalid"):
|
|
envelope._host_probe_create_args("snapshot", image_id, "/mnt/shared/code")
|
|
|
|
|
|
def test_nested_docker_pid_is_not_reinterpreted_as_host_pid(monkeypatch, tmp_path):
|
|
envelope = _module()
|
|
family_fd = os.open(tmp_path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
dind_id = "7" * 64
|
|
workload_id = "8" * 64
|
|
dind_path = f"/itworx-ci/{dind_id}"
|
|
workload_path = f"{dind_path}/docker/{workload_id}"
|
|
monkeypatch.setattr(
|
|
envelope,
|
|
"_probe_walk",
|
|
lambda _fd: [
|
|
{"path": "/itworx-ci", "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3)},
|
|
{"path": dind_path, "cpuMax": "600000 100000", "memoryMax": str(12 * 1024**3)},
|
|
{"path": workload_path, "cpuMax": "100000 100000", "memoryMax": str(1024**3)},
|
|
],
|
|
)
|
|
monkeypatch.setattr(envelope, "_probe_open_relative", lambda _fd, _path: os.dup(family_fd))
|
|
# The host cgroup contains global PID 4321; nested Docker reports PID 7
|
|
# inside DinD's private namespace. Membership is proven by ID path and a
|
|
# populated cgroup, never by comparing those unrelated PID numbers.
|
|
monkeypatch.setattr(envelope, "_probe_read", lambda _fd, name, **_kw:
|
|
"0" if name in {"cgroup.procs", "cgroup.threads"} else
|
|
"2" if name == "pids.current" else "populated 1\nfrozen 0")
|
|
monkeypatch.setattr(envelope, "_probe_population", lambda fd, **_kwargs: {
|
|
"processCount": 0 if fd == family_fd else 1, "threadCount": 0 if fd == family_fd else 1,
|
|
"hierarchicalThreads": 2, "populated": True,
|
|
})
|
|
outer = []
|
|
for index, name in enumerate(envelope.FIXED_RUNNER_AND_DIND_NAMES, start=1):
|
|
if name == envelope.DIND_NAME:
|
|
outer.append({"name": name, "id": dind_id, "running": True, "processCount": 2, "threadCount": 2})
|
|
else:
|
|
outer.append(
|
|
{
|
|
"name": name,
|
|
"id": f"{index:064x}",
|
|
"running": False,
|
|
"pid": None,
|
|
"configuredCpuMilli": 500,
|
|
"configuredMemoryBytes": 512 * 1024**2,
|
|
}
|
|
)
|
|
try:
|
|
snapshot = envelope._probe_snapshot(
|
|
{
|
|
"op": "snapshot",
|
|
"outer": outer,
|
|
"workloads": [
|
|
{
|
|
"id": workload_id,
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 7,
|
|
"cpuMilli": 100,
|
|
"memoryBytes": 1024**3,
|
|
}
|
|
],
|
|
},
|
|
family_fd,
|
|
)
|
|
finally:
|
|
os.close(family_fd)
|
|
|
|
assert snapshot["workloads"][0]["path"] == workload_path
|
|
assert snapshot["workloads"][0]["nestedPid"] == 7
|
|
assert snapshot["containers"][-1]["path"] == dind_path
|
|
assert envelope._nested_workload_cgroup_path(dind_path, workload_id, "docker") == workload_path
|
|
for invalid_parent in ("/", "../docker", "docker/escape", "../../itworx-ci"):
|
|
try:
|
|
envelope._nested_workload_cgroup_path(dind_path, workload_id, invalid_parent)
|
|
except RuntimeError as exc:
|
|
assert str(exc) == "resource_cgroup_workload_identity_invalid"
|
|
else:
|
|
raise AssertionError("unmapped nested Docker cgroup parent was accepted")
|
|
|
|
|
|
def test_attestor_rejects_unowned_descendants_below_dind():
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
snapshot["groups"].append(
|
|
_group(f"{dind_path}/unknown-child", "100000 100000", 1024**3)
|
|
)
|
|
snapshot["workloads"] = [
|
|
{"path": f"{dind_path}/docker/job-1"}
|
|
]
|
|
|
|
result = envelope._evaluate_cgroup_snapshot(
|
|
["checks"],
|
|
snapshot,
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is False
|
|
assert result["reason"] == "resource_cgroup_unmapped_descendant"
|
|
|
|
|
|
def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
family_path = str(snapshot["family"]["path"])
|
|
dind_path = str(snapshot["dind"]["path"])
|
|
probe_id = "f" * 64
|
|
snapshot["containers"] = [
|
|
{
|
|
**item,
|
|
"running": item["name"] == envelope.DIND_NAME,
|
|
"path": item["path"] if item["name"] == envelope.DIND_NAME else None,
|
|
}
|
|
for item in snapshot["containers"]
|
|
]
|
|
snapshot["groups"] = [
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"] == family_path or group["path"] == dind_path
|
|
]
|
|
workload_path = f"{dind_path}/docker/{probe_id}"
|
|
snapshot["groups"].append(_group(workload_path, "400000 100000", 6 * 1024**3))
|
|
snapshot["workloads"] = [{
|
|
"id": probe_id,
|
|
"path": workload_path,
|
|
"nestedPid": 77,
|
|
"cpuMilli": 4000,
|
|
"memoryBytes": 6 * 1024**3,
|
|
}]
|
|
monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot)
|
|
|
|
result = envelope.attest_build_probe_workload(probe_id, 77)
|
|
|
|
assert result["ready"] is True, result
|
|
assert result["runningRunnerContainers"] == 0
|
|
assert result["probeWorkload"] == {
|
|
"id": probe_id,
|
|
"path": workload_path,
|
|
"nestedPid": 77,
|
|
"cpuMilli": 4000,
|
|
"memoryBytes": 6 * 1024**3,
|
|
"descendantGroupCount": 0,
|
|
}
|
|
assert "snapshot" not in result
|
|
assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False
|
|
|
|
|
|
def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch):
|
|
envelope = _module()
|
|
snapshot = _snapshot()
|
|
profile_names = {
|
|
profile.container_name: profile.id
|
|
for profile in envelope.RUNNER_PROFILES.values()
|
|
}
|
|
ids = {row["name"]: row["id"] for row in snapshot["containers"]}
|
|
pids = {name: 2000 + index for index, name in enumerate(ids)}
|
|
outer_items = {}
|
|
for name in envelope.FIXED_RUNNER_AND_DIND_NAMES:
|
|
if name == envelope.DIND_NAME:
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": "runner-dind",
|
|
}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 6_000_000_000,
|
|
"Memory": 12 * 1024**3,
|
|
}
|
|
state = {"Running": True, "Pid": pids[name]}
|
|
else:
|
|
profile_id = profile_names[name]
|
|
labels = {
|
|
"com.docker.compose.project": "gitea-deploy-runners",
|
|
"com.docker.compose.service": f"runner-{profile_id}",
|
|
"tech.itworx.role": "gitea-actions-runner",
|
|
"tech.itworx.ci.profile": profile_id,
|
|
}
|
|
is_running = name in {"itworx-ci-checks", "itworx-ci-backend"}
|
|
host = {
|
|
"CgroupParent": "itworx-ci",
|
|
"NanoCpus": 500_000_000,
|
|
"Memory": 512 * 1024**2,
|
|
}
|
|
state = {"Running": is_running, "Pid": pids[name] if is_running else 0}
|
|
outer_items[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": ids[name],
|
|
"Config": {"Labels": labels},
|
|
"State": state,
|
|
"HostConfig": host,
|
|
}
|
|
|
|
docker_commands = []
|
|
|
|
def command(args, **_kwargs):
|
|
docker_commands.append(args)
|
|
if "info" in args:
|
|
return "2|cgroupfs|true" if "exec" in args else "2|cgroupfs"
|
|
if "docker" in args and "ps" in args:
|
|
return ""
|
|
raise AssertionError(f"unexpected fixed Docker command: {args}")
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
monkeypatch.setattr(envelope, "_inspect_container", lambda name: outer_items[name])
|
|
probe_calls = []
|
|
|
|
def fake_probe(operation, payload=None):
|
|
probe_calls.append((operation, payload))
|
|
return snapshot
|
|
|
|
monkeypatch.setattr(envelope, "_run_host_probe", fake_probe)
|
|
_fixture_outer_namespace(envelope, monkeypatch)
|
|
real_read_text = Path.read_text
|
|
|
|
def reject_local_proc_or_sys(path, *args, **kwargs):
|
|
if str(path).startswith(("/proc/", "/sys/fs/cgroup")):
|
|
raise AssertionError("the private controller must use the host helper")
|
|
return real_read_text(path, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "read_text", reject_local_proc_or_sys)
|
|
result = envelope.attest_resource_envelope(
|
|
["backend"],
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert probe_calls and probe_calls[0][0] == "snapshot"
|
|
assert result["ready"] is True
|
|
assert result["familyCpuUsageUsec"] == 1200
|
|
assert result["familyMemoryCurrentBytes"] == 4096
|
|
dind_calls = [args for args in docker_commands if "exec" in args]
|
|
assert len(dind_calls) == 2
|
|
assert all(
|
|
f"DOCKER_HOST={envelope.DIND_DOCKER_HOST}" in args
|
|
for args in dind_calls
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false"])
|
|
def test_collect_snapshot_rejects_dind_without_resource_enforcement(monkeypatch, support):
|
|
envelope = _module()
|
|
_install_classic_builder_collection(
|
|
envelope, monkeypatch, {"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000}
|
|
)
|
|
original = envelope._run
|
|
|
|
def command(args, **kwargs):
|
|
if "exec" in args and "info" in args:
|
|
return support
|
|
return original(args, **kwargs)
|
|
|
|
monkeypatch.setattr(envelope, "_run", command)
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_dind_driver_unsupported"):
|
|
envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES)
|
|
|
|
|
|
def test_collect_snapshot_bridges_classic_build_quota_to_measured_cgroup_caps(monkeypatch):
|
|
envelope = _module()
|
|
snapshot, nested_items, probe_calls = _install_classic_builder_collection(
|
|
envelope,
|
|
monkeypatch,
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
)
|
|
|
|
result = envelope.attest_resource_envelope(
|
|
["build"],
|
|
expected_host_budget={"cpuMilli": 8000, "memoryMiB": 16384},
|
|
)
|
|
|
|
assert result["ready"] is True, result
|
|
assert probe_calls[0][0] == "snapshot"
|
|
assert probe_calls[0][1]["workloads"] == [
|
|
{
|
|
"id": nested_items[0]["Id"],
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 70,
|
|
"cpuMilli": 1000,
|
|
"memoryBytes": 1024**3,
|
|
},
|
|
{
|
|
"id": nested_items[1]["Id"],
|
|
"cgroupParent": "docker",
|
|
"nestedPid": 71,
|
|
"cpuMilli": 3000,
|
|
"memoryBytes": 5 * 1024**3,
|
|
},
|
|
]
|
|
assert snapshot["workloads"] == [
|
|
{
|
|
"id": nested_items[0]["Id"],
|
|
"path": f"{snapshot['dind']['path']}/docker/{nested_items[0]['Id']}",
|
|
"nestedPid": 70,
|
|
"cpuMilli": 1000,
|
|
"memoryBytes": 1024**3,
|
|
},
|
|
{
|
|
"id": nested_items[1]["Id"],
|
|
"path": f"{snapshot['dind']['path']}/docker/{nested_items[1]['Id']}",
|
|
"nestedPid": 71,
|
|
"cpuMilli": 3000,
|
|
"memoryBytes": 5 * 1024**3,
|
|
},
|
|
]
|
|
builder_group = next(
|
|
group
|
|
for group in snapshot["groups"]
|
|
if group["path"].endswith("/docker/" + nested_items[1]["Id"])
|
|
)
|
|
assert builder_group["cpuMax"] == "300000 100000"
|
|
assert builder_group["memoryMax"] == str(5 * 1024**3)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"builder_cpu_config",
|
|
[
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 0, "CpuPeriod": 100_000},
|
|
id="zero-quota",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 0},
|
|
id="zero-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000},
|
|
id="missing-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": True, "CpuPeriod": 100_000},
|
|
id="boolean-quota",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": True},
|
|
id="boolean-period",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_000, "CpuPeriod": 1_000_001},
|
|
id="period-out-of-range",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 0, "CpuQuota": 300_001, "CpuPeriod": 100_000},
|
|
id="fractional-millicpu",
|
|
),
|
|
pytest.param(
|
|
{"NanoCpus": 3_000_000_000, "CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
id="conflicting-encodings",
|
|
),
|
|
pytest.param(
|
|
{"CpuQuota": 300_000, "CpuPeriod": 100_000},
|
|
id="missing-nanocpus-field",
|
|
),
|
|
],
|
|
)
|
|
def test_collect_snapshot_rejects_unbounded_or_ambiguous_cpu_configurations(
|
|
monkeypatch, builder_cpu_config
|
|
):
|
|
envelope = _module()
|
|
_snapshot_data, _nested_items, probe_calls = _install_classic_builder_collection(
|
|
envelope, monkeypatch, builder_cpu_config
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="resource_cgroup_workload_unbounded"):
|
|
envelope._collect_snapshot(envelope.FIXED_RUNNER_AND_DIND_NAMES)
|
|
|
|
assert probe_calls == []
|