from __future__ import annotations import base64 import copy import hashlib import importlib.util import io import json import os import stat import subprocess import sys import tarfile from contextlib import contextmanager, nullcontext from pathlib import Path from types import SimpleNamespace import pytest @pytest.fixture def controller_only_source(specialize, tmp_path, monkeypatch): image = "sha256:" + "a" * 64 code = f"{specialize.RELEASE_VOLUME_HOST_ROOT}/controller-artifacts/{'b' * 40}" model = {"name": specialize.PROJECT, "services": {"deploy-control": { "container_name": specialize.CONTROLLER_CONTAINER, "image": image, "volumes": [{"type": "bind", "source": code, "target": "/opt/gitea-deploy", "read_only": True, "bind": {"propagation": "rslave"}}], }}} compose = tmp_path / "docker-compose.yml" archive_dir = tmp_path / "backups" / "controller-v2-fixed" archive_dir.mkdir(parents=True, mode=0o700) archive_dir.parent.chmod(0o700) archive = archive_dir / "compose-before.bin" archived = b"reviewed complete legacy v1 source" archive.write_bytes(archived) archive.chmod(0o600) monkeypatch.setattr(specialize, "ROOT", tmp_path) monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) monkeypatch.setattr(specialize, "LEGACY_V1_COMPOSE_BACKUP", archive) monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(archived).hexdigest()) monkeypatch.setattr(specialize, "_controller_runtime_identity", lambda: (image, code)) return model, compose, archive, archived def test_controller_only_source_uses_fixed_pinned_archive_without_rewriting_live_compose( specialize, controller_only_source, ): model, compose, _archive, archived = controller_only_source current = json.dumps(model).encode() compose.write_bytes(current) result, current_sha = specialize._read_fleet_base_compose() assert result == archived assert current_sha == hashlib.sha256(current).hexdigest() assert compose.read_bytes() == current @pytest.mark.parametrize("mutation", ["project", "image", "container", "code", "writable", "shared", "extra_code"]) def test_controller_only_source_rejects_identity_and_code_binding_drift( specialize, controller_only_source, mutation, ): model, compose, _archive, _archived = controller_only_source controller = model["services"]["deploy-control"] mount = controller["volumes"][0] if mutation == "project": model["name"] = "other" elif mutation == "image": controller["image"] = "sha256:" + "c" * 64 elif mutation == "container": controller["container_name"] = "other" elif mutation == "code": mount["source"] = "/mnt/shared/code" elif mutation == "writable": mount["read_only"] = False elif mutation == "shared": mount["bind"]["propagation"] = "rshared" else: controller["volumes"].append(copy.deepcopy(mount)) compose.write_text(json.dumps(model)) with pytest.raises(specialize.RunnerFleetSpecializeError, match="controller_compose_untrusted"): specialize._read_fleet_base_compose() def test_controller_only_source_rejects_modified_fixed_archive(specialize, controller_only_source): model, compose, archive, _archived = controller_only_source compose.write_text(json.dumps(model)) archive.write_bytes(b"different legacy source") with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_backup_changed"): specialize._read_fleet_base_compose() def test_legacy_live_source_does_not_open_promotion_backup(specialize, monkeypatch): current = b"services:\n runner: {}\n" monkeypatch.setattr(specialize, "_read_source_file", lambda *_args, **_kwargs: current) monkeypatch.setattr(specialize, "_trusted_private_file", lambda *_args, **_kwargs: pytest.fail("archive must remain unopened")) assert specialize._read_fleet_base_compose() == (current, None) @pytest.mark.parametrize("separate_controller_hash", [False, True]) def test_controller_compose_cas_blocks_rollback_before_any_lifecycle_action( specialize, controller_only_source, monkeypatch, separate_controller_hash, ): monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) model, compose, _archive, _archived = controller_only_source compose.write_text(json.dumps(model)) original_sha = hashlib.sha256(compose.read_bytes()).hexdigest() journal = {"baseComposeSha256": original_sha} if separate_controller_hash: journal["controllerComposeSha256"] = original_sha compose.write_bytes(compose.read_bytes() + b"\n") monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda *_args: pytest.fail("must reject before lifecycle")) assert specialize._rollback_before_activation(journal) is False @pytest.mark.parametrize("reviewed,user,image,expected", [ (True, None, True, "rootless"), (False, None, True, None), (True, "root", True, "root"), (True, None, False, None), ]) def test_only_byte_pinned_v1_pinned_image_default_user_is_normalized( specialize, monkeypatch, reviewed, user, image, expected, ): source = b"reviewed v1 source" if reviewed else b"different source" monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(b"reviewed v1 source").hexdigest()) model = {"services": {"runner-dind": {"image": specialize._DIND_IMAGE_REFERENCE if image else "unknown", "user": user}}} monkeypatch.setattr(specialize, "_run_compose", lambda *_args: json.dumps(model)) assert specialize._compose_model(source)["services"]["runner-dind"]["user"] == expected ROOT = Path(__file__).resolve().parents[2] BROKER = ROOT / "ops" / "gitea-deploy-broker" SPECIALIZE_PATH = BROKER / "runner_fleet_specialize.py" @pytest.fixture def specialize(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) spec = importlib.util.spec_from_file_location("runner_fleet_specialize_test", SPECIALIZE_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module @contextmanager def _fixture_rollback_maintenance(): yield lambda: None @pytest.fixture def task_archive_source(specialize, tmp_path, monkeypatch): state = tmp_path / "fleet-state" state.mkdir(mode=0o700) state.chmod(0o700) workdir = state / ".specialization-images-reviewed" workdir.mkdir(mode=0o700) workdir.chmod(0o700) archive = workdir / "task-images.tar" payload = b"verified archive bytes\x00" * 1024 archive.write_bytes(payload) archive.chmod(0o600) manifest = _valid_task_image_manifest(specialize) manifest.update(dockerTarBytes=len(payload), dockerTarSha256=hashlib.sha256(payload).hexdigest()) monkeypatch.setattr(specialize, "SPECIALIZED_STATE_ROOT", state) return archive, manifest, payload @pytest.mark.parametrize("outcome", ["success", "nonzero", "timeout", "verify_failed", "source_changed"]) def test_image_load_streams_exact_fd_over_unix_without_temporary_container_file( specialize, task_archive_source, monkeypatch, outcome, ): archive, manifest, payload = task_archive_source calls = [] checks = [] # File ownership is separately qualified under real POSIX metadata below. @contextmanager def opened(path, value): assert path == archive and value is manifest with archive.open("rb") as stream: def current(): checks.append(True) if outcome == "source_changed" and len(checks) > 1: raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_image_archive_source_untrusted") yield stream, current current() def run(args, **kwargs): calls.append(args) assert args == ["docker", "exec", "-i", "-e", f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}", specialize.DIND_NAME, "docker", "image", "load"] assert kwargs["stdin"].tell() == 0 assert kwargs["stdin"].read() == payload assert "input" not in kwargs and "shell" not in kwargs assert kwargs["stdout"] == kwargs["stderr"] == subprocess.DEVNULL assert kwargs["timeout"] == 900 and kwargs["check"] is False if outcome == "timeout": raise subprocess.TimeoutExpired(args, 900, stderr="secret must stay redacted") return SimpleNamespace(returncode=1 if outcome == "nonzero" else 0) verified = [] monkeypatch.setattr(specialize, "_open_verified_task_archive", opened) monkeypatch.setattr(specialize.subprocess, "run", run) monkeypatch.setattr(specialize, "_run", lambda *_a, **_k: pytest.fail("no cp, temp or rm command allowed")) monkeypatch.setattr(specialize, "_verify_manifest_images_in_dind", lambda images: verified.append(images) or outcome != "verify_failed") if outcome == "success": specialize._load_images_into_dind(archive, manifest) assert verified == [manifest["images"]] else: with pytest.raises(specialize.RunnerFleetSpecializeError) as error: specialize._load_images_into_dind(archive, manifest) assert "secret" not in str(error.value) assert bool(verified) is (outcome == "verify_failed") assert len(calls) == 1 @pytest.mark.parametrize("directory", [False, True]) @pytest.mark.parametrize("fault", [None, "uid", "gid", "type", "mode", "special"]) def test_task_archive_identity_requires_exact_private_root_metadata(specialize, directory, fault): values = dict(st_mode=(stat.S_IFDIR | 0o700) if directory else (stat.S_IFREG | 0o600), st_uid=0, st_gid=0, st_nlink=2 if directory else 1, st_dev=1, st_ino=2, st_size=123, st_mtime_ns=4, st_ctime_ns=5) if fault in {"uid", "gid"}: values[f"st_{fault}"] = 1000 elif fault == "type": values["st_mode"] = stat.S_IFLNK | 0o700 elif fault == "mode": values["st_mode"] |= 0o010 elif fault == "special": values["st_mode"] |= stat.S_ISUID if fault is None: assert specialize._task_archive_identity(SimpleNamespace(**values), directory=directory) else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="source_untrusted"): specialize._task_archive_identity(SimpleNamespace(**values), directory=directory) def test_task_archive_identity_rejects_multiple_links(specialize): info = SimpleNamespace(st_mode=stat.S_IFREG | 0o600, st_uid=0, st_gid=0, st_nlink=2) with pytest.raises(specialize.RunnerFleetSpecializeError, match="source_untrusted"): specialize._task_archive_identity(info) @pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") @pytest.mark.parametrize("fault", [None, "missing", "symlink", "directory", "hardlink", "file_mode", "parent_mode", "owner", "parent_owner", "size", "digest", "foreign_path", "replace", "changed_fd"]) def test_source_image_archive_is_verified_and_same_fd_pinned_before_any_load( specialize, task_archive_source, monkeypatch, fault, ): archive, manifest, payload = task_archive_source if fault == "missing": archive.unlink() elif fault == "symlink": original = archive.with_name("owned-original") archive.rename(original) archive.symlink_to(original) elif fault == "directory": archive.unlink() archive.mkdir(mode=0o600) elif fault == "hardlink": os.link(archive, archive.with_name("extra-link")) elif fault == "file_mode": archive.chmod(0o640) elif fault == "parent_mode": archive.parent.chmod(0o710) elif fault == "owner": os.chown(archive, 1000, 0) elif fault == "parent_owner": os.chown(archive.parent, 1000, 0) elif fault == "size": manifest["dockerTarBytes"] += 1 elif fault == "digest": manifest["dockerTarSha256"] = "e" * 64 elif fault == "foreign_path": archive = archive.with_name("foreign.tar") loaded = [] def run(args, **kwargs): loaded.append(args) assert kwargs["stdin"].read() == payload if fault == "replace": replacement = archive.with_name("replacement") replacement.write_bytes(payload) replacement.chmod(0o600) replacement.replace(archive) elif fault == "changed_fd": before = archive.stat() with archive.open("r+b") as writer: writer.write(b"X") # This case tests metadata drift. Two fast writes can share the # filesystem's timestamp tick, so make that injected drift explicit. os.utime(archive, ns=(before.st_atime_ns, before.st_mtime_ns + 1_000_000_000)) return SimpleNamespace(returncode=0) verified = [] monkeypatch.setattr(specialize.subprocess, "run", run) monkeypatch.setattr(specialize, "_verify_manifest_images_in_dind", lambda images: verified.append(images) or True) if fault is None: specialize._load_images_into_dind(archive, manifest) assert len(loaded) == 1 and verified == [manifest["images"]] else: with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._load_images_into_dind(archive, manifest) assert len(loaded) == (1 if fault in {"replace", "changed_fd"} else 0) assert verified == [] def _valid_task_image_manifest(specialize, revision="a" * 40, tree="b" * 40): source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" return { "apiVersion": specialize.IMAGE_MANIFEST_SCHEMA, "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": revision, "sourceTree": tree, "workflowPath": specialize.IMAGE_WORKFLOW, "workflowRunId": "12", "fullRunId": "13", "artifactName": specialize.IMAGE_ARTIFACT_PREFIX + revision, "dockerTarSha256": "c" * 64, "dockerTarBytes": 1, "images": [ { "profile": profile_id, "reference": f"localhost/itworx-ci-{profile_id}:sha-{revision}", "imageId": f"sha256:{str(index + 1) * 64}", "architecture": "amd64", "dockerfileSha256": "d" * 64, "labels": { "org.opencontainers.image.source": source, "org.opencontainers.image.revision": revision, "tech.itworx.source-tree": tree, "tech.itworx.ci.profile": profile_id, }, } for index, profile_id in enumerate(specialize.PROFILE_IDS) ], } def _build_probe_record(specialize, *, container_id=None, status="intent"): revision = "a" * 40 tree = "b" * 40 image_labels = { "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", "org.opencontainers.image.revision": revision, "tech.itworx.source-tree": tree, "tech.itworx.ci.profile": "build", } record = specialize._new_build_probe_record( {"sourceRevision": revision, "sourceTree": tree}, { "taskImageId": "sha256:" + "1" * 64, "dockerfileSha256": "d" * 64, "imageLabels": image_labels, }, ) record["status"] = status record["containerId"] = container_id return record def _build_probe_inspect(specialize, record, *, container_id=None, changes=None): image_environment = ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] item = { "Id": container_id or "e" * 64, "Name": f"/{record['name']}", "Image": record["taskImageId"], "Config": { "Image": record["taskImageId"], "Entrypoint": [specialize._BUILD_PROBE_EXECUTABLE], "Cmd": [ record["sourceRevision"], record["sourceTree"], "--hold-seconds", str(specialize._BUILD_PROBE_HOLD_SECONDS), ], "Env": image_environment, "Labels": specialize._build_probe_container_labels(record), "Volumes": {"/home/itworx-builder/.local/share/docker": {}}, }, "HostConfig": { "NetworkMode": "bridge", "PidMode": "", "IpcMode": "private", "CgroupnsMode": "private", "CgroupParent": "docker", "NanoCpus": 4_000_000_000, "Memory": 6 * 1024**3, "PidsLimit": 1024, "BlkioWeight": 100, "Privileged": False, "CapAdd": [], "CapDrop": [], "SecurityOpt": ["seccomp=unconfined", "systempaths=unconfined"], "MaskedPaths": [], "ReadonlyPaths": [], "Devices": [{ "PathOnHost": "/dev/net/tun", "PathInContainer": "/dev/net/tun", "CgroupPermissions": "rwm", }], "DeviceRequests": [], "DeviceCgroupRules": [], "ExtraHosts": ["gitea.example.com:198.19.104.1"], "PortBindings": {}, "Binds": [], "Mounts": [], "VolumesFrom": [], "Tmpfs": {}, "RestartPolicy": {"Name": "no"}, }, "Mounts": [{ "Type": "volume", "Name": "9" * 64, "Source": f"/var/lib/docker/volumes/{'9' * 64}/_data", "Destination": "/home/itworx-builder/.local/share/docker", "Driver": "local", "Mode": "", "RW": True, "Propagation": "", }], "State": {"Running": True, "Pid": 77}, } if changes: for section, values in changes.items(): item[section].update(values) return item, image_environment def _write_task_image_archive(specialize, path: Path, *, extra_label: bool = False): images = _valid_task_image_manifest(specialize)["images"] docker_manifest = [] members: dict[str, bytes] = {} for index, image in enumerate(images): labels = specialize._runtime_task_image_labels(image) assert labels is not None if extra_label and index == 0: labels["untrusted.extra"] = "value" config = { "architecture": "amd64", "os": "linux", "config": {"Labels": labels}, } config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() config_name = f"{image['profile']}.json" layer_name = f"{image['profile']}/layer.tar" image["imageId"] = "sha256:" + hashlib.sha256(config_bytes).hexdigest() members[config_name] = config_bytes members[layer_name] = b"layer" docker_manifest.append( { "Config": config_name, "RepoTags": [image["reference"]], "Layers": [layer_name], } ) members["manifest.json"] = json.dumps(docker_manifest).encode() with tarfile.open(path, mode="w:") as archive: for name, payload in members.items(): info = tarfile.TarInfo(name) info.size = len(payload) archive.addfile(info, io.BytesIO(payload)) return images def _active_recovery_fixture(specialize): rows = [] registration_ids = {} originals = [] containers = {} for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=100): registration_ids[profile.id] = index rows.append( { "name": profile.container_name, "displayName": profile.display_name, "id": index, "labels": sorted({profile.runner_label, *profile.legacy_labels}), "disabled": True, "busy": False, "status": "offline", } ) containers[profile.container_name] = {"State": {"Running": False}} for index, name in enumerate(specialize._LEGACY_V1_NAMES, start=9): labels = sorted(specialize._legacy_allowed_labels("v1", name)) legacy_row = { "name": name, "id": index, "labels": labels, "disabled": True, "busy": False, "status": "offline", } rows.append(legacy_row) originals.append({"name": name, "id": index, "labels": labels, "disabled": True}) dind = {"State": {"Running": False, "Health": {"Status": "none"}}} containers[specialize.DIND_NAME] = dind journal = { "legacyTopology": "v1", "newRegistrationIds": registration_ids, "originalRegistrations": originals, } model = {"services": {"runner-dind": {"privileged": True}}} return rows, containers, dind, journal, model def _legacy_dind_mounts(specialize): return [ { "type": "bind", "source": specialize.LEGACY_DIND_STORAGE_ROOT.as_posix(), "target": "/home/rootless/.local/share/docker", "read_only": False, }, { "type": "bind", "source": "/mnt/cache/appdata/itworx-security-runner-workspace", "target": "/workspace", "read_only": False, }, { "type": "bind", "source": "/mnt/cache/appdata/itworx-validation-runner-2-workspace", "target": "/workspace-secondary", "read_only": False, }, ] def _specialized_dind_mounts(specialize): return [ { "type": "bind", "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "target": "/var/lib/docker", "read_only": False, }, { "type": "bind", "source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "target": specialize._DIND_CONTROL_CONTAINER_ROOT, "read_only": False, }, { "type": "bind", "source": str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "target": "/home/rootless/.local/share/docker", "read_only": True, }, *[ { "type": "bind", "source": str( specialize._host_volume_path( specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name ) ), "target": f"/{profile.container_name}-workspace", "read_only": False, } for profile in specialize.RUNNER_PROFILES.values() ], ] def _base_v1_compose_model(specialize): services = { "deploy-control": { "container_name": specialize.CONTROLLER_CONTAINER, "image": "itworx/appops-deploy-control:fixed", }, "runner-dind": { "container_name": specialize.DIND_NAME, "image": "docker@sha256:7c3e797187e43738220462658f4586572cbd3bf009f728b21e34d9c5c06ce431", "user": "rootless", "privileged": True, "cpus": 6, "mem_limit": "12g", "restart": "unless-stopped", "security_opt": ["label=disable"], "command": ["--default-address-pool=base=198.19.0.0/16,size=27"], "volumes": _legacy_dind_mounts(specialize), }, } for name, service_name in specialize._OLD_SERVICES.items(): if name not in specialize._LEGACY_V1_NAMES: continue services[service_name] = { "container_name": name, "image": specialize.RUNNER_IMAGE, "profiles": ["runner"], "cpus": 1, "mem_limit": "1g", "volumes": [ { "type": "bind", "source": source, "target": target, "read_only": read_only, } for source, target, read_only in specialize._expected_legacy_mounts("v1", name) ], "labels": {"tech.itworx.role": "gitea-actions-runner"}, } return { "services": services, "networks": { "runner-sandbox": {"name": specialize._RUNNER_NETWORK_NAME} }, } def _gitea_28_run( specialize, revision, *, run_id=21533, attempt=1, status="completed", conclusion="success" ): filename = specialize.FULL_WORKFLOW.rsplit("/", 1)[-1] return { "id": run_id, "run_attempt": attempt, "path": f"{filename}@refs/heads/main", "status": status, "conclusion": conclusion, "event": "push", "head_branch": "main", "head_sha": revision, "repository": {"id": 9, "full_name": "Jens/Unraid-AppOps-Gateway"}, "head_repository": None, } def _gitea_workflow_api(specialize, rows, *, run_pages=None, calls=None): run_pages = run_pages or {1: rows} calls = calls if calls is not None else [] def api(path): calls.append(path) if path.endswith("/actions/workflows?limit=100&page=1"): return { "workflows": [ { "id": workflow.rsplit("/", 1)[-1], "path": workflow, "state": "active", } for workflow in (specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW) ] } if "/actions/runs?" in path: page = int(path.rsplit("page=", 1)[-1]) return {"workflow_runs": run_pages.get(page, [])} raise AssertionError(f"unexpected fixed Gitea API path: {path}") return api, calls def test_gitea_workflow_inventory_uses_exact_active_filename_and_path( specialize, monkeypatch ): workflow = specialize.FULL_WORKFLOW filename = workflow.rsplit("/", 1)[-1] inventory = { "workflows": [ {"id": filename, "path": workflow, "state": "active"}, { "id": specialize.IMAGE_WORKFLOW.rsplit("/", 1)[-1], "path": specialize.IMAGE_WORKFLOW, "state": "active", }, ] } monkeypatch.setattr(specialize, "_api_json", lambda _path: inventory) assert specialize._workflow_id(workflow) == filename for bad in ( {"id": 34, "path": workflow, "state": "active"}, {"id": filename, "path": ".gitea/workflows/other.yml", "state": "active"}, {"id": filename, "path": workflow, "state": "enabled"}, ): monkeypatch.setattr( specialize, "_api_json", lambda _path, row=bad: {"workflows": [row]} ) with pytest.raises( specialize.RunnerFleetSpecializeError, match="workflow_unavailable" ): specialize._workflow_id(workflow) def test_successful_main_run_accepts_actual_gitea_run_without_workflow_id( specialize, monkeypatch ): revision = "a" * 40 run = _gitea_28_run(specialize, revision) api, calls = _gitea_workflow_api(specialize, [run]) monkeypatch.setattr(specialize, "_api_json", api) selected = specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) assert selected is run assert "workflow_id" not in selected assert selected["head_repository"] is None assert calls[0].endswith("/actions/workflows?limit=100&page=1") assert calls[1].endswith("/actions/runs?branch=main&limit=100&page=1") def test_source_image_fetch_accepts_actual_gitea_commit_tree_shape( specialize, monkeypatch, tmp_path ): revision = "a" * 40 tree = "b" * 40 calls = [] workflow_calls = [] def api(path): calls.append(path) if path.endswith("/branches/main"): return {"commit": {"id": revision}} if path.endswith(f"/git/commits/{revision}"): return {"sha": revision, "commit": {"tree": {"sha": tree}}} if "/actions/runs/" in path and "/artifacts?" in path: return { "artifacts": [ { "name": specialize.IMAGE_ARTIFACT_PREFIX + revision, "expired": False, "id": 44, "size_in_bytes": 1, } ] } raise AssertionError(f"unexpected fixed Gitea API path: {path}") def successful_run(workflow_path, requested_revision): assert requested_revision == revision workflow_calls.append(workflow_path) return {"id": 41 if workflow_path == specialize.IMAGE_WORKFLOW else 42} class ReachedArtifactDownload(Exception): pass def download(_artifact_id, _destination): raise ReachedArtifactDownload monkeypatch.setattr(specialize, "_api_json", api) monkeypatch.setattr(specialize, "_successful_main_run", successful_run) monkeypatch.setattr(specialize, "_download_artifact_zip", download) with pytest.raises(ReachedArtifactDownload): specialize._fetch_source_image_artifact(revision, tree, tmp_path) assert workflow_calls == [specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW] assert calls == [ f"/repos/{specialize.SOURCE_REPOSITORY_API}/branches/main", f"/repos/{specialize.SOURCE_REPOSITORY_API}/git/commits/{revision}", f"/repos/{specialize.SOURCE_REPOSITORY_API}/actions/runs/41/artifacts?name={specialize.IMAGE_ARTIFACT_PREFIX}{revision}", ] @pytest.mark.parametrize( ("commit_value", "error"), [ ({"sha": "a" * 40}, "source_tree_mismatch"), ({"sha": "a" * 40, "commit": {}}, "source_tree_mismatch"), ( {"sha": "a" * 40, "commit": {"tree": {"sha": "c" * 40}}}, "source_tree_mismatch", ), ( {"sha": "a" * 40, "tree": {"sha": "b" * 40}}, "source_tree_mismatch", ), ( {"sha": "c" * 40, "commit": {"tree": {"sha": "b" * 40}}}, "source_revision_mismatch", ), ( {"commit": {"tree": {"sha": "b" * 40}}}, "source_revision_mismatch", ), ], ) def test_source_image_fetch_rejects_noncanonical_gitea_commit_payload_before_artifact_lookup( specialize, monkeypatch, tmp_path, commit_value, error ): revision = "a" * 40 calls = [] workflow_calls = [] def api(path): calls.append(path) if path.endswith("/branches/main"): return {"commit": {"id": revision}} if path.endswith(f"/git/commits/{revision}"): return commit_value raise AssertionError(f"unexpected fixed Gitea API path: {path}") def unexpected_workflow(*_args): workflow_calls.append(True) raise AssertionError("workflow lookup must not follow an untrusted commit response") monkeypatch.setattr(specialize, "_api_json", api) monkeypatch.setattr(specialize, "_successful_main_run", unexpected_workflow) monkeypatch.setattr( specialize, "_download_artifact_zip", lambda *_args: pytest.fail("artifact download must not follow an untrusted commit response"), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): specialize._fetch_source_image_artifact(revision, "b" * 40, tmp_path) assert calls == [ f"/repos/{specialize.SOURCE_REPOSITORY_API}/branches/main", f"/repos/{specialize.SOURCE_REPOSITORY_API}/git/commits/{revision}", ] assert workflow_calls == [] @pytest.mark.parametrize( "changes", [ {"path": "managed-validation.yml@refs/heads/feature"}, {"repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, {"head_repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, {"head_sha": "b" * 40}, {"event": "workflow_dispatch"}, {"head_branch": "feature"}, {"pull_requests": [{"number": 123}]}, ], ) def test_successful_main_run_rejects_untrusted_gitea_identity( specialize, monkeypatch, changes ): revision = "a" * 40 run = {**_gitea_28_run(specialize, revision), **changes} api, _calls = _gitea_workflow_api(specialize, [run]) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="protected_validation_missing", ): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) @pytest.mark.parametrize( ("latest", "error"), [ ({"status": "in_progress", "conclusion": None}, "protected_validation_missing"), ({"status": "completed", "conclusion": "failure"}, "protected_validation_missing"), ( {"run_attempt": 2, "status": "completed", "conclusion": "failure"}, "protected_validation_missing", ), ({"run_attempt": None}, "workflow_runs_invalid"), ], ) def test_newest_matching_gitea_run_or_attempt_supersedes_prior_success( specialize, monkeypatch, latest, error ): revision = "a" * 40 successful = _gitea_28_run(specialize, revision, run_id=21532) newer = { **_gitea_28_run(specialize, revision, run_id=21533), **latest, } if latest.get("run_attempt") == 2: newer["id"] = successful["id"] api, _calls = _gitea_workflow_api(specialize, [successful, newer]) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) def test_workflow_run_pagination_fails_closed_at_fixed_page_limit(specialize, monkeypatch): revision = "a" * 40 wrong = {"path": "unrelated.yml@refs/heads/main"} pages = {page: [wrong] * 100 for page in range(1, 11)} api, calls = _gitea_workflow_api(specialize, [], run_pages=pages) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="workflow_runs_invalid" ): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) assert len([path for path in calls if "/actions/runs?" in path]) == 10 def test_archive_verification_hashes_large_tar_without_read_bytes(specialize, tmp_path, monkeypatch): archive_path = tmp_path / "images.tar" payload = b"bounded-stream" * 4096 with tarfile.open(archive_path, "w") as archive: member = tarfile.TarInfo("layer.tar") member.size = len(payload) archive.addfile(member, io.BytesIO(payload)) digest = hashlib.sha256(archive_path.open("rb").read()).hexdigest() def forbidden_read_bytes(_path): raise AssertionError("archive verifier must stream the tar file") monkeypatch.setattr(Path, "read_bytes", forbidden_read_bytes) specialize._verify_docker_archive(archive_path, archive_path.stat().st_size, digest) def test_archive_verifier_requires_exact_pinned_inherited_labels( specialize, tmp_path ): archive_path = tmp_path / "task-images.tar" images = _write_task_image_archive(specialize, archive_path) specialize._verify_docker_archive( archive_path, archive_path.stat().st_size, hashlib.sha256(archive_path.read_bytes()).hexdigest(), images, ) images = _write_task_image_archive(specialize, archive_path, extra_label=True) with pytest.raises( specialize.RunnerFleetSpecializeError, match="image_archive_identity_invalid", ): specialize._verify_docker_archive( archive_path, archive_path.stat().st_size, hashlib.sha256(archive_path.read_bytes()).hexdigest(), images, ) def test_installed_image_manifest_requires_complete_source_and_profile_pins(specialize): manifest = _valid_task_image_manifest(specialize) assert specialize._valid_installed_manifest(manifest) is True manifest["images"][0]["reference"] = "localhost/itworx-ci-checks:latest" assert specialize._valid_installed_manifest(manifest) is False def test_task_image_runtime_verifier_accepts_only_exact_pinned_inherited_labels( specialize, monkeypatch ): expected = _valid_task_image_manifest(specialize)["images"][0] runtime_labels = specialize._runtime_task_image_labels(expected) assert runtime_labels is not None assert len(runtime_labels) == 14 actual = { "Id": expected["imageId"], "Os": "linux", "Architecture": "amd64", "Config": {"Labels": runtime_labels}, } monkeypatch.setattr( specialize, "_run", lambda *_args, **_kwargs: json.dumps([actual]), ) assert specialize._verify_manifest_images_in_dind([expected]) is True actual["Config"]["Labels"]["untrusted.extra"] = "value" assert specialize._verify_manifest_images_in_dind([expected]) is False del actual["Config"]["Labels"]["untrusted.extra"] del expected["labels"]["tech.itworx.source-tree"] assert specialize._verify_manifest_images_in_dind([expected]) is False def test_bootstrap_receipt_is_source_bound_private_and_cas_consumed( specialize, tmp_path, monkeypatch ): state_root = tmp_path / "runner-fleet-state" state_root.mkdir(mode=0o700) os.chmod(state_root, 0o700) manifest_path = state_root / "task-image-manifest.json" marker_path = state_root / "specialized.json" receipt_path = state_root / "admission-bootstrap.json" manifest_bytes = json.dumps( _valid_task_image_manifest(specialize), sort_keys=True, separators=(",", ":") ).encode() + b"\n" manifest_path.write_bytes(manifest_bytes) os.chmod(manifest_path, 0o600) manifest_sha = hashlib.sha256(manifest_bytes).hexdigest() marker = { "schemaVersion": 1, "status": "active", "sourceScriptSha256": "e" * 64, "imageManifestSha256": manifest_sha, "imageSourceRevision": "a" * 40, "imageSourceTree": "b" * 40, "verifiedRunners": list(specialize._SPECIALIZED_NAMES), "resourceEnvelopeDigest": "f" * 64, "updatedAt": "2026-10-01T12:00:00Z", } marker_path.write_text(json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n") os.chmod(marker_path, 0o600) registrations = [] for profile in specialize.RUNNER_PROFILES.values(): registrations.append( { "profileId": profile.id, "displayName": profile.display_name, "containerName": profile.container_name, "id": len(registrations) + 10, "labels": sorted({profile.runner_label, *profile.legacy_labels}), } ) receipt = { "schemaVersion": 1, "status": "pending", "sourceScriptSha256": marker["sourceScriptSha256"], "imageManifestSha256": manifest_sha, "imageSourceRevision": "a" * 40, "imageSourceTree": "b" * 40, "verifiedRunners": list(specialize._SPECIALIZED_NAMES), "registrations": registrations, "resourceEnvelopeDigest": marker["resourceEnvelopeDigest"], "createdAt": "2026-10-01T12:00:00Z", } receipt_path.write_text(json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n") os.chmod(receipt_path, 0o600) assert specialize._valid_admission_bootstrap_receipt(receipt) validated = specialize.read_admission_bootstrap_receipt( receipt_path, marker_path=marker_path, image_manifest_path=manifest_path ) assert validated is not None pending_sha = validated[1] assert validated[0]["status"] == "pending" assert specialize.consume_admission_bootstrap_receipt( pending_sha, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) consumed = specialize.read_admission_bootstrap_receipt( receipt_path, marker_path=marker_path, image_manifest_path=manifest_path ) assert consumed is not None assert consumed[0]["status"] == "consumed" assert consumed[1] == pending_sha assert specialize.consume_admission_bootstrap_receipt( pending_sha, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) assert not specialize.consume_admission_bootstrap_receipt( "0" * 64, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) def test_runner_config_is_profile_locked_and_only_backend_frontend_keep_legacy_labels(specialize): revision = "a" * 40 checks = specialize._render_runner_config("checks", revision).decode() backend = specialize._render_runner_config("backend", revision).decode() frontend = specialize._render_runner_config("frontend", revision).decode() security = specialize._render_runner_config("security", revision).decode() assert "itworx-ci-checks:docker://localhost/itworx-ci-checks:sha-" in checks assert "linux-validation" not in checks and "ubuntu-latest" not in checks assert "linux-validation" in backend and "ubuntu-latest" in backend assert "linux-validation" in frontend and "ubuntu-latest" in frontend assert "linux-validation" not in security and "ubuntu-latest" not in security for profile_id, profile in specialize.RUNNER_PROFILES.items(): config = specialize._render_runner_config(profile_id, revision).decode() task_image = specialize._task_image_reference(profile_id, revision) job_cpu_milli, job_memory_mib = specialize._runner_job_limits(profile_id) cpu = f"{job_cpu_milli / 1000:.3f}".rstrip("0").rstrip(".") assert f"--cpus={cpu} --memory={job_memory_mib}m" in config assert f"source=itworx-ci-{profile_id}-cache,target=/opt/itworx-ci-cache" in config assert f'valid_volumes:\n - "itworx-ci-{profile_id}-cache"' in config assert "[0-9]*/**" not in config assert "DOCKER_HOST:" not in config assert 'docker_host: "-"' in config assert "host.docker.internal" not in config assert all( f"docker://{task_image}" in line for line in config.splitlines() if ":docker://" in line ) if profile_id == "build": assert "--security-opt=seccomp=unconfined" in config assert "--security-opt=systempaths=unconfined" in config assert "--device=/dev/net/tun:/dev/net/tun:rwm" in config else: assert "--security-opt=" not in config assert "--device=" not in config def test_fixed_workload_reservations_are_subtracted_from_aggregate_profiles(specialize): for profile_id in ("backend", "frontend", "browser"): profile = specialize.RUNNER_PROFILES[profile_id] service_cpu, service_memory = specialize.FIXED_WORKLOAD_RESERVATION[profile_id] job_cpu, job_memory = specialize._runner_job_limits(profile_id) assert job_cpu + service_cpu == profile.cpu_milli assert job_memory + service_memory == profile.memory_mib config = specialize._render_runner_config(profile_id, "a" * 40).decode() cpu = f"{job_cpu / 1000:.3f}".rstrip("0").rstrip(".") assert f"--cpus={cpu} --memory={job_memory}m" in config for profile_id in ("checks", "security"): profile = specialize.RUNNER_PROFILES[profile_id] assert specialize._runner_job_limits(profile_id) == ( profile.cpu_milli, profile.memory_mib, ) assert specialize._runner_job_limits("build") == (4000, 6144) def test_base_compose_model_explicitly_resolves_the_runner_profile(specialize, monkeypatch): compose = b"captured v1 source" base = _base_v1_compose_model(specialize) monkeypatch.setattr( specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest(), ) calls = [] def run_compose(_compose_bytes, *arguments, **_kwargs): calls.append(arguments) return json.dumps(base) monkeypatch.setattr(specialize, "_run_compose", run_compose) assert specialize._validate_base_composition(compose) == "v1" assert calls == [("--profile", "runner", "config", "--format", "json")] def test_rendered_compose_uses_fixed_rootful_dind_and_shared_workspaces(specialize): base = _base_v1_compose_model(specialize) rendered = json.loads(specialize._render_specialized_compose(base)) services = rendered["services"] dind = services["runner-dind"] assert services["deploy-control"] == base["services"]["deploy-control"] assert dind["user"] == "0" assert dind["command"][0] == "dockerd" assert dind["command"][1] == f"--host={specialize._DIND_DOCKER_HOST}" assert "--data-root=/var/lib/docker" in dind["command"] assert "--exec-opt=native.cgroupdriver=cgroupfs" in dind["command"] assert dind["environment"] == {"DOCKER_HOST": specialize._DIND_DOCKER_HOST} assert _mounts(specialize, dind) == { (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, False, ), ( str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "/home/rootless/.local/share/docker", True, ), *{ ( str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False, ) for profile in specialize.RUNNER_PROFILES.values() }, } assert len([volume for volume in dind["volumes"] if volume["type"] == "bind"]) == 9 expected_services = {"deploy-control", "runner-dind"} expected_services.update( specialize._OLD_SERVICES[name] for name in specialize._LEGACY_V1_NAMES ) expected_services.update(f"runner-{profile_id}" for profile_id in specialize.PROFILE_IDS) assert set(services) == expected_services for profile_id in specialize.PROFILE_IDS: service = services[f"runner-{profile_id}"] assert service["environment"]["DOCKER_HOST"] == specialize._DIND_DOCKER_HOST assert service["extra_hosts"] == ["gitea.example.com:198.19.104.1"] assert ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True, ) in specialize._compose_mounts(service) @pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") def test_profile_state_creation_uses_effective_runner_image_owners_and_resumes( specialize, tmp_path, monkeypatch ): state = tmp_path / "state" state.mkdir(mode=0o700) data_root = state / "runner-data" workspace_root = state / "workspaces" config_root = state / "runner-configs" image_root = state / "task-images" dind_root = state / "dind" / "docker" shadow_root = state / "dind" / "rootless-unused" control_root = state / "dind-control" for name, value in ( ("_SPECIALIZED_DATA_ROOT", data_root), ("_SPECIALIZED_WORKSPACE_ROOT", workspace_root), ("RUNNER_CONFIG_ROOT", config_root), ("TASK_IMAGE_ROOT", image_root), ("DIND_STORAGE_ROOT", dind_root), ("_DIND_ROOTLESS_SHADOW_ROOT", shadow_root), ("_DIND_CONTROL_ROOT", control_root), ): monkeypatch.setattr(specialize, name, value) control_root.mkdir(mode=0o700) specialize._prepare_profile_state("a" * 40) for profile_id, profile in specialize.RUNNER_PROFILES.items(): data_path = data_root / profile_id workspace_path = workspace_root / profile.container_name assert data_path.stat().st_uid == 0 assert workspace_path.stat().st_uid == 1000 assert control_root.stat().st_uid == 0 assert os.stat(control_root).st_mode & 0o777 == 0o700 config_path = config_root / "checks.yaml" expected = specialize._render_runner_config("checks", "a" * 40) invalid_configs = ( expected.replace( b'valid_volumes:\n - "itworx-ci-checks-cache"', b'valid_volumes:\n - "/itworx-ci-checks-workspace/[0-9]*/**"', 1, ), expected.replace( b'valid_volumes:\n - "itworx-ci-checks-cache"', b'valid_volumes:\n - "/run/itworx-ci-control/**"', 1, ), expected.replace( b' docker_host: "-"', b' docker_host: "tcp://runner-dind:2375"', 1, ), expected.replace( b" envs:\n", b" envs:\n DOCKER_HOST: tcp://runner-dind:2375\n", 1, ), ) for invalid_config in invalid_configs: config_path.write_bytes(invalid_config) os.chmod(config_path, 0o600) with pytest.raises( specialize.RunnerFleetSpecializeError, match="profile_config_adoption_rejected", ): specialize._prepare_profile_state("a" * 40) config_path.write_bytes(expected) os.chmod(config_path, 0o600) # Docker normalizes its own data-root to 0710. Rollback preserves this # directory; preparation must resume without changing daemon-owned bytes. dind_root.chmod(0o710) specialize._prepare_profile_state("a" * 40) assert dind_root.stat().st_mode & 0o7777 == 0o710 @pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") @pytest.mark.parametrize("fault", [None, "fresh", "mode711", "mode755", "mode770", "mode701", "special", "owner", "group", "symlink", "file", "parent_mode", "parent_owner", "parent_group", "parent_symlink"]) def test_dind_data_root_accepts_only_daemon_mode_in_private_root_parent( specialize, tmp_path, monkeypatch, fault, ): parent = tmp_path / "dind" parent.mkdir(mode=0o700) root = parent / "docker" root.mkdir(mode=0o710) monkeypatch.setattr(specialize, "DIND_STORAGE_ROOT", root) if fault == "fresh": root.chmod(0o700) elif fault and fault.startswith("mode"): root.chmod(int(fault[4:], 8)) elif fault == "special": root.chmod(0o1710) elif fault in {"owner", "group"}: os.chown(root, 1000 if fault == "owner" else 0, 1000 if fault == "group" else 0) elif fault in {"symlink", "file"}: root.rmdir() if fault == "symlink": root.symlink_to(tmp_path, target_is_directory=True) else: root.write_bytes(b"not a directory") elif fault == "parent_mode": parent.chmod(0o710) elif fault in {"parent_owner", "parent_group"}: os.chown(parent, 1000 if fault == "parent_owner" else 0, 1000 if fault == "parent_group" else 0) elif fault == "parent_symlink": real_parent = tmp_path / "real-dind" parent.rename(real_parent) parent.symlink_to(real_parent, target_is_directory=True) if fault in {None, "fresh"}: specialize._trusted_dind_data_root() if fault is None: with pytest.raises(specialize.RunnerFleetSpecializeError, match="path_untrusted"): specialize._trusted_directory(root) else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="path_untrusted"): specialize._trusted_dind_data_root() def _mounts(specialize, service): return specialize._compose_mounts(service) def test_rendered_compose_uses_only_canonical_protected_volume_sources(specialize): base = _base_v1_compose_model(specialize) rendered = json.loads(specialize._render_specialized_compose(base)) services = rendered["services"] assert specialize._compose_mounts(services["runner-dind"]) == { (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, False, ), ( str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "/home/rootless/.local/share/docker", True, ), *{ ( str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False, ) for profile in specialize.RUNNER_PROFILES.values() }, } assert services["runner-dind"]["restart"] == "no" for profile_id, profile in specialize.RUNNER_PROFILES.items(): service = services[f"runner-{profile_id}"] assert specialize._compose_mounts(service) == { (str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "/data", False), (str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "/runner-config.yaml", True), (str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True), (str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False), } assert "-f" in specialize._compose_command("up", "-d") assert specialize._compose_command("up", "-d")[specialize._compose_command("up", "-d").index("-f") + 1] == "-" def test_specialized_compose_validation_rejects_socket_and_gateway_broadening( specialize, monkeypatch ): rendered = json.loads( specialize._render_specialized_compose(_base_v1_compose_model(specialize)) ) mutations = ( lambda model: model["services"]["runner-dind"]["environment"].update( DOCKER_HOST="tcp://0.0.0.0:2375" ), lambda model: next( volume for volume in model["services"]["runner-dind"]["volumes"] if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT ).update(read_only=True), lambda model: next( volume for volume in model["services"]["runner-frontend"]["volumes"] if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT ).update(read_only=False), lambda model: model["services"]["runner-frontend"]["extra_hosts"].append( "host.docker.internal:host-gateway" ), lambda model: model["services"]["runner-frontend"].update( network_mode="host" ), lambda model: model["services"]["runner-frontend"].update( privileged=True ), lambda model: model["networks"]["runner-sandbox"].update( name="runner-sandbox" ), ) for mutate in mutations: candidate = copy.deepcopy(rendered) mutate(candidate) monkeypatch.setattr(specialize, "_compose_model", lambda _raw, value=candidate: value) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._validate_composition(b"candidate") def test_specialized_compose_validation_accepts_real_compose_normalization( specialize, monkeypatch ): normalized = json.loads( specialize._render_specialized_compose(_base_v1_compose_model(specialize)) ) service = normalized["services"]["runner-frontend"] service.pop("privileged") service["extra_hosts"] = ["gitea.example.com=198.19.104.1"] service["networks"] = {"runner-sandbox": {"aliases": ["itworx-ci-frontend"]}} monkeypatch.setattr(specialize, "_compose_model", lambda _raw: normalized) assert specialize._validate_composition(b"normalized candidate") def test_specialized_dind_healthcheck_replaces_captured_tcp_endpoint(specialize, monkeypatch): base = _base_v1_compose_model(specialize) base["services"]["runner-dind"]["healthcheck"] = { "test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"], "interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s", } rendered = json.loads(specialize._render_specialized_compose(base)) check = rendered["services"]["runner-dind"]["healthcheck"] assert check == { "test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"], "interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s", } assert base["services"]["runner-dind"]["healthcheck"]["test"][1].startswith("DOCKER_HOST=tcp:") monkeypatch.setattr(specialize, "_compose_model", lambda _raw: rendered) assert specialize._validate_composition(b"reviewed model") @pytest.mark.parametrize("check", [None, {"disable": True}, {"test": ["NONE"]}, {"disable": True, "test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"]}, {"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]}, {"test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]}, {"test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]}, ]) def test_specialized_compose_rejects_healthcheck_endpoint_drift(specialize, monkeypatch, check): model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize))) model["services"]["runner-dind"]["healthcheck"] = check monkeypatch.setattr(specialize, "_compose_model", lambda _raw: model) with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_compose_invalid"): specialize._validate_composition(b"foreign healthcheck") @pytest.mark.parametrize("check", [None, {"Test": ["NONE"]}, {"Test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]}, {"Test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]}, {"Test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]}, ]) def test_specialized_dind_runtime_rejects_healthcheck_drift(specialize, check): base = _base_v1_compose_model(specialize) model = json.loads(specialize._render_specialized_compose(base)) service = model["services"]["runner-dind"] item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no") item["Config"].update(User="0", Cmd=service["command"], Env=[f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"], Healthcheck={"Test": service["healthcheck"]["test"]}) item["HostConfig"].update(CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={}) item["Mounts"] = [{"Type": "bind", "Source": mount["source"], "Destination": mount["target"], "RW": not mount["read_only"]} for mount in service["volumes"]] assert specialize._validate_dind_outer(item, model) changed = copy.deepcopy(item) changed["Config"]["Healthcheck"] = check assert specialize._validate_dind_outer(changed, model) is False changed_model = copy.deepcopy(model) changed_model["services"]["runner-dind"]["healthcheck"] = { "test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"], } assert specialize._validate_dind_outer(item, changed_model) is False def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize): base = _base_v1_compose_model(specialize) item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="unless-stopped") assert specialize._legacy_dind_observation(item, base)["running"] is True for key, value in (("User", "0"), ("Cmd", ["--data-root=/var/lib/docker"]), ("Entrypoint", ["sh"])): changed = copy.deepcopy(item) changed["Config"][key] = value with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): specialize._legacy_dind_observation(changed, base) changed = copy.deepcopy(item) changed["Mounts"][-1]["Source"] = "/untrusted/docker" with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): specialize._legacy_dind_observation(changed, base) @pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false", "2|cgroupfs|true"]) def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces( specialize, monkeypatch, support ): model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize))) service = model["services"]["runner-dind"] item = _legacy_dind_fixture(specialize, _base_v1_compose_model(specialize), image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no") item["Config"].update(User="0", Cmd=service["command"]) item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"] item["Config"]["Healthcheck"] = {"Test": service["healthcheck"]["test"]} item["HostConfig"].update( CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={} ) item["Mounts"] = [{"Type": "bind", "Source": m["source"], "Destination": m["target"], "RW": not m["read_only"]} for m in service["volumes"]] assert specialize._validate_dind_outer(item, model) is True monkeypatch.setattr(specialize, "_inspect", lambda _name: item) commands = [] def run(args, **_kwargs): commands.append(args) return support monkeypatch.setattr(specialize, "_run", run) if support == "2|cgroupfs|true": specialize._require_dind_resource_support(model) assert f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}" in commands[-1] else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_limits_unsupported"): specialize._require_dind_resource_support(model) missing_workspace = copy.deepcopy(item) missing_workspace["Mounts"].pop() assert specialize._validate_dind_outer(missing_workspace, model) is False changed_cases = [] tcp_host = copy.deepcopy(item) tcp_host["Config"]["Env"] = ["DOCKER_HOST=tcp://0.0.0.0:2375"] changed_cases.append(tcp_host) tcp_command = copy.deepcopy(item) tcp_command["Config"]["Cmd"][1] = "--host=tcp://0.0.0.0:2375" changed_cases.append(tcp_command) published_port = copy.deepcopy(item) published_port["HostConfig"]["PortBindings"] = { "2375/tcp": [{"HostIp": "0.0.0.0", "HostPort": "2375"}] } changed_cases.append(published_port) for changed in changed_cases: assert specialize._validate_dind_outer(changed, model) is False def test_outer_runner_must_match_fixed_compose_mount_and_image_identity(specialize): profile_id = "frontend" profile = specialize.RUNNER_PROFILES[profile_id] name = profile.container_name image = { "Name": f"/{name}", "Image": specialize.RUNNER_IMAGE_ID, "Config": { "Image": specialize.RUNNER_IMAGE, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, }, "Env": [ "CONFIG_FILE=/runner-config.yaml", f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}", ], }, "State": {"Running": False}, "HostConfig": { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * specialize._MIB, "PidsLimit": 192, "PidMode": "", "NetworkMode": specialize._RUNNER_NETWORK_NAME, "Privileged": False, "ExtraHosts": ["gitea.example.com:198.19.104.1"], "RestartPolicy": {"Name": "no"}, }, "Mounts": [ {"Source": str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "Destination": "/data", "RW": True}, {"Source": str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "Destination": "/runner-config.yaml", "RW": False}, {"Source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "Destination": specialize._DIND_CONTROL_CONTAINER_ROOT, "RW": False}, { "Source": str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / name)), "Destination": f"/{name}-workspace", "RW": True, }, ], } assert specialize._validate_outer_item(name, image, profile_id) unqualified_network = copy.deepcopy(image) unqualified_network["HostConfig"]["NetworkMode"] = "runner-sandbox" assert not specialize._validate_outer_item(name, unqualified_network, profile_id) tcp_env = copy.deepcopy(image) tcp_env["Config"]["Env"][-1] = "DOCKER_HOST=tcp://runner-dind:2375" assert not specialize._validate_outer_item(name, tcp_env, profile_id) writable_socket = copy.deepcopy(image) next( mount for mount in writable_socket["Mounts"] if mount["Destination"] == specialize._DIND_CONTROL_CONTAINER_ROOT )["RW"] = True assert not specialize._validate_outer_item(name, writable_socket, profile_id) host_gateway = copy.deepcopy(image) host_gateway["HostConfig"]["ExtraHosts"].append("host.docker.internal:host-gateway") assert not specialize._validate_outer_item(name, host_gateway, profile_id) image["Image"] = "sha256:" + "0" * 64 assert not specialize._validate_outer_item(name, image, profile_id) def test_atomic_guard_replace_uses_expected_hash_as_compare_and_swap(specialize, tmp_path): path = tmp_path / "guard.sh" path.write_bytes(b"original guard\n") original_sha = hashlib.sha256(path.read_bytes()).hexdigest() specialize._atomic_write_owned( path, b"patched guard\n", mode=0o600, expected_sha256=original_sha, ) assert path.read_bytes() == b"patched guard\n" with pytest.raises(specialize.RunnerFleetSpecializeError, match="compare_and_swap_failed"): specialize._atomic_write_owned( path, b"stale write\n", mode=0o600, expected_sha256=original_sha, ) assert path.read_bytes() == b"patched guard\n" def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(specialize): source = ( "stack_recovery=0\n" "set_managed_runners_disabled false\n" "unexpected=0\n" ' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' ) patched = specialize._patch_runner_guard( source, "fixed-module --state", 'fixed-module --guard-allows-container "$name"', ) assert "--fence-legacy" in patched assert "--guard-allows-legacy-container" in patched assert 'if [ "$specialization_state" = "legacy" ]; then' in patched assert 'elif [ "$specialization_state" = "active" ]' in patched assert '--guard-allows-container "$name"' in patched assert " set_managed_runners_disabled false\n" in patched with pytest.raises(specialize.RunnerFleetSpecializeError, match="runner_guard_shape_invalid"): specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow") def _resource_guard_source(kind): prefix = '''#!/bin/bash set -eu RUNNER="legacy-primary" SECONDARY_RUNNER="legacy-secondary" RUNNER_DIND="private-dind" DEPLOY_RUNNER="deploy-runner" RUNNER_PAUSE_FILE="/nonexistent-test-pause" RUNNER_MARKER="/nonexistent-test-marker" controller() { printf "controller:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = --state ]; then printf "%s" "$GUARD_STATE"; fi; } timeout() { shift; "$@"; } docker() { printf "docker:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = inspect ]; then printf legacy; fi; } enforce_limits() { printf "enforce:%s\\n" "$*" >> "$EVENTS"; } ''' if kind == "runner": return prefix + '''# The sandbox is the aggregate resource envelope for all jobs and services. if [ -n "$(timeout 10 docker inspect -f '{{.Id}}' "$RUNNER_DIND" 2>/dev/null || true)" ]; then dind_limits="$(timeout 10 docker inspect -f '{{.HostConfig.NanoCpus}}' "$RUNNER_DIND")" if [ "$dind_limits" != "expected" ]; then timeout 20 docker update --restart unless-stopped "$RUNNER_DIND" fi if [ "$dind_limits" = "legacy" ]; then timeout 20 docker update --cpuset-cpus 0-5 "$RUNNER_DIND" fi fi enforce_limits "$DEPLOY_RUNNER" stack_recovery=0 : unexpected=0 for name in "$DEPLOY_RUNNER"; do [ "$name" = "$DEPLOY_RUNNER" ] && continue done ''' return prefix + '''# ITWORX_RUNNER_CAPACITY_V2_BEGIN : # ITWORX_RUNNER_CAPACITY_V2_END if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then enforce_limits "$RUNNER" no enforce_limits "$SECONDARY_RUNNER" no enforce_limits "$RUNNER_DIND" no else enforce_limits "$RUNNER" unless-stopped enforce_limits "$SECONDARY_RUNNER" unless-stopped enforce_limits "$RUNNER_DIND" unless-stopped fi # The deploy runner is deliberately tiny and is required for deployment and enforce_limits "$DEPLOY_RUNNER" enforce_limits tdarr enforce_limits controller enforce_limits containment-pressure runner_healthy=1 if [ "$runner_healthy" -eq 0 ]; then if [ "$sandbox_healthy" -eq 1 ] && \\ timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then : fi fi ''' @pytest.mark.parametrize("kind", ["runner", "host"]) def test_resource_writer_is_gated_before_any_legacy_mutation(specialize, kind): source = _resource_guard_source(kind) if kind == "runner": patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow") writer = "# The sandbox is the aggregate resource envelope" else: patched = specialize._patch_host_guard(source, "controller --state") writer = 'if [ -e "$RUNNER_PAUSE_FILE" ]' assert patched.index('specialization_state="$(controller --state') < patched.index(writer) assert 'if [ "$specialization_state" = "legacy" ]; then\n ' + writer in patched assert 'enforce_limits "$DEPLOY_RUNNER"' in patched @pytest.mark.skipif(os.name != "posix", reason="requires actual Bash stub execution") @pytest.mark.parametrize("kind", ["runner", "host"]) @pytest.mark.parametrize("state", ["legacy", "blocked", "active"]) def test_guard_bash_skips_only_legacy_resource_writer_in_cutover(specialize, tmp_path, kind, state): source = _resource_guard_source(kind) if kind == "runner": patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow") else: patched = specialize._patch_host_guard(source, "controller --state") events = tmp_path / "events" result = subprocess.run(["bash", "-c", patched], capture_output=True, check=False, timeout=5, env={"PATH": "/usr/bin:/bin", "EVENTS": str(events), "GUARD_STATE": state}) assert result.returncode == 0 rows = events.read_text().splitlines() assert rows[0] == "controller:--state" legacy_writes = [row for row in rows if row.startswith("docker:update") or row.startswith("enforce:legacy-") or row.startswith("enforce:private-dind")] assert bool(legacy_writes) is (state == "legacy") assert "enforce:deploy-runner" in rows if kind == "host": assert {"enforce:tdarr", "enforce:controller", "enforce:containment-pressure"}.issubset(rows) @pytest.mark.parametrize("kind", ["runner", "host"]) def test_resource_writer_shape_drift_is_rejected(specialize, kind): source = _resource_guard_source(kind) anchor = ("# The sandbox is the aggregate resource envelope for all jobs and services.\n" if kind == "runner" else 'if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then\n') patch = (lambda text: specialize._patch_runner_guard(text, "controller --state", "controller --allow")) if kind == "runner" else ( lambda text: specialize._patch_host_guard(text, "controller --state")) with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_shape_invalid"): patch(source.replace(anchor, anchor + anchor, 1)) def test_historical_orphan_transform_does_not_acquire_new_resource_gate(specialize): source = _resource_guard_source("runner") historical = specialize._patch_runner_guard_historical(source, "controller --state", "controller --allow") patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow") writer = historical.index("# The sandbox is the aggregate resource envelope") assert 'specialization_state="$(controller --state' not in historical[:writer] assert 'specialization_state="$(controller --state' in patched[:patched.index("# The sandbox is the aggregate resource envelope")] def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest()) real_os = os class RootPosixOS: name = "posix" @staticmethod def geteuid(): return 0 def __getattr__(self, name): return getattr(real_os, name) monkeypatch.setattr(specialize, "os", RootPosixOS()) monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False}) shell_calls = [] monkeypatch.setattr(specialize, "_run_approved_shell_script", shell_calls.append) with pytest.raises(specialize.RunnerFleetSpecializeError, match="script_not_approved"): specialize.execute(script + b"# altered\n") assert shell_calls == [] def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_edits( specialize, tmp_path, monkeypatch ): runner = tmp_path / "runner-guard" host = tmp_path / "host-guard" runner.write_bytes( b"stack_recovery=0\n" b" set_managed_runners_disabled false\n" b"unexpected=0\n" b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' ) host.write_bytes( b"# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" b"old capacity writer\n" b"# ITWORX_RUNNER_CAPACITY_V2_END\n" b"runner_healthy=1\n" b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' b" recover legacy stack\n" b" fi\n" ) runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() host_sha = hashlib.sha256(host.read_bytes()).hexdigest() monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) monkeypatch.setattr(specialize, "RUNNER_GUARD_SOURCE_SHA256", runner_sha) monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", host_sha) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) backup = tmp_path / "guard-backup" backup.mkdir(mode=0o700) for path, name in ((runner, "gitea-runner-guard.sh"), (host, "host-resource-guard.sh")): target = backup / name target.write_bytes(path.read_bytes()) target.chmod(0o600) journal = { "backupDirectory": str(backup), "guardFiles": [ {"path": str(runner), "originalSha256": runner_sha, "installedSha256": None}, {"path": str(host), "originalSha256": host_sha, "installedSha256": None}, ] } specialize._service_guard_scripts(journal) installed_runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() assert journal["guardFiles"][0]["installedSha256"] == installed_runner_sha assert "--fence-legacy" in runner.read_text(encoding="utf-8") assert "# ITWORX_RUNNER_CAPACITY_V2_BEGIN" not in host.read_text(encoding="utf-8") host.write_text("operator-owned guard revision\n", encoding="utf-8") with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_source_changed"): specialize._service_guard_scripts(journal) assert host.read_text(encoding="utf-8") == "operator-owned guard revision\n" @pytest.fixture def guard_transaction(specialize, tmp_path, monkeypatch): monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) runner = tmp_path / "runner-guard" host = tmp_path / "host-guard" originals = ( (b'#!/bin/bash\nstack_recovery=0\nset_managed_runners_disabled false\nunexpected=0\n' b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n'), (b'#!/bin/bash\n# ITWORX_RUNNER_CAPACITY_V2_BEGIN\nold capacity writer\n' b'# ITWORX_RUNNER_CAPACITY_V2_END\nrunner_healthy=1\n' b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' b' recover legacy stack\n fi\n'), ) backup = tmp_path / "backups" backup.mkdir(mode=0o700) for path, original, name in zip((runner, host), originals, ("gitea-runner-guard.sh", "host-resource-guard.sh"), strict=True): path.write_bytes(original) target = backup / name target.write_bytes(original) target.chmod(0o600) monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) for name, original in zip(("RUNNER_GUARD_SOURCE_SHA256", "HOST_GUARD_SOURCE_SHA256"), originals, strict=True): monkeypatch.setattr(specialize, name, hashlib.sha256(original).hexdigest()) journal = {"backupDirectory": str(backup), "guardFiles": [ {"path": str(path), "originalSha256": hashlib.sha256(original).hexdigest(), "installedSha256": None} for path, original in zip((runner, host), originals, strict=True) ]} monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) monkeypatch.setattr(specialize, "_set_stage", lambda _journal, _stage: None) monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None) return journal, runner, host, originals @pytest.mark.parametrize("invalid", [0, 1]) def test_both_candidate_syntax_checks_precede_any_intent_or_guard_write(specialize, guard_transaction, monkeypatch, invalid): journal, runner, host, originals = guard_transaction checked = [] def syntax(encoded): checked.append(encoded) if len(checked) == invalid + 1: raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_guard_syntax_invalid") monkeypatch.setattr(specialize, "_validate_guard_syntax", syntax) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: pytest.fail("invalid candidate published intent")) monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("invalid candidate installed")) with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_syntax_invalid"): specialize._service_guard_scripts(journal) assert (runner.read_bytes(), host.read_bytes()) == originals assert all(row["installedSha256"] is None for row in journal["guardFiles"]) @pytest.mark.parametrize("boundary", ["intent_failed", "after_intent", "after_runner", "after_host"]) def test_guard_installation_durable_crash_boundaries_restore_only_owned_bytes(specialize, guard_transaction, monkeypatch, boundary): journal, runner, host, originals = guard_transaction persisted = [] real_write = specialize._atomic_write_owned writes = [] def journal_write(value): assert all(row["installedSha256"] for row in value["guardFiles"]) assert (runner.read_bytes(), host.read_bytes()) == originals if boundary == "intent_failed": raise OSError("private canary write failure") persisted.append(copy.deepcopy(value)) if boundary == "after_intent": raise OSError("private canary crash after intent") def install(path, encoded, **kwargs): assert persisted and persisted[0]["guardFiles"] == journal["guardFiles"] real_write(path, encoded, **kwargs) writes.append(path) if boundary == "after_runner" or boundary == "after_host" and len(writes) == 2: raise OSError("private canary crash after replace") monkeypatch.setattr(specialize, "_write_journal", journal_write) monkeypatch.setattr(specialize, "_atomic_write_owned", install) with pytest.raises(OSError): specialize._service_guard_scripts(journal) monkeypatch.setattr(specialize, "_atomic_write_owned", real_write) specialize._restore_guard_scripts(persisted[0] if persisted else journal) assert (runner.read_bytes(), host.read_bytes()) == originals specialize._restore_guard_scripts(journal) assert (runner.read_bytes(), host.read_bytes()) == originals @pytest.mark.parametrize("mutation", ["duplicate", "foreign_path", "original_hash", "planned_hash", "backup", "foreign_edit"]) def test_guard_restore_rejects_foreign_journal_backup_or_live_bytes_before_any_write(specialize, guard_transaction, monkeypatch, mutation): journal, runner, host, _originals = guard_transaction specialize._service_guard_scripts(journal) if mutation == "duplicate": journal["guardFiles"][1] = copy.deepcopy(journal["guardFiles"][0]) elif mutation == "foreign_path": journal["guardFiles"][1]["path"] = str(host.with_name("foreign")) elif mutation == "original_hash": journal["guardFiles"][0]["originalSha256"] = "f" * 64 elif mutation == "planned_hash": journal["guardFiles"][0]["installedSha256"] = "f" * 64 elif mutation == "backup": (Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign private canary") else: host.write_bytes(b"foreign private canary") before = runner.read_bytes(), host.read_bytes() monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("foreign state restored")) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._restore_guard_scripts(journal) assert (runner.read_bytes(), host.read_bytes()) == before def test_guard_syntax_checker_never_exposes_bash_output(specialize, monkeypatch): def parse(args, **kwargs): assert args == ["bash", "-n"] and kwargs["input"] == b"candidate" assert kwargs["timeout"] == 15 and kwargs["capture_output"] is True return SimpleNamespace(returncode=2, stdout=b"private canary", stderr=b"private canary") monkeypatch.setattr(specialize.subprocess, "run", parse) with pytest.raises(specialize.RunnerFleetSpecializeError, match="^runner_fleet_specialization_guard_syntax_invalid$"): specialize._validate_guard_syntax(b"candidate") @pytest.mark.parametrize("mutation", [None, "foreign", "begin_only", "end_only", "repeated"]) def test_host_guard_without_capacity_stanza_requires_exact_approved_bytes(specialize, guard_transaction, monkeypatch, mutation): _journal, _runner, _host, originals = guard_transaction text = originals[1].decode() start = text.index("# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n") end = text.index("# ITWORX_RUNNER_CAPACITY_V2_END\n") + len("# ITWORX_RUNNER_CAPACITY_V2_END\n") approved = text[:start] + text[end:] monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", hashlib.sha256(approved.encode()).hexdigest()) if mutation == "foreign": approved += "# operator edit\n" elif mutation == "begin_only": approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" elif mutation == "end_only": approved += "# ITWORX_RUNNER_CAPACITY_V2_END\n" elif mutation == "repeated": approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n# ITWORX_RUNNER_CAPACITY_V2_END\n" * 2 if mutation is None: result = specialize._patch_host_guard(approved, "fixed --state") assert 'specialization_state="$(fixed --state' in result assert '"$specialization_state" != "legacy"' in result else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="capacity_block_invalid"): specialize._patch_host_guard(approved, "fixed --state") @pytest.mark.parametrize("mutation", ["path", "backup", "foreign_edit"]) def test_guard_rollback_refusal_preserves_marker_journal_before_lifecycle_effects(specialize, guard_transaction, monkeypatch, mutation): journal, runner, host, _originals = guard_transaction specialize._service_guard_scripts(journal) live_compose = runner.with_name("compose") live_compose.write_bytes(b"fixed compose") monkeypatch.setattr(specialize, "COMPOSE_PATH", live_compose) journal["baseComposeSha256"] = hashlib.sha256(live_compose.read_bytes()).hexdigest() monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: pytest.fail("refusal reached lifecycle effects")) marker, durable = runner.with_name("marker"), runner.with_name("journal") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) if mutation == "path": journal["guardFiles"][0]["path"] = str(runner.with_name("foreign")) elif mutation == "backup": (Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign canary") elif mutation == "foreign_edit": host.write_bytes(b"foreign canary") assert specialize._rollback_before_activation(journal) is False assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" @pytest.mark.parametrize("failure", ["cleanup", "restore_cas", None]) def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, guard_transaction, monkeypatch, failure): journal, runner, host, originals = guard_transaction specialize._service_guard_scripts(journal) fenced = runner.read_bytes(), host.read_bytes() compose, marker, durable = runner.with_name("compose"), runner.with_name("marker"), runner.with_name("journal") compose.write_bytes(b"fixed compose") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(), originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None}, originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupFamilyInitiallyPresent=False, cgroupRootSubtree=[]) monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose") monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) monkeypatch.setattr(specialize, "_inspect", lambda _name: None) events = [] def cleanup(_journal): assert (runner.read_bytes(), host.read_bytes()) == fenced events.append("cleanup") if failure == "cleanup": raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_command_failed") monkeypatch.setattr(specialize, "_cleanup_new_containers", cleanup) def cgroup(**_kwargs): assert (runner.read_bytes(), host.read_bytes()) == fenced events.append("cgroup") return {"familyExists": False} monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) write = specialize._atomic_write_owned def restore(*args, **kwargs): assert events == ["cleanup", "cgroup"] assert marker.exists() and durable.exists() if failure == "restore_cas": raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed") write(*args, **kwargs) monkeypatch.setattr(specialize, "_atomic_write_owned", restore) assert specialize._rollback_before_activation(journal) is (failure is None) if failure: assert (runner.read_bytes(), host.read_bytes()) == fenced assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" else: assert (runner.read_bytes(), host.read_bytes()) == originals assert not marker.exists() and not durable.exists() @pytest.fixture def rollback_empty_family(specialize, tmp_path, monkeypatch): compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal" compose.write_bytes(b"fixed compose") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") for name, value in (("COMPOSE_PATH", compose), ("SPECIALIZED_STATE_PATH", marker), ("MIGRATION_JOURNAL_PATH", durable)): monkeypatch.setattr(specialize, name, value) journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, "originalContainers": [], "originalRegistrations": [], "guardFiles": [], "cgroupCreated": True, "cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": ["cpu", "cpuset", "hugetlb", "io", "memory", "pids"]} family = {"exists": True, "groupCount": 1, "hasProcesses": False, "familyCpuMax": "max 100000", "familyMemoryMax": "max", "familySubtreeControllers": list(journal["cgroupRootSubtree"]), "rootSubtreeControllers": list(journal["cgroupRootSubtree"])} rebound = {"created": False, "rootSubtreeBefore": list(journal["cgroupRootSubtree"])} events = [] @contextmanager def maintenance(): yield lambda: events.append("barrier") monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance) monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: None) monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose") monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) monkeypatch.setattr(specialize, "_inspect", lambda _name: None) monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: events.append("snapshot") or family) def ensure(**kwargs): assert kwargs == {"allow_existing": True} events.append("ensure") return rebound def rollback(**kwargs): assert kwargs == {"created": True, "initial_controllers": tuple(journal["cgroupRootSubtree"])} events.append("rollback") return {"removed": True, "familyExists": False} monkeypatch.setattr(specialize, "_ensure_cgroup_family_details", ensure) monkeypatch.setattr(specialize, "rollback_host_cgroup_family", rollback) return journal, family, rebound, events, marker, durable @pytest.mark.parametrize("pending", [False, True]) @pytest.mark.parametrize("cpu", ["max", "max 100000"]) def test_rollback_rebinds_only_owned_empty_default_family_before_strict_removal( specialize, rollback_empty_family, pending, cpu, ): journal, family, _rebound, events, marker, durable = rollback_empty_family journal.update(cgroupCreated=not pending, cgroupEnsurePending=pending) family["familyCpuMax"] = cpu original = copy.deepcopy(journal) assert specialize._rollback_before_activation(journal) is True assert journal == original snapshot = events.index("snapshot") assert events[snapshot:snapshot + 6] == ["snapshot", "barrier", "ensure", "barrier", "barrier", "rollback"] assert not marker.exists() and not durable.exists() @pytest.mark.parametrize("absent", [False, True]) def test_rollback_never_ensures_absent_or_already_bounded_family(specialize, rollback_empty_family, absent): journal, family, _rebound, events, _marker, _durable = rollback_empty_family if absent: family.clear() family["exists"] = False else: family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) assert specialize._rollback_before_activation(journal) is True assert "ensure" not in events and "rollback" in events @pytest.mark.parametrize("mutation", ["initially_present", "initially_unknown", "foreign_cpu", "foreign_memory", "partial_default", "processes", "unknown_processes", "children", "bool_count", "newly_created", "root_changed"]) def test_empty_family_rebind_refusal_retains_marker_and_journal(specialize, rollback_empty_family, mutation): journal, family, rebound, events, marker, durable = rollback_empty_family if mutation == "initially_present": journal["cgroupFamilyInitiallyPresent"] = True elif mutation == "initially_unknown": journal.pop("cgroupFamilyInitiallyPresent") elif mutation == "foreign_cpu": family["familyCpuMax"] = "400000 100000" elif mutation == "foreign_memory": family["familyMemoryMax"] = str(8 * 1024**3) elif mutation == "partial_default": family["familyCpuMax"] = "800000 100000" elif mutation == "processes": family["hasProcesses"] = True elif mutation == "unknown_processes": family.pop("hasProcesses") elif mutation == "children": family["groupCount"] = 2 elif mutation == "bool_count": family["groupCount"] = True elif mutation == "newly_created": rebound["created"] = True else: rebound["rootSubtreeBefore"] = ["cpu", "memory", "pids"] assert specialize._rollback_before_activation(journal) is False assert "rollback" not in events assert ("ensure" in events) is (mutation in {"newly_created", "root_changed"}) assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" def test_rebind_barrier_loss_prevents_cgroup_removal_and_keeps_durable_state( specialize, rollback_empty_family, monkeypatch, ): journal, _family, _rebound, events, marker, durable = rollback_empty_family @contextmanager def maintenance(): def barrier(): if events and events[-1] == "ensure": raise RuntimeError("maintenance identity changed") events.append("barrier") yield barrier monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance) assert specialize._rollback_before_activation(journal) is False assert "ensure" in events and "rollback" not in events assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" @pytest.mark.parametrize("bounded", [False, True]) def test_family_rollback_accepts_only_required_controllers_for_minimal_initial_root( specialize, rollback_empty_family, bounded, ): journal, family, rebound, events, _marker, _durable = rollback_empty_family initial = ["cpu", "memory", "pids"] journal["cgroupRootSubtree"] = list(initial) family["rootSubtreeControllers"] = list(initial) family["familySubtreeControllers"] = list(initial) rebound["rootSubtreeBefore"] = list(initial) if bounded: family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) assert specialize._rollback_before_activation(journal) is True assert ("ensure" in events) is not bounded assert "rollback" in events @pytest.mark.parametrize("bounded", [False, True]) @pytest.mark.parametrize("mutation", ["missing", "null", "string", "duplicate", "unknown", "nonstring", "required_missing", "unowned_extra"]) def test_family_subtree_drift_refuses_before_ensure_or_removal( specialize, rollback_empty_family, bounded, mutation, ): journal, family, _rebound, events, marker, durable = rollback_empty_family if bounded: family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) if mutation == "missing": family.pop("familySubtreeControllers") elif mutation == "null": family["familySubtreeControllers"] = None elif mutation == "string": family["familySubtreeControllers"] = "cpu memory pids" elif mutation == "duplicate": family["familySubtreeControllers"].append("cpu") elif mutation == "unknown": family["familySubtreeControllers"].append("foreign-controller") elif mutation == "nonstring": family["familySubtreeControllers"].append(["cpu"]) elif mutation == "required_missing": family["familySubtreeControllers"].remove("pids") else: journal["cgroupRootSubtree"] = ["cpu", "memory", "pids"] assert specialize._rollback_before_activation(journal) is False assert "ensure" not in events and "rollback" not in events assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch): marker, journal = tmp_path / "marker", tmp_path / "journal" marker.write_bytes(b"owned marker") journal.write_bytes(b"owned journal") @contextmanager def busy(): raise RuntimeError("private canary lock refusal") yield monkeypatch.setattr(specialize, "_rollback_maintenance", busy) monkeypatch.setattr(specialize, "_rollback_before_activation_locked", lambda *_args: pytest.fail("busy barrier reached effects")) assert specialize._rollback_before_activation({}) is False assert marker.read_bytes() == b"owned marker" and journal.read_bytes() == b"owned journal" def test_rollback_barrier_drift_refuses_before_guards_or_cleanup(specialize, guard_transaction, monkeypatch): journal, _runner, _host, _originals = guard_transaction @contextmanager def changed(): def refuse(): raise RuntimeError("private canary changed inode") yield refuse monkeypatch.setattr(specialize, "_rollback_maintenance", changed) monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: pytest.fail("drift restored guard")) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda *_args: pytest.fail("drift changed fleet")) assert specialize._rollback_before_activation(journal) is False @pytest.mark.skipif(os.name != "posix", reason="real common flock contract requires POSIX") def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_finishes(specialize, tmp_path, monkeypatch): if os.geteuid() != 0: pytest.skip("protected maintenance lock requires root") import runner_acceptance_quiescence as quiescence lock = tmp_path / "maintenance.lock" lock.touch(mode=0o600) monkeypatch.setattr(quiescence, "MAINTENANCE_LOCK", lock) monkeypatch.setattr(quiescence, "_LOCK_PROBE", quiescence._LOCK_PROBE.replace( "/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock", str(lock))) monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=dict)) probe = ( "import fcntl,os,sys\nfd=os.open(sys.argv[1],os.O_RDWR|os.O_NOFOLLOW)\n" "try: fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)\n" "except BlockingIOError: sys.exit(0)\n" "sys.exit(1)\n" ) events = [] def held(phase): result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) assert result.returncode == 0 and not result.stdout and not result.stderr events.append(phase) compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal" compose.write_bytes(b"captured compose") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_guard_script_plan", lambda _journal: []) monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: held("guards-restored")) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: held("cleanup")) monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"captured compose") monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) monkeypatch.setattr(specialize, "_inspect", lambda _name: None) def cgroup(**_kwargs): held("cgroup") assert marker.exists() and durable.exists() return {"familyExists": False} monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}], "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, "originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": []} monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) assert specialize._rollback_before_activation(journal) is True assert events == ["cleanup", "cgroup", "guards-restored"] assert not marker.exists() and not durable.exists() # The independent participant can acquire only after the wrapper returns. result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) assert result.returncode == 1 and not result.stdout and not result.stderr @pytest.fixture def owned_orphan(specialize, guard_transaction, tmp_path, monkeypatch): import runner_acceptance_quiescence as quiescence journal, runner, host, originals = guard_transaction corrected = specialize._guard_script_plan(journal)[0][3] closing = f'\nfi\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode() orphan = corrected.replace(closing, f'\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode(), 1) runner.write_bytes(orphan) monkeypatch.setattr(specialize, "_OWNED_MALFORMED_RUNNER_GUARD_SHA256", hashlib.sha256(orphan).hexdigest()) receipt = { "schema": quiescence.SCHEMA, "revision": "0ea6f267c28d51ac7113e6d943c89857d3a189ab", "manifestDigest": quiescence._digest(quiescence.OBSERVATION), "phase": "complete", "progress": {row["id"]: "quiesced" for row in quiescence._fixed_rows()}, "registrationProgress": {"9": "restored", "22": "restored"}, "guardProgress": {"runner": "restored", "host": "restored"}, "guards": {name: { "original": base64.b64encode(original).decode(), "originalSha256": hashlib.sha256(original).hexdigest(), "wrappedSha256": hashlib.sha256(quiescence.ControllerDriver._wrapped_guard(original)).hexdigest(), "mode": path.stat().st_mode & 0o777, } for name, path, original in zip(("runner", "host"), (runner, host), originals, strict=True)}, "baseline": {"rows": [{"state": {"Running": False}} for _row in quiescence._fixed_rows()]}, } events = [] for name in ("SPECIALIZED_STATE_PATH", "MIGRATION_JOURNAL_PATH"): monkeypatch.setattr(specialize, name, tmp_path / name) monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: False) monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) monkeypatch.setattr(specialize, "_read_fleet_base_compose", lambda: (b"source", None)) monkeypatch.setattr(specialize, "_validate_base_composition", lambda _source: "v1") monkeypatch.setattr(specialize, "_runner_rows", lambda: [{"id": 9}]) monkeypatch.setattr(specialize, "_validate_legacy_registrations", lambda *_args: [{"id": 9}]) legacy_item = {"Id": "a" * 64, "Image": "sha256:" + "b" * 64, "Created": "fixed", "Name": "/fixed", "Path": "/fixed-entrypoint", "Args": [], "Config": {"Labels": {}}, "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, "Mounts": [], "NetworkSettings": {"Networks": {}}, "RestartCount": 0, "State": {"Running": True, "Paused": False, "Restarting": False, "Dead": False, "Pid": 123, "StartedAt": "fixed", "FinishedAt": "zero", "ExitCode": 0}} monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: ({"fixed": copy.deepcopy(legacy_item)}, [])) monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=lambda: {"fixed": "registry"})) monkeypatch.setattr(quiescence, "_read_receipt", lambda: copy.deepcopy(receipt)) monkeypatch.setattr(quiescence, "_idle", lambda *_args: events.append("idle")) def attest(snapshot, *, baseline=None, progress=None): if baseline is None: assert snapshot == receipt["baseline"] events.append("original-attested") return assert baseline == receipt["baseline"] and progress == receipt["progress"] assert snapshot["rows"] == baseline["rows"] events.append("attested") monkeypatch.setattr(quiescence, "attest_snapshot", attest) class Driver: _wrapped_guard = staticmethod(quiescence.ControllerDriver._wrapped_guard) def __init__(self, projects): assert projects == {"fixed": "registry"} @contextmanager def maintenance(self): with (tmp_path / "fixture-lock").open("wb") as lock: self._maintenance_fd = lock.fileno() events.append("locked") yield events.append("unlocked") def assert_barrier(self): events.append("barrier") def snapshot(self): self._guard_binding() events.append("snapshot") return copy.deepcopy(receipt["baseline"]) monkeypatch.setattr(quiescence, "ControllerDriver", Driver) return quiescence, receipt, runner, host, originals, orphan, events, Driver def test_owned_orphan_recovery_uses_full_repeated_admission_and_cas_then_is_idempotent(specialize, owned_orphan, monkeypatch): _q, receipt, runner, host, originals, orphan, events, _driver = owned_orphan before = copy.deepcopy(receipt) write = specialize._atomic_write_owned def restore(path, data, **kwargs): assert events.count("attested") == 2 and events.count("idle") == 3 assert path == runner and data == originals[0] assert kwargs["expected_sha256"] == hashlib.sha256(orphan).hexdigest() events.append("restore") write(path, data, **kwargs) monkeypatch.setattr(specialize, "_atomic_write_owned", restore) specialize._recover_owned_orphan_guard() assert (runner.read_bytes(), host.read_bytes()) == originals and receipt == before assert events.index("locked") < events.index("restore") < events.index("unlocked") specialize._recover_owned_orphan_guard() assert events.count("restore") == 1 @pytest.mark.parametrize("mutation", ["revision", "manifest", "phase", "progress", "registration", "guard_progress", "original", "wrapped", "foreign_runner", "foreign_host", "marker", "journal", "storage", "family", "idle", "lock", "origin", "changed_receipt", "container", "cas"]) def test_orphan_recovery_refuses_unowned_or_changed_state_without_write(specialize, owned_orphan, monkeypatch, mutation): q, receipt, runner, host, _originals, _orphan, _events, driver = owned_orphan if mutation in {"revision", "manifest", "phase"}: receipt[{"revision": "revision", "manifest": "manifestDigest", "phase": "phase"}[mutation]] = "foreign" elif mutation == "progress": receipt["progress"].pop(next(iter(receipt["progress"]))) elif mutation == "registration": receipt["registrationProgress"]["9"] = "disabled" elif mutation == "guard_progress": receipt["guardProgress"]["host"] = "installed" elif mutation in {"original", "wrapped"}: receipt["guards"]["runner"]["originalSha256" if mutation == "original" else "wrappedSha256"] = "f" * 64 elif mutation in {"foreign_runner", "foreign_host"}: (runner if mutation == "foreign_runner" else host).write_bytes(b"private foreign canary") elif mutation in {"marker", "journal"}: (specialize.SPECIALIZED_STATE_PATH if mutation == "marker" else specialize.MIGRATION_JOURNAL_PATH).touch() elif mutation == "storage": monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: True) elif mutation == "family": monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": True}) elif mutation == "idle": monkeypatch.setattr(q, "_idle", lambda *_args: (_ for _ in ()).throw(q.QuiescenceError("global_ci_work_present"))) elif mutation == "lock": monkeypatch.setattr(driver, "assert_barrier", lambda _self: (_ for _ in ()).throw(q.QuiescenceError("dispatch_maintenance_busy"))) elif mutation == "origin": monkeypatch.setattr(q, "attest_snapshot", lambda *_args, **_kwargs: (_ for _ in ()).throw(q.QuiescenceError("origin_changed"))) elif mutation == "changed_receipt": reads = [] def changed(): reads.append(True) result = copy.deepcopy(receipt) if len(reads) > 1: result["revision"] = "foreign" return result monkeypatch.setattr(q, "_read_receipt", changed) elif mutation == "container": reads = [] original = specialize._validate_legacy_containers def changing(*args): reads.append(True) items, summaries = original(*args) items["fixed"]["Id"] = str(len(reads)) return items, summaries monkeypatch.setattr(specialize, "_validate_legacy_containers", changing) else: monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: (_ for _ in ()).throw( specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed"))) before = runner.read_bytes(), host.read_bytes() if mutation != "cas": monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("unowned orphan restored")) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._recover_owned_orphan_guard() assert (runner.read_bytes(), host.read_bytes()) == before @pytest.fixture def completed_orphan_engine(specialize): import runner_acceptance_quiescence as q rows = copy.deepcopy(q._fixed_rows()) for row in rows: row["ownerLabelDigest"] = "b" * 64 if row["ownerLabelPresent"] else None row["labelDigest"] = q._digest(row["composeLabels"]) origins = {} for project, claim in q.ORIGINS.items(): attempt = claim["attempt"] if claim["attempt"] is not None else 0 origins[project] = {"repository": "jens/workmesh", "run": claim["run"], "head": claim["head"], "status": "completed", "conclusion": "failure", "attempt": attempt, "task": claim["task"], "taskHead": claim["head"], "taskAttempt": attempt, "jobsTerminal": True} project = next(iter(q.ORIGINS)) networks = [{"Id": f"{index + 1:064x}", "Name": f"{project}_net{index}", "Created": "fixed", "Driver": "bridge", "Containers": {}, "Labels": {"com.docker.compose.project": project}} for index in range(8)] volumes = [{"Name": f"{project}_vol{index}", "CreatedAt": "fixed", "Driver": "local", "Options": None, "Labels": {"com.docker.compose.project": project}, "Mountpoint": f"{q.ROOTLESS_STORAGE}/volumes/{project}_vol{index}/_data"} for index in range(10)] resources = {"networks": [{"id": item["Id"], "name": item["Name"], "created": item["Created"], "project": project, "driver": "bridge", "ownerDigest": q._digest(None)} for item in networks], "volumes": [{"name": item["Name"], "created": item["CreatedAt"], "project": project, "driver": "local", "ownerDigest": q._digest(None)} for item in volumes], "attachmentsVerified": True, "ownershipVerified": True} baseline = {"rows": rows, "origins": origins, "resources": resources, "daemon": {"legacyAttested": True, "engineId": "fixed-engine", "imageId": "sha256:" + "a" * 64, "outerId": "a" * 64, "created": "original", "labelsDigest": "a" * 64, "dockerRootDir": q.ROOTLESS_STORAGE, "rootless": True, "mounts": [], "networks": {}}, "productBinding": {"exact": "production"}, "guardBinding": {"exact": "guards"}, "productExclusionVerified": True, "dispatchBarrierVerified": True} snapshot = copy.deepcopy(baseline) snapshot["rows"] = [] snapshot["resources"] = {"networks": [], "volumes": [], "attachmentsVerified": True, "ownershipVerified": True} snapshot["daemon"].update(outerId="c" * 64, created="recreated", labelsDigest="c" * 64) class Driver: def __init__(self): self.ps = "" self.effects = [] self.inspect_mutation = None def assert_barrier(self): return None def _run(self, args, **kwargs): assert args[:7] == ["docker", "exec", "-e", "DOCKER_HOST=tcp://127.0.0.1:2375", specialize.DIND_NAME, "docker", args[6]] command = args[6:] if command == ["ps", "-a", "--no-trunc", "-q"]: return self.ps kind = command[0] assert command in (["network", "ls", "--quiet", "--no-trunc"], ["volume", "ls", "--quiet"]) return "\n".join(item["Id" if kind == "network" else "Name"] for item in (networks if kind == "network" else volumes)) def _docker(self, kind, action, *ids): assert action == "inspect" result = [copy.deepcopy(item) for item in (networks if kind == "network" else volumes) if item["Id" if kind == "network" else "Name"] in ids] return self.inspect_mutation(result) if self.inspect_mutation else result return q, baseline, snapshot, networks, volumes, Driver def test_completed_orphan_admission_requires_empty_whole_engine_and_preserved_resources(specialize, completed_orphan_engine): q, baseline, snapshot, networks, volumes, driver_type = completed_orphan_engine before = copy.deepcopy(baseline) # Include unrelated resources: complete inventories cannot assume an empty # resource store when the container engine is empty. networks.append({"Id": "f" * 64, "Name": "unrelated", "Labels": {}}) for index in range(634): volumes.append({"Name": f"unrelated{index}", "Labels": {}}) driver = driver_type() specialize._attest_completed_orphan_engine(driver, snapshot, {"baseline": baseline}, q) assert baseline == before and driver.effects == [] and len(volumes) == 644 with pytest.raises(q.QuiescenceError, match="project_inventory_changed"): q.attest_snapshot(snapshot, baseline=baseline) @pytest.mark.parametrize("field", ["engineId", "imageId", "dockerRootDir", "rootless", "legacyAttested", "mounts", "networks"]) def test_completed_orphan_refuses_stable_daemon_changes(specialize, completed_orphan_engine, field): q, baseline, snapshot, *_rest = completed_orphan_engine snapshot["daemon"][field] = "foreign" with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._attest_completed_orphan_snapshot(snapshot, baseline, q) @pytest.mark.parametrize("field", ["origins", "productBinding", "guardBinding", "productExclusionVerified", "dispatchBarrierVerified", "rows", "resources"]) def test_completed_orphan_refuses_changed_proof(specialize, completed_orphan_engine, field): q, baseline, snapshot, *_rest = completed_orphan_engine snapshot[field] = [{"foreign": True}] if field == "rows" else {} with pytest.raises((specialize.RunnerFleetSpecializeError, q.QuiescenceError)): specialize._attest_completed_orphan_snapshot(snapshot, baseline, q) @pytest.mark.parametrize("mutation", ["network_id", "network_name", "network_created", "attachment", "volume_name", "volume_created", "mountpoint", "options", "driver", "owner", "project", "missing", "extra_origin", "lost_labels"]) def test_completed_orphan_refuses_resource_drift(specialize, completed_orphan_engine, mutation): q, baseline, _snapshot, networks, volumes, _driver = completed_orphan_engine if mutation == "attachment": networks[0]["Containers"] = {"a" * 64: {}} elif mutation == "missing": volumes.pop() elif mutation == "extra_origin": extra = copy.deepcopy(volumes[0]) extra["Name"] += "extra" volumes.append(extra) elif mutation == "lost_labels": networks[0]["Labels"] = {} elif mutation in {"owner", "project"}: volumes[0]["Labels"][q.OWNER_LABEL if mutation == "owner" else "com.docker.compose.project"] = "foreign" else: item, key = {"network_id": (networks[0], "Id"), "network_name": (networks[0], "Name"), "network_created": (networks[0], "Created"), "volume_name": (volumes[0], "Name"), "volume_created": (volumes[0], "CreatedAt"), "mountpoint": (volumes[0], "Mountpoint"), "options": (volumes[0], "Options"), "driver": (volumes[0], "Driver")}[mutation] item[key] = "foreign" with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._orphan_resource_projection(networks, volumes, baseline, q) @pytest.mark.parametrize("mutation", ["foreign_container", "omitted_inspect", "duplicate_inspect", "duplicate_inventory", "oversize_inventory", "new_container_during_inspect"]) def test_completed_orphan_rejects_incomplete_or_racing_whole_inventory(specialize, completed_orphan_engine, mutation): q, baseline, snapshot, networks, volumes, driver_type = completed_orphan_engine driver = driver_type() if mutation == "foreign_container": driver.ps = "f" * 64 elif mutation == "omitted_inspect": driver.inspect_mutation = lambda items: items[:-1] elif mutation == "duplicate_inspect": driver.inspect_mutation = lambda items: items[:1] * len(items) elif mutation == "duplicate_inventory": networks.append(copy.deepcopy(networks[0])) elif mutation == "oversize_inventory": volumes.extend({"Name": f"unrelated{index}"} for index in range(991)) else: def race(items): driver.ps = "f" * 64 return items driver.inspect_mutation = race with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._attest_completed_orphan_engine(driver, snapshot, {"baseline": baseline}, q) assert driver.effects == [] @pytest.mark.parametrize("change", [None, "origin", "resource", "container", "idle"]) def test_empty_engine_owned_orphan_recovery_rechecks_every_admission_before_cas( specialize, owned_orphan, completed_orphan_engine, monkeypatch, change): q, receipt, runner, host, originals, orphan, events, driver_type = owned_orphan _q, baseline, snapshot, networks, _volumes, engine_type = completed_orphan_engine receipt["baseline"] = copy.deepcopy(baseline) engine = engine_type() snapshots = [] def current(self): self._guard_binding() snapshots.append(True) if len(snapshots) == 2: if change == "origin": snapshot["origins"][next(iter(q.ORIGINS))]["head"] = "f" * 40 elif change == "resource": networks[0]["Created"] = "replaced" elif change == "container": engine.ps = "f" * 64 return copy.deepcopy(snapshot) monkeypatch.setattr(driver_type, "snapshot", current) monkeypatch.setattr(driver_type, "_run", lambda self, *args, **kwargs: engine._run(*args, **kwargs), raising=False) monkeypatch.setattr(driver_type, "_docker", lambda self, *args: engine._docker(*args), raising=False) idle = q._idle def recheck_idle(*args): if change == "idle" and events.count("idle") == 2: raise q.QuiescenceError("global_ci_work_present") idle(*args) monkeypatch.setattr(q, "_idle", recheck_idle) write = specialize._atomic_write_owned writes = [] def cas(*args, **kwargs): assert change is None and len(snapshots) == 2 and events.count("idle") == 3 assert events.count("original-attested") == 2 writes.append(True) return write(*args, **kwargs) monkeypatch.setattr(specialize, "_atomic_write_owned", cas) before = copy.deepcopy(receipt) if change is None: specialize._recover_owned_orphan_guard() assert runner.read_bytes() == originals[0] and writes == [True] specialize._recover_owned_orphan_guard() assert writes == [True] else: with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._recover_owned_orphan_guard() assert runner.read_bytes() == orphan and writes == [] assert receipt == before and host.read_bytes() == originals[1] and engine.effects == [] def test_orphan_legacy_identity_ignores_only_health_observations(specialize, owned_orphan): q, *_rest = owned_orphan containers = specialize._validate_legacy_containers("v1") containers[0]["fixed"]["State"]["Health"] = {"Status": "healthy", "FailingStreak": 0, "Log": []} before = specialize._orphan_legacy_identity(containers, q) containers[0]["fixed"]["State"]["Health"]["Log"] = [{"Start": "new", "End": "new"}] assert specialize._orphan_legacy_identity(containers, q) == before for field in ("Id", "Image", "Created", "Name", "Path", "Args", "Config", "HostConfig", "Mounts", "NetworkSettings", "RestartCount"): changed = copy.deepcopy(containers) if field == "Mounts": changed[0]["fixed"][field] = [{"Destination": "/foreign", "Type": "bind", "Source": "/foreign"}] elif field == "NetworkSettings": changed[0]["fixed"][field] = {"Networks": {"foreign": {"NetworkID": "f" * 64}}} else: changed[0]["fixed"][field] = "foreign" assert specialize._orphan_legacy_identity(changed, q) != before changed = copy.deepcopy(containers) changed[0]["fixed"]["State"]["Health"]["Status"] = "unhealthy" assert specialize._orphan_legacy_identity(changed, q) != before for field in ("Running", "Paused", "Restarting", "Dead", "Pid", "StartedAt", "FinishedAt", "ExitCode"): changed = copy.deepcopy(containers) changed[0]["fixed"]["State"][field] = "foreign" assert specialize._orphan_legacy_identity(changed, q) != before def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() digest = hashlib.sha256(script).hexdigest() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", digest) real_os = os class RootPosixOS: name = "posix" @staticmethod def geteuid(): return 0 def __getattr__(self, name): return getattr(real_os, name) monkeypatch.setattr(specialize, "os", RootPosixOS()) monkeypatch.setattr(specialize, "_migration_lock", nullcontext) monkeypatch.setattr(specialize, "specialization_state", lambda: "legacy") monkeypatch.setitem( sys.modules, "status_server", type("StatusServerStub", (), {"_require_controller_release": staticmethod(lambda: { "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": "a" * 40, "sourceTree": "b" * 40, })}), ) shell_calls: list[bytes] = [] state = {"active": False} def run_approved(received): shell_calls.append(received) state["active"] = True monkeypatch.setattr(specialize, "_run_approved_shell_script", run_approved) monkeypatch.setattr( specialize, "verified_topology", lambda: { "ready": state["active"], "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if state["active"] else [], "resourceEnvelope": {"ready": state["active"], "digest": "c" * 64}, }, ) result = specialize.execute(script) assert result["status"] == "specialized" assert result["verifiedRunners"] == list(specialize._SPECIALIZED_NAMES) assert shell_calls == [script] @pytest.fixture def profile_registration(specialize, monkeypatch): rows = [] commands = [] journals = [] state = {"check_host_paths": False, "missing_label": None} token = "private-registration-canary" monkeypatch.setattr(specialize, "_runner_rows", lambda: copy.deepcopy(rows)) monkeypatch.setattr(specialize, "_registration_token", lambda: token) def register(args, data, **kwargs): assert kwargs == {"timeout": 90, "max_output": 32 * 1024} assert data == (token + "\n").encode() and token not in " ".join(args) profile_id = next(key for key, value in specialize.RUNNER_PROFILES.items() if value.container_name == args[args.index("--name") + 1]) profile = specialize.RUNNER_PROFILES[profile_id] if state["check_host_paths"]: mounts = [args[index + 1] for index, argument in enumerate(args) if argument == "--volume"] assert mounts == [f"{specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)}:/data", f"{specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / (profile_id + '.yaml'))}:/runner-config.yaml:ro"] assert all("/controller-hub-releases/" not in mount.replace("\\", "/") for mount in mounts) assert args[-2] == profile.display_name assert args[-1] == specialize._profile_labels_for_registration(profile_id, "a" * 40) required = {profile.runner_label, *profile.legacy_labels} if state["missing_label"] == profile_id: required.remove(profile.runner_label) rows.append({"id": 100 + len(rows), "name": profile.container_name, "displayName": profile.display_name, "labels": sorted(required), "disabled": False, "busy": False}) commands.append(profile_id) return "registered" monkeypatch.setattr(specialize, "_run_with_input", register) def disable(identifier, disabled): assert disabled is True next(row for row in rows if row["id"] == identifier)["disabled"] = True monkeypatch.setattr(specialize, "_disable_registration", disable) def journal(value): for profile_id, identifier in value["newRegistrationIds"].items(): matching = next(row for row in rows if row["id"] == identifier) assert matching["disabled"] is True and matching["busy"] is False assert matching["name"] == specialize.RUNNER_PROFILES[profile_id].container_name journals.append(copy.deepcopy(value)) monkeypatch.setattr(specialize, "_write_journal", journal) return rows, commands, journals, state def test_new_profile_registration_uses_daemon_visible_volume_children(specialize, profile_registration): rows, commands, journals, state = profile_registration state["check_host_paths"] = True journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} specialize._register_profiles(journal) assert commands == list(specialize.PROFILE_IDS) assert len(rows) == 6 and set(journal["newRegistrationIds"]) == set(specialize.PROFILE_IDS) assert journals[-1]["stage"] == "registrations_created" @pytest.mark.parametrize("existing", [False, True]) def test_new_registration_label_verification_is_initialized_for_each_profile(specialize, profile_registration, existing): rows, commands, journals, _state = profile_registration journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} if existing: key = specialize.PROFILE_IDS[0] profile = specialize.RUNNER_PROFILES[key] rows.append({"id": 9, "name": profile.container_name, "displayName": profile.display_name, "labels": sorted({profile.runner_label, *profile.legacy_labels}), "disabled": True, "busy": False}) journal["newRegistrationIds"][key] = 9 specialize._register_profiles(journal) assert len(rows) == 6 and len(commands) == (5 if existing else 6) assert set(journal["newRegistrationIds"]) == set(specialize.PROFILE_IDS) assert journals[-1]["stage"] == "registrations_created" before = copy.deepcopy(journal) specialize._register_profiles(journal) assert journal == before and len(commands) == (5 if existing else 6) @pytest.mark.parametrize("profile_id", ["checks", "backend", "frontend", "security", "browser", "build"]) def test_new_profile_missing_own_required_label_is_not_recorded(specialize, profile_registration, profile_id): _rows, commands, journals, state = profile_registration state["missing_label"] = profile_id journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} with pytest.raises(specialize.RunnerFleetSpecializeError, match="registration_create_unverified"): specialize._register_profiles(journal) assert profile_id == commands[-1] and profile_id not in journal["newRegistrationIds"] assert all(profile_id not in value["newRegistrationIds"] for value in journals) @pytest.mark.parametrize("reason", ["command_failed", "command_unavailable"]) def test_registration_command_failure_is_finite_and_does_not_record_registration( specialize, profile_registration, monkeypatch, reason, ): def fail(*_args, **_kwargs): raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_" + reason) monkeypatch.setattr(specialize, "_run_with_input", fail) journal = {"sourceRevision": "a" * 40, "newRegistrationIds": {}} with pytest.raises(specialize.RunnerFleetSpecializeError) as caught: specialize._register_profiles(journal) assert str(caught.value) == "runner_fleet_specialization_registration_command_failed" assert str(caught.value) in specialize._PUBLIC_SPECIALIZATION_FAILURES assert caught.value.__suppress_context__ is True assert journal["newRegistrationIds"] == {} @pytest.fixture def cleanup_workspace(specialize, tmp_path, monkeypatch): workspace = tmp_path / "workspaces" profile = workspace / specialize.RUNNER_PROFILES["checks"].container_name nested = profile / "checkout" nested.mkdir(parents=True) (nested / "output").write_bytes(b"preserved-until-owned-cleanup") monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", workspace) owners = {profile: 1000, nested: 1000} original = Path.lstat def owned(path, *args, **kwargs): value = list(original(path, *args, **kwargs)) value[4] = owners.get(path, 0) return os.stat_result(value) monkeypatch.setattr(Path, "lstat", owned) return workspace, profile, nested, owners def test_owned_profile_workspace_cleanup_accepts_prepared_uid1000_directories(specialize, cleanup_workspace): workspace, _profile, _nested, _owners = cleanup_workspace specialize._remove_fixed_private_tree(workspace) assert not workspace.exists() @pytest.mark.parametrize("mutation", ["root_owner", "profile_owner", "nested_owner", "foreign_profile", "outside_workspace"]) def test_workspace_cleanup_keeps_exact_root_profile_and_owner_boundary(specialize, cleanup_workspace, mutation): workspace, profile, nested, owners = cleanup_workspace target = workspace if mutation == "root_owner": owners[workspace] = 1000 elif mutation == "profile_owner": owners[profile] = 0 elif mutation == "nested_owner": owners[nested] = 1001 elif mutation == "foreign_profile": foreign = workspace / "foreign-runner" profile.rename(foreign) owners[foreign] = 1000 else: target = workspace.parent / "runner-data" target.mkdir() child = target / "checks" child.mkdir() owners[child] = 1000 with pytest.raises(specialize.RunnerFleetSpecializeError, match="rollback_path_untrusted"): specialize._remove_fixed_private_tree(target) assert target.exists() @pytest.mark.skipif(os.name != "posix", reason="requires POSIX symlinks") def test_workspace_cleanup_refuses_symlink_without_touching_target(specialize, cleanup_workspace, tmp_path): workspace, _profile, nested, _owners = cleanup_workspace outside = tmp_path / "outside" outside.mkdir() preserved = outside / "preserved" preserved.write_bytes(b"foreign") (nested / "foreign-link").symlink_to(outside, target_is_directory=True) with pytest.raises(specialize.RunnerFleetSpecializeError, match="rollback_path_untrusted"): specialize._remove_fixed_private_tree(workspace) assert preserved.read_bytes() == b"foreign" @pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") def test_workspace_cleanup_removes_real_prepared_runner_owned_tree(specialize, tmp_path, monkeypatch): workspace = tmp_path / "workspaces" nested = workspace / specialize.RUNNER_PROFILES["checks"].container_name / "checkout" nested.mkdir(parents=True, mode=0o700) os.chown(nested.parent, 1000, 1000) os.chown(nested, 1000, 1000) monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", workspace) specialize._remove_fixed_private_tree(workspace) assert not workspace.exists() def test_run_with_input_keeps_registration_secret_out_of_command_arguments(specialize, monkeypatch): observed = {} def fake_run(args, **kwargs): observed["args"] = args observed["input"] = kwargs["input"] observed["capture_output"] = kwargs["capture_output"] return type("Result", (), {"returncode": 0, "stdout": b"registered"})() monkeypatch.setattr(specialize.subprocess, "run", fake_run) result = specialize._run_with_input(["docker", "run", "fixed"], b"sensitive-runner-token\n") assert result == "registered" assert observed["args"] == ["docker", "run", "fixed"] assert observed["input"] == b"sensitive-runner-token\n" assert "sensitive-runner-token" not in " ".join(observed["args"]) def test_build_probe_create_command_is_fixed_and_has_no_mount_or_shared_socket(specialize): record = _build_probe_record(specialize) args = specialize._build_probe_create_args(record) assert args[:8] == [ "create", "--name", record["name"], "--restart=no", "--cgroup-parent=docker", "--network=bridge", "--cgroupns=private", "--label", ] assert record["taskImageId"] in args assert "--cpus=4" in args assert "--memory=6144m" in args assert "--pids-limit=1024" in args assert "--blkio-weight=100" in args assert "--security-opt=seccomp=unconfined" in args assert "--security-opt=systempaths=unconfined" in args assert "--device=/dev/net/tun:/dev/net/tun:rwm" in args assert "--entrypoint" in args assert specialize._BUILD_PROBE_EXECUTABLE in args assert record["sourceRevision"] in args assert record["sourceTree"] in args assert not any( value == "--privileged" or value == "-e" or value.startswith(("--cap-add", "--volume", "--mount", "--env")) for value in args ) assert not any("docker.sock" in value or "host.docker.internal" in value for value in args) def test_build_probe_creation_inspect_requires_stopped_zero_pid_and_exact_inherited_labels( specialize, ): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["State"] = {"Running": False, "Pid": 0} assert specialize._validate_build_probe_container( item, record, image_environment=environment, require_running=False, require_created_stopped=True, ) labels = specialize._build_probe_container_labels(record) assert set(specialize.PINNED_TASK_BASE_INHERITED_LABELS).issubset(labels) assert record["imageLabels"].items() <= labels.items() assert item["Config"]["Labels"] == labels running, _ = _build_probe_inspect(specialize, record) assert not specialize._validate_build_probe_container( running, record, image_environment=environment, require_running=False, require_created_stopped=True, ) bad_pid = copy.deepcopy(item) bad_pid["State"]["Pid"] = 9 assert not specialize._validate_build_probe_container( bad_pid, record, image_environment=environment, require_running=False, require_created_stopped=True, ) foreign_label = copy.deepcopy(item) foreign_label["Config"]["Labels"]["untrusted.extra"] = "true" assert not specialize._validate_build_probe_container( foreign_label, record, image_environment=environment, require_running=False, require_created_stopped=True, ) def test_build_probe_container_inspect_accepts_only_fixed_private_build_job(specialize): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) assert specialize._validate_build_probe_container(item, record, image_environment=environment) for section, changes in ( ("HostConfig", {"Privileged": True}), ("HostConfig", {"PidMode": "host"}), ("HostConfig", {"NetworkMode": "host"}), ("HostConfig", {"CapAdd": ["SYS_ADMIN"]}), ("HostConfig", {"Devices": [ *item["HostConfig"]["Devices"], {"PathOnHost": "/dev/sda", "PathInContainer": "/dev/sda", "CgroupPermissions": "rwm"}, ]}), ("HostConfig", {"Mounts": [{"Type": "bind", "Source": "/host", "Target": "/host"}]}), ("HostConfig", {"SecurityOpt": ["seccomp=unconfined"], "MaskedPaths": ["/proc/kcore"]}), ("Config", {"Env": [*environment, "DOCKER_HOST=tcp://shared-dind:2375"]}), ("Config", {"Image": "localhost/itworx-ci-build:latest"}), ): changed, expected_environment = _build_probe_inspect(specialize, record) changed[section].update(changes) assert not specialize._validate_build_probe_container( changed, record, image_environment=expected_environment, ) @pytest.mark.parametrize("field,value", [ ("Type", "bind"), ("Name", "persistent-builder-cache"), ("Source", "/var/run/docker.sock"), ("Destination", "/host"), ("Driver", "nfs"), ("RW", False), ("Propagation", "rshared"), ]) def test_build_probe_rejects_foreign_or_shared_data_mount(specialize, field, value): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["Mounts"][0][field] = value assert not specialize._validate_build_probe_container(item, record, image_environment=environment) @pytest.mark.parametrize("security_options", [ ["seccomp=unconfined"], ["seccomp=unconfined", "systempaths=unconfined"], ]) def test_build_probe_accepts_only_verified_effective_unconfined_system_paths(specialize, security_options): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["HostConfig"]["SecurityOpt"] = security_options assert specialize._validate_build_probe_container(item, record, image_environment=environment) for field in ("MaskedPaths", "ReadonlyPaths"): for value in (None, ["/proc/kcore"], "", {}): changed = copy.deepcopy(item) changed["HostConfig"][field] = value assert not specialize._validate_build_probe_container(changed, record, image_environment=environment) missing = copy.deepcopy(item) del missing["HostConfig"][field] assert not specialize._validate_build_probe_container(missing, record, image_environment=environment) for options in (None, [], ["systempaths=unconfined"], ["seccomp=unconfined", "apparmor=unconfined"]): changed = copy.deepcopy(item) changed["HostConfig"]["SecurityOpt"] = options assert not specialize._validate_build_probe_container(changed, record, image_environment=environment) def test_build_probe_volume_identity_detects_same_name_recreation(specialize, monkeypatch): name = "9" * 64 volume = { "Name": name, "Driver": "local", "Scope": "local", "Mountpoint": f"/var/lib/docker/volumes/{name}/_data", "Labels": {"com.docker.volume.anonymous": ""}, "Options": None, "CreatedAt": "2026-10-03T00:00:00Z", } monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: json.dumps([volume])) identity = specialize._attest_build_probe_data_volume(name) volume["Labels"]["foreign.owner"] = "another-job" with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): specialize._attest_build_probe_data_volume(name, identity) volume["Labels"] = {"com.docker.volume.anonymous": ""} volume["CreatedAt"] = "2026-10-03T00:00:01Z" with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): specialize._attest_build_probe_data_volume(name, identity) def test_build_probe_volume_recovery_never_removes_volume_used_by_another_container(specialize, monkeypatch): identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} mutations = [] def run(*args, **kwargs): if args[:2] == ("volume", "ls"): return identity["name"] if args[:2] == ("ps", "-a"): return "f" * 64 mutations.append(args) raise AssertionError("unexpected volume mutation") monkeypatch.setattr(specialize, "_run_dind_docker", run) monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_in_use"): specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) assert mutations == [] def test_build_probe_volume_recovery_removes_only_exact_journal_owned_volume(specialize, monkeypatch): identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} mutations = [] present = True def run(*args, **kwargs): nonlocal present if args[:2] == ("volume", "ls"): assert args == ("volume", "ls", "--quiet", "--filter", f"name={identity['name']}") return identity["name"] if present else "" if args[:2] == ("ps", "-a"): return "" assert args == ("volume", "rm", identity["name"]) mutations.append(args) present = False return identity["name"] monkeypatch.setattr(specialize, "_run_dind_docker", run) monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) assert mutations == [("volume", "rm", identity["name"])] def test_build_probe_result_requires_exact_marker_and_every_rootless_tool_check(specialize): record = _build_probe_record(specialize) payload = { "schema": specialize._BUILD_PROBE_SCHEMA, "status": "passed", "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "engineVersion": "29.8.2", "engineRootless": True, "engineStorageDriver": "overlay2", "engineCgroupDriver": "none", "engineSocket": "unix:///run/itworx-job-docker/docker.sock", "engineSocketUid": 1000, "composeVersion": "5.5.1", "buildRun": True, "composeHealthy": True, "composeDnsResolved": True, "archiveNormalized": True, "archiveLoaded": True, "imageId": "sha256:" + "2" * 64, "imageLabels": record["imageLabels"], "archiveSha256": "sha256:" + "3" * 64, "holdSeconds": 30, } line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( payload, sort_keys=True, separators=(",", ":"), ) assert specialize._validate_build_probe_result(line, record=record) == payload with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_missing"): specialize._validate_build_probe_result(line + "\n" + line, record=record) for key, value in ( ("engineRootless", False), ("engineSocket", "tcp://0.0.0.0:2375"), ("composeHealthy", False), ("holdSeconds", 0), ): changed = {**payload, key: value} changed_line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( changed, sort_keys=True, separators=(",", ":"), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_untrusted"): specialize._validate_build_probe_result(changed_line, record=record) def test_build_probe_live_process_inventory_requires_daemon_and_compose_children(specialize): output = "\n".join( ( "PID PPID COMMAND", "1 0 python3 /usr/local/bin/itworx-ci-job-docker-probe", "10 1 rootlesskit --net=slirp4netns", "11 10 dockerd --host=unix:///run/itworx-job-docker/docker.sock --storage-driver=overlay2", "12 11 containerd --config /run/containerd/config.toml", "21 20 sleep 300", ) ) result = specialize._validate_build_probe_processes(output) assert result == { "processCount": 5, "probe": True, "rootlesskit": True, "dockerd": True, "containerd": True, "composeService": True, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): specialize._validate_build_probe_processes( output.replace("21 20 sleep 300", "21 20 cat /etc/os-release") ) with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): specialize._validate_build_probe_processes(output.replace("containerd --config", "unknown --config")) def test_build_probe_intent_binds_revision_tree_image_nonce_and_exact_container_id(specialize): record = _build_probe_record(specialize) assert specialize._validated_build_probe_record( record, source_revision="a" * 40, source_tree="b" * 40, ) == record invalid = dict(record, name=record["name"] + "-foreign") with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): specialize._validated_build_probe_record( invalid, source_revision="a" * 40, source_tree="b" * 40, ) def _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, ids): image_environment = item["Config"]["Env"] present = list(ids) removed = [] monkeypatch.setattr( specialize, "_inspect", lambda name: {"State": {"Running": True}} if name == specialize.DIND_NAME else None, ) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) monkeypatch.setattr( specialize, "_build_probe_ids", lambda *, label=True, name=None: list(present) if label or name == record["name"] and present else [], ) monkeypatch.setattr(specialize, "_inspect_dind_container", lambda _cid: item) monkeypatch.setattr( specialize, "_build_probe_image_environment", lambda _record: image_environment, ) def run_dind(*args, **_kwargs): if args[:3] == ("rm", "--force", "--volumes"): removed.append(args[3]) present.clear() return args[3] if args[:2] == ("volume", "ls"): return "" raise AssertionError(f"unexpected private-DinD command: {args}") monkeypatch.setattr(specialize, "_run_dind_docker", run_dind) monkeypatch.setattr( specialize, "_attest_build_probe_data_volume", lambda name, expected=None: {"name": name, "createdAt": "2026-10-03T00:00:00Z"}, ) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) return removed @pytest.mark.parametrize("running", [True, False]) def test_build_probe_running_gate_distinguishes_owned_exit_and_preserves_identity(specialize, running): record = _build_probe_record(specialize) item, environment = _build_probe_inspect(specialize, record) item["State"]["Running"] = running item["State"]["Pid"] = 123 if running else 0 if running: specialize._require_running_build_probe(item, record, environment) else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_exited_early"): specialize._require_running_build_probe(item, record, environment) item["HostConfig"]["Privileged"] = True with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_container_untrusted"): specialize._require_running_build_probe(item, record, environment) def test_build_probe_recovery_cleans_only_intent_owned_container_after_create_crash( specialize, monkeypatch, ): record = _build_probe_record(specialize, status="intent") cid = "e" * 64 item, _environment = _build_probe_inspect(specialize, record, container_id=cid) removed = _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, [cid]) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } specialize._cleanup_build_probe(journal) assert removed == [cid] assert journal["buildProbe"]["containerId"] == cid assert journal["buildProbe"]["status"] == "cleaned" def test_build_probe_recovery_refuses_foreign_cid_even_with_matching_name_and_nonce( specialize, monkeypatch, ): expected_cid = "d" * 64 foreign_cid = "e" * 64 record = _build_probe_record( specialize, container_id=expected_cid, status="started", ) item, _environment = _build_probe_inspect(specialize, record, container_id=foreign_cid) removed = _install_build_probe_cleanup_fakes( specialize, monkeypatch, record, item, [foreign_cid], ) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_container_untrusted"): specialize._cleanup_build_probe(journal) assert removed == [] def test_build_probe_recovery_refuses_ambiguous_owned_label_inventory( specialize, monkeypatch, ): record = _build_probe_record(specialize, status="intent") cid = "e" * 64 item, _environment = _build_probe_inspect(specialize, record, container_id=cid) removed = _install_build_probe_cleanup_fakes( specialize, monkeypatch, record, item, [cid, "f" * 64], ) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_ambiguous_containers"): specialize._cleanup_build_probe(journal) assert removed == [] invalid = dict(record, status="started") with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): specialize._validated_build_probe_record( invalid, source_revision="a" * 40, source_tree="b" * 40, ) @pytest.mark.parametrize("recovering_cgroup_intent", [False, True]) @pytest.mark.parametrize("probe_fails", [False, True]) def test_specialization_stage_fixture_quiesces_old_before_registering_and_never_runs_over_two( specialize, tmp_path, monkeypatch, recovering_cgroup_intent, probe_fails ): revision = "a" * 40 tree = "b" * 40 manifest_sha = "d" * 64 manifest = { "images": [ {"profile": profile_id, "reference": specialize._task_image_reference(profile_id, revision)} for profile_id in specialize.PROFILE_IDS ] } archive = tmp_path / "task-images.tar" archive.write_bytes(b"fixture") journal = { "status": "in_progress", "stage": "prepared", "legacyTopology": "v1", "sourceRevision": revision, "sourceTree": tree, "cgroupCreated": False, "cgroupEnsurePending": recovering_cgroup_intent, "baseComposeSha256": hashlib.sha256(b"captured-base").hexdigest(), "baseComposeModelSha256": hashlib.sha256(b"base-model").hexdigest(), "specializedComposeSha256": hashlib.sha256(b"specialized-model").hexdigest(), "backupDirectory": str(tmp_path / "specialized-test"), "newRegistrationIds": {}, } events: list[str] = [] running: set[str] = set() maximum_running = 0 marker_active = {"value": False} def rows(): return [ { "name": specialize.RUNNER_PROFILES[profile_id].container_name, "id": index + 1, "labels": sorted( { specialize.RUNNER_PROFILES[profile_id].runner_label, *specialize.RUNNER_PROFILES[profile_id].legacy_labels, } ), "disabled": True, "busy": False, } for index, profile_id in enumerate(specialize.PROFILE_IDS) ] def inventory(): return ( { name: {"name": name, "State": {"Running": name in running}} for name in (*specialize._SPECIALIZED_NAMES, specialize.DIND_NAME) }, [], ) def resource_proof(_claims, _containers, _legacy): return { "ready": True, "digest": "e" * 64, "runningRunnerContainers": len(running), } monkeypatch.setattr(specialize, "_captured_base_source", lambda _journal: b"captured-base") monkeypatch.setattr( specialize, "_captured_compose_model", lambda _journal, *, specialized: b"specialized-model" if specialized else b"base-model", ) monkeypatch.setattr(specialize, "_read_source_file", lambda *_args, **_kwargs: b"captured-base") monkeypatch.setattr(specialize, "_validate_base_composition", lambda _bytes=None: "v1") monkeypatch.setattr(specialize, "_validate_composition", lambda raw: raw) monkeypatch.setattr(specialize, "_run_specialized_compose", lambda _journal, *args, **_kwargs: events.append(f"compose:{args[-1]}")) monkeypatch.setattr(specialize, "_wait_guards_quiet", lambda: events.append("guards-quiet")) monkeypatch.setattr(specialize, "_service_guard_scripts", lambda _journal: events.append("guards-fenced")) monkeypatch.setattr(specialize, "_stop_legacy_capacity_daemon", lambda: events.append("capacity-stopped")) monkeypatch.setattr(specialize, "_set_legacy_disabled", lambda _journal: events.append("legacy-disabled")) monkeypatch.setattr(specialize, "_check_legacy_queued_jobs", lambda: None) monkeypatch.setattr(specialize, "_stop_remove_legacy_containers", lambda _journal: events.append("legacy-removed")) monkeypatch.setattr( specialize, "_cgroup_family_snapshot", lambda: { "exists": recovering_cgroup_intent, "rootSubtreeControllers": ["cpu", "memory", "pids"], }, ) def ensure_cgroup_family(current_journal): assert current_journal["cgroupEnsurePending"] is True assert "compose:runner-dind" in events events.append("cgroup-ensured") monkeypatch.setattr( specialize, "_bind_owned_dind_family", ensure_cgroup_family, ) monkeypatch.setattr(specialize, "_prepare_profile_state", lambda _revision: None) monkeypatch.setattr(specialize, "_write_private_atomic", lambda *_args, **_kwargs: None) def register(current_journal): assert "legacy-disabled" in events events.append("registrations-created") current_journal["newRegistrationIds"] = { profile_id: index + 1 for index, profile_id in enumerate(specialize.PROFILE_IDS) } specialize._set_stage(current_journal, "registrations_created") monkeypatch.setattr(specialize, "_register_profiles", register) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: None) monkeypatch.setattr(specialize, "_inspect", lambda _name: {"State": {"Running": True}}) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: False) monkeypatch.setattr(specialize, "_is_captured_original_dind", lambda *_args: True) monkeypatch.setattr(specialize, "_require_dind_resource_support", lambda _model: None) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {"services": {}}) monkeypatch.setattr(specialize, "_load_images_into_dind", lambda *_args: None) monkeypatch.setattr(specialize, "_verify_manifest_images_in_dind", lambda _items: True) monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) monkeypatch.setattr(specialize, "_resource_proof", resource_proof) monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) monkeypatch.setattr( specialize, "_trusted_private_file", lambda path, **_kwargs: specialize._render_runner_config("build", revision) if str(path).endswith("build.yaml") else b"fixture", ) monkeypatch.setattr(specialize, "_cleanup_build_probe", lambda _journal: None) def run_build_probe(_journal, _manifest): assert not running events.append("build-probed") if probe_fails: raise specialize.RunnerFleetSpecializeError( "runner_fleet_specialization_build_probe_failed" ) monkeypatch.setattr(specialize, "_run_build_probe", run_build_probe) monkeypatch.setattr(specialize, "_runner_rows", rows) def write_journal(current_journal): if current_journal.get("cgroupEnsurePending") is True: events.append("cgroup-intent-durable") if current_journal.get("cgroupCreated") is True: assert current_journal["stage"] not in {"prepared", "state_prepared", "legacy_removed"} events.append("cgroup-result-durable") if current_journal.get("status") == "active": events.append("journal-active-durable") monkeypatch.setattr(specialize, "_write_journal", write_journal) batches: list[tuple[str, ...]] = [] def set_containers(profiles, start): nonlocal maximum_running names = {specialize.RUNNER_PROFILES[item].container_name for item in profiles} if start: assert len(running | names) <= 2 running.update(names) maximum_running = max(maximum_running, len(running)) batches.append(tuple(profiles)) events.append("batch-started") else: running.difference_update(names) events.append("batch-stopped") monkeypatch.setattr(specialize, "_set_runner_containers_running", set_containers) monkeypatch.setattr(specialize, "_bootstrap_registrations", lambda: [{"name": name} for name in specialize._SPECIALIZED_NAMES]) monkeypatch.setattr(specialize, "_read_installed_manifest_sha256", lambda: manifest_sha) monkeypatch.setattr(specialize, "_write_pending_bootstrap_receipt", lambda *_args: events.append("receipt-written")) crash = {"next_active_write": True} class SimulatedPowerLoss(BaseException): pass def write_marker(**kwargs): assert kwargs["status"] == "active" assert not running assert "receipt-written" in events events.append("marker-active-attempt") if crash["next_active_write"]: crash["next_active_write"] = False raise SimulatedPowerLoss marker_active["value"] = True events.append("marker-active") monkeypatch.setattr(specialize, "_write_marker", write_marker) monkeypatch.setattr(specialize, "specialization_state", lambda: "active" if marker_active["value"] else "blocked") if probe_fails: monkeypatch.setattr(specialize, "_record_migration_failure", lambda _journal, _error: events.append("failure-recorded")) def rollback(_journal): assert "failure-recorded" in events events.append("rollback") return True monkeypatch.setattr(specialize, "_rollback_before_activation", rollback) with pytest.raises( specialize.RunnerFleetSpecializeError, match="failed_rolled_back", ): specialize._perform_migration_stages(journal, manifest, archive) assert "build-probed" in events assert "receipt-written" not in events assert "marker-active-attempt" not in events assert not marker_active["value"] return with pytest.raises(SimulatedPowerLoss): specialize._perform_migration_stages(journal, manifest, archive) assert journal["status"] == "active" assert not marker_active["value"] assert events.index("journal-active-durable") < events.index("marker-active-attempt") marker = { "status": "in_progress", "sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256, "imageSourceRevision": revision, "imageSourceTree": tree, } script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_read_json_file", lambda *_args, **_kwargs: marker) monkeypatch.setattr(specialize, "_valid_marker", lambda _marker: True) monkeypatch.setattr(specialize, "_migration_lock", lambda: nullcontext()) monkeypatch.setattr( specialize, "verified_topology", lambda: { "ready": marker_active["value"], "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if marker_active["value"] else [], "resourceEnvelope": {"ready": marker_active["value"], "digest": "e" * 64}, }, ) monkeypatch.setitem( sys.modules, "status_server", type( "StatusServerStub", (), { "_require_controller_release": staticmethod( lambda: { "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": revision, "sourceTree": tree, } ) }, ), ) monkeypatch.setattr( specialize, "_run_approved_shell_script", lambda _script: specialize._perform_migration_stages(journal, manifest, archive), ) result = specialize.execute(script) assert result["status"] == "specialized" assert maximum_running == 2 assert events.index("cgroup-intent-durable") < events.index("cgroup-ensured") assert events.index("cgroup-intent-durable") < events.index("compose:runner-dind") assert events.index("compose:runner-dind") < events.index("cgroup-ensured") assert events.index("cgroup-ensured") < events.index("registrations-created") assert events.index("cgroup-ensured") < events.index("cgroup-result-durable") assert batches == [ tuple(specialize.PROFILE_IDS[0:2]), tuple(specialize.PROFILE_IDS[2:4]), tuple(specialize.PROFILE_IDS[4:6]), ] assert events.index("legacy-disabled") < events.index("registrations-created") assert events.index("batch-stopped") < events.index("build-probed") assert events.index("build-probed") < events.index("receipt-written") assert events.index("receipt-written") < events.index("marker-active") assert journal["cgroupCreated"] is True assert journal["status"] == "active" assert not running @pytest.mark.parametrize("defect", [None, "resume_same", "resume_wrong_pid", "foreign_cid", "restarted", "foreign_model", "workload", "bad_top", "duplicate_pid", "wrong_pid", "wrong_controls", "helper_refusal", "bad_result"]) def test_owned_dind_family_binding_requires_fresh_exact_identity(specialize, monkeypatch, defect): cid = "a" * 64 state = {"Running": True, "Pid": 1234, "StartedAt": "2026-10-11T00:00:00Z"} before = {"Id": cid, "State": state, "Image": specialize._DIND_IMAGE_ID} after = copy.deepcopy(before) if defect == "foreign_cid": after["Id"] = "b" * 64 if defect == "restarted": after["State"]["StartedAt"] = "2026-10-11T00:01:00Z" snapshots = iter([before, after]) monkeypatch.setattr(specialize, "_inspect", lambda _name: next(snapshots)) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: defect != "foreign_model") monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: cid if defect == "workload" else "") monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\ninvalid" if defect == "bad_top" else "PID\n1234\n1234" if defect == "duplicate_pid" else "PID\n9999" if defect == "wrong_pid" else "PID\n1234\n1235") calls = [] controls = ["cpu", "memory", "pids"] def bind(**kwargs): calls.append(kwargs) if defect == "helper_refusal": raise RuntimeError("resource_cgroup_foreign_process") if defect == "bad_result": return None return {"created": False, "rootSubtreeBefore": [] if defect == "wrong_controls" else controls} monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", bind) journal = {"cgroupRootSubtree": controls, "cgroupEnsurePending": True, "cgroupFamilyInitiallyPresent": False, "cgroupDindBinding": None} if defect in {"resume_same", "resume_wrong_pid"}: journal["cgroupDindBinding"] = {"containerId": cid, "hostPid": 9999 if defect == "resume_wrong_pid" else 1234, "startedAt": state["StartedAt"], "imageId": specialize._DIND_IMAGE_ID} if defect in {None, "resume_same"}: specialize._bind_owned_dind_family(journal) assert calls == [{"expected_container_id": cid, "expected_host_pid": 1234, "expected_process_ids": [1234, 1235], "allow_owned_partial": defect == "resume_same"}] assert journal["cgroupDindBinding"] == {"containerId": cid, "hostPid": 1234, "startedAt": state["StartedAt"], "imageId": specialize._DIND_IMAGE_ID} else: with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._bind_owned_dind_family(journal) if defect in {"foreign_model", "workload", "bad_top", "duplicate_pid", "wrong_pid", "resume_wrong_pid"}: assert calls == [] @pytest.mark.parametrize("original_running", [False, True]) def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( specialize, tmp_path, monkeypatch, original_running ): monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) image_id = specialize._DIND_IMAGE_ID base_model = _base_v1_compose_model(specialize) specialized_model = { "services": { "runner-dind": { **base_model["services"]["runner-dind"], "cgroup_parent": "itworx-ci", "restart": "no", "volumes": [ { "type": "bind", "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "target": "/var/lib/docker", "read_only": False, } ], } } } journal = { "originalDind": { "present": True, "running": original_running, "restartPolicy": "unless-stopped", "imageId": image_id, }, "originalContainers": [], "originalRegistrations": [], "newRegistrationIds": {}, "guardFiles": [], "cgroupCreated": True, "cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": ["cpu", "memory", "pids"], "backupDirectory": str(tmp_path / "specialized-test"), } events = [] current = {"kind": "specialized", "State": {"Running": True}} current_marker = {"value": current} def inspect(name): assert name == specialize.DIND_NAME or name in specialize._SPECIALIZED_NAMES return current_marker["value"] if name == specialize.DIND_NAME else None def run(args, **_kwargs): events.append(tuple(args)) if args[:2] == ["docker", "update"] and args[-1] == specialize.DIND_NAME: if args[2] == "--restart=no": return "" current_marker["value"]["HostConfig"]["RestartPolicy"]["Name"] = args[2].partition("=")[2] return "" if args[:2] == ["docker", "stop"]: current_marker["value"]["State"]["Running"] = False return "" if args[:2] == ["docker", "rm"]: current_marker["value"] = None return "" return "" def run_compose(_compose, *args, **_kwargs): events.append(("compose", *args)) assert args[-1] == "runner-dind" restored_running = original_running if args[0] == "up" else False current_marker["value"] = _legacy_dind_fixture( specialize, base_model, image_id=image_id, running=restored_running, restart_policy="no", ) base_bytes = json.dumps(base_model, sort_keys=True, separators=(",", ":")).encode() specialized_bytes = json.dumps(specialized_model, sort_keys=True, separators=(",", ":")).encode() live_compose = tmp_path / "docker-compose.yml" live_compose.write_bytes(base_bytes) monkeypatch.setattr(specialize, "COMPOSE_PATH", live_compose) journal["baseComposeSha256"] = hashlib.sha256(base_bytes).hexdigest() monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr( specialize, "_captured_compose_model", lambda _journal, *, specialized: specialized_bytes if specialized else base_bytes, ) monkeypatch.setattr( specialize, "_validate_dind_outer", lambda item, _model: item.get("kind") == "specialized", ) monkeypatch.setattr(specialize, "_inspect", inspect) monkeypatch.setattr(specialize, "_run", run) monkeypatch.setattr(specialize, "_run_compose", run_compose) monkeypatch.setattr(specialize, "_runner_rows", lambda: []) monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) monkeypatch.setattr( specialize, "rollback_host_cgroup_family", lambda **_kwargs: events.append("cgroup-cleanup") or {"familyExists": False}, ) monkeypatch.setattr(specialize, "SPECIALIZED_COMPOSE_PATH", tmp_path / "specialized-compose.yml") monkeypatch.setattr(specialize, "TASK_IMAGE_MANIFEST_PATH", tmp_path / "task-image-manifest.json") monkeypatch.setattr(specialize, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "receipt.json") monkeypatch.setattr(specialize, "RUNNER_CONFIG_ROOT", tmp_path / "runner-configs") monkeypatch.setattr(specialize, "_SPECIALIZED_DATA_ROOT", tmp_path / "runner-data") monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", tmp_path / "workspaces") monkeypatch.setattr(specialize, "TASK_IMAGE_ROOT", tmp_path / "task-images") monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", tmp_path / "specialized.json") monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", tmp_path / "journal.json") assert specialize._rollback_before_activation(journal) is True restored = current_marker["value"] assert restored["Image"] == image_id assert restored["State"]["Running"] is original_running assert restored["HostConfig"]["RestartPolicy"]["Name"] == "unless-stopped" compose = next(event for event in events if isinstance(event, tuple) and event[0] == "compose") assert compose[1] == ("up" if original_running else "create") remove = next(index for index, event in enumerate(events) if isinstance(event, tuple) and event[:2] == ("docker", "rm")) assert remove < events.index(compose) < events.index("cgroup-cleanup") def _legacy_dind_fixture( specialize, base_model, *, image_id, running, restart_policy ): service = base_model["services"]["runner-dind"] return { "Name": f"/{specialize.DIND_NAME}", "Image": image_id, "Config": { "Image": service["image"], "User": "rootless", "Entrypoint": ["dockerd-entrypoint.sh"], "Cmd": ["--default-address-pool=base=198.19.0.0/16,size=27"], "Volumes": {"/home/rootless/.local/share/docker": {}, "/var/lib/docker": {}}, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": "runner-dind", }, }, "State": {"Running": running}, "HostConfig": { "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, "PidMode": "", "Privileged": True, "RestartPolicy": {"Name": restart_policy}, }, "Mounts": [ { "Type": "bind", "Source": mount["source"], "Destination": mount["target"], "RW": not mount.get("read_only", False), } for mount in service["volumes"] ] + [{ "Type": "volume", "Driver": "local", "Name": "f" * 64, "Source": "/var/lib/docker/volumes/" + "f" * 64 + "/_data", "Destination": "/var/lib/docker", "RW": True, }], } def test_known_v1_and_v2_topologies_are_explicit_and_v1_does_not_require_absent_v2_services( specialize, monkeypatch ): compose = b"known Tower v1 compose fixture" monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest()) v1_services = { service_name: {"container_name": name} for name, service_name in specialize._OLD_SERVICES.items() if name in specialize._LEGACY_V1_NAMES } assert specialize._classify_legacy_topology(compose, v1_services) == "v1" v2_services = {} for name in specialize._LEGACY_V2_NAMES: service_name = specialize._OLD_SERVICES[name] burst = name.endswith("burst-1") v2_services[service_name] = { "container_name": name, "image": specialize.RUNNER_IMAGE, "profiles": ["runner"], "labels": { "tech.itworx.role": "gitea-actions-burst-runner" if burst else "gitea-actions-runner" }, **({"cpus": 0.5, "mem_limit": "512m"} if name in specialize._LEGACY_V2_NAMES[2:] else {}), } assert specialize._classify_legacy_topology(b"known v2 compose", v2_services) == "v2" partial_v2 = dict(v1_services) partial_v2["runner-3"] = v2_services["runner-3"] with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_compose_invalid"): specialize._classify_legacy_topology(compose, partial_v2) def test_exact_live_v1_registration_and_container_contract_accepts_only_two_known_runners(specialize): rows = [ { "name": name, "id": runner_id, "labels": sorted(specialize._OLD_LABELS[name]), "disabled": False, "busy": False, } for runner_id, name in zip((9, 22), specialize._LEGACY_V1_NAMES, strict=True) ] rows.extend( [ {"name": "itworx-unraid-deploy", "id": 13, "labels": ["unraid-deploy"], "disabled": False, "busy": False}, {"name": "itworx-windows11-native", "id": 14, "labels": ["windows-native"], "disabled": False, "busy": False}, ] ) assert len(specialize._validate_legacy_registrations(rows, "v1")) == 2 name = "itworx-security-runner" mount_entries = [ {"Source": source, "Destination": target, "RW": True} for source, target, _read_only in specialize._V1_RUNNER_VOLUMES[name] ] item = { "Image": specialize.RUNNER_IMAGE_ID, "Config": { "Image": specialize.RUNNER_IMAGE, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": "runner", "tech.itworx.role": "gitea-actions-runner", }, }, "State": {"Running": True}, "HostConfig": { "NanoCpus": 1_000_000_000, "Memory": 1024 * specialize._MIB, "PidMode": "", "RestartPolicy": {"Name": "unless-stopped"}, }, "Mounts": mount_entries, } assert specialize._validate_legacy_container_item("v1", name, item)["running"] is True item["HostConfig"]["PidMode"] = "host" with pytest.raises(specialize.RunnerFleetSpecializeError, match="v1_container_budget_invalid"): specialize._validate_legacy_container_item("v1", name, item) extra_known_runner = [*rows, {"name": "itworx-validation-runner-3", "id": 30}] with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_registration_invalid"): specialize._validate_legacy_registrations(extra_known_runner, "v1") def test_queue_gate_allows_only_legacy_labels_covered_by_new_profiles(specialize): assert specialize._queued_job_is_covered(["self-hosted", "linux", "linux-validation", "ubuntu-latest"]) assert specialize._queued_job_is_covered(["itworx-ci-backend"]) assert specialize._queued_job_is_covered(["docker-validation"]) assert not specialize._queued_job_is_covered(["mcp-validation", "unraid"]) def _legacy_queue_job(job_id, status, labels, *, run_id=21500): return {"id": job_id, "run_id": run_id, "status": status, "labels": labels} def test_legacy_queue_gate_reads_queued_and_requested_admin_inventories(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=pending" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(102, "requested", ["linux-validation", "ubuntu-latest"])], } if "status=queued" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(101, "queued", ["itworx-ci-backend"])], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) specialize._check_legacy_queued_jobs() assert len(calls) == 2 assert "status=pending&limit=50&page=1&sort=id&order=asc" in calls[0] assert "status=queued&limit=50&page=1&sort=id&order=asc" in calls[1] def test_legacy_queue_gate_blocks_unsupported_requested_dependency_job(specialize, monkeypatch): def api(path): if "status=queued" in path: return {"total_count": 0, "jobs": []} if "status=pending" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(201, "requested", ["mcp-validation", "unraid"])], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_catches_requested_to_queued_transition_between_filters( specialize, monkeypatch ): state = "requested" calls = [] def api(path): nonlocal state calls.append(path) if "status=pending" in path: # The job changes status immediately before this filtered read. state = "queued" return {"total_count": 0, "jobs": []} if "status=queued" in path: jobs = ( [_legacy_queue_job(250, "queued", ["mcp-validation", "unraid"])] if state == "queued" else [] ) return {"total_count": len(jobs), "jobs": jobs} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert "status=pending" in calls[0] assert "status=queued" in calls[1] def test_legacy_queue_gate_paginates_51_jobs_and_blocks_unsupported_page_two_job(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) if page == 1: return { "total_count": 51, "jobs": [ _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) for index in range(50) ], } return { "total_count": 51, "jobs": [_legacy_queue_job(51, "queued", ["mcp-validation", "unraid"])], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert any("status=queued&limit=50&page=2" in path for path in calls) def test_legacy_queue_gate_paginates_101_jobs_and_checks_unsupported_last_page(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) start_id = (page - 1) * 50 + 1 page_size = min(50, 101 - (page - 1) * 50) labels = ["mcp-validation", "unraid"] if page == 3 else ["itworx-ci-backend"] return { "total_count": 101, "jobs": [ _legacy_queue_job(start_id + offset, "queued", labels) for offset in range(page_size) ], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert any("status=queued&limit=50&page=3" in path for path in calls) def test_legacy_queue_gate_rejects_total_count_changes_between_pages(specialize, monkeypatch): def api(path): if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) if page == 1: return { "total_count": 51, "jobs": [ _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) for index in range(50) ], } return { "total_count": 102, "jobs": [_legacy_queue_job(51, "queued", ["itworx-ci-backend"])], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_accepts_maximum_1000_job_inventory(specialize, monkeypatch): calls = [] def api(path): calls.append(path) page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) start_id = (page - 1) * 50 + 1 return { "total_count": 1000, "jobs": [ _legacy_queue_job(start_id + offset, "queued", ["itworx-ci-backend"]) for offset in range(50) ], } monkeypatch.setattr(specialize, "_api_json", api) rows = specialize._legacy_action_jobs("queued") assert len(rows) == 1000 assert len(calls) == 20 assert "limit=50&page=20" in calls[-1] def test_legacy_queue_gate_rejects_inventory_over_1000_jobs(specialize, monkeypatch): calls = [] def api(path): calls.append(path) return {"total_count": 1001, "jobs": []} monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._legacy_action_jobs("queued") assert len(calls) == 1 @pytest.mark.parametrize( "payload", [ {"total_count": 1, "jobs": []}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "requested", ["itworx-ci-backend"])]}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", [""])]}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"], run_id=0)]}, {"total_count": 2, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"])]}, { "total_count": 2, "jobs": [ _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), ], }, { "total_count": 2, "jobs": [ _legacy_queue_job(2, "queued", ["itworx-ci-backend"]), _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), ], }, ], ) def test_legacy_queue_gate_rejects_malformed_or_truncated_inventory(specialize, monkeypatch, payload): def api(path): if "status=queued" in path: return payload if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specialize, monkeypatch): row = _legacy_queue_job(301, "queued", ["itworx-ci-backend"]) def api(path): if "status=queued" in path: return {"total_count": 1, "jobs": [row]} if "status=pending" in path: return { "total_count": 1, "jobs": [{**row, "status": "requested"}], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url(): workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8") build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8") frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8") assert "github-server-url: http://gitea-runner-internal:3000" in workflow assert "runs-on: linux-validation" in workflow assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output"' in workflow assert "runs-on: itworx-ci-build" not in workflow assert "--job-local" not in workflow assert "jens/unraid-appops-gateway" in build assert "Jens/Unraid-AppOps-Gateway" in build assert "corepack prepare pnpm@11.15.0 --activate" in frontend assert "test \"$(pnpm --version)\" = 11.15.0" in frontend def test_specialization_shell_entrypoint_is_fixed_and_has_no_caller_selected_target(): script = ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh" content = script.read_text(encoding="utf-8") assert content.startswith("#!/usr/bin/env bash\n") assert "SPECIALIZE_CI_RUNNER_FLEET" in content assert "--apply-fixed" in content assert "eval " not in content assert "docker run" not in content assert "docker compose" not in content def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity( specialize, monkeypatch ): registrations = [ { "name": profile.display_name, "id": index, "labels": [profile.runner_label, *profile.legacy_labels], "disabled": True, "busy": False, } for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=10) ] monkeypatch.setattr(specialize, "_api_json", lambda _path: {"runners": registrations}) normalized = specialize._runner_rows() assert [row["name"] for row in normalized] == list(specialize._SPECIALIZED_NAMES) assert [row["displayName"] for row in normalized] == [ profile.display_name for profile in specialize.RUNNER_PROFILES.values() ] registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].container_name with pytest.raises( specialize.RunnerFleetSpecializeError, match="registration_name_invalid", ): specialize._runner_rows() registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].display_name registrations[1]["name"] = specialize.RUNNER_PROFILES["checks"].display_name with pytest.raises( specialize.RunnerFleetSpecializeError, match="registration_duplicate", ): specialize._runner_rows() def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): rows, containers, dind, journal, model = _active_recovery_fixture(specialize) events = [] class Volume: def __enter__(self): return self def __exit__(self, *_args): return False def assert_current(self): events.append("volume-current") monkeypatch.setattr(specialize, "_migration_lock", lambda **_kwargs: nullcontext()) monkeypatch.setattr(specialize, "_open_fixed_release_volume", lambda: Volume()) monkeypatch.setattr( specialize, "_active_release_bundle", lambda: ({"status": "active"}, journal, model, b"captured-compose"), ) monkeypatch.setattr(specialize, "_runner_rows", lambda: rows) monkeypatch.setattr(specialize, "_docker_outer_inventory", lambda: (containers, [])) monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) monkeypatch.setattr( specialize, "_cgroup_family_snapshot", lambda: { "exists": family_exists, "safeToStartDind": True, "rootSubtreeControllers": ["cpu", "memory", "pids"], }, ) monkeypatch.setattr( specialize, "_fresh_recovery_host_gate", lambda: events.append("fresh-host-gate"), ) monkeypatch.setattr( specialize, "_ensure_cgroup_family_details", lambda **_kwargs: events.append("cgroup-create") or {"created": True, "rootSubtreeBefore": ["cpu", "memory", "pids"]}, ) monkeypatch.setattr( specialize, "_run_compose", lambda compose, *args, **kwargs: events.append(("compose", compose, args, kwargs)), ) monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy")) monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True) monkeypatch.setattr( specialize, "_resource_proof", lambda *_args: {"ready": True, "runningRunnerContainers": 0}, ) def inspect(name): if name == specialize.DIND_NAME: dind["State"] = {"Running": True, "Health": {"Status": "healthy"}} return dind return None def inventory(): return containers, [] monkeypatch.setattr(specialize, "_inspect", inspect) monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) return events, rows, containers, dind def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( specialize, monkeypatch, family_exists=True ) result = specialize.ensure_active_fleet_ready() assert result == {"ready": True, "reason": None} compose_events = [item for item in events if isinstance(item, tuple) and item[0] == "compose"] assert len(compose_events) == 1 _, compose_bytes, args, kwargs = compose_events[0] assert compose_bytes == b"captured-compose" assert args == ("up", "-d", "--no-deps", "runner-dind") assert kwargs["timeout"] <= 30 assert events.count("fresh-host-gate") == 2 assert events.index("fresh-host-gate") < events.index("healthy") def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( specialize, monkeypatch, family_exists=False ) gate_count = {"value": 0} def fresh_gate(): gate_count["value"] += 1 events.append(f"fresh-host-gate-{gate_count['value']}") if gate_count["value"] == 2: raise specialize.RunnerFleetSpecializeError( "runner_fleet_specialization_host_guard_paused" ) rollback = [] monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fresh_gate) monkeypatch.setattr( specialize, "rollback_host_cgroup_family", lambda **kwargs: rollback.append(kwargs) or {"removed": True, "familyExists": False}, ) result = specialize.ensure_active_fleet_ready() assert result == { "ready": False, "reason": "runner_fleet_specialization_host_guard_paused", } assert rollback == [ {"created": True, "initial_controllers": ["cpu", "memory", "pids"]} ] assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) def test_active_recovery_accepts_only_disabled_legacy_registrations_after_containers_removed( specialize, monkeypatch ): events, rows, _containers, dind = _stub_active_recovery( specialize, monkeypatch, family_exists=True ) rows[-1]["disabled"] = False result = specialize.ensure_active_fleet_ready() assert result["ready"] is False assert result["reason"] == "runner_fleet_specialization_legacy_registration_untrusted" assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) assert dind["State"]["Running"] is False def test_active_recovery_cli_reports_only_sanitized_ready_or_reason( specialize, monkeypatch, capsys ): monkeypatch.setattr( specialize, "ensure_active_fleet_ready", lambda: {"ready": False, "reason": "runner_fleet_specialization_host_guard_paused"}, ) assert specialize._main(("--ensure-active-fleet-ready",)) == 1 assert capsys.readouterr().err.strip() == "runner_fleet_specialization_host_guard_paused" monkeypatch.setattr( specialize, "ensure_active_fleet_ready", lambda: {"ready": True, "reason": None}, ) assert specialize._main(("--ensure-active-fleet-ready",)) == 0 assert capsys.readouterr().out.strip() == "ready" def test_active_recovery_host_gate_rechecks_guard_after_fresh_sample( specialize, monkeypatch ): now = specialize.time.time() sample = { "loadRatio": 0.2, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapSampleAt": now, "swapCritical": False, } guard_states = iter([("clear", ""), ("paused", "guard marker appeared")]) class HostSampler: def host_snapshot(self): return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_fleet", type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: next(guard_states))}), ) monkeypatch.setitem( sys.modules, "runner_capacity_controller", type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), ) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_host_guard_paused", ): specialize._fresh_recovery_host_gate() @pytest.mark.parametrize( "sample,expected_reason", [ (None, "runner_fleet_specialization_host_telemetry_unavailable"), ( { "loadRatio": 0.2, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapSampleAt": 1, "swapCritical": False, }, "runner_fleet_specialization_host_headroom_insufficient", ), ], ) def test_active_recovery_host_gate_fails_closed_on_missing_or_stale_sample( specialize, monkeypatch, sample, expected_reason ): class HostSampler: def host_snapshot(self): if sample is None: return {"status": "unavailable", "host": None} return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_fleet", type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: ("clear", ""))}), ) monkeypatch.setitem( sys.modules, "runner_capacity_controller", type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match=expected_reason): specialize._fresh_recovery_host_gate()