from __future__ import annotations import base64 import copy import hashlib import importlib.util import io import json import os import subprocess import sys import tarfile from contextlib import contextmanager, nullcontext from pathlib import Path from types import SimpleNamespace import pytest @pytest.fixture def controller_only_source(specialize, tmp_path, monkeypatch): image = "sha256:" + "a" * 64 code = f"{specialize.RELEASE_VOLUME_HOST_ROOT}/controller-artifacts/{'b' * 40}" model = {"name": specialize.PROJECT, "services": {"deploy-control": { "container_name": specialize.CONTROLLER_CONTAINER, "image": image, "volumes": [{"type": "bind", "source": code, "target": "/opt/gitea-deploy", "read_only": True, "bind": {"propagation": "rslave"}}], }}} compose = tmp_path / "docker-compose.yml" archive_dir = tmp_path / "backups" / "controller-v2-fixed" archive_dir.mkdir(parents=True, mode=0o700) archive_dir.parent.chmod(0o700) archive = archive_dir / "compose-before.bin" archived = b"reviewed complete legacy v1 source" archive.write_bytes(archived) archive.chmod(0o600) monkeypatch.setattr(specialize, "ROOT", tmp_path) monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) monkeypatch.setattr(specialize, "LEGACY_V1_COMPOSE_BACKUP", archive) monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(archived).hexdigest()) monkeypatch.setattr(specialize, "_controller_runtime_identity", lambda: (image, code)) return model, compose, archive, archived def test_controller_only_source_uses_fixed_pinned_archive_without_rewriting_live_compose( specialize, controller_only_source, ): model, compose, _archive, archived = controller_only_source current = json.dumps(model).encode() compose.write_bytes(current) result, current_sha = specialize._read_fleet_base_compose() assert result == archived assert current_sha == hashlib.sha256(current).hexdigest() assert compose.read_bytes() == current @pytest.mark.parametrize("mutation", ["project", "image", "container", "code", "writable", "shared", "extra_code"]) def test_controller_only_source_rejects_identity_and_code_binding_drift( specialize, controller_only_source, mutation, ): model, compose, _archive, _archived = controller_only_source controller = model["services"]["deploy-control"] mount = controller["volumes"][0] if mutation == "project": model["name"] = "other" elif mutation == "image": controller["image"] = "sha256:" + "c" * 64 elif mutation == "container": controller["container_name"] = "other" elif mutation == "code": mount["source"] = "/mnt/shared/code" elif mutation == "writable": mount["read_only"] = False elif mutation == "shared": mount["bind"]["propagation"] = "rshared" else: controller["volumes"].append(copy.deepcopy(mount)) compose.write_text(json.dumps(model)) with pytest.raises(specialize.RunnerFleetSpecializeError, match="controller_compose_untrusted"): specialize._read_fleet_base_compose() def test_controller_only_source_rejects_modified_fixed_archive(specialize, controller_only_source): model, compose, archive, _archived = controller_only_source compose.write_text(json.dumps(model)) archive.write_bytes(b"different legacy source") with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_backup_changed"): specialize._read_fleet_base_compose() def test_legacy_live_source_does_not_open_promotion_backup(specialize, monkeypatch): current = b"services:\n runner: {}\n" monkeypatch.setattr(specialize, "_read_source_file", lambda *_args, **_kwargs: current) monkeypatch.setattr(specialize, "_trusted_private_file", lambda *_args, **_kwargs: pytest.fail("archive must remain unopened")) assert specialize._read_fleet_base_compose() == (current, None) @pytest.mark.parametrize("separate_controller_hash", [False, True]) def test_controller_compose_cas_blocks_rollback_before_any_lifecycle_action( specialize, controller_only_source, monkeypatch, separate_controller_hash, ): monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) model, compose, _archive, _archived = controller_only_source compose.write_text(json.dumps(model)) original_sha = hashlib.sha256(compose.read_bytes()).hexdigest() journal = {"baseComposeSha256": original_sha} if separate_controller_hash: journal["controllerComposeSha256"] = original_sha compose.write_bytes(compose.read_bytes() + b"\n") monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda *_args: pytest.fail("must reject before lifecycle")) assert specialize._rollback_before_activation(journal) is False @pytest.mark.parametrize("reviewed,user,image,expected", [ (True, None, True, "rootless"), (False, None, True, None), (True, "root", True, "root"), (True, None, False, None), ]) def test_only_byte_pinned_v1_pinned_image_default_user_is_normalized( specialize, monkeypatch, reviewed, user, image, expected, ): source = b"reviewed v1 source" if reviewed else b"different source" monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(b"reviewed v1 source").hexdigest()) model = {"services": {"runner-dind": {"image": specialize._DIND_IMAGE_REFERENCE if image else "unknown", "user": user}}} monkeypatch.setattr(specialize, "_run_compose", lambda *_args: json.dumps(model)) assert specialize._compose_model(source)["services"]["runner-dind"]["user"] == expected ROOT = Path(__file__).resolve().parents[2] BROKER = ROOT / "ops" / "gitea-deploy-broker" SPECIALIZE_PATH = BROKER / "runner_fleet_specialize.py" @pytest.fixture def specialize(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) spec = importlib.util.spec_from_file_location("runner_fleet_specialize_test", SPECIALIZE_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module @contextmanager def _fixture_rollback_maintenance(): yield lambda: None def _valid_task_image_manifest(specialize, revision="a" * 40, tree="b" * 40): source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" return { "apiVersion": specialize.IMAGE_MANIFEST_SCHEMA, "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": revision, "sourceTree": tree, "workflowPath": specialize.IMAGE_WORKFLOW, "workflowRunId": "12", "fullRunId": "13", "artifactName": specialize.IMAGE_ARTIFACT_PREFIX + revision, "dockerTarSha256": "c" * 64, "dockerTarBytes": 1, "images": [ { "profile": profile_id, "reference": f"localhost/itworx-ci-{profile_id}:sha-{revision}", "imageId": f"sha256:{str(index + 1) * 64}", "architecture": "amd64", "dockerfileSha256": "d" * 64, "labels": { "org.opencontainers.image.source": source, "org.opencontainers.image.revision": revision, "tech.itworx.source-tree": tree, "tech.itworx.ci.profile": profile_id, }, } for index, profile_id in enumerate(specialize.PROFILE_IDS) ], } def _build_probe_record(specialize, *, container_id=None, status="intent"): revision = "a" * 40 tree = "b" * 40 image_labels = { "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", "org.opencontainers.image.revision": revision, "tech.itworx.source-tree": tree, "tech.itworx.ci.profile": "build", } record = specialize._new_build_probe_record( {"sourceRevision": revision, "sourceTree": tree}, { "taskImageId": "sha256:" + "1" * 64, "dockerfileSha256": "d" * 64, "imageLabels": image_labels, }, ) record["status"] = status record["containerId"] = container_id return record def _build_probe_inspect(specialize, record, *, container_id=None, changes=None): image_environment = ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] item = { "Id": container_id or "e" * 64, "Name": f"/{record['name']}", "Image": record["taskImageId"], "Config": { "Image": record["taskImageId"], "Entrypoint": [specialize._BUILD_PROBE_EXECUTABLE], "Cmd": [ record["sourceRevision"], record["sourceTree"], "--hold-seconds", str(specialize._BUILD_PROBE_HOLD_SECONDS), ], "Env": image_environment, "Labels": specialize._build_probe_container_labels(record), "Volumes": {"/home/itworx-builder/.local/share/docker": {}}, }, "HostConfig": { "NetworkMode": "bridge", "PidMode": "", "IpcMode": "private", "CgroupnsMode": "private", "CgroupParent": "docker", "NanoCpus": 4_000_000_000, "Memory": 6 * 1024**3, "PidsLimit": 1024, "BlkioWeight": 100, "Privileged": False, "CapAdd": [], "CapDrop": [], "SecurityOpt": ["seccomp=unconfined", "systempaths=unconfined"], "Devices": [{ "PathOnHost": "/dev/net/tun", "PathInContainer": "/dev/net/tun", "CgroupPermissions": "rwm", }], "DeviceRequests": [], "DeviceCgroupRules": [], "ExtraHosts": ["gitea.example.com:198.19.104.1"], "PortBindings": {}, "Binds": [], "Mounts": [], "VolumesFrom": [], "Tmpfs": {}, "RestartPolicy": {"Name": "no"}, }, "Mounts": [{ "Type": "volume", "Name": "9" * 64, "Source": f"/var/lib/docker/volumes/{'9' * 64}/_data", "Destination": "/home/itworx-builder/.local/share/docker", "Driver": "local", "Mode": "", "RW": True, "Propagation": "", }], "State": {"Running": True, "Pid": 77}, } if changes: for section, values in changes.items(): item[section].update(values) return item, image_environment def _write_task_image_archive(specialize, path: Path, *, extra_label: bool = False): images = _valid_task_image_manifest(specialize)["images"] docker_manifest = [] members: dict[str, bytes] = {} for index, image in enumerate(images): labels = specialize._runtime_task_image_labels(image) assert labels is not None if extra_label and index == 0: labels["untrusted.extra"] = "value" config = { "architecture": "amd64", "os": "linux", "config": {"Labels": labels}, } config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() config_name = f"{image['profile']}.json" layer_name = f"{image['profile']}/layer.tar" image["imageId"] = "sha256:" + hashlib.sha256(config_bytes).hexdigest() members[config_name] = config_bytes members[layer_name] = b"layer" docker_manifest.append( { "Config": config_name, "RepoTags": [image["reference"]], "Layers": [layer_name], } ) members["manifest.json"] = json.dumps(docker_manifest).encode() with tarfile.open(path, mode="w:") as archive: for name, payload in members.items(): info = tarfile.TarInfo(name) info.size = len(payload) archive.addfile(info, io.BytesIO(payload)) return images def _active_recovery_fixture(specialize): rows = [] registration_ids = {} originals = [] containers = {} for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=100): registration_ids[profile.id] = index rows.append( { "name": profile.container_name, "displayName": profile.display_name, "id": index, "labels": sorted({profile.runner_label, *profile.legacy_labels}), "disabled": True, "busy": False, "status": "offline", } ) containers[profile.container_name] = {"State": {"Running": False}} for index, name in enumerate(specialize._LEGACY_V1_NAMES, start=9): labels = sorted(specialize._legacy_allowed_labels("v1", name)) legacy_row = { "name": name, "id": index, "labels": labels, "disabled": True, "busy": False, "status": "offline", } rows.append(legacy_row) originals.append({"name": name, "id": index, "labels": labels, "disabled": True}) dind = {"State": {"Running": False, "Health": {"Status": "none"}}} containers[specialize.DIND_NAME] = dind journal = { "legacyTopology": "v1", "newRegistrationIds": registration_ids, "originalRegistrations": originals, } model = {"services": {"runner-dind": {"privileged": True}}} return rows, containers, dind, journal, model def _legacy_dind_mounts(specialize): return [ { "type": "bind", "source": specialize.LEGACY_DIND_STORAGE_ROOT.as_posix(), "target": "/home/rootless/.local/share/docker", "read_only": False, }, { "type": "bind", "source": "/mnt/cache/appdata/itworx-security-runner-workspace", "target": "/workspace", "read_only": False, }, { "type": "bind", "source": "/mnt/cache/appdata/itworx-validation-runner-2-workspace", "target": "/workspace-secondary", "read_only": False, }, ] def _specialized_dind_mounts(specialize): return [ { "type": "bind", "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "target": "/var/lib/docker", "read_only": False, }, { "type": "bind", "source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "target": specialize._DIND_CONTROL_CONTAINER_ROOT, "read_only": False, }, { "type": "bind", "source": str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "target": "/home/rootless/.local/share/docker", "read_only": True, }, *[ { "type": "bind", "source": str( specialize._host_volume_path( specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name ) ), "target": f"/{profile.container_name}-workspace", "read_only": False, } for profile in specialize.RUNNER_PROFILES.values() ], ] def _base_v1_compose_model(specialize): services = { "deploy-control": { "container_name": specialize.CONTROLLER_CONTAINER, "image": "itworx/appops-deploy-control:fixed", }, "runner-dind": { "container_name": specialize.DIND_NAME, "image": "docker@sha256:7c3e797187e43738220462658f4586572cbd3bf009f728b21e34d9c5c06ce431", "user": "rootless", "privileged": True, "cpus": 6, "mem_limit": "12g", "restart": "unless-stopped", "security_opt": ["label=disable"], "command": ["--default-address-pool=base=198.19.0.0/16,size=27"], "volumes": _legacy_dind_mounts(specialize), }, } for name, service_name in specialize._OLD_SERVICES.items(): if name not in specialize._LEGACY_V1_NAMES: continue services[service_name] = { "container_name": name, "image": specialize.RUNNER_IMAGE, "profiles": ["runner"], "cpus": 1, "mem_limit": "1g", "volumes": [ { "type": "bind", "source": source, "target": target, "read_only": read_only, } for source, target, read_only in specialize._expected_legacy_mounts("v1", name) ], "labels": {"tech.itworx.role": "gitea-actions-runner"}, } return { "services": services, "networks": { "runner-sandbox": {"name": specialize._RUNNER_NETWORK_NAME} }, } def _gitea_28_run( specialize, revision, *, run_id=21533, attempt=1, status="completed", conclusion="success" ): filename = specialize.FULL_WORKFLOW.rsplit("/", 1)[-1] return { "id": run_id, "run_attempt": attempt, "path": f"{filename}@refs/heads/main", "status": status, "conclusion": conclusion, "event": "push", "head_branch": "main", "head_sha": revision, "repository": {"id": 9, "full_name": "Jens/Unraid-AppOps-Gateway"}, "head_repository": None, } def _gitea_workflow_api(specialize, rows, *, run_pages=None, calls=None): run_pages = run_pages or {1: rows} calls = calls if calls is not None else [] def api(path): calls.append(path) if path.endswith("/actions/workflows?limit=100&page=1"): return { "workflows": [ { "id": workflow.rsplit("/", 1)[-1], "path": workflow, "state": "active", } for workflow in (specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW) ] } if "/actions/runs?" in path: page = int(path.rsplit("page=", 1)[-1]) return {"workflow_runs": run_pages.get(page, [])} raise AssertionError(f"unexpected fixed Gitea API path: {path}") return api, calls def test_gitea_workflow_inventory_uses_exact_active_filename_and_path( specialize, monkeypatch ): workflow = specialize.FULL_WORKFLOW filename = workflow.rsplit("/", 1)[-1] inventory = { "workflows": [ {"id": filename, "path": workflow, "state": "active"}, { "id": specialize.IMAGE_WORKFLOW.rsplit("/", 1)[-1], "path": specialize.IMAGE_WORKFLOW, "state": "active", }, ] } monkeypatch.setattr(specialize, "_api_json", lambda _path: inventory) assert specialize._workflow_id(workflow) == filename for bad in ( {"id": 34, "path": workflow, "state": "active"}, {"id": filename, "path": ".gitea/workflows/other.yml", "state": "active"}, {"id": filename, "path": workflow, "state": "enabled"}, ): monkeypatch.setattr( specialize, "_api_json", lambda _path, row=bad: {"workflows": [row]} ) with pytest.raises( specialize.RunnerFleetSpecializeError, match="workflow_unavailable" ): specialize._workflow_id(workflow) def test_successful_main_run_accepts_actual_gitea_run_without_workflow_id( specialize, monkeypatch ): revision = "a" * 40 run = _gitea_28_run(specialize, revision) api, calls = _gitea_workflow_api(specialize, [run]) monkeypatch.setattr(specialize, "_api_json", api) selected = specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) assert selected is run assert "workflow_id" not in selected assert selected["head_repository"] is None assert calls[0].endswith("/actions/workflows?limit=100&page=1") assert calls[1].endswith("/actions/runs?branch=main&limit=100&page=1") def test_source_image_fetch_accepts_actual_gitea_commit_tree_shape( specialize, monkeypatch, tmp_path ): revision = "a" * 40 tree = "b" * 40 calls = [] workflow_calls = [] def api(path): calls.append(path) if path.endswith("/branches/main"): return {"commit": {"id": revision}} if path.endswith(f"/git/commits/{revision}"): return {"sha": revision, "commit": {"tree": {"sha": tree}}} if "/actions/runs/" in path and "/artifacts?" in path: return { "artifacts": [ { "name": specialize.IMAGE_ARTIFACT_PREFIX + revision, "expired": False, "id": 44, "size_in_bytes": 1, } ] } raise AssertionError(f"unexpected fixed Gitea API path: {path}") def successful_run(workflow_path, requested_revision): assert requested_revision == revision workflow_calls.append(workflow_path) return {"id": 41 if workflow_path == specialize.IMAGE_WORKFLOW else 42} class ReachedArtifactDownload(Exception): pass def download(_artifact_id, _destination): raise ReachedArtifactDownload monkeypatch.setattr(specialize, "_api_json", api) monkeypatch.setattr(specialize, "_successful_main_run", successful_run) monkeypatch.setattr(specialize, "_download_artifact_zip", download) with pytest.raises(ReachedArtifactDownload): specialize._fetch_source_image_artifact(revision, tree, tmp_path) assert workflow_calls == [specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW] assert calls == [ f"/repos/{specialize.SOURCE_REPOSITORY_API}/branches/main", f"/repos/{specialize.SOURCE_REPOSITORY_API}/git/commits/{revision}", f"/repos/{specialize.SOURCE_REPOSITORY_API}/actions/runs/41/artifacts?name={specialize.IMAGE_ARTIFACT_PREFIX}{revision}", ] @pytest.mark.parametrize( ("commit_value", "error"), [ ({"sha": "a" * 40}, "source_tree_mismatch"), ({"sha": "a" * 40, "commit": {}}, "source_tree_mismatch"), ( {"sha": "a" * 40, "commit": {"tree": {"sha": "c" * 40}}}, "source_tree_mismatch", ), ( {"sha": "a" * 40, "tree": {"sha": "b" * 40}}, "source_tree_mismatch", ), ( {"sha": "c" * 40, "commit": {"tree": {"sha": "b" * 40}}}, "source_revision_mismatch", ), ( {"commit": {"tree": {"sha": "b" * 40}}}, "source_revision_mismatch", ), ], ) def test_source_image_fetch_rejects_noncanonical_gitea_commit_payload_before_artifact_lookup( specialize, monkeypatch, tmp_path, commit_value, error ): revision = "a" * 40 calls = [] workflow_calls = [] def api(path): calls.append(path) if path.endswith("/branches/main"): return {"commit": {"id": revision}} if path.endswith(f"/git/commits/{revision}"): return commit_value raise AssertionError(f"unexpected fixed Gitea API path: {path}") def unexpected_workflow(*_args): workflow_calls.append(True) raise AssertionError("workflow lookup must not follow an untrusted commit response") monkeypatch.setattr(specialize, "_api_json", api) monkeypatch.setattr(specialize, "_successful_main_run", unexpected_workflow) monkeypatch.setattr( specialize, "_download_artifact_zip", lambda *_args: pytest.fail("artifact download must not follow an untrusted commit response"), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): specialize._fetch_source_image_artifact(revision, "b" * 40, tmp_path) assert calls == [ f"/repos/{specialize.SOURCE_REPOSITORY_API}/branches/main", f"/repos/{specialize.SOURCE_REPOSITORY_API}/git/commits/{revision}", ] assert workflow_calls == [] @pytest.mark.parametrize( "changes", [ {"path": "managed-validation.yml@refs/heads/feature"}, {"repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, {"head_repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, {"head_sha": "b" * 40}, {"event": "workflow_dispatch"}, {"head_branch": "feature"}, {"pull_requests": [{"number": 123}]}, ], ) def test_successful_main_run_rejects_untrusted_gitea_identity( specialize, monkeypatch, changes ): revision = "a" * 40 run = {**_gitea_28_run(specialize, revision), **changes} api, _calls = _gitea_workflow_api(specialize, [run]) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="protected_validation_missing", ): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) @pytest.mark.parametrize( ("latest", "error"), [ ({"status": "in_progress", "conclusion": None}, "protected_validation_missing"), ({"status": "completed", "conclusion": "failure"}, "protected_validation_missing"), ( {"run_attempt": 2, "status": "completed", "conclusion": "failure"}, "protected_validation_missing", ), ({"run_attempt": None}, "workflow_runs_invalid"), ], ) def test_newest_matching_gitea_run_or_attempt_supersedes_prior_success( specialize, monkeypatch, latest, error ): revision = "a" * 40 successful = _gitea_28_run(specialize, revision, run_id=21532) newer = { **_gitea_28_run(specialize, revision, run_id=21533), **latest, } if latest.get("run_attempt") == 2: newer["id"] = successful["id"] api, _calls = _gitea_workflow_api(specialize, [successful, newer]) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) def test_workflow_run_pagination_fails_closed_at_fixed_page_limit(specialize, monkeypatch): revision = "a" * 40 wrong = {"path": "unrelated.yml@refs/heads/main"} pages = {page: [wrong] * 100 for page in range(1, 11)} api, calls = _gitea_workflow_api(specialize, [], run_pages=pages) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="workflow_runs_invalid" ): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) assert len([path for path in calls if "/actions/runs?" in path]) == 10 def test_archive_verification_hashes_large_tar_without_read_bytes(specialize, tmp_path, monkeypatch): archive_path = tmp_path / "images.tar" payload = b"bounded-stream" * 4096 with tarfile.open(archive_path, "w") as archive: member = tarfile.TarInfo("layer.tar") member.size = len(payload) archive.addfile(member, io.BytesIO(payload)) digest = hashlib.sha256(archive_path.open("rb").read()).hexdigest() def forbidden_read_bytes(_path): raise AssertionError("archive verifier must stream the tar file") monkeypatch.setattr(Path, "read_bytes", forbidden_read_bytes) specialize._verify_docker_archive(archive_path, archive_path.stat().st_size, digest) def test_archive_verifier_requires_exact_pinned_inherited_labels( specialize, tmp_path ): archive_path = tmp_path / "task-images.tar" images = _write_task_image_archive(specialize, archive_path) specialize._verify_docker_archive( archive_path, archive_path.stat().st_size, hashlib.sha256(archive_path.read_bytes()).hexdigest(), images, ) images = _write_task_image_archive(specialize, archive_path, extra_label=True) with pytest.raises( specialize.RunnerFleetSpecializeError, match="image_archive_identity_invalid", ): specialize._verify_docker_archive( archive_path, archive_path.stat().st_size, hashlib.sha256(archive_path.read_bytes()).hexdigest(), images, ) def test_installed_image_manifest_requires_complete_source_and_profile_pins(specialize): manifest = _valid_task_image_manifest(specialize) assert specialize._valid_installed_manifest(manifest) is True manifest["images"][0]["reference"] = "localhost/itworx-ci-checks:latest" assert specialize._valid_installed_manifest(manifest) is False def test_task_image_runtime_verifier_accepts_only_exact_pinned_inherited_labels( specialize, monkeypatch ): expected = _valid_task_image_manifest(specialize)["images"][0] runtime_labels = specialize._runtime_task_image_labels(expected) assert runtime_labels is not None assert len(runtime_labels) == 14 actual = { "Id": expected["imageId"], "Os": "linux", "Architecture": "amd64", "Config": {"Labels": runtime_labels}, } monkeypatch.setattr( specialize, "_run", lambda *_args, **_kwargs: json.dumps([actual]), ) assert specialize._verify_manifest_images_in_dind([expected]) is True actual["Config"]["Labels"]["untrusted.extra"] = "value" assert specialize._verify_manifest_images_in_dind([expected]) is False del actual["Config"]["Labels"]["untrusted.extra"] del expected["labels"]["tech.itworx.source-tree"] assert specialize._verify_manifest_images_in_dind([expected]) is False def test_bootstrap_receipt_is_source_bound_private_and_cas_consumed( specialize, tmp_path, monkeypatch ): state_root = tmp_path / "runner-fleet-state" state_root.mkdir(mode=0o700) os.chmod(state_root, 0o700) manifest_path = state_root / "task-image-manifest.json" marker_path = state_root / "specialized.json" receipt_path = state_root / "admission-bootstrap.json" manifest_bytes = json.dumps( _valid_task_image_manifest(specialize), sort_keys=True, separators=(",", ":") ).encode() + b"\n" manifest_path.write_bytes(manifest_bytes) os.chmod(manifest_path, 0o600) manifest_sha = hashlib.sha256(manifest_bytes).hexdigest() marker = { "schemaVersion": 1, "status": "active", "sourceScriptSha256": "e" * 64, "imageManifestSha256": manifest_sha, "imageSourceRevision": "a" * 40, "imageSourceTree": "b" * 40, "verifiedRunners": list(specialize._SPECIALIZED_NAMES), "resourceEnvelopeDigest": "f" * 64, "updatedAt": "2026-10-01T12:00:00Z", } marker_path.write_text(json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n") os.chmod(marker_path, 0o600) registrations = [] for profile in specialize.RUNNER_PROFILES.values(): registrations.append( { "profileId": profile.id, "displayName": profile.display_name, "containerName": profile.container_name, "id": len(registrations) + 10, "labels": sorted({profile.runner_label, *profile.legacy_labels}), } ) receipt = { "schemaVersion": 1, "status": "pending", "sourceScriptSha256": marker["sourceScriptSha256"], "imageManifestSha256": manifest_sha, "imageSourceRevision": "a" * 40, "imageSourceTree": "b" * 40, "verifiedRunners": list(specialize._SPECIALIZED_NAMES), "registrations": registrations, "resourceEnvelopeDigest": marker["resourceEnvelopeDigest"], "createdAt": "2026-10-01T12:00:00Z", } receipt_path.write_text(json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n") os.chmod(receipt_path, 0o600) assert specialize._valid_admission_bootstrap_receipt(receipt) validated = specialize.read_admission_bootstrap_receipt( receipt_path, marker_path=marker_path, image_manifest_path=manifest_path ) assert validated is not None pending_sha = validated[1] assert validated[0]["status"] == "pending" assert specialize.consume_admission_bootstrap_receipt( pending_sha, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) consumed = specialize.read_admission_bootstrap_receipt( receipt_path, marker_path=marker_path, image_manifest_path=manifest_path ) assert consumed is not None assert consumed[0]["status"] == "consumed" assert consumed[1] == pending_sha assert specialize.consume_admission_bootstrap_receipt( pending_sha, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) assert not specialize.consume_admission_bootstrap_receipt( "0" * 64, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) def test_runner_config_is_profile_locked_and_only_backend_frontend_keep_legacy_labels(specialize): revision = "a" * 40 checks = specialize._render_runner_config("checks", revision).decode() backend = specialize._render_runner_config("backend", revision).decode() frontend = specialize._render_runner_config("frontend", revision).decode() security = specialize._render_runner_config("security", revision).decode() assert "itworx-ci-checks:docker://localhost/itworx-ci-checks:sha-" in checks assert "linux-validation" not in checks and "ubuntu-latest" not in checks assert "linux-validation" in backend and "ubuntu-latest" in backend assert "linux-validation" in frontend and "ubuntu-latest" in frontend assert "linux-validation" not in security and "ubuntu-latest" not in security for profile_id, profile in specialize.RUNNER_PROFILES.items(): config = specialize._render_runner_config(profile_id, revision).decode() task_image = specialize._task_image_reference(profile_id, revision) job_cpu_milli, job_memory_mib = specialize._runner_job_limits(profile_id) cpu = f"{job_cpu_milli / 1000:.3f}".rstrip("0").rstrip(".") assert f"--cpus={cpu} --memory={job_memory_mib}m" in config assert f"source=itworx-ci-{profile_id}-cache,target=/opt/itworx-ci-cache" in config assert f'valid_volumes:\n - "itworx-ci-{profile_id}-cache"' in config assert "[0-9]*/**" not in config assert "DOCKER_HOST:" not in config assert 'docker_host: "-"' in config assert "host.docker.internal" not in config assert all( f"docker://{task_image}" in line for line in config.splitlines() if ":docker://" in line ) if profile_id == "build": assert "--security-opt=seccomp=unconfined" in config assert "--security-opt=systempaths=unconfined" in config assert "--device=/dev/net/tun:/dev/net/tun:rwm" in config else: assert "--security-opt=" not in config assert "--device=" not in config def test_fixed_workload_reservations_are_subtracted_from_aggregate_profiles(specialize): for profile_id in ("backend", "frontend", "browser"): profile = specialize.RUNNER_PROFILES[profile_id] service_cpu, service_memory = specialize.FIXED_WORKLOAD_RESERVATION[profile_id] job_cpu, job_memory = specialize._runner_job_limits(profile_id) assert job_cpu + service_cpu == profile.cpu_milli assert job_memory + service_memory == profile.memory_mib config = specialize._render_runner_config(profile_id, "a" * 40).decode() cpu = f"{job_cpu / 1000:.3f}".rstrip("0").rstrip(".") assert f"--cpus={cpu} --memory={job_memory}m" in config for profile_id in ("checks", "security"): profile = specialize.RUNNER_PROFILES[profile_id] assert specialize._runner_job_limits(profile_id) == ( profile.cpu_milli, profile.memory_mib, ) assert specialize._runner_job_limits("build") == (4000, 6144) def test_base_compose_model_explicitly_resolves_the_runner_profile(specialize, monkeypatch): compose = b"captured v1 source" base = _base_v1_compose_model(specialize) monkeypatch.setattr( specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest(), ) calls = [] def run_compose(_compose_bytes, *arguments, **_kwargs): calls.append(arguments) return json.dumps(base) monkeypatch.setattr(specialize, "_run_compose", run_compose) assert specialize._validate_base_composition(compose) == "v1" assert calls == [("--profile", "runner", "config", "--format", "json")] def test_rendered_compose_uses_fixed_rootful_dind_and_shared_workspaces(specialize): base = _base_v1_compose_model(specialize) rendered = json.loads(specialize._render_specialized_compose(base)) services = rendered["services"] dind = services["runner-dind"] assert services["deploy-control"] == base["services"]["deploy-control"] assert dind["user"] == "0" assert dind["command"][0] == "dockerd" assert dind["command"][1] == f"--host={specialize._DIND_DOCKER_HOST}" assert "--data-root=/var/lib/docker" in dind["command"] assert "--exec-opt=native.cgroupdriver=cgroupfs" in dind["command"] assert dind["environment"] == {"DOCKER_HOST": specialize._DIND_DOCKER_HOST} assert _mounts(specialize, dind) == { (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, False, ), ( str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "/home/rootless/.local/share/docker", True, ), *{ ( str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False, ) for profile in specialize.RUNNER_PROFILES.values() }, } assert len([volume for volume in dind["volumes"] if volume["type"] == "bind"]) == 9 expected_services = {"deploy-control", "runner-dind"} expected_services.update( specialize._OLD_SERVICES[name] for name in specialize._LEGACY_V1_NAMES ) expected_services.update(f"runner-{profile_id}" for profile_id in specialize.PROFILE_IDS) assert set(services) == expected_services for profile_id in specialize.PROFILE_IDS: service = services[f"runner-{profile_id}"] assert service["environment"]["DOCKER_HOST"] == specialize._DIND_DOCKER_HOST assert service["extra_hosts"] == ["gitea.example.com:198.19.104.1"] assert ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True, ) in specialize._compose_mounts(service) @pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") def test_profile_state_creation_uses_effective_runner_image_owners_and_resumes( specialize, tmp_path, monkeypatch ): state = tmp_path / "state" state.mkdir(mode=0o700) data_root = state / "runner-data" workspace_root = state / "workspaces" config_root = state / "runner-configs" image_root = state / "task-images" dind_root = state / "dind" / "docker" shadow_root = state / "dind" / "rootless-unused" control_root = state / "dind-control" for name, value in ( ("_SPECIALIZED_DATA_ROOT", data_root), ("_SPECIALIZED_WORKSPACE_ROOT", workspace_root), ("RUNNER_CONFIG_ROOT", config_root), ("TASK_IMAGE_ROOT", image_root), ("DIND_STORAGE_ROOT", dind_root), ("_DIND_ROOTLESS_SHADOW_ROOT", shadow_root), ("_DIND_CONTROL_ROOT", control_root), ): monkeypatch.setattr(specialize, name, value) control_root.mkdir(mode=0o700) specialize._prepare_profile_state("a" * 40) for profile_id, profile in specialize.RUNNER_PROFILES.items(): data_path = data_root / profile_id workspace_path = workspace_root / profile.container_name assert data_path.stat().st_uid == 0 assert workspace_path.stat().st_uid == 1000 assert control_root.stat().st_uid == 0 assert os.stat(control_root).st_mode & 0o777 == 0o700 config_path = config_root / "checks.yaml" expected = specialize._render_runner_config("checks", "a" * 40) invalid_configs = ( expected.replace( b'valid_volumes:\n - "itworx-ci-checks-cache"', b'valid_volumes:\n - "/itworx-ci-checks-workspace/[0-9]*/**"', 1, ), expected.replace( b'valid_volumes:\n - "itworx-ci-checks-cache"', b'valid_volumes:\n - "/run/itworx-ci-control/**"', 1, ), expected.replace( b' docker_host: "-"', b' docker_host: "tcp://runner-dind:2375"', 1, ), expected.replace( b" envs:\n", b" envs:\n DOCKER_HOST: tcp://runner-dind:2375\n", 1, ), ) for invalid_config in invalid_configs: config_path.write_bytes(invalid_config) os.chmod(config_path, 0o600) with pytest.raises( specialize.RunnerFleetSpecializeError, match="profile_config_adoption_rejected", ): specialize._prepare_profile_state("a" * 40) config_path.write_bytes(expected) os.chmod(config_path, 0o600) specialize._prepare_profile_state("a" * 40) def _mounts(specialize, service): return specialize._compose_mounts(service) def test_rendered_compose_uses_only_canonical_protected_volume_sources(specialize): base = _base_v1_compose_model(specialize) rendered = json.loads(specialize._render_specialized_compose(base)) services = rendered["services"] assert specialize._compose_mounts(services["runner-dind"]) == { (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, False, ), ( str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "/home/rootless/.local/share/docker", True, ), *{ ( str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False, ) for profile in specialize.RUNNER_PROFILES.values() }, } assert services["runner-dind"]["restart"] == "no" for profile_id, profile in specialize.RUNNER_PROFILES.items(): service = services[f"runner-{profile_id}"] assert specialize._compose_mounts(service) == { (str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "/data", False), (str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "/runner-config.yaml", True), (str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True), (str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False), } assert "-f" in specialize._compose_command("up", "-d") assert specialize._compose_command("up", "-d")[specialize._compose_command("up", "-d").index("-f") + 1] == "-" def test_specialized_compose_validation_rejects_socket_and_gateway_broadening( specialize, monkeypatch ): rendered = json.loads( specialize._render_specialized_compose(_base_v1_compose_model(specialize)) ) mutations = ( lambda model: model["services"]["runner-dind"]["environment"].update( DOCKER_HOST="tcp://0.0.0.0:2375" ), lambda model: next( volume for volume in model["services"]["runner-dind"]["volumes"] if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT ).update(read_only=True), lambda model: next( volume for volume in model["services"]["runner-frontend"]["volumes"] if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT ).update(read_only=False), lambda model: model["services"]["runner-frontend"]["extra_hosts"].append( "host.docker.internal:host-gateway" ), lambda model: model["services"]["runner-frontend"].update( network_mode="host" ), lambda model: model["services"]["runner-frontend"].update( privileged=True ), lambda model: model["networks"]["runner-sandbox"].update( name="runner-sandbox" ), ) for mutate in mutations: candidate = copy.deepcopy(rendered) mutate(candidate) monkeypatch.setattr(specialize, "_compose_model", lambda _raw, value=candidate: value) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._validate_composition(b"candidate") def test_specialized_compose_validation_accepts_real_compose_normalization( specialize, monkeypatch ): normalized = json.loads( specialize._render_specialized_compose(_base_v1_compose_model(specialize)) ) service = normalized["services"]["runner-frontend"] service.pop("privileged") service["extra_hosts"] = ["gitea.example.com=198.19.104.1"] service["networks"] = {"runner-sandbox": {"aliases": ["itworx-ci-frontend"]}} monkeypatch.setattr(specialize, "_compose_model", lambda _raw: normalized) assert specialize._validate_composition(b"normalized candidate") def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize): base = _base_v1_compose_model(specialize) item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="unless-stopped") assert specialize._legacy_dind_observation(item, base)["running"] is True for key, value in (("User", "0"), ("Cmd", ["--data-root=/var/lib/docker"]), ("Entrypoint", ["sh"])): changed = copy.deepcopy(item) changed["Config"][key] = value with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): specialize._legacy_dind_observation(changed, base) changed = copy.deepcopy(item) changed["Mounts"][-1]["Source"] = "/untrusted/docker" with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): specialize._legacy_dind_observation(changed, base) @pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false", "2|cgroupfs|true"]) def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces( specialize, monkeypatch, support ): model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize))) service = model["services"]["runner-dind"] item = _legacy_dind_fixture(specialize, _base_v1_compose_model(specialize), image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no") item["Config"].update(User="0", Cmd=service["command"]) item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"] item["HostConfig"].update( CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={} ) item["Mounts"] = [{"Type": "bind", "Source": m["source"], "Destination": m["target"], "RW": not m["read_only"]} for m in service["volumes"]] assert specialize._validate_dind_outer(item, model) is True monkeypatch.setattr(specialize, "_inspect", lambda _name: item) commands = [] def run(args, **_kwargs): commands.append(args) return support monkeypatch.setattr(specialize, "_run", run) if support == "2|cgroupfs|true": specialize._require_dind_resource_support(model) assert f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}" in commands[-1] else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_limits_unsupported"): specialize._require_dind_resource_support(model) missing_workspace = copy.deepcopy(item) missing_workspace["Mounts"].pop() assert specialize._validate_dind_outer(missing_workspace, model) is False changed_cases = [] tcp_host = copy.deepcopy(item) tcp_host["Config"]["Env"] = ["DOCKER_HOST=tcp://0.0.0.0:2375"] changed_cases.append(tcp_host) tcp_command = copy.deepcopy(item) tcp_command["Config"]["Cmd"][1] = "--host=tcp://0.0.0.0:2375" changed_cases.append(tcp_command) published_port = copy.deepcopy(item) published_port["HostConfig"]["PortBindings"] = { "2375/tcp": [{"HostIp": "0.0.0.0", "HostPort": "2375"}] } changed_cases.append(published_port) for changed in changed_cases: assert specialize._validate_dind_outer(changed, model) is False def test_outer_runner_must_match_fixed_compose_mount_and_image_identity(specialize): profile_id = "frontend" profile = specialize.RUNNER_PROFILES[profile_id] name = profile.container_name image = { "Name": f"/{name}", "Image": specialize.RUNNER_IMAGE_ID, "Config": { "Image": specialize.RUNNER_IMAGE, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, }, "Env": [ "CONFIG_FILE=/runner-config.yaml", f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}", ], }, "State": {"Running": False}, "HostConfig": { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * specialize._MIB, "PidsLimit": 192, "PidMode": "", "NetworkMode": specialize._RUNNER_NETWORK_NAME, "Privileged": False, "ExtraHosts": ["gitea.example.com:198.19.104.1"], "RestartPolicy": {"Name": "no"}, }, "Mounts": [ {"Source": str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "Destination": "/data", "RW": True}, {"Source": str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "Destination": "/runner-config.yaml", "RW": False}, {"Source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "Destination": specialize._DIND_CONTROL_CONTAINER_ROOT, "RW": False}, { "Source": str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / name)), "Destination": f"/{name}-workspace", "RW": True, }, ], } assert specialize._validate_outer_item(name, image, profile_id) unqualified_network = copy.deepcopy(image) unqualified_network["HostConfig"]["NetworkMode"] = "runner-sandbox" assert not specialize._validate_outer_item(name, unqualified_network, profile_id) tcp_env = copy.deepcopy(image) tcp_env["Config"]["Env"][-1] = "DOCKER_HOST=tcp://runner-dind:2375" assert not specialize._validate_outer_item(name, tcp_env, profile_id) writable_socket = copy.deepcopy(image) next( mount for mount in writable_socket["Mounts"] if mount["Destination"] == specialize._DIND_CONTROL_CONTAINER_ROOT )["RW"] = True assert not specialize._validate_outer_item(name, writable_socket, profile_id) host_gateway = copy.deepcopy(image) host_gateway["HostConfig"]["ExtraHosts"].append("host.docker.internal:host-gateway") assert not specialize._validate_outer_item(name, host_gateway, profile_id) image["Image"] = "sha256:" + "0" * 64 assert not specialize._validate_outer_item(name, image, profile_id) def test_atomic_guard_replace_uses_expected_hash_as_compare_and_swap(specialize, tmp_path): path = tmp_path / "guard.sh" path.write_bytes(b"original guard\n") original_sha = hashlib.sha256(path.read_bytes()).hexdigest() specialize._atomic_write_owned( path, b"patched guard\n", mode=0o600, expected_sha256=original_sha, ) assert path.read_bytes() == b"patched guard\n" with pytest.raises(specialize.RunnerFleetSpecializeError, match="compare_and_swap_failed"): specialize._atomic_write_owned( path, b"stale write\n", mode=0o600, expected_sha256=original_sha, ) assert path.read_bytes() == b"patched guard\n" def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(specialize): source = ( "stack_recovery=0\n" "set_managed_runners_disabled false\n" "unexpected=0\n" ' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' ) patched = specialize._patch_runner_guard( source, "fixed-module --state", 'fixed-module --guard-allows-container "$name"', ) assert "--fence-legacy" in patched assert "--guard-allows-legacy-container" in patched assert 'if [ "$specialization_state" = "legacy" ]; then' in patched assert 'elif [ "$specialization_state" = "active" ]' in patched assert '--guard-allows-container "$name"' in patched assert " set_managed_runners_disabled false\n" in patched with pytest.raises(specialize.RunnerFleetSpecializeError, match="runner_guard_shape_invalid"): specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow") def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest()) real_os = os class RootPosixOS: name = "posix" @staticmethod def geteuid(): return 0 def __getattr__(self, name): return getattr(real_os, name) monkeypatch.setattr(specialize, "os", RootPosixOS()) monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False}) shell_calls = [] monkeypatch.setattr(specialize, "_run_approved_shell_script", shell_calls.append) with pytest.raises(specialize.RunnerFleetSpecializeError, match="script_not_approved"): specialize.execute(script + b"# altered\n") assert shell_calls == [] def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_edits( specialize, tmp_path, monkeypatch ): runner = tmp_path / "runner-guard" host = tmp_path / "host-guard" runner.write_bytes( b"stack_recovery=0\n" b" set_managed_runners_disabled false\n" b"unexpected=0\n" b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' ) host.write_bytes( b"# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" b"old capacity writer\n" b"# ITWORX_RUNNER_CAPACITY_V2_END\n" b"runner_healthy=1\n" b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' b" recover legacy stack\n" b" fi\n" ) runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() host_sha = hashlib.sha256(host.read_bytes()).hexdigest() monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) monkeypatch.setattr(specialize, "RUNNER_GUARD_SOURCE_SHA256", runner_sha) monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", host_sha) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) backup = tmp_path / "guard-backup" backup.mkdir(mode=0o700) for path, name in ((runner, "gitea-runner-guard.sh"), (host, "host-resource-guard.sh")): target = backup / name target.write_bytes(path.read_bytes()) target.chmod(0o600) journal = { "backupDirectory": str(backup), "guardFiles": [ {"path": str(runner), "originalSha256": runner_sha, "installedSha256": None}, {"path": str(host), "originalSha256": host_sha, "installedSha256": None}, ] } specialize._service_guard_scripts(journal) installed_runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() assert journal["guardFiles"][0]["installedSha256"] == installed_runner_sha assert "--fence-legacy" in runner.read_text(encoding="utf-8") assert "# ITWORX_RUNNER_CAPACITY_V2_BEGIN" not in host.read_text(encoding="utf-8") host.write_text("operator-owned guard revision\n", encoding="utf-8") with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_source_changed"): specialize._service_guard_scripts(journal) assert host.read_text(encoding="utf-8") == "operator-owned guard revision\n" @pytest.fixture def guard_transaction(specialize, tmp_path, monkeypatch): monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) runner = tmp_path / "runner-guard" host = tmp_path / "host-guard" originals = ( (b'#!/bin/bash\nstack_recovery=0\nset_managed_runners_disabled false\nunexpected=0\n' b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n'), (b'#!/bin/bash\n# ITWORX_RUNNER_CAPACITY_V2_BEGIN\nold capacity writer\n' b'# ITWORX_RUNNER_CAPACITY_V2_END\nrunner_healthy=1\n' b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' b' recover legacy stack\n fi\n'), ) backup = tmp_path / "backups" backup.mkdir(mode=0o700) for path, original, name in zip((runner, host), originals, ("gitea-runner-guard.sh", "host-resource-guard.sh"), strict=True): path.write_bytes(original) target = backup / name target.write_bytes(original) target.chmod(0o600) monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) for name, original in zip(("RUNNER_GUARD_SOURCE_SHA256", "HOST_GUARD_SOURCE_SHA256"), originals, strict=True): monkeypatch.setattr(specialize, name, hashlib.sha256(original).hexdigest()) journal = {"backupDirectory": str(backup), "guardFiles": [ {"path": str(path), "originalSha256": hashlib.sha256(original).hexdigest(), "installedSha256": None} for path, original in zip((runner, host), originals, strict=True) ]} monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) monkeypatch.setattr(specialize, "_set_stage", lambda _journal, _stage: None) monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None) return journal, runner, host, originals @pytest.mark.parametrize("invalid", [0, 1]) def test_both_candidate_syntax_checks_precede_any_intent_or_guard_write(specialize, guard_transaction, monkeypatch, invalid): journal, runner, host, originals = guard_transaction checked = [] def syntax(encoded): checked.append(encoded) if len(checked) == invalid + 1: raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_guard_syntax_invalid") monkeypatch.setattr(specialize, "_validate_guard_syntax", syntax) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: pytest.fail("invalid candidate published intent")) monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("invalid candidate installed")) with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_syntax_invalid"): specialize._service_guard_scripts(journal) assert (runner.read_bytes(), host.read_bytes()) == originals assert all(row["installedSha256"] is None for row in journal["guardFiles"]) @pytest.mark.parametrize("boundary", ["intent_failed", "after_intent", "after_runner", "after_host"]) def test_guard_installation_durable_crash_boundaries_restore_only_owned_bytes(specialize, guard_transaction, monkeypatch, boundary): journal, runner, host, originals = guard_transaction persisted = [] real_write = specialize._atomic_write_owned writes = [] def journal_write(value): assert all(row["installedSha256"] for row in value["guardFiles"]) assert (runner.read_bytes(), host.read_bytes()) == originals if boundary == "intent_failed": raise OSError("private canary write failure") persisted.append(copy.deepcopy(value)) if boundary == "after_intent": raise OSError("private canary crash after intent") def install(path, encoded, **kwargs): assert persisted and persisted[0]["guardFiles"] == journal["guardFiles"] real_write(path, encoded, **kwargs) writes.append(path) if boundary == "after_runner" or boundary == "after_host" and len(writes) == 2: raise OSError("private canary crash after replace") monkeypatch.setattr(specialize, "_write_journal", journal_write) monkeypatch.setattr(specialize, "_atomic_write_owned", install) with pytest.raises(OSError): specialize._service_guard_scripts(journal) monkeypatch.setattr(specialize, "_atomic_write_owned", real_write) specialize._restore_guard_scripts(persisted[0] if persisted else journal) assert (runner.read_bytes(), host.read_bytes()) == originals specialize._restore_guard_scripts(journal) assert (runner.read_bytes(), host.read_bytes()) == originals @pytest.mark.parametrize("mutation", ["duplicate", "foreign_path", "original_hash", "planned_hash", "backup", "foreign_edit"]) def test_guard_restore_rejects_foreign_journal_backup_or_live_bytes_before_any_write(specialize, guard_transaction, monkeypatch, mutation): journal, runner, host, _originals = guard_transaction specialize._service_guard_scripts(journal) if mutation == "duplicate": journal["guardFiles"][1] = copy.deepcopy(journal["guardFiles"][0]) elif mutation == "foreign_path": journal["guardFiles"][1]["path"] = str(host.with_name("foreign")) elif mutation == "original_hash": journal["guardFiles"][0]["originalSha256"] = "f" * 64 elif mutation == "planned_hash": journal["guardFiles"][0]["installedSha256"] = "f" * 64 elif mutation == "backup": (Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign private canary") else: host.write_bytes(b"foreign private canary") before = runner.read_bytes(), host.read_bytes() monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("foreign state restored")) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._restore_guard_scripts(journal) assert (runner.read_bytes(), host.read_bytes()) == before def test_guard_syntax_checker_never_exposes_bash_output(specialize, monkeypatch): def parse(args, **kwargs): assert args == ["bash", "-n"] and kwargs["input"] == b"candidate" assert kwargs["timeout"] == 15 and kwargs["capture_output"] is True return SimpleNamespace(returncode=2, stdout=b"private canary", stderr=b"private canary") monkeypatch.setattr(specialize.subprocess, "run", parse) with pytest.raises(specialize.RunnerFleetSpecializeError, match="^runner_fleet_specialization_guard_syntax_invalid$"): specialize._validate_guard_syntax(b"candidate") @pytest.mark.parametrize("mutation", [None, "foreign", "begin_only", "end_only", "repeated"]) def test_host_guard_without_capacity_stanza_requires_exact_approved_bytes(specialize, guard_transaction, monkeypatch, mutation): _journal, _runner, _host, originals = guard_transaction text = originals[1].decode() start = text.index("# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n") end = text.index("# ITWORX_RUNNER_CAPACITY_V2_END\n") + len("# ITWORX_RUNNER_CAPACITY_V2_END\n") approved = text[:start] + text[end:] monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", hashlib.sha256(approved.encode()).hexdigest()) if mutation == "foreign": approved += "# operator edit\n" elif mutation == "begin_only": approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" elif mutation == "end_only": approved += "# ITWORX_RUNNER_CAPACITY_V2_END\n" elif mutation == "repeated": approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n# ITWORX_RUNNER_CAPACITY_V2_END\n" * 2 if mutation is None: result = specialize._patch_host_guard(approved, "fixed --state") assert 'specialization_state="$(fixed --state' in result assert '"$specialization_state" != "legacy"' in result else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="capacity_block_invalid"): specialize._patch_host_guard(approved, "fixed --state") @pytest.mark.parametrize("mutation", ["path", "backup", "foreign_edit"]) def test_guard_rollback_refusal_preserves_marker_journal_before_lifecycle_effects(specialize, guard_transaction, monkeypatch, mutation): journal, runner, host, _originals = guard_transaction specialize._service_guard_scripts(journal) live_compose = runner.with_name("compose") live_compose.write_bytes(b"fixed compose") monkeypatch.setattr(specialize, "COMPOSE_PATH", live_compose) journal["baseComposeSha256"] = hashlib.sha256(live_compose.read_bytes()).hexdigest() monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: pytest.fail("refusal reached lifecycle effects")) marker, durable = runner.with_name("marker"), runner.with_name("journal") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) if mutation == "path": journal["guardFiles"][0]["path"] = str(runner.with_name("foreign")) elif mutation == "backup": (Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign canary") elif mutation == "foreign_edit": host.write_bytes(b"foreign canary") assert specialize._rollback_before_activation(journal) is False assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" @pytest.mark.parametrize("failure", ["cleanup", "restore_cas", None]) def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, guard_transaction, monkeypatch, failure): journal, runner, host, originals = guard_transaction specialize._service_guard_scripts(journal) fenced = runner.read_bytes(), host.read_bytes() compose, marker, durable = runner.with_name("compose"), runner.with_name("marker"), runner.with_name("journal") compose.write_bytes(b"fixed compose") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(), originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None}, originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupRootSubtree=[]) monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose") monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) monkeypatch.setattr(specialize, "_inspect", lambda _name: None) events = [] def cleanup(_journal): assert (runner.read_bytes(), host.read_bytes()) == fenced events.append("cleanup") if failure == "cleanup": raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_command_failed") monkeypatch.setattr(specialize, "_cleanup_new_containers", cleanup) def cgroup(**_kwargs): assert (runner.read_bytes(), host.read_bytes()) == fenced events.append("cgroup") return {"familyExists": False} monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) write = specialize._atomic_write_owned def restore(*args, **kwargs): assert events == ["cleanup", "cgroup"] assert marker.exists() and durable.exists() if failure == "restore_cas": raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed") write(*args, **kwargs) monkeypatch.setattr(specialize, "_atomic_write_owned", restore) assert specialize._rollback_before_activation(journal) is (failure is None) if failure: assert (runner.read_bytes(), host.read_bytes()) == fenced assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" else: assert (runner.read_bytes(), host.read_bytes()) == originals assert not marker.exists() and not durable.exists() def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch): marker, journal = tmp_path / "marker", tmp_path / "journal" marker.write_bytes(b"owned marker") journal.write_bytes(b"owned journal") @contextmanager def busy(): raise RuntimeError("private canary lock refusal") yield monkeypatch.setattr(specialize, "_rollback_maintenance", busy) monkeypatch.setattr(specialize, "_rollback_before_activation_locked", lambda *_args: pytest.fail("busy barrier reached effects")) assert specialize._rollback_before_activation({}) is False assert marker.read_bytes() == b"owned marker" and journal.read_bytes() == b"owned journal" def test_rollback_barrier_drift_refuses_before_guards_or_cleanup(specialize, guard_transaction, monkeypatch): journal, _runner, _host, _originals = guard_transaction @contextmanager def changed(): def refuse(): raise RuntimeError("private canary changed inode") yield refuse monkeypatch.setattr(specialize, "_rollback_maintenance", changed) monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: pytest.fail("drift restored guard")) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda *_args: pytest.fail("drift changed fleet")) assert specialize._rollback_before_activation(journal) is False @pytest.mark.skipif(os.name != "posix", reason="real common flock contract requires POSIX") def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_finishes(specialize, tmp_path, monkeypatch): if os.geteuid() != 0: pytest.skip("protected maintenance lock requires root") import runner_acceptance_quiescence as quiescence lock = tmp_path / "maintenance.lock" lock.touch(mode=0o600) monkeypatch.setattr(quiescence, "MAINTENANCE_LOCK", lock) monkeypatch.setattr(quiescence, "_LOCK_PROBE", quiescence._LOCK_PROBE.replace( "/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock", str(lock))) monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=dict)) probe = ( "import fcntl,os,sys\nfd=os.open(sys.argv[1],os.O_RDWR|os.O_NOFOLLOW)\n" "try: fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)\n" "except BlockingIOError: sys.exit(0)\n" "sys.exit(1)\n" ) events = [] def held(phase): result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) assert result.returncode == 0 and not result.stdout and not result.stderr events.append(phase) compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal" compose.write_bytes(b"captured compose") marker.write_bytes(b"owned marker") durable.write_bytes(b"owned journal") monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_guard_script_plan", lambda _journal: []) monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: held("guards-restored")) monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: held("cleanup")) monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"captured compose") monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) monkeypatch.setattr(specialize, "_inspect", lambda _name: None) def cgroup(**_kwargs): held("cgroup") assert marker.exists() and durable.exists() return {"familyExists": False} monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}], "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, "originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupRootSubtree": []} assert specialize._rollback_before_activation(journal) is True assert events == ["cleanup", "cgroup", "guards-restored"] assert not marker.exists() and not durable.exists() # The independent participant can acquire only after the wrapper returns. result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) assert result.returncode == 1 and not result.stdout and not result.stderr @pytest.fixture def owned_orphan(specialize, guard_transaction, tmp_path, monkeypatch): import runner_acceptance_quiescence as quiescence journal, runner, host, originals = guard_transaction corrected = specialize._guard_script_plan(journal)[0][3] closing = f'\nfi\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode() orphan = corrected.replace(closing, f'\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode(), 1) runner.write_bytes(orphan) monkeypatch.setattr(specialize, "_OWNED_MALFORMED_RUNNER_GUARD_SHA256", hashlib.sha256(orphan).hexdigest()) receipt = { "schema": quiescence.SCHEMA, "revision": "0ea6f267c28d51ac7113e6d943c89857d3a189ab", "manifestDigest": quiescence._digest(quiescence.OBSERVATION), "phase": "complete", "progress": {row["id"]: "quiesced" for row in quiescence._fixed_rows()}, "registrationProgress": {"9": "restored", "22": "restored"}, "guardProgress": {"runner": "restored", "host": "restored"}, "guards": {name: { "original": base64.b64encode(original).decode(), "originalSha256": hashlib.sha256(original).hexdigest(), "wrappedSha256": hashlib.sha256(quiescence.ControllerDriver._wrapped_guard(original)).hexdigest(), "mode": path.stat().st_mode & 0o777, } for name, path, original in zip(("runner", "host"), (runner, host), originals, strict=True)}, "baseline": {"rows": [{"state": {"Running": False}} for _row in quiescence._fixed_rows()]}, } events = [] for name in ("SPECIALIZED_STATE_PATH", "MIGRATION_JOURNAL_PATH"): monkeypatch.setattr(specialize, name, tmp_path / name) monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: False) monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) monkeypatch.setattr(specialize, "_read_fleet_base_compose", lambda: (b"source", None)) monkeypatch.setattr(specialize, "_validate_base_composition", lambda _source: "v1") monkeypatch.setattr(specialize, "_runner_rows", lambda: [{"id": 9}]) monkeypatch.setattr(specialize, "_validate_legacy_registrations", lambda *_args: [{"id": 9}]) monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: ("v1", [{"id": "fixed"}])) monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=lambda: {"fixed": "registry"})) monkeypatch.setattr(quiescence, "_read_receipt", lambda: copy.deepcopy(receipt)) monkeypatch.setattr(quiescence, "_idle", lambda *_args: events.append("idle")) def attest(snapshot, *, baseline, progress): assert baseline == receipt["baseline"] and progress == receipt["progress"] assert snapshot["rows"] == baseline["rows"] events.append("attested") monkeypatch.setattr(quiescence, "attest_snapshot", attest) class Driver: _wrapped_guard = staticmethod(quiescence.ControllerDriver._wrapped_guard) def __init__(self, projects): assert projects == {"fixed": "registry"} @contextmanager def maintenance(self): with (tmp_path / "fixture-lock").open("wb") as lock: self._maintenance_fd = lock.fileno() events.append("locked") yield events.append("unlocked") def assert_barrier(self): events.append("barrier") def snapshot(self): self._guard_binding() events.append("snapshot") return copy.deepcopy(receipt["baseline"]) monkeypatch.setattr(quiescence, "ControllerDriver", Driver) return quiescence, receipt, runner, host, originals, orphan, events, Driver def test_owned_orphan_recovery_uses_full_repeated_admission_and_cas_then_is_idempotent(specialize, owned_orphan, monkeypatch): _q, receipt, runner, host, originals, orphan, events, _driver = owned_orphan before = copy.deepcopy(receipt) write = specialize._atomic_write_owned def restore(path, data, **kwargs): assert events.count("attested") == 2 and events.count("idle") == 3 assert path == runner and data == originals[0] assert kwargs["expected_sha256"] == hashlib.sha256(orphan).hexdigest() events.append("restore") write(path, data, **kwargs) monkeypatch.setattr(specialize, "_atomic_write_owned", restore) specialize._recover_owned_orphan_guard() assert (runner.read_bytes(), host.read_bytes()) == originals and receipt == before assert events.index("locked") < events.index("restore") < events.index("unlocked") specialize._recover_owned_orphan_guard() assert events.count("restore") == 1 @pytest.mark.parametrize("mutation", ["revision", "manifest", "phase", "progress", "registration", "guard_progress", "original", "wrapped", "foreign_runner", "foreign_host", "marker", "journal", "storage", "family", "idle", "lock", "origin", "changed_receipt", "container", "cas"]) def test_orphan_recovery_refuses_unowned_or_changed_state_without_write(specialize, owned_orphan, monkeypatch, mutation): q, receipt, runner, host, _originals, _orphan, _events, driver = owned_orphan if mutation in {"revision", "manifest", "phase"}: receipt[{"revision": "revision", "manifest": "manifestDigest", "phase": "phase"}[mutation]] = "foreign" elif mutation == "progress": receipt["progress"].pop(next(iter(receipt["progress"]))) elif mutation == "registration": receipt["registrationProgress"]["9"] = "disabled" elif mutation == "guard_progress": receipt["guardProgress"]["host"] = "installed" elif mutation in {"original", "wrapped"}: receipt["guards"]["runner"]["originalSha256" if mutation == "original" else "wrappedSha256"] = "f" * 64 elif mutation in {"foreign_runner", "foreign_host"}: (runner if mutation == "foreign_runner" else host).write_bytes(b"private foreign canary") elif mutation in {"marker", "journal"}: (specialize.SPECIALIZED_STATE_PATH if mutation == "marker" else specialize.MIGRATION_JOURNAL_PATH).touch() elif mutation == "storage": monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: True) elif mutation == "family": monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": True}) elif mutation == "idle": monkeypatch.setattr(q, "_idle", lambda *_args: (_ for _ in ()).throw(q.QuiescenceError("global_ci_work_present"))) elif mutation == "lock": monkeypatch.setattr(driver, "assert_barrier", lambda _self: (_ for _ in ()).throw(q.QuiescenceError("dispatch_maintenance_busy"))) elif mutation == "origin": monkeypatch.setattr(q, "attest_snapshot", lambda *_args, **_kwargs: (_ for _ in ()).throw(q.QuiescenceError("origin_changed"))) elif mutation == "changed_receipt": reads = [] def changed(): reads.append(True) result = copy.deepcopy(receipt) if len(reads) > 1: result["revision"] = "foreign" return result monkeypatch.setattr(q, "_read_receipt", changed) elif mutation == "container": reads = [] monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: reads.append(True) or ("v1", [{"id": str(len(reads))}])) else: monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: (_ for _ in ()).throw( specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed"))) before = runner.read_bytes(), host.read_bytes() if mutation != "cas": monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("unowned orphan restored")) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._recover_owned_orphan_guard() assert (runner.read_bytes(), host.read_bytes()) == before def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() digest = hashlib.sha256(script).hexdigest() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", digest) real_os = os class RootPosixOS: name = "posix" @staticmethod def geteuid(): return 0 def __getattr__(self, name): return getattr(real_os, name) monkeypatch.setattr(specialize, "os", RootPosixOS()) monkeypatch.setattr(specialize, "_migration_lock", nullcontext) monkeypatch.setattr(specialize, "specialization_state", lambda: "legacy") monkeypatch.setitem( sys.modules, "status_server", type("StatusServerStub", (), {"_require_controller_release": staticmethod(lambda: { "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": "a" * 40, "sourceTree": "b" * 40, })}), ) shell_calls: list[bytes] = [] state = {"active": False} def run_approved(received): shell_calls.append(received) state["active"] = True monkeypatch.setattr(specialize, "_run_approved_shell_script", run_approved) monkeypatch.setattr( specialize, "verified_topology", lambda: { "ready": state["active"], "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if state["active"] else [], "resourceEnvelope": {"ready": state["active"], "digest": "c" * 64}, }, ) result = specialize.execute(script) assert result["status"] == "specialized" assert result["verifiedRunners"] == list(specialize._SPECIALIZED_NAMES) assert shell_calls == [script] def test_run_with_input_keeps_registration_secret_out_of_command_arguments(specialize, monkeypatch): observed = {} def fake_run(args, **kwargs): observed["args"] = args observed["input"] = kwargs["input"] observed["capture_output"] = kwargs["capture_output"] return type("Result", (), {"returncode": 0, "stdout": b"registered"})() monkeypatch.setattr(specialize.subprocess, "run", fake_run) result = specialize._run_with_input(["docker", "run", "fixed"], b"sensitive-runner-token\n") assert result == "registered" assert observed["args"] == ["docker", "run", "fixed"] assert observed["input"] == b"sensitive-runner-token\n" assert "sensitive-runner-token" not in " ".join(observed["args"]) def test_build_probe_create_command_is_fixed_and_has_no_mount_or_shared_socket(specialize): record = _build_probe_record(specialize) args = specialize._build_probe_create_args(record) assert args[:8] == [ "create", "--name", record["name"], "--restart=no", "--cgroup-parent=docker", "--network=bridge", "--cgroupns=private", "--label", ] assert record["taskImageId"] in args assert "--cpus=4" in args assert "--memory=6144m" in args assert "--pids-limit=1024" in args assert "--blkio-weight=100" in args assert "--security-opt=seccomp=unconfined" in args assert "--security-opt=systempaths=unconfined" in args assert "--device=/dev/net/tun:/dev/net/tun:rwm" in args assert "--entrypoint" in args assert specialize._BUILD_PROBE_EXECUTABLE in args assert record["sourceRevision"] in args assert record["sourceTree"] in args assert not any( value == "--privileged" or value == "-e" or value.startswith(("--cap-add", "--volume", "--mount", "--env")) for value in args ) assert not any("docker.sock" in value or "host.docker.internal" in value for value in args) def test_build_probe_creation_inspect_requires_stopped_zero_pid_and_exact_inherited_labels( specialize, ): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["State"] = {"Running": False, "Pid": 0} assert specialize._validate_build_probe_container( item, record, image_environment=environment, require_running=False, require_created_stopped=True, ) labels = specialize._build_probe_container_labels(record) assert set(specialize.PINNED_TASK_BASE_INHERITED_LABELS).issubset(labels) assert record["imageLabels"].items() <= labels.items() assert item["Config"]["Labels"] == labels running, _ = _build_probe_inspect(specialize, record) assert not specialize._validate_build_probe_container( running, record, image_environment=environment, require_running=False, require_created_stopped=True, ) bad_pid = copy.deepcopy(item) bad_pid["State"]["Pid"] = 9 assert not specialize._validate_build_probe_container( bad_pid, record, image_environment=environment, require_running=False, require_created_stopped=True, ) foreign_label = copy.deepcopy(item) foreign_label["Config"]["Labels"]["untrusted.extra"] = "true" assert not specialize._validate_build_probe_container( foreign_label, record, image_environment=environment, require_running=False, require_created_stopped=True, ) def test_build_probe_container_inspect_accepts_only_fixed_private_build_job(specialize): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) assert specialize._validate_build_probe_container(item, record, image_environment=environment) for section, changes in ( ("HostConfig", {"Privileged": True}), ("HostConfig", {"PidMode": "host"}), ("HostConfig", {"NetworkMode": "host"}), ("HostConfig", {"CapAdd": ["SYS_ADMIN"]}), ("HostConfig", {"Devices": [ *item["HostConfig"]["Devices"], {"PathOnHost": "/dev/sda", "PathInContainer": "/dev/sda", "CgroupPermissions": "rwm"}, ]}), ("HostConfig", {"Mounts": [{"Type": "bind", "Source": "/host", "Target": "/host"}]}), ("HostConfig", {"SecurityOpt": ["seccomp=unconfined"]}), ("Config", {"Env": [*environment, "DOCKER_HOST=tcp://shared-dind:2375"]}), ("Config", {"Image": "localhost/itworx-ci-build:latest"}), ): changed, expected_environment = _build_probe_inspect(specialize, record) changed[section].update(changes) assert not specialize._validate_build_probe_container( changed, record, image_environment=expected_environment, ) @pytest.mark.parametrize("field,value", [ ("Type", "bind"), ("Name", "persistent-builder-cache"), ("Source", "/var/run/docker.sock"), ("Destination", "/host"), ("Driver", "nfs"), ("RW", False), ("Propagation", "rshared"), ]) def test_build_probe_rejects_foreign_or_shared_data_mount(specialize, field, value): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["Mounts"][0][field] = value assert not specialize._validate_build_probe_container(item, record, image_environment=environment) def test_build_probe_volume_identity_detects_same_name_recreation(specialize, monkeypatch): name = "9" * 64 volume = { "Name": name, "Driver": "local", "Scope": "local", "Mountpoint": f"/var/lib/docker/volumes/{name}/_data", "Labels": {"com.docker.volume.anonymous": ""}, "Options": None, "CreatedAt": "2026-10-03T00:00:00Z", } monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: json.dumps([volume])) identity = specialize._attest_build_probe_data_volume(name) volume["Labels"]["foreign.owner"] = "another-job" with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): specialize._attest_build_probe_data_volume(name, identity) volume["Labels"] = {"com.docker.volume.anonymous": ""} volume["CreatedAt"] = "2026-10-03T00:00:01Z" with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): specialize._attest_build_probe_data_volume(name, identity) def test_build_probe_volume_recovery_never_removes_volume_used_by_another_container(specialize, monkeypatch): identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} mutations = [] def run(*args, **kwargs): if args[:2] == ("volume", "ls"): return identity["name"] if args[:2] == ("ps", "-a"): return "f" * 64 mutations.append(args) raise AssertionError("unexpected volume mutation") monkeypatch.setattr(specialize, "_run_dind_docker", run) monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_in_use"): specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) assert mutations == [] def test_build_probe_volume_recovery_removes_only_exact_journal_owned_volume(specialize, monkeypatch): identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} mutations = [] present = True def run(*args, **kwargs): nonlocal present if args[:2] == ("volume", "ls"): assert args == ("volume", "ls", "--quiet", "--filter", f"name={identity['name']}") return identity["name"] if present else "" if args[:2] == ("ps", "-a"): return "" assert args == ("volume", "rm", identity["name"]) mutations.append(args) present = False return identity["name"] monkeypatch.setattr(specialize, "_run_dind_docker", run) monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) assert mutations == [("volume", "rm", identity["name"])] def test_build_probe_result_requires_exact_marker_and_every_rootless_tool_check(specialize): record = _build_probe_record(specialize) payload = { "schema": specialize._BUILD_PROBE_SCHEMA, "status": "passed", "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "engineVersion": "29.8.2", "engineRootless": True, "engineStorageDriver": "overlay2", "engineCgroupDriver": "none", "engineSocket": "unix:///run/itworx-job-docker/docker.sock", "engineSocketUid": 1000, "composeVersion": "5.5.1", "buildRun": True, "composeHealthy": True, "composeDnsResolved": True, "archiveNormalized": True, "archiveLoaded": True, "imageId": "sha256:" + "2" * 64, "imageLabels": record["imageLabels"], "archiveSha256": "sha256:" + "3" * 64, "holdSeconds": 30, } line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( payload, sort_keys=True, separators=(",", ":"), ) assert specialize._validate_build_probe_result(line, record=record) == payload with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_missing"): specialize._validate_build_probe_result(line + "\n" + line, record=record) for key, value in ( ("engineRootless", False), ("engineSocket", "tcp://0.0.0.0:2375"), ("composeHealthy", False), ("holdSeconds", 0), ): changed = {**payload, key: value} changed_line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( changed, sort_keys=True, separators=(",", ":"), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_untrusted"): specialize._validate_build_probe_result(changed_line, record=record) def test_build_probe_live_process_inventory_requires_daemon_and_compose_children(specialize): output = "\n".join( ( "PID PPID COMMAND", "1 0 python3 /usr/local/bin/itworx-ci-job-docker-probe", "10 1 rootlesskit --net=slirp4netns", "11 10 dockerd --host=unix:///run/itworx-job-docker/docker.sock --storage-driver=overlay2", "12 11 containerd --config /run/containerd/config.toml", "21 20 sleep 300", ) ) result = specialize._validate_build_probe_processes(output) assert result == { "processCount": 5, "probe": True, "rootlesskit": True, "dockerd": True, "containerd": True, "composeService": True, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): specialize._validate_build_probe_processes( output.replace("21 20 sleep 300", "21 20 cat /etc/os-release") ) with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): specialize._validate_build_probe_processes(output.replace("containerd --config", "unknown --config")) def test_build_probe_intent_binds_revision_tree_image_nonce_and_exact_container_id(specialize): record = _build_probe_record(specialize) assert specialize._validated_build_probe_record( record, source_revision="a" * 40, source_tree="b" * 40, ) == record invalid = dict(record, name=record["name"] + "-foreign") with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): specialize._validated_build_probe_record( invalid, source_revision="a" * 40, source_tree="b" * 40, ) def _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, ids): image_environment = item["Config"]["Env"] present = list(ids) removed = [] monkeypatch.setattr( specialize, "_inspect", lambda name: {"State": {"Running": True}} if name == specialize.DIND_NAME else None, ) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) monkeypatch.setattr( specialize, "_build_probe_ids", lambda *, label=True, name=None: list(present) if label or name == record["name"] and present else [], ) monkeypatch.setattr(specialize, "_inspect_dind_container", lambda _cid: item) monkeypatch.setattr( specialize, "_build_probe_image_environment", lambda _record: image_environment, ) def run_dind(*args, **_kwargs): if args[:3] == ("rm", "--force", "--volumes"): removed.append(args[3]) present.clear() return args[3] if args[:2] == ("volume", "ls"): return "" raise AssertionError(f"unexpected private-DinD command: {args}") monkeypatch.setattr(specialize, "_run_dind_docker", run_dind) monkeypatch.setattr( specialize, "_attest_build_probe_data_volume", lambda name, expected=None: {"name": name, "createdAt": "2026-10-03T00:00:00Z"}, ) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) return removed def test_build_probe_recovery_cleans_only_intent_owned_container_after_create_crash( specialize, monkeypatch, ): record = _build_probe_record(specialize, status="intent") cid = "e" * 64 item, _environment = _build_probe_inspect(specialize, record, container_id=cid) removed = _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, [cid]) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } specialize._cleanup_build_probe(journal) assert removed == [cid] assert journal["buildProbe"]["containerId"] == cid assert journal["buildProbe"]["status"] == "cleaned" def test_build_probe_recovery_refuses_foreign_cid_even_with_matching_name_and_nonce( specialize, monkeypatch, ): expected_cid = "d" * 64 foreign_cid = "e" * 64 record = _build_probe_record( specialize, container_id=expected_cid, status="started", ) item, _environment = _build_probe_inspect(specialize, record, container_id=foreign_cid) removed = _install_build_probe_cleanup_fakes( specialize, monkeypatch, record, item, [foreign_cid], ) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_container_untrusted"): specialize._cleanup_build_probe(journal) assert removed == [] def test_build_probe_recovery_refuses_ambiguous_owned_label_inventory( specialize, monkeypatch, ): record = _build_probe_record(specialize, status="intent") cid = "e" * 64 item, _environment = _build_probe_inspect(specialize, record, container_id=cid) removed = _install_build_probe_cleanup_fakes( specialize, monkeypatch, record, item, [cid, "f" * 64], ) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_ambiguous_containers"): specialize._cleanup_build_probe(journal) assert removed == [] invalid = dict(record, status="started") with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): specialize._validated_build_probe_record( invalid, source_revision="a" * 40, source_tree="b" * 40, ) @pytest.mark.parametrize("recovering_cgroup_intent", [False, True]) @pytest.mark.parametrize("probe_fails", [False, True]) def test_specialization_stage_fixture_quiesces_old_before_registering_and_never_runs_over_two( specialize, tmp_path, monkeypatch, recovering_cgroup_intent, probe_fails ): revision = "a" * 40 tree = "b" * 40 manifest_sha = "d" * 64 manifest = { "images": [ {"profile": profile_id, "reference": specialize._task_image_reference(profile_id, revision)} for profile_id in specialize.PROFILE_IDS ] } archive = tmp_path / "task-images.tar" archive.write_bytes(b"fixture") journal = { "status": "in_progress", "stage": "prepared", "legacyTopology": "v1", "sourceRevision": revision, "sourceTree": tree, "cgroupCreated": False, "cgroupEnsurePending": recovering_cgroup_intent, "baseComposeSha256": hashlib.sha256(b"captured-base").hexdigest(), "baseComposeModelSha256": hashlib.sha256(b"base-model").hexdigest(), "specializedComposeSha256": hashlib.sha256(b"specialized-model").hexdigest(), "backupDirectory": str(tmp_path / "specialized-test"), "newRegistrationIds": {}, } events: list[str] = [] running: set[str] = set() maximum_running = 0 marker_active = {"value": False} def rows(): return [ { "name": specialize.RUNNER_PROFILES[profile_id].container_name, "id": index + 1, "labels": sorted( { specialize.RUNNER_PROFILES[profile_id].runner_label, *specialize.RUNNER_PROFILES[profile_id].legacy_labels, } ), "disabled": True, "busy": False, } for index, profile_id in enumerate(specialize.PROFILE_IDS) ] def inventory(): return ( { name: {"name": name, "State": {"Running": name in running}} for name in (*specialize._SPECIALIZED_NAMES, specialize.DIND_NAME) }, [], ) def resource_proof(_claims, _containers, _legacy): return { "ready": True, "digest": "e" * 64, "runningRunnerContainers": len(running), } monkeypatch.setattr(specialize, "_captured_base_source", lambda _journal: b"captured-base") monkeypatch.setattr( specialize, "_captured_compose_model", lambda _journal, *, specialized: b"specialized-model" if specialized else b"base-model", ) monkeypatch.setattr(specialize, "_read_source_file", lambda *_args, **_kwargs: b"captured-base") monkeypatch.setattr(specialize, "_validate_base_composition", lambda _bytes=None: "v1") monkeypatch.setattr(specialize, "_validate_composition", lambda raw: raw) monkeypatch.setattr(specialize, "_run_specialized_compose", lambda _journal, *args, **_kwargs: events.append(f"compose:{args[-1]}")) monkeypatch.setattr(specialize, "_wait_guards_quiet", lambda: events.append("guards-quiet")) monkeypatch.setattr(specialize, "_service_guard_scripts", lambda _journal: events.append("guards-fenced")) monkeypatch.setattr(specialize, "_stop_legacy_capacity_daemon", lambda: events.append("capacity-stopped")) monkeypatch.setattr(specialize, "_set_legacy_disabled", lambda _journal: events.append("legacy-disabled")) monkeypatch.setattr(specialize, "_check_legacy_queued_jobs", lambda: None) monkeypatch.setattr(specialize, "_stop_remove_legacy_containers", lambda _journal: events.append("legacy-removed")) monkeypatch.setattr( specialize, "_cgroup_family_snapshot", lambda: { "exists": recovering_cgroup_intent, "rootSubtreeControllers": ["cpu", "memory", "pids"], }, ) def ensure_cgroup_family(**kwargs): assert kwargs["allow_existing"] is recovering_cgroup_intent events.append("cgroup-ensured") return not recovering_cgroup_intent monkeypatch.setattr( specialize, "_ensure_cgroup_family", ensure_cgroup_family, ) monkeypatch.setattr(specialize, "_prepare_profile_state", lambda _revision: None) monkeypatch.setattr(specialize, "_write_private_atomic", lambda *_args, **_kwargs: None) def register(current_journal): assert "legacy-disabled" in events events.append("registrations-created") current_journal["newRegistrationIds"] = { profile_id: index + 1 for index, profile_id in enumerate(specialize.PROFILE_IDS) } specialize._set_stage(current_journal, "registrations_created") monkeypatch.setattr(specialize, "_register_profiles", register) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: None) monkeypatch.setattr(specialize, "_inspect", lambda _name: {"State": {"Running": True}}) monkeypatch.setattr(specialize, "_require_dind_resource_support", lambda _model: None) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {"services": {}}) monkeypatch.setattr(specialize, "_load_images_into_dind", lambda *_args: None) monkeypatch.setattr(specialize, "_verify_manifest_images_in_dind", lambda _items: True) monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) monkeypatch.setattr(specialize, "_resource_proof", resource_proof) monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) monkeypatch.setattr( specialize, "_trusted_private_file", lambda path, **_kwargs: specialize._render_runner_config("build", revision) if str(path).endswith("build.yaml") else b"fixture", ) monkeypatch.setattr(specialize, "_cleanup_build_probe", lambda _journal: None) def run_build_probe(_journal, _manifest): assert not running events.append("build-probed") if probe_fails: raise specialize.RunnerFleetSpecializeError( "runner_fleet_specialization_build_probe_failed" ) monkeypatch.setattr(specialize, "_run_build_probe", run_build_probe) monkeypatch.setattr(specialize, "_runner_rows", rows) def write_journal(current_journal): if current_journal.get("cgroupEnsurePending") is True: events.append("cgroup-intent-durable") if current_journal.get("cgroupCreated") is True: events.append("cgroup-result-durable") if current_journal.get("status") == "active": events.append("journal-active-durable") monkeypatch.setattr(specialize, "_write_journal", write_journal) batches: list[tuple[str, ...]] = [] def set_containers(profiles, start): nonlocal maximum_running names = {specialize.RUNNER_PROFILES[item].container_name for item in profiles} if start: assert len(running | names) <= 2 running.update(names) maximum_running = max(maximum_running, len(running)) batches.append(tuple(profiles)) events.append("batch-started") else: running.difference_update(names) events.append("batch-stopped") monkeypatch.setattr(specialize, "_set_runner_containers_running", set_containers) monkeypatch.setattr(specialize, "_bootstrap_registrations", lambda: [{"name": name} for name in specialize._SPECIALIZED_NAMES]) monkeypatch.setattr(specialize, "_read_installed_manifest_sha256", lambda: manifest_sha) monkeypatch.setattr(specialize, "_write_pending_bootstrap_receipt", lambda *_args: events.append("receipt-written")) crash = {"next_active_write": True} class SimulatedPowerLoss(BaseException): pass def write_marker(**kwargs): assert kwargs["status"] == "active" assert not running assert "receipt-written" in events events.append("marker-active-attempt") if crash["next_active_write"]: crash["next_active_write"] = False raise SimulatedPowerLoss marker_active["value"] = True events.append("marker-active") monkeypatch.setattr(specialize, "_write_marker", write_marker) monkeypatch.setattr(specialize, "specialization_state", lambda: "active" if marker_active["value"] else "blocked") if probe_fails: monkeypatch.setattr(specialize, "_rollback_before_activation", lambda _journal: True) with pytest.raises( specialize.RunnerFleetSpecializeError, match="failed_rolled_back", ): specialize._perform_migration_stages(journal, manifest, archive) assert "build-probed" in events assert "receipt-written" not in events assert "marker-active-attempt" not in events assert not marker_active["value"] return with pytest.raises(SimulatedPowerLoss): specialize._perform_migration_stages(journal, manifest, archive) assert journal["status"] == "active" assert not marker_active["value"] assert events.index("journal-active-durable") < events.index("marker-active-attempt") marker = { "status": "in_progress", "sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256, "imageSourceRevision": revision, "imageSourceTree": tree, } script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_read_json_file", lambda *_args, **_kwargs: marker) monkeypatch.setattr(specialize, "_valid_marker", lambda _marker: True) monkeypatch.setattr(specialize, "_migration_lock", lambda: nullcontext()) monkeypatch.setattr( specialize, "verified_topology", lambda: { "ready": marker_active["value"], "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if marker_active["value"] else [], "resourceEnvelope": {"ready": marker_active["value"], "digest": "e" * 64}, }, ) monkeypatch.setitem( sys.modules, "status_server", type( "StatusServerStub", (), { "_require_controller_release": staticmethod( lambda: { "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": revision, "sourceTree": tree, } ) }, ), ) monkeypatch.setattr( specialize, "_run_approved_shell_script", lambda _script: specialize._perform_migration_stages(journal, manifest, archive), ) result = specialize.execute(script) assert result["status"] == "specialized" assert maximum_running == 2 assert events.index("cgroup-intent-durable") < events.index("cgroup-ensured") assert events.index("cgroup-ensured") < events.index("cgroup-result-durable") assert batches == [ tuple(specialize.PROFILE_IDS[0:2]), tuple(specialize.PROFILE_IDS[2:4]), tuple(specialize.PROFILE_IDS[4:6]), ] assert events.index("legacy-disabled") < events.index("registrations-created") assert events.index("batch-stopped") < events.index("build-probed") assert events.index("build-probed") < events.index("receipt-written") assert events.index("receipt-written") < events.index("marker-active") assert journal["cgroupCreated"] is True assert journal["status"] == "active" assert not running @pytest.mark.parametrize("original_running", [False, True]) def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( specialize, tmp_path, monkeypatch, original_running ): monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) image_id = specialize._DIND_IMAGE_ID base_model = _base_v1_compose_model(specialize) specialized_model = { "services": { "runner-dind": { **base_model["services"]["runner-dind"], "cgroup_parent": "itworx-ci", "restart": "no", "volumes": [ { "type": "bind", "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "target": "/var/lib/docker", "read_only": False, } ], } } } journal = { "originalDind": { "present": True, "running": original_running, "restartPolicy": "unless-stopped", "imageId": image_id, }, "originalContainers": [], "originalRegistrations": [], "newRegistrationIds": {}, "guardFiles": [], "cgroupCreated": True, "cgroupEnsurePending": False, "cgroupRootSubtree": ["cpu", "memory", "pids"], "backupDirectory": str(tmp_path / "specialized-test"), } events = [] current = {"kind": "specialized", "State": {"Running": True}} current_marker = {"value": current} def inspect(name): assert name == specialize.DIND_NAME or name in specialize._SPECIALIZED_NAMES return current_marker["value"] if name == specialize.DIND_NAME else None def run(args, **_kwargs): events.append(tuple(args)) if args[:2] == ["docker", "update"] and args[-1] == specialize.DIND_NAME: if args[2] == "--restart=no": return "" current_marker["value"]["HostConfig"]["RestartPolicy"]["Name"] = args[2].partition("=")[2] return "" if args[:2] == ["docker", "stop"]: current_marker["value"]["State"]["Running"] = False return "" if args[:2] == ["docker", "rm"]: current_marker["value"] = None return "" return "" def run_compose(_compose, *args, **_kwargs): events.append(("compose", *args)) assert args[-1] == "runner-dind" restored_running = original_running if args[0] == "up" else False current_marker["value"] = _legacy_dind_fixture( specialize, base_model, image_id=image_id, running=restored_running, restart_policy="no", ) base_bytes = json.dumps(base_model, sort_keys=True, separators=(",", ":")).encode() specialized_bytes = json.dumps(specialized_model, sort_keys=True, separators=(",", ":")).encode() live_compose = tmp_path / "docker-compose.yml" live_compose.write_bytes(base_bytes) monkeypatch.setattr(specialize, "COMPOSE_PATH", live_compose) journal["baseComposeSha256"] = hashlib.sha256(base_bytes).hexdigest() monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr( specialize, "_captured_compose_model", lambda _journal, *, specialized: specialized_bytes if specialized else base_bytes, ) monkeypatch.setattr( specialize, "_validate_dind_outer", lambda item, _model: item.get("kind") == "specialized", ) monkeypatch.setattr(specialize, "_inspect", inspect) monkeypatch.setattr(specialize, "_run", run) monkeypatch.setattr(specialize, "_run_compose", run_compose) monkeypatch.setattr(specialize, "_runner_rows", lambda: []) monkeypatch.setattr( specialize, "rollback_host_cgroup_family", lambda **_kwargs: events.append("cgroup-cleanup") or {"familyExists": False}, ) monkeypatch.setattr(specialize, "SPECIALIZED_COMPOSE_PATH", tmp_path / "specialized-compose.yml") monkeypatch.setattr(specialize, "TASK_IMAGE_MANIFEST_PATH", tmp_path / "task-image-manifest.json") monkeypatch.setattr(specialize, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "receipt.json") monkeypatch.setattr(specialize, "RUNNER_CONFIG_ROOT", tmp_path / "runner-configs") monkeypatch.setattr(specialize, "_SPECIALIZED_DATA_ROOT", tmp_path / "runner-data") monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", tmp_path / "workspaces") monkeypatch.setattr(specialize, "TASK_IMAGE_ROOT", tmp_path / "task-images") monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", tmp_path / "specialized.json") monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", tmp_path / "journal.json") assert specialize._rollback_before_activation(journal) is True restored = current_marker["value"] assert restored["Image"] == image_id assert restored["State"]["Running"] is original_running assert restored["HostConfig"]["RestartPolicy"]["Name"] == "unless-stopped" compose = next(event for event in events if isinstance(event, tuple) and event[0] == "compose") assert compose[1] == ("up" if original_running else "create") remove = next(index for index, event in enumerate(events) if isinstance(event, tuple) and event[:2] == ("docker", "rm")) assert remove < events.index(compose) < events.index("cgroup-cleanup") def _legacy_dind_fixture( specialize, base_model, *, image_id, running, restart_policy ): service = base_model["services"]["runner-dind"] return { "Name": f"/{specialize.DIND_NAME}", "Image": image_id, "Config": { "Image": service["image"], "User": "rootless", "Entrypoint": ["dockerd-entrypoint.sh"], "Cmd": ["--default-address-pool=base=198.19.0.0/16,size=27"], "Volumes": {"/home/rootless/.local/share/docker": {}, "/var/lib/docker": {}}, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": "runner-dind", }, }, "State": {"Running": running}, "HostConfig": { "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, "PidMode": "", "Privileged": True, "RestartPolicy": {"Name": restart_policy}, }, "Mounts": [ { "Type": "bind", "Source": mount["source"], "Destination": mount["target"], "RW": not mount.get("read_only", False), } for mount in service["volumes"] ] + [{ "Type": "volume", "Driver": "local", "Name": "f" * 64, "Source": "/var/lib/docker/volumes/" + "f" * 64 + "/_data", "Destination": "/var/lib/docker", "RW": True, }], } def test_known_v1_and_v2_topologies_are_explicit_and_v1_does_not_require_absent_v2_services( specialize, monkeypatch ): compose = b"known Tower v1 compose fixture" monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest()) v1_services = { service_name: {"container_name": name} for name, service_name in specialize._OLD_SERVICES.items() if name in specialize._LEGACY_V1_NAMES } assert specialize._classify_legacy_topology(compose, v1_services) == "v1" v2_services = {} for name in specialize._LEGACY_V2_NAMES: service_name = specialize._OLD_SERVICES[name] burst = name.endswith("burst-1") v2_services[service_name] = { "container_name": name, "image": specialize.RUNNER_IMAGE, "profiles": ["runner"], "labels": { "tech.itworx.role": "gitea-actions-burst-runner" if burst else "gitea-actions-runner" }, **({"cpus": 0.5, "mem_limit": "512m"} if name in specialize._LEGACY_V2_NAMES[2:] else {}), } assert specialize._classify_legacy_topology(b"known v2 compose", v2_services) == "v2" partial_v2 = dict(v1_services) partial_v2["runner-3"] = v2_services["runner-3"] with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_compose_invalid"): specialize._classify_legacy_topology(compose, partial_v2) def test_exact_live_v1_registration_and_container_contract_accepts_only_two_known_runners(specialize): rows = [ { "name": name, "id": runner_id, "labels": sorted(specialize._OLD_LABELS[name]), "disabled": False, "busy": False, } for runner_id, name in zip((9, 22), specialize._LEGACY_V1_NAMES, strict=True) ] rows.extend( [ {"name": "itworx-unraid-deploy", "id": 13, "labels": ["unraid-deploy"], "disabled": False, "busy": False}, {"name": "itworx-windows11-native", "id": 14, "labels": ["windows-native"], "disabled": False, "busy": False}, ] ) assert len(specialize._validate_legacy_registrations(rows, "v1")) == 2 name = "itworx-security-runner" mount_entries = [ {"Source": source, "Destination": target, "RW": True} for source, target, _read_only in specialize._V1_RUNNER_VOLUMES[name] ] item = { "Image": specialize.RUNNER_IMAGE_ID, "Config": { "Image": specialize.RUNNER_IMAGE, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": "runner", "tech.itworx.role": "gitea-actions-runner", }, }, "State": {"Running": True}, "HostConfig": { "NanoCpus": 1_000_000_000, "Memory": 1024 * specialize._MIB, "PidMode": "", "RestartPolicy": {"Name": "unless-stopped"}, }, "Mounts": mount_entries, } assert specialize._validate_legacy_container_item("v1", name, item)["running"] is True item["HostConfig"]["PidMode"] = "host" with pytest.raises(specialize.RunnerFleetSpecializeError, match="v1_container_budget_invalid"): specialize._validate_legacy_container_item("v1", name, item) extra_known_runner = [*rows, {"name": "itworx-validation-runner-3", "id": 30}] with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_registration_invalid"): specialize._validate_legacy_registrations(extra_known_runner, "v1") def test_queue_gate_allows_only_legacy_labels_covered_by_new_profiles(specialize): assert specialize._queued_job_is_covered(["self-hosted", "linux", "linux-validation", "ubuntu-latest"]) assert specialize._queued_job_is_covered(["itworx-ci-backend"]) assert specialize._queued_job_is_covered(["docker-validation"]) assert not specialize._queued_job_is_covered(["mcp-validation", "unraid"]) def _legacy_queue_job(job_id, status, labels, *, run_id=21500): return {"id": job_id, "run_id": run_id, "status": status, "labels": labels} def test_legacy_queue_gate_reads_queued_and_requested_admin_inventories(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=pending" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(102, "requested", ["linux-validation", "ubuntu-latest"])], } if "status=queued" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(101, "queued", ["itworx-ci-backend"])], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) specialize._check_legacy_queued_jobs() assert len(calls) == 2 assert "status=pending&limit=50&page=1&sort=id&order=asc" in calls[0] assert "status=queued&limit=50&page=1&sort=id&order=asc" in calls[1] def test_legacy_queue_gate_blocks_unsupported_requested_dependency_job(specialize, monkeypatch): def api(path): if "status=queued" in path: return {"total_count": 0, "jobs": []} if "status=pending" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(201, "requested", ["mcp-validation", "unraid"])], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_catches_requested_to_queued_transition_between_filters( specialize, monkeypatch ): state = "requested" calls = [] def api(path): nonlocal state calls.append(path) if "status=pending" in path: # The job changes status immediately before this filtered read. state = "queued" return {"total_count": 0, "jobs": []} if "status=queued" in path: jobs = ( [_legacy_queue_job(250, "queued", ["mcp-validation", "unraid"])] if state == "queued" else [] ) return {"total_count": len(jobs), "jobs": jobs} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert "status=pending" in calls[0] assert "status=queued" in calls[1] def test_legacy_queue_gate_paginates_51_jobs_and_blocks_unsupported_page_two_job(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) if page == 1: return { "total_count": 51, "jobs": [ _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) for index in range(50) ], } return { "total_count": 51, "jobs": [_legacy_queue_job(51, "queued", ["mcp-validation", "unraid"])], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert any("status=queued&limit=50&page=2" in path for path in calls) def test_legacy_queue_gate_paginates_101_jobs_and_checks_unsupported_last_page(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) start_id = (page - 1) * 50 + 1 page_size = min(50, 101 - (page - 1) * 50) labels = ["mcp-validation", "unraid"] if page == 3 else ["itworx-ci-backend"] return { "total_count": 101, "jobs": [ _legacy_queue_job(start_id + offset, "queued", labels) for offset in range(page_size) ], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert any("status=queued&limit=50&page=3" in path for path in calls) def test_legacy_queue_gate_rejects_total_count_changes_between_pages(specialize, monkeypatch): def api(path): if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) if page == 1: return { "total_count": 51, "jobs": [ _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) for index in range(50) ], } return { "total_count": 102, "jobs": [_legacy_queue_job(51, "queued", ["itworx-ci-backend"])], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_accepts_maximum_1000_job_inventory(specialize, monkeypatch): calls = [] def api(path): calls.append(path) page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) start_id = (page - 1) * 50 + 1 return { "total_count": 1000, "jobs": [ _legacy_queue_job(start_id + offset, "queued", ["itworx-ci-backend"]) for offset in range(50) ], } monkeypatch.setattr(specialize, "_api_json", api) rows = specialize._legacy_action_jobs("queued") assert len(rows) == 1000 assert len(calls) == 20 assert "limit=50&page=20" in calls[-1] def test_legacy_queue_gate_rejects_inventory_over_1000_jobs(specialize, monkeypatch): calls = [] def api(path): calls.append(path) return {"total_count": 1001, "jobs": []} monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._legacy_action_jobs("queued") assert len(calls) == 1 @pytest.mark.parametrize( "payload", [ {"total_count": 1, "jobs": []}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "requested", ["itworx-ci-backend"])]}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", [""])]}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"], run_id=0)]}, {"total_count": 2, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"])]}, { "total_count": 2, "jobs": [ _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), ], }, { "total_count": 2, "jobs": [ _legacy_queue_job(2, "queued", ["itworx-ci-backend"]), _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), ], }, ], ) def test_legacy_queue_gate_rejects_malformed_or_truncated_inventory(specialize, monkeypatch, payload): def api(path): if "status=queued" in path: return payload if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specialize, monkeypatch): row = _legacy_queue_job(301, "queued", ["itworx-ci-backend"]) def api(path): if "status=queued" in path: return {"total_count": 1, "jobs": [row]} if "status=pending" in path: return { "total_count": 1, "jobs": [{**row, "status": "requested"}], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url(): workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8") build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8") frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8") assert "github-server-url: http://gitea-runner-internal:3000" in workflow assert "runs-on: linux-validation" in workflow assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output"' in workflow assert "runs-on: itworx-ci-build" not in workflow assert "--job-local" not in workflow assert "jens/unraid-appops-gateway" in build assert "Jens/Unraid-AppOps-Gateway" in build assert "corepack prepare pnpm@11.15.0 --activate" in frontend assert "test \"$(pnpm --version)\" = 11.15.0" in frontend def test_specialization_shell_entrypoint_is_fixed_and_has_no_caller_selected_target(): script = ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh" content = script.read_text(encoding="utf-8") assert content.startswith("#!/usr/bin/env bash\n") assert "SPECIALIZE_CI_RUNNER_FLEET" in content assert "--apply-fixed" in content assert "eval " not in content assert "docker run" not in content assert "docker compose" not in content def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity( specialize, monkeypatch ): registrations = [ { "name": profile.display_name, "id": index, "labels": [profile.runner_label, *profile.legacy_labels], "disabled": True, "busy": False, } for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=10) ] monkeypatch.setattr(specialize, "_api_json", lambda _path: {"runners": registrations}) normalized = specialize._runner_rows() assert [row["name"] for row in normalized] == list(specialize._SPECIALIZED_NAMES) assert [row["displayName"] for row in normalized] == [ profile.display_name for profile in specialize.RUNNER_PROFILES.values() ] registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].container_name with pytest.raises( specialize.RunnerFleetSpecializeError, match="registration_name_invalid", ): specialize._runner_rows() registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].display_name registrations[1]["name"] = specialize.RUNNER_PROFILES["checks"].display_name with pytest.raises( specialize.RunnerFleetSpecializeError, match="registration_duplicate", ): specialize._runner_rows() def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): rows, containers, dind, journal, model = _active_recovery_fixture(specialize) events = [] class Volume: def __enter__(self): return self def __exit__(self, *_args): return False def assert_current(self): events.append("volume-current") monkeypatch.setattr(specialize, "_migration_lock", lambda **_kwargs: nullcontext()) monkeypatch.setattr(specialize, "_open_fixed_release_volume", lambda: Volume()) monkeypatch.setattr( specialize, "_active_release_bundle", lambda: ({"status": "active"}, journal, model, b"captured-compose"), ) monkeypatch.setattr(specialize, "_runner_rows", lambda: rows) monkeypatch.setattr(specialize, "_docker_outer_inventory", lambda: (containers, [])) monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) monkeypatch.setattr( specialize, "_cgroup_family_snapshot", lambda: { "exists": family_exists, "safeToStartDind": True, "rootSubtreeControllers": ["cpu", "memory", "pids"], }, ) monkeypatch.setattr( specialize, "_fresh_recovery_host_gate", lambda: events.append("fresh-host-gate"), ) monkeypatch.setattr( specialize, "_ensure_cgroup_family_details", lambda **_kwargs: events.append("cgroup-create") or {"created": True, "rootSubtreeBefore": ["cpu", "memory", "pids"]}, ) monkeypatch.setattr( specialize, "_run_compose", lambda compose, *args, **kwargs: events.append(("compose", compose, args, kwargs)), ) monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy")) monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True) monkeypatch.setattr( specialize, "_resource_proof", lambda *_args: {"ready": True, "runningRunnerContainers": 0}, ) def inspect(name): if name == specialize.DIND_NAME: dind["State"] = {"Running": True, "Health": {"Status": "healthy"}} return dind return None def inventory(): return containers, [] monkeypatch.setattr(specialize, "_inspect", inspect) monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) return events, rows, containers, dind def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( specialize, monkeypatch, family_exists=True ) result = specialize.ensure_active_fleet_ready() assert result == {"ready": True, "reason": None} compose_events = [item for item in events if isinstance(item, tuple) and item[0] == "compose"] assert len(compose_events) == 1 _, compose_bytes, args, kwargs = compose_events[0] assert compose_bytes == b"captured-compose" assert args == ("up", "-d", "--no-deps", "runner-dind") assert kwargs["timeout"] <= 30 assert events.count("fresh-host-gate") == 2 assert events.index("fresh-host-gate") < events.index("healthy") def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( specialize, monkeypatch, family_exists=False ) gate_count = {"value": 0} def fresh_gate(): gate_count["value"] += 1 events.append(f"fresh-host-gate-{gate_count['value']}") if gate_count["value"] == 2: raise specialize.RunnerFleetSpecializeError( "runner_fleet_specialization_host_guard_paused" ) rollback = [] monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fresh_gate) monkeypatch.setattr( specialize, "rollback_host_cgroup_family", lambda **kwargs: rollback.append(kwargs) or {"removed": True, "familyExists": False}, ) result = specialize.ensure_active_fleet_ready() assert result == { "ready": False, "reason": "runner_fleet_specialization_host_guard_paused", } assert rollback == [ {"created": True, "initial_controllers": ["cpu", "memory", "pids"]} ] assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) def test_active_recovery_accepts_only_disabled_legacy_registrations_after_containers_removed( specialize, monkeypatch ): events, rows, _containers, dind = _stub_active_recovery( specialize, monkeypatch, family_exists=True ) rows[-1]["disabled"] = False result = specialize.ensure_active_fleet_ready() assert result["ready"] is False assert result["reason"] == "runner_fleet_specialization_legacy_registration_untrusted" assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) assert dind["State"]["Running"] is False def test_active_recovery_cli_reports_only_sanitized_ready_or_reason( specialize, monkeypatch, capsys ): monkeypatch.setattr( specialize, "ensure_active_fleet_ready", lambda: {"ready": False, "reason": "runner_fleet_specialization_host_guard_paused"}, ) assert specialize._main(("--ensure-active-fleet-ready",)) == 1 assert capsys.readouterr().err.strip() == "runner_fleet_specialization_host_guard_paused" monkeypatch.setattr( specialize, "ensure_active_fleet_ready", lambda: {"ready": True, "reason": None}, ) assert specialize._main(("--ensure-active-fleet-ready",)) == 0 assert capsys.readouterr().out.strip() == "ready" def test_active_recovery_host_gate_rechecks_guard_after_fresh_sample( specialize, monkeypatch ): now = specialize.time.time() sample = { "loadRatio": 0.2, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapSampleAt": now, "swapCritical": False, } guard_states = iter([("clear", ""), ("paused", "guard marker appeared")]) class HostSampler: def host_snapshot(self): return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_fleet", type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: next(guard_states))}), ) monkeypatch.setitem( sys.modules, "runner_capacity_controller", type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), ) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_host_guard_paused", ): specialize._fresh_recovery_host_gate() @pytest.mark.parametrize( "sample,expected_reason", [ (None, "runner_fleet_specialization_host_telemetry_unavailable"), ( { "loadRatio": 0.2, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapSampleAt": 1, "swapCritical": False, }, "runner_fleet_specialization_host_headroom_insufficient", ), ], ) def test_active_recovery_host_gate_fails_closed_on_missing_or_stale_sample( specialize, monkeypatch, sample, expected_reason ): class HostSampler: def host_snapshot(self): if sample is None: return {"status": "unavailable", "host": None} return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_fleet", type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: ("clear", ""))}), ) monkeypatch.setitem( sys.modules, "runner_capacity_controller", type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match=expected_reason): specialize._fresh_recovery_host_gate()