from __future__ import annotations import hashlib import copy import importlib.util import io import json import os import sys import tarfile from contextlib import nullcontext from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] BROKER = ROOT / "ops" / "gitea-deploy-broker" SPECIALIZE_PATH = BROKER / "runner_fleet_specialize.py" @pytest.fixture def specialize(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) spec = importlib.util.spec_from_file_location("runner_fleet_specialize_test", SPECIALIZE_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _valid_task_image_manifest(specialize, revision="a" * 40, tree="b" * 40): source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" return { "apiVersion": specialize.IMAGE_MANIFEST_SCHEMA, "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": revision, "sourceTree": tree, "workflowPath": specialize.IMAGE_WORKFLOW, "workflowRunId": "12", "fullRunId": "13", "artifactName": specialize.IMAGE_ARTIFACT_PREFIX + revision, "dockerTarSha256": "c" * 64, "dockerTarBytes": 1, "images": [ { "profile": profile_id, "reference": f"localhost/itworx-ci-{profile_id}:sha-{revision}", "imageId": f"sha256:{str(index + 1) * 64}", "architecture": "amd64", "dockerfileSha256": "d" * 64, "labels": { "org.opencontainers.image.source": source, "org.opencontainers.image.revision": revision, "tech.itworx.source-tree": tree, "tech.itworx.ci.profile": profile_id, }, } for index, profile_id in enumerate(specialize.PROFILE_IDS) ], } def _build_probe_record(specialize, *, container_id=None, status="intent"): revision = "a" * 40 tree = "b" * 40 image_labels = { "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", "org.opencontainers.image.revision": revision, "tech.itworx.source-tree": tree, "tech.itworx.ci.profile": "build", } record = specialize._new_build_probe_record( {"sourceRevision": revision, "sourceTree": tree}, { "taskImageId": "sha256:" + "1" * 64, "dockerfileSha256": "d" * 64, "imageLabels": image_labels, }, ) record["status"] = status record["containerId"] = container_id return record def _build_probe_inspect(specialize, record, *, container_id=None, changes=None): image_environment = ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"] item = { "Id": container_id or "e" * 64, "Name": f"/{record['name']}", "Image": record["taskImageId"], "Config": { "Image": record["taskImageId"], "Entrypoint": [specialize._BUILD_PROBE_EXECUTABLE], "Cmd": [ record["sourceRevision"], record["sourceTree"], "--hold-seconds", str(specialize._BUILD_PROBE_HOLD_SECONDS), ], "Env": image_environment, "Labels": specialize._build_probe_container_labels(record), "Volumes": {"/home/itworx-builder/.local/share/docker": {}}, }, "HostConfig": { "NetworkMode": "bridge", "PidMode": "", "IpcMode": "private", "CgroupnsMode": "private", "CgroupParent": "docker", "NanoCpus": 4_000_000_000, "Memory": 6 * 1024**3, "PidsLimit": 1024, "BlkioWeight": 100, "Privileged": False, "CapAdd": [], "CapDrop": [], "SecurityOpt": ["seccomp=unconfined", "systempaths=unconfined"], "Devices": [{ "PathOnHost": "/dev/net/tun", "PathInContainer": "/dev/net/tun", "CgroupPermissions": "rwm", }], "DeviceRequests": [], "DeviceCgroupRules": [], "ExtraHosts": ["gitea.example.com:198.19.104.1"], "PortBindings": {}, "Binds": [], "Mounts": [], "VolumesFrom": [], "Tmpfs": {}, "RestartPolicy": {"Name": "no"}, }, "Mounts": [{ "Type": "volume", "Name": "9" * 64, "Source": f"/var/lib/docker/volumes/{'9' * 64}/_data", "Destination": "/home/itworx-builder/.local/share/docker", "Driver": "local", "Mode": "", "RW": True, "Propagation": "", }], "State": {"Running": True, "Pid": 77}, } if changes: for section, values in changes.items(): item[section].update(values) return item, image_environment def _write_task_image_archive(specialize, path: Path, *, extra_label: bool = False): images = _valid_task_image_manifest(specialize)["images"] docker_manifest = [] members: dict[str, bytes] = {} for index, image in enumerate(images): labels = specialize._runtime_task_image_labels(image) assert labels is not None if extra_label and index == 0: labels["untrusted.extra"] = "value" config = { "architecture": "amd64", "os": "linux", "config": {"Labels": labels}, } config_bytes = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() config_name = f"{image['profile']}.json" layer_name = f"{image['profile']}/layer.tar" image["imageId"] = "sha256:" + hashlib.sha256(config_bytes).hexdigest() members[config_name] = config_bytes members[layer_name] = b"layer" docker_manifest.append( { "Config": config_name, "RepoTags": [image["reference"]], "Layers": [layer_name], } ) members["manifest.json"] = json.dumps(docker_manifest).encode() with tarfile.open(path, mode="w:") as archive: for name, payload in members.items(): info = tarfile.TarInfo(name) info.size = len(payload) archive.addfile(info, io.BytesIO(payload)) return images def _active_recovery_fixture(specialize): rows = [] registration_ids = {} originals = [] containers = {} for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=100): registration_ids[profile.id] = index rows.append( { "name": profile.container_name, "displayName": profile.display_name, "id": index, "labels": sorted({profile.runner_label, *profile.legacy_labels}), "disabled": True, "busy": False, "status": "offline", } ) containers[profile.container_name] = {"State": {"Running": False}} for index, name in enumerate(specialize._LEGACY_V1_NAMES, start=9): labels = sorted(specialize._legacy_allowed_labels("v1", name)) legacy_row = { "name": name, "id": index, "labels": labels, "disabled": True, "busy": False, "status": "offline", } rows.append(legacy_row) originals.append({"name": name, "id": index, "labels": labels, "disabled": True}) dind = {"State": {"Running": False, "Health": {"Status": "none"}}} containers[specialize.DIND_NAME] = dind journal = { "legacyTopology": "v1", "newRegistrationIds": registration_ids, "originalRegistrations": originals, } model = {"services": {"runner-dind": {"privileged": True}}} return rows, containers, dind, journal, model def _legacy_dind_mounts(specialize): return [ { "type": "bind", "source": specialize.LEGACY_DIND_STORAGE_ROOT.as_posix(), "target": "/home/rootless/.local/share/docker", "read_only": False, }, { "type": "bind", "source": "/mnt/cache/appdata/itworx-security-runner-workspace", "target": "/workspace", "read_only": False, }, { "type": "bind", "source": "/mnt/cache/appdata/itworx-validation-runner-2-workspace", "target": "/workspace-secondary", "read_only": False, }, ] def _specialized_dind_mounts(specialize): return [ { "type": "bind", "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "target": "/var/lib/docker", "read_only": False, }, { "type": "bind", "source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "target": specialize._DIND_CONTROL_CONTAINER_ROOT, "read_only": False, }, { "type": "bind", "source": str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "target": "/home/rootless/.local/share/docker", "read_only": True, }, *[ { "type": "bind", "source": str( specialize._host_volume_path( specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name ) ), "target": f"/{profile.container_name}-workspace", "read_only": False, } for profile in specialize.RUNNER_PROFILES.values() ], ] def _base_v1_compose_model(specialize): services = { "deploy-control": { "container_name": specialize.CONTROLLER_CONTAINER, "image": "itworx/appops-deploy-control:fixed", }, "runner-dind": { "container_name": specialize.DIND_NAME, "image": "docker@sha256:7c3e797187e43738220462658f4586572cbd3bf009f728b21e34d9c5c06ce431", "user": "rootless", "privileged": True, "cpus": 6, "mem_limit": "12g", "restart": "unless-stopped", "security_opt": ["label=disable"], "command": ["--default-address-pool=base=198.19.0.0/16,size=27"], "volumes": _legacy_dind_mounts(specialize), }, } for name, service_name in specialize._OLD_SERVICES.items(): if name not in specialize._LEGACY_V1_NAMES: continue services[service_name] = { "container_name": name, "image": specialize.RUNNER_IMAGE, "profiles": ["runner"], "cpus": 1, "mem_limit": "1g", "volumes": [ { "type": "bind", "source": source, "target": target, "read_only": read_only, } for source, target, read_only in specialize._expected_legacy_mounts("v1", name) ], "labels": {"tech.itworx.role": "gitea-actions-runner"}, } return { "services": services, "networks": { "runner-sandbox": {"name": specialize._RUNNER_NETWORK_NAME} }, } def _gitea_28_run( specialize, revision, *, run_id=21533, attempt=1, status="completed", conclusion="success" ): filename = specialize.FULL_WORKFLOW.rsplit("/", 1)[-1] return { "id": run_id, "run_attempt": attempt, "path": f"{filename}@refs/heads/main", "status": status, "conclusion": conclusion, "event": "push", "head_branch": "main", "head_sha": revision, "repository": {"id": 9, "full_name": "Jens/Unraid-AppOps-Gateway"}, "head_repository": None, } def _gitea_workflow_api(specialize, rows, *, run_pages=None, calls=None): run_pages = run_pages or {1: rows} calls = calls if calls is not None else [] def api(path): calls.append(path) if path.endswith("/actions/workflows?limit=100&page=1"): return { "workflows": [ { "id": workflow.rsplit("/", 1)[-1], "path": workflow, "state": "active", } for workflow in (specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW) ] } if "/actions/runs?" in path: page = int(path.rsplit("page=", 1)[-1]) return {"workflow_runs": run_pages.get(page, [])} raise AssertionError(f"unexpected fixed Gitea API path: {path}") return api, calls def test_gitea_workflow_inventory_uses_exact_active_filename_and_path( specialize, monkeypatch ): workflow = specialize.FULL_WORKFLOW filename = workflow.rsplit("/", 1)[-1] inventory = { "workflows": [ {"id": filename, "path": workflow, "state": "active"}, { "id": specialize.IMAGE_WORKFLOW.rsplit("/", 1)[-1], "path": specialize.IMAGE_WORKFLOW, "state": "active", }, ] } monkeypatch.setattr(specialize, "_api_json", lambda _path: inventory) assert specialize._workflow_id(workflow) == filename for bad in ( {"id": 34, "path": workflow, "state": "active"}, {"id": filename, "path": ".gitea/workflows/other.yml", "state": "active"}, {"id": filename, "path": workflow, "state": "enabled"}, ): monkeypatch.setattr( specialize, "_api_json", lambda _path, row=bad: {"workflows": [row]} ) with pytest.raises( specialize.RunnerFleetSpecializeError, match="workflow_unavailable" ): specialize._workflow_id(workflow) def test_successful_main_run_accepts_actual_gitea_run_without_workflow_id( specialize, monkeypatch ): revision = "a" * 40 run = _gitea_28_run(specialize, revision) api, calls = _gitea_workflow_api(specialize, [run]) monkeypatch.setattr(specialize, "_api_json", api) selected = specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) assert selected is run assert "workflow_id" not in selected assert selected["head_repository"] is None assert calls[0].endswith("/actions/workflows?limit=100&page=1") assert calls[1].endswith("/actions/runs?branch=main&limit=100&page=1") def test_source_image_fetch_accepts_actual_gitea_commit_tree_shape( specialize, monkeypatch, tmp_path ): revision = "a" * 40 tree = "b" * 40 calls = [] workflow_calls = [] def api(path): calls.append(path) if path.endswith("/branches/main"): return {"commit": {"id": revision}} if path.endswith(f"/git/commits/{revision}"): return {"sha": revision, "commit": {"tree": {"sha": tree}}} if "/actions/runs/" in path and "/artifacts?" in path: return { "artifacts": [ { "name": specialize.IMAGE_ARTIFACT_PREFIX + revision, "expired": False, "id": 44, "size_in_bytes": 1, } ] } raise AssertionError(f"unexpected fixed Gitea API path: {path}") def successful_run(workflow_path, requested_revision): assert requested_revision == revision workflow_calls.append(workflow_path) return {"id": 41 if workflow_path == specialize.IMAGE_WORKFLOW else 42} class ReachedArtifactDownload(Exception): pass def download(_artifact_id, _destination): raise ReachedArtifactDownload monkeypatch.setattr(specialize, "_api_json", api) monkeypatch.setattr(specialize, "_successful_main_run", successful_run) monkeypatch.setattr(specialize, "_download_artifact_zip", download) with pytest.raises(ReachedArtifactDownload): specialize._fetch_source_image_artifact(revision, tree, tmp_path) assert workflow_calls == [specialize.IMAGE_WORKFLOW, specialize.FULL_WORKFLOW] assert calls == [ f"/repos/{specialize.SOURCE_REPOSITORY_API}/branches/main", f"/repos/{specialize.SOURCE_REPOSITORY_API}/git/commits/{revision}", f"/repos/{specialize.SOURCE_REPOSITORY_API}/actions/runs/41/artifacts?name={specialize.IMAGE_ARTIFACT_PREFIX}{revision}", ] @pytest.mark.parametrize( ("commit_value", "error"), [ ({"sha": "a" * 40}, "source_tree_mismatch"), ({"sha": "a" * 40, "commit": {}}, "source_tree_mismatch"), ( {"sha": "a" * 40, "commit": {"tree": {"sha": "c" * 40}}}, "source_tree_mismatch", ), ( {"sha": "a" * 40, "tree": {"sha": "b" * 40}}, "source_tree_mismatch", ), ( {"sha": "c" * 40, "commit": {"tree": {"sha": "b" * 40}}}, "source_revision_mismatch", ), ( {"commit": {"tree": {"sha": "b" * 40}}}, "source_revision_mismatch", ), ], ) def test_source_image_fetch_rejects_noncanonical_gitea_commit_payload_before_artifact_lookup( specialize, monkeypatch, tmp_path, commit_value, error ): revision = "a" * 40 calls = [] workflow_calls = [] def api(path): calls.append(path) if path.endswith("/branches/main"): return {"commit": {"id": revision}} if path.endswith(f"/git/commits/{revision}"): return commit_value raise AssertionError(f"unexpected fixed Gitea API path: {path}") def unexpected_workflow(*_args): workflow_calls.append(True) raise AssertionError("workflow lookup must not follow an untrusted commit response") monkeypatch.setattr(specialize, "_api_json", api) monkeypatch.setattr(specialize, "_successful_main_run", unexpected_workflow) monkeypatch.setattr( specialize, "_download_artifact_zip", lambda *_args: pytest.fail("artifact download must not follow an untrusted commit response"), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): specialize._fetch_source_image_artifact(revision, "b" * 40, tmp_path) assert calls == [ f"/repos/{specialize.SOURCE_REPOSITORY_API}/branches/main", f"/repos/{specialize.SOURCE_REPOSITORY_API}/git/commits/{revision}", ] assert workflow_calls == [] @pytest.mark.parametrize( "changes", [ {"path": "managed-validation.yml@refs/heads/feature"}, {"repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, {"head_repository": {"full_name": "attacker/Unraid-AppOps-Gateway"}}, {"head_sha": "b" * 40}, {"event": "workflow_dispatch"}, {"head_branch": "feature"}, {"pull_requests": [{"number": 123}]}, ], ) def test_successful_main_run_rejects_untrusted_gitea_identity( specialize, monkeypatch, changes ): revision = "a" * 40 run = {**_gitea_28_run(specialize, revision), **changes} api, _calls = _gitea_workflow_api(specialize, [run]) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="protected_validation_missing", ): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) @pytest.mark.parametrize( ("latest", "error"), [ ({"status": "in_progress", "conclusion": None}, "protected_validation_missing"), ({"status": "completed", "conclusion": "failure"}, "protected_validation_missing"), ( {"run_attempt": 2, "status": "completed", "conclusion": "failure"}, "protected_validation_missing", ), ({"run_attempt": None}, "workflow_runs_invalid"), ], ) def test_newest_matching_gitea_run_or_attempt_supersedes_prior_success( specialize, monkeypatch, latest, error ): revision = "a" * 40 successful = _gitea_28_run(specialize, revision, run_id=21532) newer = { **_gitea_28_run(specialize, revision, run_id=21533), **latest, } if latest.get("run_attempt") == 2: newer["id"] = successful["id"] api, _calls = _gitea_workflow_api(specialize, [successful, newer]) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) def test_workflow_run_pagination_fails_closed_at_fixed_page_limit(specialize, monkeypatch): revision = "a" * 40 wrong = {"path": "unrelated.yml@refs/heads/main"} pages = {page: [wrong] * 100 for page in range(1, 11)} api, calls = _gitea_workflow_api(specialize, [], run_pages=pages) monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="workflow_runs_invalid" ): specialize._successful_main_run(specialize.FULL_WORKFLOW, revision) assert len([path for path in calls if "/actions/runs?" in path]) == 10 def test_archive_verification_hashes_large_tar_without_read_bytes(specialize, tmp_path, monkeypatch): archive_path = tmp_path / "images.tar" payload = b"bounded-stream" * 4096 with tarfile.open(archive_path, "w") as archive: member = tarfile.TarInfo("layer.tar") member.size = len(payload) archive.addfile(member, io.BytesIO(payload)) digest = hashlib.sha256(archive_path.open("rb").read()).hexdigest() def forbidden_read_bytes(_path): raise AssertionError("archive verifier must stream the tar file") monkeypatch.setattr(Path, "read_bytes", forbidden_read_bytes) specialize._verify_docker_archive(archive_path, archive_path.stat().st_size, digest) def test_archive_verifier_requires_exact_pinned_inherited_labels( specialize, tmp_path ): archive_path = tmp_path / "task-images.tar" images = _write_task_image_archive(specialize, archive_path) specialize._verify_docker_archive( archive_path, archive_path.stat().st_size, hashlib.sha256(archive_path.read_bytes()).hexdigest(), images, ) images = _write_task_image_archive(specialize, archive_path, extra_label=True) with pytest.raises( specialize.RunnerFleetSpecializeError, match="image_archive_identity_invalid", ): specialize._verify_docker_archive( archive_path, archive_path.stat().st_size, hashlib.sha256(archive_path.read_bytes()).hexdigest(), images, ) def test_installed_image_manifest_requires_complete_source_and_profile_pins(specialize): manifest = _valid_task_image_manifest(specialize) assert specialize._valid_installed_manifest(manifest) is True manifest["images"][0]["reference"] = "localhost/itworx-ci-checks:latest" assert specialize._valid_installed_manifest(manifest) is False def test_task_image_runtime_verifier_accepts_only_exact_pinned_inherited_labels( specialize, monkeypatch ): expected = _valid_task_image_manifest(specialize)["images"][0] runtime_labels = specialize._runtime_task_image_labels(expected) assert runtime_labels is not None assert len(runtime_labels) == 14 actual = { "Id": expected["imageId"], "Os": "linux", "Architecture": "amd64", "Config": {"Labels": runtime_labels}, } monkeypatch.setattr( specialize, "_run", lambda *_args, **_kwargs: json.dumps([actual]), ) assert specialize._verify_manifest_images_in_dind([expected]) is True actual["Config"]["Labels"]["untrusted.extra"] = "value" assert specialize._verify_manifest_images_in_dind([expected]) is False del actual["Config"]["Labels"]["untrusted.extra"] del expected["labels"]["tech.itworx.source-tree"] assert specialize._verify_manifest_images_in_dind([expected]) is False def test_bootstrap_receipt_is_source_bound_private_and_cas_consumed( specialize, tmp_path, monkeypatch ): state_root = tmp_path / "runner-fleet-state" state_root.mkdir(mode=0o700) os.chmod(state_root, 0o700) manifest_path = state_root / "task-image-manifest.json" marker_path = state_root / "specialized.json" receipt_path = state_root / "admission-bootstrap.json" manifest_bytes = json.dumps( _valid_task_image_manifest(specialize), sort_keys=True, separators=(",", ":") ).encode() + b"\n" manifest_path.write_bytes(manifest_bytes) os.chmod(manifest_path, 0o600) manifest_sha = hashlib.sha256(manifest_bytes).hexdigest() marker = { "schemaVersion": 1, "status": "active", "sourceScriptSha256": "e" * 64, "imageManifestSha256": manifest_sha, "imageSourceRevision": "a" * 40, "imageSourceTree": "b" * 40, "verifiedRunners": list(specialize._SPECIALIZED_NAMES), "resourceEnvelopeDigest": "f" * 64, "updatedAt": "2026-10-01T12:00:00Z", } marker_path.write_text(json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n") os.chmod(marker_path, 0o600) registrations = [] for profile in specialize.RUNNER_PROFILES.values(): registrations.append( { "profileId": profile.id, "displayName": profile.display_name, "containerName": profile.container_name, "id": len(registrations) + 10, "labels": sorted({profile.runner_label, *profile.legacy_labels}), } ) receipt = { "schemaVersion": 1, "status": "pending", "sourceScriptSha256": marker["sourceScriptSha256"], "imageManifestSha256": manifest_sha, "imageSourceRevision": "a" * 40, "imageSourceTree": "b" * 40, "verifiedRunners": list(specialize._SPECIALIZED_NAMES), "registrations": registrations, "resourceEnvelopeDigest": marker["resourceEnvelopeDigest"], "createdAt": "2026-10-01T12:00:00Z", } receipt_path.write_text(json.dumps(receipt, sort_keys=True, separators=(",", ":")) + "\n") os.chmod(receipt_path, 0o600) assert specialize._valid_admission_bootstrap_receipt(receipt) validated = specialize.read_admission_bootstrap_receipt( receipt_path, marker_path=marker_path, image_manifest_path=manifest_path ) assert validated is not None pending_sha = validated[1] assert validated[0]["status"] == "pending" assert specialize.consume_admission_bootstrap_receipt( pending_sha, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) consumed = specialize.read_admission_bootstrap_receipt( receipt_path, marker_path=marker_path, image_manifest_path=manifest_path ) assert consumed is not None assert consumed[0]["status"] == "consumed" assert consumed[1] == pending_sha assert specialize.consume_admission_bootstrap_receipt( pending_sha, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) assert not specialize.consume_admission_bootstrap_receipt( "0" * 64, receipt_path, marker_path=marker_path, image_manifest_path=manifest_path, ) def test_runner_config_is_profile_locked_and_only_backend_frontend_keep_legacy_labels(specialize): revision = "a" * 40 checks = specialize._render_runner_config("checks", revision).decode() backend = specialize._render_runner_config("backend", revision).decode() frontend = specialize._render_runner_config("frontend", revision).decode() security = specialize._render_runner_config("security", revision).decode() assert "itworx-ci-checks:docker://localhost/itworx-ci-checks:sha-" in checks assert "linux-validation" not in checks and "ubuntu-latest" not in checks assert "linux-validation" in backend and "ubuntu-latest" in backend assert "linux-validation" in frontend and "ubuntu-latest" in frontend assert "linux-validation" not in security and "ubuntu-latest" not in security for profile_id, profile in specialize.RUNNER_PROFILES.items(): config = specialize._render_runner_config(profile_id, revision).decode() task_image = specialize._task_image_reference(profile_id, revision) job_cpu_milli, job_memory_mib = specialize._runner_job_limits(profile_id) cpu = f"{job_cpu_milli / 1000:.3f}".rstrip("0").rstrip(".") assert f"--cpus={cpu} --memory={job_memory_mib}m" in config assert f"source=itworx-ci-{profile_id}-cache,target=/opt/itworx-ci-cache" in config assert f'valid_volumes:\n - "itworx-ci-{profile_id}-cache"' in config assert "[0-9]*/**" not in config assert "DOCKER_HOST:" not in config assert 'docker_host: "-"' in config assert "host.docker.internal" not in config assert all( f"docker://{task_image}" in line for line in config.splitlines() if ":docker://" in line ) if profile_id == "build": assert "--security-opt=seccomp=unconfined" in config assert "--security-opt=systempaths=unconfined" in config assert "--device=/dev/net/tun:/dev/net/tun:rwm" in config else: assert "--security-opt=" not in config assert "--device=" not in config def test_fixed_workload_reservations_are_subtracted_from_aggregate_profiles(specialize): for profile_id in ("backend", "frontend", "browser"): profile = specialize.RUNNER_PROFILES[profile_id] service_cpu, service_memory = specialize.FIXED_WORKLOAD_RESERVATION[profile_id] job_cpu, job_memory = specialize._runner_job_limits(profile_id) assert job_cpu + service_cpu == profile.cpu_milli assert job_memory + service_memory == profile.memory_mib config = specialize._render_runner_config(profile_id, "a" * 40).decode() cpu = f"{job_cpu / 1000:.3f}".rstrip("0").rstrip(".") assert f"--cpus={cpu} --memory={job_memory}m" in config for profile_id in ("checks", "security"): profile = specialize.RUNNER_PROFILES[profile_id] assert specialize._runner_job_limits(profile_id) == ( profile.cpu_milli, profile.memory_mib, ) assert specialize._runner_job_limits("build") == (4000, 6144) def test_base_compose_model_explicitly_resolves_the_runner_profile(specialize, monkeypatch): compose = b"captured v1 source" base = _base_v1_compose_model(specialize) monkeypatch.setattr( specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest(), ) calls = [] def run_compose(_compose_bytes, *arguments, **_kwargs): calls.append(arguments) return json.dumps(base) monkeypatch.setattr(specialize, "_run_compose", run_compose) assert specialize._validate_base_composition(compose) == "v1" assert calls == [("--profile", "runner", "config", "--format", "json")] def test_rendered_compose_uses_fixed_rootful_dind_and_shared_workspaces(specialize): base = _base_v1_compose_model(specialize) rendered = json.loads(specialize._render_specialized_compose(base)) services = rendered["services"] dind = services["runner-dind"] assert services["deploy-control"] == base["services"]["deploy-control"] assert dind["user"] == "0" assert dind["command"][0] == "dockerd" assert dind["command"][1] == f"--host={specialize._DIND_DOCKER_HOST}" assert "--data-root=/var/lib/docker" in dind["command"] assert "--exec-opt=native.cgroupdriver=cgroupfs" in dind["command"] assert dind["environment"] == {"DOCKER_HOST": specialize._DIND_DOCKER_HOST} assert _mounts(specialize, dind) == { (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, False, ), ( str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "/home/rootless/.local/share/docker", True, ), *{ ( str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False, ) for profile in specialize.RUNNER_PROFILES.values() }, } assert len([volume for volume in dind["volumes"] if volume["type"] == "bind"]) == 9 expected_services = {"deploy-control", "runner-dind"} expected_services.update( specialize._OLD_SERVICES[name] for name in specialize._LEGACY_V1_NAMES ) expected_services.update(f"runner-{profile_id}" for profile_id in specialize.PROFILE_IDS) assert set(services) == expected_services for profile_id in specialize.PROFILE_IDS: service = services[f"runner-{profile_id}"] assert service["environment"]["DOCKER_HOST"] == specialize._DIND_DOCKER_HOST assert service["extra_hosts"] == ["gitea.example.com:198.19.104.1"] assert ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True, ) in specialize._compose_mounts(service) @pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="requires POSIX root ownership") def test_profile_state_creation_uses_effective_runner_image_owners_and_resumes( specialize, tmp_path, monkeypatch ): state = tmp_path / "state" state.mkdir(mode=0o700) data_root = state / "runner-data" workspace_root = state / "workspaces" config_root = state / "runner-configs" image_root = state / "task-images" dind_root = state / "dind" / "docker" shadow_root = state / "dind" / "rootless-unused" control_root = state / "dind-control" for name, value in ( ("_SPECIALIZED_DATA_ROOT", data_root), ("_SPECIALIZED_WORKSPACE_ROOT", workspace_root), ("RUNNER_CONFIG_ROOT", config_root), ("TASK_IMAGE_ROOT", image_root), ("DIND_STORAGE_ROOT", dind_root), ("_DIND_ROOTLESS_SHADOW_ROOT", shadow_root), ("_DIND_CONTROL_ROOT", control_root), ): monkeypatch.setattr(specialize, name, value) control_root.mkdir(mode=0o700) specialize._prepare_profile_state("a" * 40) for profile_id, profile in specialize.RUNNER_PROFILES.items(): data_path = data_root / profile_id workspace_path = workspace_root / profile.container_name assert data_path.stat().st_uid == 0 assert workspace_path.stat().st_uid == 1000 assert control_root.stat().st_uid == 0 assert os.stat(control_root).st_mode & 0o777 == 0o700 config_path = config_root / "checks.yaml" expected = specialize._render_runner_config("checks", "a" * 40) invalid_configs = ( expected.replace( b'valid_volumes:\n - "itworx-ci-checks-cache"', b'valid_volumes:\n - "/itworx-ci-checks-workspace/[0-9]*/**"', 1, ), expected.replace( b'valid_volumes:\n - "itworx-ci-checks-cache"', b'valid_volumes:\n - "/run/itworx-ci-control/**"', 1, ), expected.replace( b' docker_host: "-"', b' docker_host: "tcp://runner-dind:2375"', 1, ), expected.replace( b" envs:\n", b" envs:\n DOCKER_HOST: tcp://runner-dind:2375\n", 1, ), ) for invalid_config in invalid_configs: config_path.write_bytes(invalid_config) os.chmod(config_path, 0o600) with pytest.raises( specialize.RunnerFleetSpecializeError, match="profile_config_adoption_rejected", ): specialize._prepare_profile_state("a" * 40) config_path.write_bytes(expected) os.chmod(config_path, 0o600) specialize._prepare_profile_state("a" * 40) def _mounts(specialize, service): return specialize._compose_mounts(service) def test_rendered_compose_uses_only_canonical_protected_volume_sources(specialize): base = _base_v1_compose_model(specialize) rendered = json.loads(specialize._render_specialized_compose(base)) services = rendered["services"] assert specialize._compose_mounts(services["runner-dind"]) == { (str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "/var/lib/docker", False), ( str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, False, ), ( str(specialize._host_volume_path(specialize._DIND_ROOTLESS_SHADOW_ROOT)), "/home/rootless/.local/share/docker", True, ), *{ ( str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False, ) for profile in specialize.RUNNER_PROFILES.values() }, } assert services["runner-dind"]["restart"] == "no" for profile_id, profile in specialize.RUNNER_PROFILES.items(): service = services[f"runner-{profile_id}"] assert specialize._compose_mounts(service) == { (str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "/data", False), (str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "/runner-config.yaml", True), (str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), specialize._DIND_CONTROL_CONTAINER_ROOT, True), (str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / profile.container_name)), f"/{profile.container_name}-workspace", False), } assert "-f" in specialize._compose_command("up", "-d") assert specialize._compose_command("up", "-d")[specialize._compose_command("up", "-d").index("-f") + 1] == "-" def test_specialized_compose_validation_rejects_socket_and_gateway_broadening( specialize, monkeypatch ): rendered = json.loads( specialize._render_specialized_compose(_base_v1_compose_model(specialize)) ) mutations = ( lambda model: model["services"]["runner-dind"]["environment"].update( DOCKER_HOST="tcp://0.0.0.0:2375" ), lambda model: next( volume for volume in model["services"]["runner-dind"]["volumes"] if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT ).update(read_only=True), lambda model: next( volume for volume in model["services"]["runner-frontend"]["volumes"] if volume["target"] == specialize._DIND_CONTROL_CONTAINER_ROOT ).update(read_only=False), lambda model: model["services"]["runner-frontend"]["extra_hosts"].append( "host.docker.internal:host-gateway" ), lambda model: model["services"]["runner-frontend"].update( network_mode="host" ), lambda model: model["services"]["runner-frontend"].update( privileged=True ), lambda model: model["networks"]["runner-sandbox"].update( name="runner-sandbox" ), ) for mutate in mutations: candidate = copy.deepcopy(rendered) mutate(candidate) monkeypatch.setattr(specialize, "_compose_model", lambda _raw, value=candidate: value) with pytest.raises(specialize.RunnerFleetSpecializeError): specialize._validate_composition(b"candidate") def test_specialized_compose_validation_accepts_real_compose_normalization( specialize, monkeypatch ): normalized = json.loads( specialize._render_specialized_compose(_base_v1_compose_model(specialize)) ) service = normalized["services"]["runner-frontend"] service.pop("privileged") service["extra_hosts"] = ["gitea.example.com=198.19.104.1"] service["networks"] = {"runner-sandbox": {"aliases": ["itworx-ci-frontend"]}} monkeypatch.setattr(specialize, "_compose_model", lambda _raw: normalized) assert specialize._validate_composition(b"normalized candidate") def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize): base = _base_v1_compose_model(specialize) item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="unless-stopped") assert specialize._legacy_dind_observation(item, base)["running"] is True for key, value in (("User", "0"), ("Cmd", ["--data-root=/var/lib/docker"]), ("Entrypoint", ["sh"])): changed = copy.deepcopy(item) changed["Config"][key] = value with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): specialize._legacy_dind_observation(changed, base) changed = copy.deepcopy(item) changed["Mounts"][-1]["Source"] = "/untrusted/docker" with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_dind_identity_invalid"): specialize._legacy_dind_observation(changed, base) @pytest.mark.parametrize("support", ["2|none|false", "2|cgroupfs|false", "2|cgroupfs|true"]) def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces( specialize, monkeypatch, support ): model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize))) service = model["services"]["runner-dind"] item = _legacy_dind_fixture(specialize, _base_v1_compose_model(specialize), image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no") item["Config"].update(User="0", Cmd=service["command"]) item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"] item["HostConfig"].update( CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={} ) item["Mounts"] = [{"Type": "bind", "Source": m["source"], "Destination": m["target"], "RW": not m["read_only"]} for m in service["volumes"]] assert specialize._validate_dind_outer(item, model) is True monkeypatch.setattr(specialize, "_inspect", lambda _name: item) commands = [] def run(args, **_kwargs): commands.append(args) return support monkeypatch.setattr(specialize, "_run", run) if support == "2|cgroupfs|true": specialize._require_dind_resource_support(model) assert f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}" in commands[-1] else: with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_limits_unsupported"): specialize._require_dind_resource_support(model) missing_workspace = copy.deepcopy(item) missing_workspace["Mounts"].pop() assert specialize._validate_dind_outer(missing_workspace, model) is False changed_cases = [] tcp_host = copy.deepcopy(item) tcp_host["Config"]["Env"] = ["DOCKER_HOST=tcp://0.0.0.0:2375"] changed_cases.append(tcp_host) tcp_command = copy.deepcopy(item) tcp_command["Config"]["Cmd"][1] = "--host=tcp://0.0.0.0:2375" changed_cases.append(tcp_command) published_port = copy.deepcopy(item) published_port["HostConfig"]["PortBindings"] = { "2375/tcp": [{"HostIp": "0.0.0.0", "HostPort": "2375"}] } changed_cases.append(published_port) for changed in changed_cases: assert specialize._validate_dind_outer(changed, model) is False def test_outer_runner_must_match_fixed_compose_mount_and_image_identity(specialize): profile_id = "frontend" profile = specialize.RUNNER_PROFILES[profile_id] name = profile.container_name image = { "Name": f"/{name}", "Image": specialize.RUNNER_IMAGE_ID, "Config": { "Image": specialize.RUNNER_IMAGE, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": f"runner-{profile_id}", "tech.itworx.role": "gitea-actions-runner", "tech.itworx.ci.profile": profile_id, }, "Env": [ "CONFIG_FILE=/runner-config.yaml", f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}", ], }, "State": {"Running": False}, "HostConfig": { "CgroupParent": "itworx-ci", "NanoCpus": 500_000_000, "Memory": 512 * specialize._MIB, "PidsLimit": 192, "PidMode": "", "NetworkMode": specialize._RUNNER_NETWORK_NAME, "Privileged": False, "ExtraHosts": ["gitea.example.com:198.19.104.1"], "RestartPolicy": {"Name": "no"}, }, "Mounts": [ {"Source": str(specialize._host_volume_path(specialize._SPECIALIZED_DATA_ROOT / profile_id)), "Destination": "/data", "RW": True}, {"Source": str(specialize._host_volume_path(specialize.RUNNER_CONFIG_ROOT / f"{profile_id}.yaml")), "Destination": "/runner-config.yaml", "RW": False}, {"Source": str(specialize._host_volume_path(specialize._DIND_CONTROL_ROOT)), "Destination": specialize._DIND_CONTROL_CONTAINER_ROOT, "RW": False}, { "Source": str(specialize._host_volume_path(specialize._SPECIALIZED_WORKSPACE_ROOT / name)), "Destination": f"/{name}-workspace", "RW": True, }, ], } assert specialize._validate_outer_item(name, image, profile_id) unqualified_network = copy.deepcopy(image) unqualified_network["HostConfig"]["NetworkMode"] = "runner-sandbox" assert not specialize._validate_outer_item(name, unqualified_network, profile_id) tcp_env = copy.deepcopy(image) tcp_env["Config"]["Env"][-1] = "DOCKER_HOST=tcp://runner-dind:2375" assert not specialize._validate_outer_item(name, tcp_env, profile_id) writable_socket = copy.deepcopy(image) next( mount for mount in writable_socket["Mounts"] if mount["Destination"] == specialize._DIND_CONTROL_CONTAINER_ROOT )["RW"] = True assert not specialize._validate_outer_item(name, writable_socket, profile_id) host_gateway = copy.deepcopy(image) host_gateway["HostConfig"]["ExtraHosts"].append("host.docker.internal:host-gateway") assert not specialize._validate_outer_item(name, host_gateway, profile_id) image["Image"] = "sha256:" + "0" * 64 assert not specialize._validate_outer_item(name, image, profile_id) def test_atomic_guard_replace_uses_expected_hash_as_compare_and_swap(specialize, tmp_path): path = tmp_path / "guard.sh" path.write_bytes(b"original guard\n") original_sha = hashlib.sha256(path.read_bytes()).hexdigest() specialize._atomic_write_owned( path, b"patched guard\n", mode=0o600, expected_sha256=original_sha, ) assert path.read_bytes() == b"patched guard\n" with pytest.raises(specialize.RunnerFleetSpecializeError, match="compare_and_swap_failed"): specialize._atomic_write_owned( path, b"stale write\n", mode=0o600, expected_sha256=original_sha, ) assert path.read_bytes() == b"patched guard\n" def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(specialize): source = ( "stack_recovery=0\n" "set_managed_runners_disabled false\n" "unexpected=0\n" ' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' ) patched = specialize._patch_runner_guard( source, "fixed-module --state", 'fixed-module --guard-allows-container "$name"', ) assert "--fence-legacy" in patched assert "--guard-allows-legacy-container" in patched assert 'if [ "$specialization_state" = "legacy" ]; then' in patched assert 'elif [ "$specialization_state" = "active" ]' in patched assert '--guard-allows-container "$name"' in patched assert " set_managed_runners_disabled false\n" in patched with pytest.raises(specialize.RunnerFleetSpecializeError, match="runner_guard_shape_invalid"): specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow") def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest()) real_os = os class RootPosixOS: name = "posix" @staticmethod def geteuid(): return 0 def __getattr__(self, name): return getattr(real_os, name) monkeypatch.setattr(specialize, "os", RootPosixOS()) monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False}) shell_calls = [] monkeypatch.setattr(specialize, "_run_approved_shell_script", shell_calls.append) with pytest.raises(specialize.RunnerFleetSpecializeError, match="script_not_approved"): specialize.execute(script + b"# altered\n") assert shell_calls == [] def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_edits( specialize, tmp_path, monkeypatch ): runner = tmp_path / "runner-guard" host = tmp_path / "host-guard" runner.write_bytes( b"stack_recovery=0\n" b" set_managed_runners_disabled false\n" b"unexpected=0\n" b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n' ) host.write_bytes( b"# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" b"old capacity writer\n" b"# ITWORX_RUNNER_CAPACITY_V2_END\n" b"runner_healthy=1\n" b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' b" recover legacy stack\n" b" fi\n" ) runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() host_sha = hashlib.sha256(host.read_bytes()).hexdigest() monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) monkeypatch.setattr(specialize, "RUNNER_GUARD_SOURCE_SHA256", runner_sha) monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", host_sha) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) journal = { "guardFiles": [ {"path": str(runner), "originalSha256": runner_sha, "installedSha256": None}, {"path": str(host), "originalSha256": host_sha, "installedSha256": None}, ] } specialize._service_guard_scripts(journal) installed_runner_sha = hashlib.sha256(runner.read_bytes()).hexdigest() assert journal["guardFiles"][0]["installedSha256"] == installed_runner_sha assert "--fence-legacy" in runner.read_text(encoding="utf-8") assert "# ITWORX_RUNNER_CAPACITY_V2_BEGIN" not in host.read_text(encoding="utf-8") host.write_text("operator-owned guard revision\n", encoding="utf-8") with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_source_changed"): specialize._service_guard_scripts(journal) assert host.read_text(encoding="utf-8") == "operator-owned guard revision\n" def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() digest = hashlib.sha256(script).hexdigest() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", digest) real_os = os class RootPosixOS: name = "posix" @staticmethod def geteuid(): return 0 def __getattr__(self, name): return getattr(real_os, name) monkeypatch.setattr(specialize, "os", RootPosixOS()) monkeypatch.setattr(specialize, "_migration_lock", nullcontext) monkeypatch.setattr(specialize, "specialization_state", lambda: "legacy") monkeypatch.setitem( sys.modules, "status_server", type("StatusServerStub", (), {"_require_controller_release": staticmethod(lambda: { "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": "a" * 40, "sourceTree": "b" * 40, })}), ) shell_calls: list[bytes] = [] state = {"active": False} def run_approved(received): shell_calls.append(received) state["active"] = True monkeypatch.setattr(specialize, "_run_approved_shell_script", run_approved) monkeypatch.setattr( specialize, "verified_topology", lambda: { "ready": state["active"], "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if state["active"] else [], "resourceEnvelope": {"ready": state["active"], "digest": "c" * 64}, }, ) result = specialize.execute(script) assert result["status"] == "specialized" assert result["verifiedRunners"] == list(specialize._SPECIALIZED_NAMES) assert shell_calls == [script] def test_run_with_input_keeps_registration_secret_out_of_command_arguments(specialize, monkeypatch): observed = {} def fake_run(args, **kwargs): observed["args"] = args observed["input"] = kwargs["input"] observed["capture_output"] = kwargs["capture_output"] return type("Result", (), {"returncode": 0, "stdout": b"registered"})() monkeypatch.setattr(specialize.subprocess, "run", fake_run) result = specialize._run_with_input(["docker", "run", "fixed"], b"sensitive-runner-token\n") assert result == "registered" assert observed["args"] == ["docker", "run", "fixed"] assert observed["input"] == b"sensitive-runner-token\n" assert "sensitive-runner-token" not in " ".join(observed["args"]) def test_build_probe_create_command_is_fixed_and_has_no_mount_or_shared_socket(specialize): record = _build_probe_record(specialize) args = specialize._build_probe_create_args(record) assert args[:8] == [ "create", "--name", record["name"], "--restart=no", "--cgroup-parent=docker", "--network=bridge", "--cgroupns=private", "--label", ] assert record["taskImageId"] in args assert "--cpus=4" in args assert "--memory=6144m" in args assert "--pids-limit=1024" in args assert "--blkio-weight=100" in args assert "--security-opt=seccomp=unconfined" in args assert "--security-opt=systempaths=unconfined" in args assert "--device=/dev/net/tun:/dev/net/tun:rwm" in args assert "--entrypoint" in args assert specialize._BUILD_PROBE_EXECUTABLE in args assert record["sourceRevision"] in args assert record["sourceTree"] in args assert not any( value == "--privileged" or value == "-e" or value.startswith(("--cap-add", "--volume", "--mount", "--env")) for value in args ) assert not any("docker.sock" in value or "host.docker.internal" in value for value in args) def test_build_probe_creation_inspect_requires_stopped_zero_pid_and_exact_inherited_labels( specialize, ): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["State"] = {"Running": False, "Pid": 0} assert specialize._validate_build_probe_container( item, record, image_environment=environment, require_running=False, require_created_stopped=True, ) labels = specialize._build_probe_container_labels(record) assert set(specialize.PINNED_TASK_BASE_INHERITED_LABELS).issubset(labels) assert record["imageLabels"].items() <= labels.items() assert item["Config"]["Labels"] == labels running, _ = _build_probe_inspect(specialize, record) assert not specialize._validate_build_probe_container( running, record, image_environment=environment, require_running=False, require_created_stopped=True, ) bad_pid = copy.deepcopy(item) bad_pid["State"]["Pid"] = 9 assert not specialize._validate_build_probe_container( bad_pid, record, image_environment=environment, require_running=False, require_created_stopped=True, ) foreign_label = copy.deepcopy(item) foreign_label["Config"]["Labels"]["untrusted.extra"] = "true" assert not specialize._validate_build_probe_container( foreign_label, record, image_environment=environment, require_running=False, require_created_stopped=True, ) def test_build_probe_container_inspect_accepts_only_fixed_private_build_job(specialize): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) assert specialize._validate_build_probe_container(item, record, image_environment=environment) for section, changes in ( ("HostConfig", {"Privileged": True}), ("HostConfig", {"PidMode": "host"}), ("HostConfig", {"NetworkMode": "host"}), ("HostConfig", {"CapAdd": ["SYS_ADMIN"]}), ("HostConfig", {"Devices": [ *item["HostConfig"]["Devices"], {"PathOnHost": "/dev/sda", "PathInContainer": "/dev/sda", "CgroupPermissions": "rwm"}, ]}), ("HostConfig", {"Mounts": [{"Type": "bind", "Source": "/host", "Target": "/host"}]}), ("HostConfig", {"SecurityOpt": ["seccomp=unconfined"]}), ("Config", {"Env": [*environment, "DOCKER_HOST=tcp://shared-dind:2375"]}), ("Config", {"Image": "localhost/itworx-ci-build:latest"}), ): changed, expected_environment = _build_probe_inspect(specialize, record) changed[section].update(changes) assert not specialize._validate_build_probe_container( changed, record, image_environment=expected_environment, ) @pytest.mark.parametrize("field,value", [ ("Type", "bind"), ("Name", "persistent-builder-cache"), ("Source", "/var/run/docker.sock"), ("Destination", "/host"), ("Driver", "nfs"), ("RW", False), ("Propagation", "rshared"), ]) def test_build_probe_rejects_foreign_or_shared_data_mount(specialize, field, value): record = _build_probe_record(specialize, container_id="e" * 64, status="created") item, environment = _build_probe_inspect(specialize, record) item["Mounts"][0][field] = value assert not specialize._validate_build_probe_container(item, record, image_environment=environment) def test_build_probe_volume_identity_detects_same_name_recreation(specialize, monkeypatch): name = "9" * 64 volume = { "Name": name, "Driver": "local", "Scope": "local", "Mountpoint": f"/var/lib/docker/volumes/{name}/_data", "Labels": {"com.docker.volume.anonymous": ""}, "Options": None, "CreatedAt": "2026-10-03T00:00:00Z", } monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: json.dumps([volume])) identity = specialize._attest_build_probe_data_volume(name) volume["Labels"]["foreign.owner"] = "another-job" with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): specialize._attest_build_probe_data_volume(name, identity) volume["Labels"] = {"com.docker.volume.anonymous": ""} volume["CreatedAt"] = "2026-10-03T00:00:01Z" with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_untrusted"): specialize._attest_build_probe_data_volume(name, identity) def test_build_probe_volume_recovery_never_removes_volume_used_by_another_container(specialize, monkeypatch): identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} mutations = [] def run(*args, **kwargs): if args[:2] == ("volume", "ls"): return identity["name"] if args[:2] == ("ps", "-a"): return "f" * 64 mutations.append(args) raise AssertionError("unexpected volume mutation") monkeypatch.setattr(specialize, "_run_dind_docker", run) monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) with pytest.raises(specialize.RunnerFleetSpecializeError, match="volume_in_use"): specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) assert mutations == [] def test_build_probe_volume_recovery_removes_only_exact_journal_owned_volume(specialize, monkeypatch): identity = {"name": "9" * 64, "createdAt": "2026-10-03T00:00:00Z"} mutations = [] present = True def run(*args, **kwargs): nonlocal present if args[:2] == ("volume", "ls"): assert args == ("volume", "ls", "--quiet", "--filter", f"name={identity['name']}") return identity["name"] if present else "" if args[:2] == ("ps", "-a"): return "" assert args == ("volume", "rm", identity["name"]) mutations.append(args) present = False return identity["name"] monkeypatch.setattr(specialize, "_run_dind_docker", run) monkeypatch.setattr(specialize, "_attest_build_probe_data_volume", lambda *_args: identity) specialize._remove_recorded_build_probe_volume({"dataVolume": identity}) assert mutations == [("volume", "rm", identity["name"])] def test_build_probe_result_requires_exact_marker_and_every_rootless_tool_check(specialize): record = _build_probe_record(specialize) payload = { "schema": specialize._BUILD_PROBE_SCHEMA, "status": "passed", "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "engineVersion": "29.8.2", "engineRootless": True, "engineStorageDriver": "overlay2", "engineCgroupDriver": "none", "engineSocket": "unix:///run/itworx-job-docker/docker.sock", "engineSocketUid": 1000, "composeVersion": "5.5.1", "buildRun": True, "composeHealthy": True, "composeDnsResolved": True, "archiveNormalized": True, "archiveLoaded": True, "imageId": "sha256:" + "2" * 64, "imageLabels": record["imageLabels"], "archiveSha256": "sha256:" + "3" * 64, "holdSeconds": 30, } line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( payload, sort_keys=True, separators=(",", ":"), ) assert specialize._validate_build_probe_result(line, record=record) == payload with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_missing"): specialize._validate_build_probe_result(line + "\n" + line, record=record) for key, value in ( ("engineRootless", False), ("engineSocket", "tcp://0.0.0.0:2375"), ("composeHealthy", False), ("holdSeconds", 0), ): changed = {**payload, key: value} changed_line = specialize._BUILD_PROBE_RESULT_PREFIX + json.dumps( changed, sort_keys=True, separators=(",", ":"), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_result_untrusted"): specialize._validate_build_probe_result(changed_line, record=record) def test_build_probe_live_process_inventory_requires_daemon_and_compose_children(specialize): output = "\n".join( ( "PID PPID COMMAND", "1 0 python3 /usr/local/bin/itworx-ci-job-docker-probe", "10 1 rootlesskit --net=slirp4netns", "11 10 dockerd --host=unix:///run/itworx-job-docker/docker.sock --storage-driver=overlay2", "12 11 containerd --config /run/containerd/config.toml", "21 20 sleep 300", ) ) result = specialize._validate_build_probe_processes(output) assert result == { "processCount": 5, "probe": True, "rootlesskit": True, "dockerd": True, "containerd": True, "composeService": True, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): specialize._validate_build_probe_processes( output.replace("21 20 sleep 300", "21 20 cat /etc/os-release") ) with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_processes_untrusted"): specialize._validate_build_probe_processes(output.replace("containerd --config", "unknown --config")) def test_build_probe_intent_binds_revision_tree_image_nonce_and_exact_container_id(specialize): record = _build_probe_record(specialize) assert specialize._validated_build_probe_record( record, source_revision="a" * 40, source_tree="b" * 40, ) == record invalid = dict(record, name=record["name"] + "-foreign") with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): specialize._validated_build_probe_record( invalid, source_revision="a" * 40, source_tree="b" * 40, ) def _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, ids): image_environment = item["Config"]["Env"] present = list(ids) removed = [] monkeypatch.setattr( specialize, "_inspect", lambda name: {"State": {"Running": True}} if name == specialize.DIND_NAME else None, ) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) monkeypatch.setattr( specialize, "_build_probe_ids", lambda *, label=True, name=None: list(present) if label or name == record["name"] and present else [], ) monkeypatch.setattr(specialize, "_inspect_dind_container", lambda _cid: item) monkeypatch.setattr( specialize, "_build_probe_image_environment", lambda _record: image_environment, ) def run_dind(*args, **_kwargs): if args[:3] == ("rm", "--force", "--volumes"): removed.append(args[3]) present.clear() return args[3] if args[:2] == ("volume", "ls"): return "" raise AssertionError(f"unexpected private-DinD command: {args}") monkeypatch.setattr(specialize, "_run_dind_docker", run_dind) monkeypatch.setattr( specialize, "_attest_build_probe_data_volume", lambda name, expected=None: {"name": name, "createdAt": "2026-10-03T00:00:00Z"}, ) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) return removed def test_build_probe_recovery_cleans_only_intent_owned_container_after_create_crash( specialize, monkeypatch, ): record = _build_probe_record(specialize, status="intent") cid = "e" * 64 item, _environment = _build_probe_inspect(specialize, record, container_id=cid) removed = _install_build_probe_cleanup_fakes(specialize, monkeypatch, record, item, [cid]) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } specialize._cleanup_build_probe(journal) assert removed == [cid] assert journal["buildProbe"]["containerId"] == cid assert journal["buildProbe"]["status"] == "cleaned" def test_build_probe_recovery_refuses_foreign_cid_even_with_matching_name_and_nonce( specialize, monkeypatch, ): expected_cid = "d" * 64 foreign_cid = "e" * 64 record = _build_probe_record( specialize, container_id=expected_cid, status="started", ) item, _environment = _build_probe_inspect(specialize, record, container_id=foreign_cid) removed = _install_build_probe_cleanup_fakes( specialize, monkeypatch, record, item, [foreign_cid], ) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_container_untrusted"): specialize._cleanup_build_probe(journal) assert removed == [] def test_build_probe_recovery_refuses_ambiguous_owned_label_inventory( specialize, monkeypatch, ): record = _build_probe_record(specialize, status="intent") cid = "e" * 64 item, _environment = _build_probe_inspect(specialize, record, container_id=cid) removed = _install_build_probe_cleanup_fakes( specialize, monkeypatch, record, item, [cid, "f" * 64], ) journal = { "sourceRevision": record["sourceRevision"], "sourceTree": record["sourceTree"], "buildProbe": record, } with pytest.raises(specialize.RunnerFleetSpecializeError, match="build_probe_ambiguous_containers"): specialize._cleanup_build_probe(journal) assert removed == [] invalid = dict(record, status="started") with pytest.raises(specialize.RunnerFleetSpecializeError, match="journal_invalid"): specialize._validated_build_probe_record( invalid, source_revision="a" * 40, source_tree="b" * 40, ) @pytest.mark.parametrize("recovering_cgroup_intent", [False, True]) @pytest.mark.parametrize("probe_fails", [False, True]) def test_specialization_stage_fixture_quiesces_old_before_registering_and_never_runs_over_two( specialize, tmp_path, monkeypatch, recovering_cgroup_intent, probe_fails ): revision = "a" * 40 tree = "b" * 40 manifest_sha = "d" * 64 manifest = { "images": [ {"profile": profile_id, "reference": specialize._task_image_reference(profile_id, revision)} for profile_id in specialize.PROFILE_IDS ] } archive = tmp_path / "task-images.tar" archive.write_bytes(b"fixture") journal = { "status": "in_progress", "stage": "prepared", "legacyTopology": "v1", "sourceRevision": revision, "sourceTree": tree, "cgroupCreated": False, "cgroupEnsurePending": recovering_cgroup_intent, "baseComposeSha256": hashlib.sha256(b"captured-base").hexdigest(), "baseComposeModelSha256": hashlib.sha256(b"base-model").hexdigest(), "specializedComposeSha256": hashlib.sha256(b"specialized-model").hexdigest(), "backupDirectory": str(tmp_path / "specialized-test"), "newRegistrationIds": {}, } events: list[str] = [] running: set[str] = set() maximum_running = 0 marker_active = {"value": False} def rows(): return [ { "name": specialize.RUNNER_PROFILES[profile_id].container_name, "id": index + 1, "labels": sorted( { specialize.RUNNER_PROFILES[profile_id].runner_label, *specialize.RUNNER_PROFILES[profile_id].legacy_labels, } ), "disabled": True, "busy": False, } for index, profile_id in enumerate(specialize.PROFILE_IDS) ] def inventory(): return ( { name: {"name": name, "State": {"Running": name in running}} for name in (*specialize._SPECIALIZED_NAMES, specialize.DIND_NAME) }, [], ) def resource_proof(_claims, _containers, _legacy): return { "ready": True, "digest": "e" * 64, "runningRunnerContainers": len(running), } monkeypatch.setattr(specialize, "_captured_base_source", lambda _journal: b"captured-base") monkeypatch.setattr( specialize, "_captured_compose_model", lambda _journal, *, specialized: b"specialized-model" if specialized else b"base-model", ) monkeypatch.setattr(specialize, "_read_source_file", lambda *_args, **_kwargs: b"captured-base") monkeypatch.setattr(specialize, "_validate_base_composition", lambda _bytes=None: "v1") monkeypatch.setattr(specialize, "_validate_composition", lambda raw: raw) monkeypatch.setattr(specialize, "_run_specialized_compose", lambda _journal, *args, **_kwargs: events.append(f"compose:{args[-1]}")) monkeypatch.setattr(specialize, "_wait_guards_quiet", lambda: events.append("guards-quiet")) monkeypatch.setattr(specialize, "_service_guard_scripts", lambda _journal: events.append("guards-fenced")) monkeypatch.setattr(specialize, "_stop_legacy_capacity_daemon", lambda: events.append("capacity-stopped")) monkeypatch.setattr(specialize, "_set_legacy_disabled", lambda _journal: events.append("legacy-disabled")) monkeypatch.setattr(specialize, "_check_legacy_queued_jobs", lambda: None) monkeypatch.setattr(specialize, "_stop_remove_legacy_containers", lambda _journal: events.append("legacy-removed")) monkeypatch.setattr( specialize, "_cgroup_family_snapshot", lambda: { "exists": recovering_cgroup_intent, "rootSubtreeControllers": ["cpu", "memory", "pids"], }, ) def ensure_cgroup_family(**kwargs): assert kwargs["allow_existing"] is recovering_cgroup_intent events.append("cgroup-ensured") return not recovering_cgroup_intent monkeypatch.setattr( specialize, "_ensure_cgroup_family", ensure_cgroup_family, ) monkeypatch.setattr(specialize, "_prepare_profile_state", lambda _revision: None) monkeypatch.setattr(specialize, "_write_private_atomic", lambda *_args, **_kwargs: None) def register(current_journal): assert "legacy-disabled" in events events.append("registrations-created") current_journal["newRegistrationIds"] = { profile_id: index + 1 for index, profile_id in enumerate(specialize.PROFILE_IDS) } specialize._set_stage(current_journal, "registrations_created") monkeypatch.setattr(specialize, "_register_profiles", register) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: None) monkeypatch.setattr(specialize, "_inspect", lambda _name: {"State": {"Running": True}}) monkeypatch.setattr(specialize, "_require_dind_resource_support", lambda _model: None) monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {"services": {}}) monkeypatch.setattr(specialize, "_load_images_into_dind", lambda *_args: None) monkeypatch.setattr(specialize, "_verify_manifest_images_in_dind", lambda _items: True) monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) monkeypatch.setattr(specialize, "_resource_proof", resource_proof) monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) monkeypatch.setattr( specialize, "_trusted_private_file", lambda path, **_kwargs: specialize._render_runner_config("build", revision) if str(path).endswith("build.yaml") else b"fixture", ) monkeypatch.setattr(specialize, "_cleanup_build_probe", lambda _journal: None) def run_build_probe(_journal, _manifest): assert not running events.append("build-probed") if probe_fails: raise specialize.RunnerFleetSpecializeError( "runner_fleet_specialization_build_probe_failed" ) monkeypatch.setattr(specialize, "_run_build_probe", run_build_probe) monkeypatch.setattr(specialize, "_runner_rows", rows) def write_journal(current_journal): if current_journal.get("cgroupEnsurePending") is True: events.append("cgroup-intent-durable") if current_journal.get("cgroupCreated") is True: events.append("cgroup-result-durable") if current_journal.get("status") == "active": events.append("journal-active-durable") monkeypatch.setattr(specialize, "_write_journal", write_journal) batches: list[tuple[str, ...]] = [] def set_containers(profiles, start): nonlocal maximum_running names = {specialize.RUNNER_PROFILES[item].container_name for item in profiles} if start: assert len(running | names) <= 2 running.update(names) maximum_running = max(maximum_running, len(running)) batches.append(tuple(profiles)) events.append("batch-started") else: running.difference_update(names) events.append("batch-stopped") monkeypatch.setattr(specialize, "_set_runner_containers_running", set_containers) monkeypatch.setattr(specialize, "_bootstrap_registrations", lambda: [{"name": name} for name in specialize._SPECIALIZED_NAMES]) monkeypatch.setattr(specialize, "_read_installed_manifest_sha256", lambda: manifest_sha) monkeypatch.setattr(specialize, "_write_pending_bootstrap_receipt", lambda *_args: events.append("receipt-written")) crash = {"next_active_write": True} class SimulatedPowerLoss(BaseException): pass def write_marker(**kwargs): assert kwargs["status"] == "active" assert not running assert "receipt-written" in events events.append("marker-active-attempt") if crash["next_active_write"]: crash["next_active_write"] = False raise SimulatedPowerLoss marker_active["value"] = True events.append("marker-active") monkeypatch.setattr(specialize, "_write_marker", write_marker) monkeypatch.setattr(specialize, "specialization_state", lambda: "active" if marker_active["value"] else "blocked") if probe_fails: monkeypatch.setattr(specialize, "_rollback_before_activation", lambda _journal: True) with pytest.raises( specialize.RunnerFleetSpecializeError, match="failed_rolled_back", ): specialize._perform_migration_stages(journal, manifest, archive) assert "build-probed" in events assert "receipt-written" not in events assert "marker-active-attempt" not in events assert not marker_active["value"] return with pytest.raises(SimulatedPowerLoss): specialize._perform_migration_stages(journal, manifest, archive) assert journal["status"] == "active" assert not marker_active["value"] assert events.index("journal-active-durable") < events.index("marker-active-attempt") marker = { "status": "in_progress", "sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256, "imageSourceRevision": revision, "imageSourceTree": tree, } script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_read_json_file", lambda *_args, **_kwargs: marker) monkeypatch.setattr(specialize, "_valid_marker", lambda _marker: True) monkeypatch.setattr(specialize, "_migration_lock", lambda: nullcontext()) monkeypatch.setattr( specialize, "verified_topology", lambda: { "ready": marker_active["value"], "verifiedRunners": list(specialize._SPECIALIZED_NAMES) if marker_active["value"] else [], "resourceEnvelope": {"ready": marker_active["value"], "digest": "e" * 64}, }, ) monkeypatch.setitem( sys.modules, "status_server", type( "StatusServerStub", (), { "_require_controller_release": staticmethod( lambda: { "sourceRepository": specialize.SOURCE_REPOSITORY, "sourceRevision": revision, "sourceTree": tree, } ) }, ), ) monkeypatch.setattr( specialize, "_run_approved_shell_script", lambda _script: specialize._perform_migration_stages(journal, manifest, archive), ) result = specialize.execute(script) assert result["status"] == "specialized" assert maximum_running == 2 assert events.index("cgroup-intent-durable") < events.index("cgroup-ensured") assert events.index("cgroup-ensured") < events.index("cgroup-result-durable") assert batches == [ tuple(specialize.PROFILE_IDS[0:2]), tuple(specialize.PROFILE_IDS[2:4]), tuple(specialize.PROFILE_IDS[4:6]), ] assert events.index("legacy-disabled") < events.index("registrations-created") assert events.index("batch-stopped") < events.index("build-probed") assert events.index("build-probed") < events.index("receipt-written") assert events.index("receipt-written") < events.index("marker-active") assert journal["cgroupCreated"] is True assert journal["status"] == "active" assert not running @pytest.mark.parametrize("original_running", [False, True]) def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( specialize, tmp_path, monkeypatch, original_running ): image_id = specialize._DIND_IMAGE_ID base_model = _base_v1_compose_model(specialize) specialized_model = { "services": { "runner-dind": { **base_model["services"]["runner-dind"], "cgroup_parent": "itworx-ci", "restart": "no", "volumes": [ { "type": "bind", "source": str(specialize._host_volume_path(specialize.DIND_STORAGE_ROOT)), "target": "/var/lib/docker", "read_only": False, } ], } } } journal = { "originalDind": { "present": True, "running": original_running, "restartPolicy": "unless-stopped", "imageId": image_id, }, "originalContainers": [], "originalRegistrations": [], "newRegistrationIds": {}, "guardFiles": [], "cgroupCreated": True, "cgroupEnsurePending": False, "cgroupRootSubtree": ["cpu", "memory", "pids"], "backupDirectory": str(tmp_path / "specialized-test"), } events = [] current = {"kind": "specialized", "State": {"Running": True}} current_marker = {"value": current} def inspect(name): assert name == specialize.DIND_NAME or name in specialize._SPECIALIZED_NAMES return current_marker["value"] if name == specialize.DIND_NAME else None def run(args, **_kwargs): events.append(tuple(args)) if args[:2] == ["docker", "update"] and args[-1] == specialize.DIND_NAME: if args[2] == "--restart=no": return "" current_marker["value"]["HostConfig"]["RestartPolicy"]["Name"] = args[2].partition("=")[2] return "" if args[:2] == ["docker", "stop"]: current_marker["value"]["State"]["Running"] = False return "" if args[:2] == ["docker", "rm"]: current_marker["value"] = None return "" return "" def run_compose(_compose, *args, **_kwargs): events.append(("compose", *args)) assert args[-1] == "runner-dind" restored_running = original_running if args[0] == "up" else False current_marker["value"] = _legacy_dind_fixture( specialize, base_model, image_id=image_id, running=restored_running, restart_policy="no", ) base_bytes = json.dumps(base_model, sort_keys=True, separators=(",", ":")).encode() specialized_bytes = json.dumps(specialized_model, sort_keys=True, separators=(",", ":")).encode() monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr( specialize, "_captured_compose_model", lambda _journal, *, specialized: specialized_bytes if specialized else base_bytes, ) monkeypatch.setattr( specialize, "_validate_dind_outer", lambda item, _model: item.get("kind") == "specialized", ) monkeypatch.setattr(specialize, "_inspect", inspect) monkeypatch.setattr(specialize, "_run", run) monkeypatch.setattr(specialize, "_run_compose", run_compose) monkeypatch.setattr(specialize, "_runner_rows", lambda: []) monkeypatch.setattr( specialize, "rollback_host_cgroup_family", lambda **_kwargs: events.append("cgroup-cleanup") or {"familyExists": False}, ) monkeypatch.setattr(specialize, "SPECIALIZED_COMPOSE_PATH", tmp_path / "specialized-compose.yml") monkeypatch.setattr(specialize, "TASK_IMAGE_MANIFEST_PATH", tmp_path / "task-image-manifest.json") monkeypatch.setattr(specialize, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "receipt.json") monkeypatch.setattr(specialize, "RUNNER_CONFIG_ROOT", tmp_path / "runner-configs") monkeypatch.setattr(specialize, "_SPECIALIZED_DATA_ROOT", tmp_path / "runner-data") monkeypatch.setattr(specialize, "_SPECIALIZED_WORKSPACE_ROOT", tmp_path / "workspaces") monkeypatch.setattr(specialize, "TASK_IMAGE_ROOT", tmp_path / "task-images") monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", tmp_path / "specialized.json") monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", tmp_path / "journal.json") assert specialize._rollback_before_activation(journal) is True restored = current_marker["value"] assert restored["Image"] == image_id assert restored["State"]["Running"] is original_running assert restored["HostConfig"]["RestartPolicy"]["Name"] == "unless-stopped" compose = next(event for event in events if isinstance(event, tuple) and event[0] == "compose") assert compose[1] == ("up" if original_running else "create") remove = next(index for index, event in enumerate(events) if isinstance(event, tuple) and event[:2] == ("docker", "rm")) assert remove < events.index(compose) < events.index("cgroup-cleanup") def _legacy_dind_fixture( specialize, base_model, *, image_id, running, restart_policy ): service = base_model["services"]["runner-dind"] return { "Name": f"/{specialize.DIND_NAME}", "Image": image_id, "Config": { "Image": service["image"], "User": "rootless", "Entrypoint": ["dockerd-entrypoint.sh"], "Cmd": ["--default-address-pool=base=198.19.0.0/16,size=27"], "Volumes": {"/home/rootless/.local/share/docker": {}, "/var/lib/docker": {}}, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": "runner-dind", }, }, "State": {"Running": running}, "HostConfig": { "NanoCpus": 6_000_000_000, "Memory": 12 * 1024**3, "PidMode": "", "Privileged": True, "RestartPolicy": {"Name": restart_policy}, }, "Mounts": [ { "Type": "bind", "Source": mount["source"], "Destination": mount["target"], "RW": not mount.get("read_only", False), } for mount in service["volumes"] ] + [{ "Type": "volume", "Driver": "local", "Name": "f" * 64, "Source": "/var/lib/docker/volumes/" + "f" * 64 + "/_data", "Destination": "/var/lib/docker", "RW": True, }], } def test_known_v1_and_v2_topologies_are_explicit_and_v1_does_not_require_absent_v2_services( specialize, monkeypatch ): compose = b"known Tower v1 compose fixture" monkeypatch.setattr(specialize, "_KNOWN_V1_COMPOSE_SHA256", hashlib.sha256(compose).hexdigest()) v1_services = { service_name: {"container_name": name} for name, service_name in specialize._OLD_SERVICES.items() if name in specialize._LEGACY_V1_NAMES } assert specialize._classify_legacy_topology(compose, v1_services) == "v1" v2_services = {} for name in specialize._LEGACY_V2_NAMES: service_name = specialize._OLD_SERVICES[name] burst = name.endswith("burst-1") v2_services[service_name] = { "container_name": name, "image": specialize.RUNNER_IMAGE, "profiles": ["runner"], "labels": { "tech.itworx.role": "gitea-actions-burst-runner" if burst else "gitea-actions-runner" }, **({"cpus": 0.5, "mem_limit": "512m"} if name in specialize._LEGACY_V2_NAMES[2:] else {}), } assert specialize._classify_legacy_topology(b"known v2 compose", v2_services) == "v2" partial_v2 = dict(v1_services) partial_v2["runner-3"] = v2_services["runner-3"] with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_compose_invalid"): specialize._classify_legacy_topology(compose, partial_v2) def test_exact_live_v1_registration_and_container_contract_accepts_only_two_known_runners(specialize): rows = [ { "name": name, "id": runner_id, "labels": sorted(specialize._OLD_LABELS[name]), "disabled": False, "busy": False, } for runner_id, name in zip((9, 22), specialize._LEGACY_V1_NAMES, strict=True) ] rows.extend( [ {"name": "itworx-unraid-deploy", "id": 13, "labels": ["unraid-deploy"], "disabled": False, "busy": False}, {"name": "itworx-windows11-native", "id": 14, "labels": ["windows-native"], "disabled": False, "busy": False}, ] ) assert len(specialize._validate_legacy_registrations(rows, "v1")) == 2 name = "itworx-security-runner" mount_entries = [ {"Source": source, "Destination": target, "RW": True} for source, target, _read_only in specialize._V1_RUNNER_VOLUMES[name] ] item = { "Image": specialize.RUNNER_IMAGE_ID, "Config": { "Image": specialize.RUNNER_IMAGE, "Labels": { "com.docker.compose.project": specialize.PROJECT, "com.docker.compose.service": "runner", "tech.itworx.role": "gitea-actions-runner", }, }, "State": {"Running": True}, "HostConfig": { "NanoCpus": 1_000_000_000, "Memory": 1024 * specialize._MIB, "PidMode": "", "RestartPolicy": {"Name": "unless-stopped"}, }, "Mounts": mount_entries, } assert specialize._validate_legacy_container_item("v1", name, item)["running"] is True item["HostConfig"]["PidMode"] = "host" with pytest.raises(specialize.RunnerFleetSpecializeError, match="v1_container_budget_invalid"): specialize._validate_legacy_container_item("v1", name, item) extra_known_runner = [*rows, {"name": "itworx-validation-runner-3", "id": 30}] with pytest.raises(specialize.RunnerFleetSpecializeError, match="legacy_registration_invalid"): specialize._validate_legacy_registrations(extra_known_runner, "v1") def test_queue_gate_allows_only_legacy_labels_covered_by_new_profiles(specialize): assert specialize._queued_job_is_covered(["self-hosted", "linux", "linux-validation", "ubuntu-latest"]) assert specialize._queued_job_is_covered(["itworx-ci-backend"]) assert specialize._queued_job_is_covered(["docker-validation"]) assert not specialize._queued_job_is_covered(["mcp-validation", "unraid"]) def _legacy_queue_job(job_id, status, labels, *, run_id=21500): return {"id": job_id, "run_id": run_id, "status": status, "labels": labels} def test_legacy_queue_gate_reads_queued_and_requested_admin_inventories(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=pending" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(102, "requested", ["linux-validation", "ubuntu-latest"])], } if "status=queued" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(101, "queued", ["itworx-ci-backend"])], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) specialize._check_legacy_queued_jobs() assert len(calls) == 2 assert "status=pending&limit=100&page=1&sort=id&order=asc" in calls[0] assert "status=queued&limit=100&page=1&sort=id&order=asc" in calls[1] def test_legacy_queue_gate_blocks_unsupported_requested_dependency_job(specialize, monkeypatch): def api(path): if "status=queued" in path: return {"total_count": 0, "jobs": []} if "status=pending" in path: return { "total_count": 1, "jobs": [_legacy_queue_job(201, "requested", ["mcp-validation", "unraid"])], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_catches_requested_to_queued_transition_between_filters( specialize, monkeypatch ): state = "requested" calls = [] def api(path): nonlocal state calls.append(path) if "status=pending" in path: # The job changes status immediately before this filtered read. state = "queued" return {"total_count": 0, "jobs": []} if "status=queued" in path: jobs = ( [_legacy_queue_job(250, "queued", ["mcp-validation", "unraid"])] if state == "queued" else [] ) return {"total_count": len(jobs), "jobs": jobs} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert "status=pending" in calls[0] assert "status=queued" in calls[1] def test_legacy_queue_gate_paginates_and_blocks_unsupported_page_two_job(specialize, monkeypatch): calls = [] def api(path): calls.append(path) if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) if page == 1: return { "total_count": 101, "jobs": [ _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) for index in range(100) ], } return { "total_count": 101, "jobs": [_legacy_queue_job(101, "queued", ["mcp-validation", "unraid"])], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_present", ): specialize._check_legacy_queued_jobs() assert any("status=queued&limit=100&page=2" in path for path in calls) def test_legacy_queue_gate_rejects_total_count_changes_between_pages(specialize, monkeypatch): def api(path): if "status=queued" in path: page = int(path.rsplit("page=", 1)[1].split("&", 1)[0]) if page == 1: return { "total_count": 101, "jobs": [ _legacy_queue_job(index + 1, "queued", ["itworx-ci-backend"]) for index in range(100) ], } return { "total_count": 102, "jobs": [_legacy_queue_job(101, "queued", ["itworx-ci-backend"])], } if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() @pytest.mark.parametrize( "payload", [ {"total_count": 1, "jobs": []}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "requested", ["itworx-ci-backend"])]}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", [""])]}, {"total_count": 1, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"], run_id=0)]}, {"total_count": 2, "jobs": [_legacy_queue_job(1, "queued", ["itworx-ci-backend"])]}, { "total_count": 2, "jobs": [ _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), ], }, { "total_count": 2, "jobs": [ _legacy_queue_job(2, "queued", ["itworx-ci-backend"]), _legacy_queue_job(1, "queued", ["itworx-ci-backend"]), ], }, ], ) def test_legacy_queue_gate_rejects_malformed_or_truncated_inventory(specialize, monkeypatch, payload): def api(path): if "status=queued" in path: return payload if "status=pending" in path: return {"total_count": 0, "jobs": []} raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specialize, monkeypatch): row = _legacy_queue_job(301, "queued", ["itworx-ci-backend"]) def api(path): if "status=queued" in path: return {"total_count": 1, "jobs": [row]} if "status=pending" in path: return { "total_count": 1, "jobs": [{**row, "status": "requested"}], } raise AssertionError(f"unexpected fixed Gitea API path: {path}") monkeypatch.setattr(specialize, "_api_json", api) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_legacy_queue_untrusted", ): specialize._check_legacy_queued_jobs() def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url(): workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8") build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8") frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8") assert "github-server-url: http://gitea-runner-internal:3000" in workflow assert "bash scripts/build_ci_task_images.sh" in workflow assert "jens/unraid-appops-gateway" in build assert "Jens/Unraid-AppOps-Gateway" in build assert "corepack prepare pnpm@11.15.0 --activate" in frontend assert "test \"$(pnpm --version)\" = 11.15.0" in frontend def test_specialization_shell_entrypoint_is_fixed_and_has_no_caller_selected_target(): script = ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh" content = script.read_text(encoding="utf-8") assert content.startswith("#!/usr/bin/env bash\n") assert "SPECIALIZE_CI_RUNNER_FLEET" in content assert "--apply-fixed" in content assert "eval " not in content assert "docker run" not in content assert "docker compose" not in content def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity( specialize, monkeypatch ): registrations = [ { "name": profile.display_name, "id": index, "labels": [profile.runner_label, *profile.legacy_labels], "disabled": True, "busy": False, } for index, profile in enumerate(specialize.RUNNER_PROFILES.values(), start=10) ] monkeypatch.setattr(specialize, "_api_json", lambda _path: {"runners": registrations}) normalized = specialize._runner_rows() assert [row["name"] for row in normalized] == list(specialize._SPECIALIZED_NAMES) assert [row["displayName"] for row in normalized] == [ profile.display_name for profile in specialize.RUNNER_PROFILES.values() ] registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].container_name with pytest.raises( specialize.RunnerFleetSpecializeError, match="registration_name_invalid", ): specialize._runner_rows() registrations[0]["name"] = specialize.RUNNER_PROFILES["checks"].display_name registrations[1]["name"] = specialize.RUNNER_PROFILES["checks"].display_name with pytest.raises( specialize.RunnerFleetSpecializeError, match="registration_duplicate", ): specialize._runner_rows() def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): rows, containers, dind, journal, model = _active_recovery_fixture(specialize) events = [] class Volume: def __enter__(self): return self def __exit__(self, *_args): return False def assert_current(self): events.append("volume-current") monkeypatch.setattr(specialize, "_migration_lock", lambda **_kwargs: nullcontext()) monkeypatch.setattr(specialize, "_open_fixed_release_volume", lambda: Volume()) monkeypatch.setattr( specialize, "_active_release_bundle", lambda: ({"status": "active"}, journal, model, b"captured-compose"), ) monkeypatch.setattr(specialize, "_runner_rows", lambda: rows) monkeypatch.setattr(specialize, "_docker_outer_inventory", lambda: (containers, [])) monkeypatch.setattr(specialize, "_validate_outer_item", lambda *_args: True) monkeypatch.setattr(specialize, "_validate_dind_outer", lambda *_args: True) monkeypatch.setattr( specialize, "_cgroup_family_snapshot", lambda: { "exists": family_exists, "safeToStartDind": True, "rootSubtreeControllers": ["cpu", "memory", "pids"], }, ) monkeypatch.setattr( specialize, "_fresh_recovery_host_gate", lambda: events.append("fresh-host-gate"), ) monkeypatch.setattr( specialize, "_ensure_cgroup_family_details", lambda **_kwargs: events.append("cgroup-create") or {"created": True, "rootSubtreeBefore": ["cpu", "memory", "pids"]}, ) monkeypatch.setattr( specialize, "_run_compose", lambda compose, *args, **kwargs: events.append(("compose", compose, args, kwargs)), ) monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy")) monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True) monkeypatch.setattr( specialize, "_resource_proof", lambda *_args: {"ready": True, "runningRunnerContainers": 0}, ) def inspect(name): if name == specialize.DIND_NAME: dind["State"] = {"Running": True, "Health": {"Status": "healthy"}} return dind return None def inventory(): return containers, [] monkeypatch.setattr(specialize, "_inspect", inspect) monkeypatch.setattr(specialize, "_docker_outer_inventory", inventory) return events, rows, containers, dind def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( specialize, monkeypatch, family_exists=True ) result = specialize.ensure_active_fleet_ready() assert result == {"ready": True, "reason": None} compose_events = [item for item in events if isinstance(item, tuple) and item[0] == "compose"] assert len(compose_events) == 1 _, compose_bytes, args, kwargs = compose_events[0] assert compose_bytes == b"captured-compose" assert args == ("up", "-d", "--no-deps", "runner-dind") assert kwargs["timeout"] <= 30 assert events.count("fresh-host-gate") == 2 assert events.index("fresh-host-gate") < events.index("healthy") def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( specialize, monkeypatch, family_exists=False ) gate_count = {"value": 0} def fresh_gate(): gate_count["value"] += 1 events.append(f"fresh-host-gate-{gate_count['value']}") if gate_count["value"] == 2: raise specialize.RunnerFleetSpecializeError( "runner_fleet_specialization_host_guard_paused" ) rollback = [] monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fresh_gate) monkeypatch.setattr( specialize, "rollback_host_cgroup_family", lambda **kwargs: rollback.append(kwargs) or {"removed": True, "familyExists": False}, ) result = specialize.ensure_active_fleet_ready() assert result == { "ready": False, "reason": "runner_fleet_specialization_host_guard_paused", } assert rollback == [ {"created": True, "initial_controllers": ["cpu", "memory", "pids"]} ] assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) def test_active_recovery_accepts_only_disabled_legacy_registrations_after_containers_removed( specialize, monkeypatch ): events, rows, _containers, dind = _stub_active_recovery( specialize, monkeypatch, family_exists=True ) rows[-1]["disabled"] = False result = specialize.ensure_active_fleet_ready() assert result["ready"] is False assert result["reason"] == "runner_fleet_specialization_legacy_registration_untrusted" assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) assert dind["State"]["Running"] is False def test_active_recovery_cli_reports_only_sanitized_ready_or_reason( specialize, monkeypatch, capsys ): monkeypatch.setattr( specialize, "ensure_active_fleet_ready", lambda: {"ready": False, "reason": "runner_fleet_specialization_host_guard_paused"}, ) assert specialize._main(("--ensure-active-fleet-ready",)) == 1 assert capsys.readouterr().err.strip() == "runner_fleet_specialization_host_guard_paused" monkeypatch.setattr( specialize, "ensure_active_fleet_ready", lambda: {"ready": True, "reason": None}, ) assert specialize._main(("--ensure-active-fleet-ready",)) == 0 assert capsys.readouterr().out.strip() == "ready" def test_active_recovery_host_gate_rechecks_guard_after_fresh_sample( specialize, monkeypatch ): now = specialize.time.time() sample = { "loadRatio": 0.2, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapSampleAt": now, "swapCritical": False, } guard_states = iter([("clear", ""), ("paused", "guard marker appeared")]) class HostSampler: def host_snapshot(self): return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_fleet", type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: next(guard_states))}), ) monkeypatch.setitem( sys.modules, "runner_capacity_controller", type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), ) with pytest.raises( specialize.RunnerFleetSpecializeError, match="runner_fleet_specialization_host_guard_paused", ): specialize._fresh_recovery_host_gate() @pytest.mark.parametrize( "sample,expected_reason", [ (None, "runner_fleet_specialization_host_telemetry_unavailable"), ( { "loadRatio": 0.2, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapSampleAt": 1, "swapCritical": False, }, "runner_fleet_specialization_host_headroom_insufficient", ), ], ) def test_active_recovery_host_gate_fails_closed_on_missing_or_stale_sample( specialize, monkeypatch, sample, expected_reason ): class HostSampler: def host_snapshot(self): if sample is None: return {"status": "unavailable", "host": None} return {"status": "observed", "generatedAt": "2026-10-02T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_fleet", type("FleetStub", (), {"_host_guard_state": staticmethod(lambda: ("clear", ""))}), ) monkeypatch.setitem( sys.modules, "runner_capacity_controller", type("CapacityStub", (), {"RunnerCapacityController": HostSampler}), ) with pytest.raises(specialize.RunnerFleetSpecializeError, match=expected_reason): specialize._fresh_recovery_host_gate()