diff --git a/.gitattributes b/.gitattributes index 0e40e39..adc4e21 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,4 @@ docker/Dockerfile text eol=lf *.yaml text eol=lf ops/gitea-deploy-broker/*.py text eol=lf scripts/*.py text eol=lf +ops/runner-launcher/trusted-systempaths.patch text eol=lf whitespace=-space-before-tab diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 95d81ca..c119145 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `1c6a58589e215177617aa20bfa9d877c67874950`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `df9538cc820b70cece4f17eb9034739917560f96`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index c9578b5..63ad069 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "1c6a58589e215177617aa20bfa9d877c67874950", + "sourceRevision": "df9538cc820b70cece4f17eb9034739917560f96", "files": [ { "path": ".dockerignore", @@ -14,8 +14,8 @@ }, { "path": ".gitattributes", - "sha256": "90bd4c4cb40336c70db4c69e6737eb8de370824aac69cbc5a624da37f0e509a2", - "bytes": 173 + "sha256": "4cdb98d45cdeb40d0460788e2c0d7603ae3e0b36fe847a3e5a80ac5d3a083d22", + "bytes": 260 }, { "path": ".gitignore", @@ -1604,8 +1604,8 @@ }, { "path": "backend/tests/test_runner_admission_controller.py", - "sha256": "05c4397dc9c155b6f5ba0721f1db6c231dfe3ef7a20bc0cc1a82569f47a6e655", - "bytes": 80570 + "sha256": "a03f9a18184149e210a3062578ade025559823050a355d55a760d861f36ae76c", + "bytes": 81756 }, { "path": "backend/tests/test_runner_admission_policy.py", @@ -1617,6 +1617,11 @@ "sha256": "618c65909b89a7c52e85c6c190bf63a4c6dc202917d60d48eb71b069e5cf30c2", "bytes": 9336 }, + { + "path": "backend/tests/test_runner_build_launcher_refresh.py", + "sha256": "cf05c9a595b126cdd140f1803c37913157104a9c5ef0c8295a258961ad413f97", + "bytes": 20307 + }, { "path": "backend/tests/test_runner_capacity_controller.py", "sha256": "f1c12a7799f11b7dcb78e79820303a5dce6828dded26b258898e4c80f80a3ed8", diff --git a/backend/tests/test_runner_admission_controller.py b/backend/tests/test_runner_admission_controller.py index a772623..9d64e57 100644 --- a/backend/tests/test_runner_admission_controller.py +++ b/backend/tests/test_runner_admission_controller.py @@ -277,6 +277,32 @@ def test_handover_cli_redacts_errors_and_does_not_misreport_unknown_writes(contr assert result["status"] == "blocked" and result["executed"] is False +@pytest.mark.parametrize("profile,claims,required_mib", [ + ("build", ["build"], 15 * 1024), + ("build", ["build", "checks"], 15 * 1024 + 1536), + ("browser", ["browser"], 20 * 1024), + ("backend", ["backend"], 15 * 1024), +]) +def test_profile_memory_boundary_keeps_other_claims_and_stability( + controller_module, monkeypatch, profile, claims, required_mib +): + controller, now, host, _ = _host_gate_sample(controller_module, monkeypatch) + required_bytes = required_mib * 1024**2 + host["availableMemoryBytes"] = required_bytes - 1 + state = {} + assert controller._fresh_mutation_gate(profile, claims, state)[0] == "runner_admission_host_headroom_insufficient" + assert state["hostStability"] is None + + host["availableMemoryBytes"] = required_bytes + assert controller._fresh_mutation_gate(profile, claims, state)[0] == "runner_admission_host_headroom_not_stable" + now[0] = 159 + host["swapSampleAt"] = 159 + assert controller._fresh_mutation_gate(profile, claims, state)[0] == "runner_admission_host_headroom_not_stable" + now[0] = 160 + host["swapSampleAt"] = 160 + assert controller._fresh_mutation_gate(profile, claims, state)[0] is None + + def test_safe_live_metric_fluctuations_do_not_restart_stable_headroom(controller_module, monkeypatch): module = controller_module controller, now, host, _ = _host_gate_sample(module, monkeypatch) diff --git a/backend/tests/test_runner_build_launcher_refresh.py b/backend/tests/test_runner_build_launcher_refresh.py new file mode 100644 index 0000000..004b6ff --- /dev/null +++ b/backend/tests/test_runner_build_launcher_refresh.py @@ -0,0 +1,541 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import sys +from contextlib import contextmanager, nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture +def launcher(monkeypatch, tmp_path): + root = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker" + monkeypatch.syspath_prepend(str(root)) + spec = importlib.util.spec_from_file_location( + "build_launcher_refresh_test", root / "runner_fleet_specialize.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module.os, "geteuid", lambda: 0, raising=False) + monkeypatch.setattr(module, "SPECIALIZED_STATE_ROOT", tmp_path) + monkeypatch.setattr(module, "SPECIALIZED_STATE_PATH", tmp_path / "marker.json") + monkeypatch.setattr(module, "MIGRATION_JOURNAL_PATH", tmp_path / "journal.json") + for path in (module.SPECIALIZED_STATE_PATH, module.MIGRATION_JOURNAL_PATH): + path.write_bytes(b"{}") + monkeypatch.setattr( + module, + "_trusted_private_file", + lambda path, **_k: path.read_bytes() if path.exists() else b"config", + ) + monkeypatch.setattr( + module, "_read_json_file", lambda path, **_k: json.loads(path.read_bytes()) + ) + monkeypatch.setattr(module, "_read_journal", lambda: {}) + monkeypatch.setattr( + module, "_write_private_atomic", lambda path, raw: path.write_bytes(raw) + ) + ids = {profile: 71 + index for index, profile in enumerate(module.PROFILE_IDS)} + monkeypatch.setattr(module, "_active_profile_registration_ids", lambda _j: ids) + containers, rows = {}, {} + for index, profile in enumerate(module.RUNNER_PROFILES.values()): + name = profile.container_name + containers[name] = { + "Name": "/" + name, + "Id": str(index + 1) * 64, + "Image": module.RUNNER_IMAGE_ID, + "Config": { + "Image": module.RUNNER_IMAGE, + "Labels": { + "com.docker.compose.project": module.PROJECT, + "com.docker.compose.service": "runner-" + profile.id, + "tech.itworx.role": "gitea-actions-runner", + "tech.itworx.ci.profile": profile.id, + }, + "Env": [ + "CONFIG_FILE=/runner-config.yaml", + "DOCKER_HOST=" + module._DIND_DOCKER_HOST, + ], + }, + "State": {"Running": False}, + "HostConfig": { + "CgroupParent": "itworx-ci", + "NanoCpus": 500_000_000, + "Memory": 512 * module._MIB, + "PidsLimit": 192, + "PidMode": "", + "NetworkMode": module._RUNNER_NETWORK_NAME, + "Privileged": False, + "ExtraHosts": ["gitea.example.com:198.19.104.1"], + "RestartPolicy": {"Name": "no"}, + }, + "Mounts": [ + { + "Source": str(module._host_volume_path(source)), + "Destination": destination, + "RW": writable, + } + for source, destination, writable in ( + (module._SPECIALIZED_DATA_ROOT / profile.id, "/data", True), + ( + module.RUNNER_CONFIG_ROOT / (profile.id + ".yaml"), + "/runner-config.yaml", + False, + ), + ( + module._DIND_CONTROL_ROOT, + module._DIND_CONTROL_CONTAINER_ROOT, + False, + ), + ( + module._SPECIALIZED_WORKSPACE_ROOT / name, + "/" + name + "-workspace", + True, + ), + ) + ], + } + rows[name] = { + "name": name, + "id": ids[profile.id], + "disabled": True, + "busy": False, + } + model = { + "services": { + "runner-" + profile: {"image": module.RUNNER_IMAGE, "unchanged": profile} + for profile in module.PROFILE_IDS + } + } + monkeypatch.setattr( + module, + "_active_release_bundle", + lambda: ({}, {}, model, module._canonical_bytes(model)), + ) + monkeypatch.setattr(module, "_docker_outer_inventory", lambda: (containers, [])) + monkeypatch.setattr(module, "_runner_rows", lambda: list(rows.values())) + monkeypatch.setattr(module, "_inspect", lambda name: containers.get(name)) + monkeypatch.setattr(module, "_guard_state_unchanged", lambda _j: True) + monkeypatch.setattr(module, "_fresh_recovery_host_gate", lambda: None) + monkeypatch.setattr(module, "_resource_proof", lambda *_a: {"ready": True}) + monkeypatch.setattr(module, "verify_profile_task_images", lambda *_a: True) + monkeypatch.setattr( + module, + "verified_topology", + lambda: { + "ready": all( + module._validate_outer_item(name, item, profile) + for profile, name in ( + (p.id, p.container_name) for p in module.RUNNER_PROFILES.values() + ) + for item in [containers[name]] + ) + }, + ) + admission = {"pending": None, "bootstrapReceiptSha256": "a" * 64, "revision": 7} + controller = SimpleNamespace( + _read_state=lambda: admission, _process_lock=lambda: nullcontext() + ) + monkeypatch.setitem( + sys.modules, + "runner_admission_controller", + SimpleNamespace(RunnerAdmissionController=lambda: controller), + ) + release = { + "sourceRevision": "b" * 40, + "sourceTree": "c" * 40, + "artifactSha256": "d" * 64, + } + monkeypatch.setitem( + sys.modules, + "status_server", + SimpleNamespace(_require_controller_release=lambda: release), + ) + monkeypatch.setattr(module, "_migration_lock", lambda **_k: nullcontext()) + monkeypatch.setattr( + module, + "_open_fixed_release_volume", + lambda: nullcontext(SimpleNamespace(assert_current=lambda: None)), + ) + + @contextmanager + def maintenance(): + yield lambda: None + + monkeypatch.setattr(module, "_rollback_maintenance", maintenance) + effects = [] + + original = copy.deepcopy(containers["itworx-ci-build"]) + + def docker(args, **_k): + if args[:3] == ["docker", "image", "inspect"]: + return json.dumps([{"Id": args[-1]}]) + if args[:2] == ["docker", "inspect"]: + result = [item for item in containers.values() if item["Id"] == args[-1]] + if not result: + raise module.RunnerFleetSpecializeError( + "runner_fleet_specialization_command_failed" + ) + return json.dumps(result) + if args[:2] == ["docker", "rename"]: + name, item = next( + (name, item) + for name, item in containers.items() + if item["Id"] == args[2] + ) + assert args[3] not in containers + containers.pop(name) + item["Name"] = "/" + args[3] + containers[args[3]] = item + effects.append(tuple(args)) + elif args[:2] == ["docker", "create"]: + effects.append(tuple(args)) + item = copy.deepcopy(original) + item["Id"] = "a" * 64 + item["Image"] = module.BUILD_LAUNCHER_IMAGE + item["Config"]["Image"] = module.BUILD_LAUNCHER_IMAGE + for index, token in enumerate(args): + if token == "--label": + key, value = args[index + 1].split("=", 1) + item["Config"]["Labels"][key] = value + assert "itworx-ci-build" not in containers + containers["itworx-ci-build"] = item + return item["Id"] + elif args[:2] == ["docker", "rm"]: + name = next( + name for name, item in containers.items() if item["Id"] == args[2] + ) + containers.pop(name) + effects.append(tuple(args)) + return "" + + monkeypatch.setattr(module, "_run", docker) + monkeypatch.setattr( + module, + "_run_compose", + lambda *_a, **_k: pytest.fail( + "Archived Compose must not recreate the launcher" + ), + ) + return module, containers, rows, admission, effects, model + + +def test_refresh_and_explicit_rollback_preserve_fleet_and_archived_model(launcher): + module, containers, rows, _, effects, model = launcher + before = copy.deepcopy(containers) + original_model = copy.deepcopy(model) + registrations = copy.deepcopy(rows) + preflight = module._build_launcher_preflight("refresh") + assert not effects and preflight["executed"] is False + result = module._apply_build_launcher("refresh", preflight["preflightSha256"]) + assert result["status"] == "applied" and result["registrationId"] == 76 + assert rows == registrations and model == original_model + assert effects[0][:3] == ("docker", "rename", before["itworx-ci-build"]["Id"]) + assert effects[1][:2] == ("docker", "create") + assert ( + module._build_launcher_original(module._build_launcher_receipt())["Id"] + == before["itworx-ci-build"]["Id"] + ) + for name in before: + if name != "itworx-ci-build": + assert containers[name] == before[name] + rollback = module._build_launcher_preflight("rollback") + result = module._apply_build_launcher("rollback", rollback["preflightSha256"]) + assert ( + result["status"] == "rolled_back" + and result["imageId"] == module.RUNNER_IMAGE_ID + ) + assert rows == registrations and model == original_model + assert module.verified_topology()["ready"] is True + + +@pytest.mark.parametrize( + "change", + [ + "busy", + "enabled", + "running", + "registration", + "image", + "pending", + "bootstrap", + "private_jobs", + "guard", + "host", + "source", + ], +) +def test_preflight_refuses_defective_conditions_without_effects( + launcher, monkeypatch, change +): + module, containers, rows, admission, effects, _ = launcher + row, item = rows["itworx-ci-build"], containers["itworx-ci-build"] + if change == "busy": + row["busy"] = True + elif change == "enabled": + row["disabled"] = False + elif change == "running": + item["State"]["Running"] = True + elif change == "registration": + row["id"] = 99 + elif change == "image": + item["Image"] = module.BUILD_LAUNCHER_IMAGE + elif change == "pending": + admission["pending"] = {"profileId": "build"} + elif change == "bootstrap": + admission["bootstrapReceiptSha256"] = None + elif change == "private_jobs": + monkeypatch.setattr(module, "_run", lambda *_a, **_k: "some-container") + elif change == "guard": + monkeypatch.setattr(module, "_guard_state_unchanged", lambda _j: False) + else: + + def refuse(*_a, **_k): + raise module.RunnerFleetSpecializeError("fixed-refusal") + + monkeypatch.setattr( + module, + "_fresh_recovery_host_gate" + if change == "host" + else "_active_release_bundle", + refuse, + ) + with pytest.raises(module.RunnerFleetSpecializeError): + module._build_launcher_preflight("refresh") + assert not effects and not module._build_launcher_receipt_path().exists() + + +def test_preflight_cas_refuses_new_admission_revision(launcher): + module, _, _, state, effects, _ = launcher + preflight = module._build_launcher_preflight("refresh") + state["revision"] += 1 + with pytest.raises(module.RunnerFleetSpecializeError, match="preflight_changed"): + module._apply_build_launcher("refresh", preflight["preflightSha256"]) + assert not effects and not module._build_launcher_receipt_path().exists() + + +@pytest.mark.parametrize("foreign", [False, True]) +def test_failed_replacement_restores_only_owned_idle_identity( + launcher, monkeypatch, foreign +): + module, containers, _, _, effects, _ = launcher + original = module._run + + def fail(args, **kwargs): + result = original(args, **kwargs) + if args[:2] == ["docker", "create"]: + if foreign: + containers["itworx-ci-build"]["HostConfig"]["Privileged"] = True + raise RuntimeError("Bearer secret-must-never-escape") + return result + + monkeypatch.setattr(module, "_run", fail) + preflight = module._build_launcher_preflight("refresh") + with pytest.raises( + module.RunnerFleetSpecializeError, + match="outcome_unknown" if foreign else "failed_rolled_back", + ): + module._apply_build_launcher("refresh", preflight["preflightSha256"]) + assert len(effects) == (2 if foreign else 4) + assert module._build_launcher_receipt()["status"] == ( + "rollback_unknown" if foreign else "rolled_back" + ) + assert module.verified_topology()["ready"] is (not foreign) + + +def test_variant_is_bound_only_to_build_and_exact_receipt_cid(launcher): + module, containers, _, _, _, _ = launcher + module._apply_build_launcher( + "refresh", module._build_launcher_preflight("refresh")["preflightSha256"] + ) + for name, item in containers.items(): + altered = copy.deepcopy(item) + altered["Image"] = module.BUILD_LAUNCHER_IMAGE + altered["Config"]["Image"] = module.BUILD_LAUNCHER_IMAGE + profile = name.removeprefix("itworx-ci-") + assert module._validate_outer_item(name, altered, profile) is ( + profile == "build" + ) + containers["itworx-ci-build"]["Id"] = "f" * 64 + assert module.verified_topology()["ready"] is False + + +@pytest.mark.parametrize( + "field,value", + [ + ("patchSha256", "e" * 64), + ("markerSha256", "e" * 64), + ("registrationId", 99), + ("status", "fake"), + ], +) +def test_corrupt_receipt_fails_closed(launcher, field, value): + module, *_ = launcher + module._apply_build_launcher( + "refresh", module._build_launcher_preflight("refresh")["preflightSha256"] + ) + path = module._build_launcher_receipt_path() + receipt = json.loads(path.read_bytes()) + receipt[field] = value + path.write_text(json.dumps(receipt)) + assert module.verified_topology()["ready"] is False + + +def test_launcher_cli_redacts_unexpected_error_and_requires_exact_confirmation( + launcher, monkeypatch, capsys +): + module, *_ = launcher + + def refused(*_a): + raise RuntimeError("Bearer secret-must-never-escape") + + monkeypatch.setattr(module, "_build_launcher_preflight", refused) + assert module._main(("--preflight-build-launcher-refresh",)) == 1 + assert "secret-must-never-escape" not in capsys.readouterr().out + assert ( + module._main( + ( + "--apply-build-launcher-refresh", + "--expected-preflight-sha256", + "a" * 64, + "--confirm", + "wrong", + ) + ) + == 64 + ) + + +def test_provider_patch_matches_controller_pin(launcher): + module, *_ = launcher + patch = ( + Path(__file__).resolve().parents[2] + / "ops/runner-launcher/trusted-systempaths.patch" + ) + assert ( + hashlib.sha256(patch.read_bytes()).hexdigest() + == module.BUILD_LAUNCHER_PATCH_SHA256 + ) + + +class Interrupted(BaseException): + pass + + +@pytest.mark.parametrize( + "point", + ["backup", "prepared", "rename", "create", "created", "before_applied", "applied"], +) +def test_each_durable_refresh_boundary_has_supported_recovery( + launcher, monkeypatch, point +): + module, containers, rows, _, effects, _ = launcher + original_id = containers["itworx-ci-build"]["Id"] + original_rows = copy.deepcopy(rows) + write, docker = module._write_private_atomic, module._run + + def interrupt_write(path, raw): + status = json.loads(raw).get("status") + if point == "before_applied" and status == "applied": + raise Interrupted() + write(path, raw) + if ( + point == "backup" + and path.name.startswith("build-launcher-before-") + or point == status + ): + raise Interrupted() + + def interrupt_docker(args, **kwargs): + result = docker(args, **kwargs) + if args[:2] == ["docker", point]: + raise Interrupted() + return result + + monkeypatch.setattr(module, "_write_private_atomic", interrupt_write) + monkeypatch.setattr(module, "_run", interrupt_docker) + preflight = module._build_launcher_preflight("refresh") + with pytest.raises(Interrupted): + module._apply_build_launcher("refresh", preflight["preflightSha256"]) + monkeypatch.setattr(module, "_write_private_atomic", write) + monkeypatch.setattr(module, "_run", docker) + if point == "backup": + assert not effects and module._build_launcher_receipt() is None + module._apply_build_launcher("refresh", preflight["preflightSha256"]) + point = "applied" + if point == "applied": + rollback = module._build_launcher_preflight("rollback") + module._apply_build_launcher("rollback", rollback["preflightSha256"]) + else: + recovery = module._build_launcher_recovery_preflight() + assert recovery["executed"] is False + module._recover_build_launcher(recovery["preflightSha256"]) + assert containers["itworx-ci-build"]["Id"] == original_id + assert module._build_launcher_receipt()["status"] == "rolled_back" + assert rows == original_rows and module.verified_topology()["ready"] is True + + +@pytest.mark.parametrize( + "point", ["rolling_back", "rm", "rename", "before_rolled_back"] +) +def test_each_rollback_boundary_recovers_original_instance( + launcher, monkeypatch, point +): + module, containers, _, _, _, _ = launcher + original_id = containers["itworx-ci-build"]["Id"] + module._apply_build_launcher( + "refresh", module._build_launcher_preflight("refresh")["preflightSha256"] + ) + write, docker = module._write_private_atomic, module._run + + def interrupt_write(path, raw): + status = json.loads(raw).get("status") + if point == "before_rolled_back" and status == "rolled_back": + raise Interrupted() + write(path, raw) + if point == status: + raise Interrupted() + + def interrupt_docker(args, **kwargs): + result = docker(args, **kwargs) + if args[:2] == ["docker", point]: + raise Interrupted() + return result + + monkeypatch.setattr(module, "_write_private_atomic", interrupt_write) + monkeypatch.setattr(module, "_run", interrupt_docker) + rollback = module._build_launcher_preflight("rollback") + with pytest.raises(Interrupted): + module._apply_build_launcher("rollback", rollback["preflightSha256"]) + monkeypatch.setattr(module, "_write_private_atomic", write) + monkeypatch.setattr(module, "_run", docker) + recovery = module._build_launcher_recovery_preflight() + module._recover_build_launcher(recovery["preflightSha256"]) + assert containers["itworx-ci-build"]["Id"] == original_id + assert module.verified_topology()["ready"] is True + + +def test_foreign_same_shape_and_image_with_new_cid_is_never_removed( + launcher, monkeypatch +): + module, containers, _, _, effects, _ = launcher + original_verify = module._resource_proof + + def replace_and_refuse(*args): + original_verify(*args) + containers["itworx-ci-build"]["Id"] = "f" * 64 + raise RuntimeError("postcondition failure") + + monkeypatch.setattr(module, "_resource_proof", replace_and_refuse) + with pytest.raises(module.RunnerFleetSpecializeError, match="outcome_unknown"): + module._apply_build_launcher( + "refresh", module._build_launcher_preflight("refresh")["preflightSha256"] + ) + assert containers["itworx-ci-build"]["Id"] == "f" * 64 + assert not any(args[:2] == ("docker", "rm") for args in effects) + with pytest.raises(module.RunnerFleetSpecializeError, match="foreign_container"): + module._build_launcher_recovery_preflight()