diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index e168c6a..43f49e1 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `ae6b891c48da412de613ce0ed52c0bc1e9908fbf`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `945a919d1a3bbd2bb5bd21eb221fc9b58d795d80`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index ed65a13..98eb4f5 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "ae6b891c48da412de613ce0ed52c0bc1e9908fbf", + "sourceRevision": "945a919d1a3bbd2bb5bd21eb221fc9b58d795d80", "files": [ { "path": ".dockerignore", @@ -1164,8 +1164,8 @@ }, { "path": "backend/tests/test_forge_secret_delivery_controller.py", - "sha256": "1695383ed99777000bec2df4ae97d252dc242b690430f102284fa101679465ed", - "bytes": 31582 + "sha256": "d5b35a6f9d188849171fed6f3cfcc813a0e999a9a821855520c3fc6a7502764b", + "bytes": 32266 }, { "path": "backend/tests/test_frontend_file_boundary.py", @@ -1204,8 +1204,8 @@ }, { "path": "backend/tests/test_hub_candidate_import.py", - "sha256": "c5eb8845b6fba09cb8d49edd06b8fad1420a8a14eac0e732198fdce0b6a0a333", - "bytes": 45629 + "sha256": "87762293c6088b5e3e71adfa4c109ec468d7931f3e10b8be6e04ce1d19f8d849", + "bytes": 47846 }, { "path": "backend/tests/test_hub_candidate_transport.py", @@ -1259,13 +1259,13 @@ }, { "path": "backend/tests/test_managed_validation_checkout.py", - "sha256": "92239d5c8760034c145a917534dd4622d507af334eaa0c10db3a310b53bae482", - "bytes": 462 + "sha256": "ada4d91ef0e22e94cf2c5f7af63f7ace8e2db1820a6b4ace7ae174ff065300fb", + "bytes": 541 }, { "path": "backend/tests/test_managed_validation_parallel.py", - "sha256": "5654211d4ab1ab2749c269c148497f9448b0257c70779ebac063ef55780e3c11", - "bytes": 2355 + "sha256": "8b3354bd6353831a081459b7398eac2ce4f8626251ff38422f6d217caf516ec3", + "bytes": 2367 }, { "path": "backend/tests/test_mcp_hub_candidate_preparation.py", @@ -1604,14 +1604,19 @@ }, { "path": "backend/tests/test_runner_admission_controller.py", - "sha256": "dcfb8e7a1d1ee86d6bfadef71bbd8be629a7f70a26baafb37bb134c61e78491d", - "bytes": 57459 + "sha256": "c05470d633c3f8c745522019724de85f470df926235fe7155f9aa754ea3e4e23", + "bytes": 69519 }, { "path": "backend/tests/test_runner_admission_policy.py", "sha256": "e4bdf5b454b5e17d1785b2db82819c1e43150fe84de1bf0fd242288352f4ee26", "bytes": 5951 }, + { + "path": "backend/tests/test_runner_build_acceptance_workflow.py", + "sha256": "618c65909b89a7c52e85c6c190bf63a4c6dc202917d60d48eb71b069e5cf30c2", + "bytes": 9336 + }, { "path": "backend/tests/test_runner_capacity_controller.py", "sha256": "f1c12a7799f11b7dcb78e79820303a5dce6828dded26b258898e4c80f80a3ed8", @@ -1659,8 +1664,8 @@ }, { "path": "backend/tests/test_runner_fleet_specialize.py", - "sha256": "5bd85d16db6ef824a8c83bdafb4e1a70cbb736064611929e538df7224748f885", - "bytes": 205798 + "sha256": "c2dd0ed7a893df8a73868e8efd27baa4ae83949dfa03f75816a3c59e66389895", + "bytes": 220641 }, { "path": "backend/tests/test_runner_fleet_v2_execution_surface.py", @@ -1724,8 +1729,8 @@ }, { "path": "backend/tests/test_runner_task_image_producer.py", - "sha256": "c55f54bf58fa81649ddb31a8953abadf299c2fe32d1b4411dd59889981cc7376", - "bytes": 12112 + "sha256": "025307274b0a93038ebe2e4ceb85579d1bf012ee855a392418f5ed7d6daab617", + "bytes": 13787 }, { "path": "backend/tests/test_secret_storage.py", diff --git a/backend/tests/test_forge_secret_delivery_controller.py b/backend/tests/test_forge_secret_delivery_controller.py index 0990297..ed3ad8f 100644 --- a/backend/tests/test_forge_secret_delivery_controller.py +++ b/backend/tests/test_forge_secret_delivery_controller.py @@ -129,8 +129,8 @@ def _signed_headers(status_module, body: bytes, *, nonce: str | None = None): } -def _post(address, path: str, body: bytes, headers: dict[str, str]): - connection = http.client.HTTPConnection(*address, timeout=3) +def _post(address, path: str, body: bytes, headers: dict[str, str], *, timeout: int = 3): + connection = http.client.HTTPConnection(*address, timeout=timeout) try: connection.request("POST", path, body=body, headers=headers) response = connection.getresponse() @@ -497,12 +497,13 @@ def test_new_nonce_retries_same_bundle_and_changed_input_is_rejected( assert READ_TOKEN.decode() not in response.decode() +@pytest.mark.parametrize("fsync_delay_seconds", [0.0, 0.04]) @pytest.mark.skipif( os.name != "posix" or getattr(os, "geteuid", lambda: 1)() != 0, reason="the real bundle helper requires POSIX root ownership", ) def test_signed_http_delivery_uses_real_bundle_helper_and_never_rotates( - status_module, endpoint, monkeypatch, tmp_path + status_module, endpoint, monkeypatch, tmp_path, fsync_delay_seconds ): _install_hmac_test(monkeypatch, status_module) import importlib @@ -511,6 +512,19 @@ def test_signed_http_delivery_uses_real_bundle_helper_and_never_rotates( assert Path(bundle_module.__file__).resolve() == ( BROKER / "forge_secret_bundle.py" ).resolve() + if fsync_delay_seconds: + real_os = bundle_module.os + + class SlowFsyncOs: + def __getattr__(self, name): + return getattr(real_os, name) + + def fsync(self, descriptor): + time.sleep(fsync_delay_seconds) + return real_os.fsync(descriptor) + + # Keep actual durable writes and all ownership checks; model valid slow I/O. + monkeypatch.setattr(bundle_module, "os", SlowFsyncOs()) root = tmp_path / "real-bundle-root" root.mkdir(mode=0o700) root.chmod(0o700) @@ -541,6 +555,8 @@ def test_signed_http_delivery_uses_real_bundle_helper_and_never_rotates( status_module.FORGE_SECRET_DELIVERY_PATH, body, _signed_headers(status_module, body), + # Match the existing bounded production HTTP budget for real fsync I/O. + timeout=10, ) status, response = send(_payload(status_module)) diff --git a/backend/tests/test_hub_candidate_import.py b/backend/tests/test_hub_candidate_import.py index 72f3edb..ccdaf90 100644 --- a/backend/tests/test_hub_candidate_import.py +++ b/backend/tests/test_hub_candidate_import.py @@ -5,6 +5,7 @@ import importlib import io import json import contextlib +import os import stat import struct import sys @@ -119,6 +120,20 @@ def _private_file(path: Path, data: bytes) -> None: path.chmod(0o600) +def _candidate_directory_link_count(monkeypatch, directory: Path, count: int) -> None: + original_lstat = Path.lstat + + def candidate_lstat(path: Path, *args, **kwargs): + info = original_lstat(path, *args, **kwargs) + if path == directory: + values = list(info) + values[3] = count + return os.stat_result(values) + return info + + monkeypatch.setattr(Path, "lstat", candidate_lstat) + + def _synthetic_scan_reports( services, images, *, created_at: str, finding_service: str | None = None ) -> dict[str, bytes]: @@ -590,8 +605,10 @@ def test_unfixable_vulnerability_finding_is_bound_to_fixed_image_reference(modul @pytest.mark.parametrize("finding_present", [False, True]) @pytest.mark.parametrize("rerun_during_load", [False, True]) +@pytest.mark.parametrize("directory_link_count", [1, 2]) def test_fake_local_import_publishes_candidate_and_requires_review_for_findings( - modules, monkeypatch, tmp_path, finding_present, rerun_during_load + modules, monkeypatch, tmp_path, finding_present, rerun_during_load, + directory_link_count, ): importer = modules.importer prepare = modules.prepare @@ -889,6 +906,7 @@ def test_fake_local_import_publishes_candidate_and_requires_review_for_findings( monkeypatch.setattr(importer, "_publish_candidate", fake_no_overwrite_publish) candidate_directory = state_root / "mcp-hub-release-candidates" / revision + _candidate_directory_link_count(monkeypatch, candidate_directory, directory_link_count) if rerun_during_load: with pytest.raises(importer.CandidateRejected, match="run changed"): importer.prepare_candidate( @@ -970,6 +988,39 @@ def test_fake_local_import_publishes_candidate_and_requires_review_for_findings( assert (candidate_directory / "import-receipt.json").read_bytes() == receipt_before_review +@pytest.mark.skipif(os.name != "posix", reason="POSIX directory inode metadata") +@pytest.mark.parametrize("directory_issue", ["zero_links", "public_mode", "foreign_owner", "symlink", "file"]) +def test_cached_candidate_rejects_unsafe_directory( + modules, monkeypatch, tmp_path, directory_issue, +): + importer, controller = modules.importer, modules.controller + revision = "a" * 40 + state_root = tmp_path / "state" + directory = state_root / "mcp-hub-release-candidates" / revision + directory.mkdir(mode=0o700, parents=True) + monkeypatch.setattr(controller, "STATE_ROOT", state_root) + monkeypatch.setattr(controller, "_root_owned", lambda *_args: directory_issue != "foreign_owner") + _candidate_directory_link_count(monkeypatch, directory, 0 if directory_issue == "zero_links" else 1) + if directory_issue == "public_mode": + directory.chmod(0o755) + elif directory_issue == "symlink": + target = directory.with_name("owned-target") + directory.rename(target) + directory.symlink_to(target, target_is_directory=True) + elif directory_issue == "file": + directory.rmdir() + _private_file(directory, b"{}") + + def reject_late_read(*_args): + raise AssertionError("Directory rejection must precede candidate evidence reads") + + monkeypatch.setattr(controller, "_mcp_hub_candidate_directory", reject_late_read) + with pytest.raises(importer.CandidateRejected, match="not.*private"): + importer._cached_candidate_result( + revision, "b" * 40, {}, "", lambda *_args: {}, lambda *_args: None, + ) + + def test_candidate_artifact_is_exact_bounded_and_unexpired(modules): importer = modules.importer revision = "c" * 40 diff --git a/backend/tests/test_managed_validation_checkout.py b/backend/tests/test_managed_validation_checkout.py index 2fae45a..4862218 100644 --- a/backend/tests/test_managed_validation_checkout.py +++ b/backend/tests/test_managed_validation_checkout.py @@ -6,8 +6,9 @@ ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml" -def test_managed_validation_uses_internal_gitea_checkout_path() -> None: +def test_managed_validation_uses_private_https_gitea_checkout_path() -> None: workflow = WORKFLOW.read_text(encoding="utf-8") assert "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" in workflow - assert "github-server-url: http://gitea-runner-internal:3000" in workflow + assert "github-server-url: https://gitea.example.com" in workflow + assert "github-server-url: http://gitea-runner-internal:3000" not in workflow diff --git a/backend/tests/test_managed_validation_parallel.py b/backend/tests/test_managed_validation_parallel.py index a082294..1f62e24 100644 --- a/backend/tests/test_managed_validation_parallel.py +++ b/backend/tests/test_managed_validation_parallel.py @@ -56,7 +56,7 @@ def test_backend_budget_is_bounded_without_weakening_validation(): line.strip() for line in backend_body.splitlines() if "-m pytest " in line ] assert pytest_commands == [ - '"${py}" -m pytest backend/tests -q --durations=20' + '"${py}" -m pytest backend/tests -q --durations=20 --maxfail=1' ] assert "continue-on-error" not in backend_body assert "--ignore" not in backend_body diff --git a/backend/tests/test_runner_admission_controller.py b/backend/tests/test_runner_admission_controller.py index 31459a0..e95f627 100644 --- a/backend/tests/test_runner_admission_controller.py +++ b/backend/tests/test_runner_admission_controller.py @@ -44,6 +44,89 @@ def controller_module(monkeypatch): ) +def _host_gate_sample(module, monkeypatch): + now = [100.0] + host = {"loadRatio": 0.62, "availableMemoryBytes": 19 * 1024**3, + "cacheFreeBytes": 140 * 1024**3, "swapCritical": False, + "swapSampleAt": now[0]} + capacity = {"host": host, "decision": {"pressure": "headroom"}} + controller = module.RunnerAdmissionController(clock=lambda: now[0]) + monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, capacity)) + return controller, now, host, capacity + + +def test_safe_live_metric_fluctuations_do_not_restart_stable_headroom(controller_module, monkeypatch): + module = controller_module + controller, now, host, _ = _host_gate_sample(module, monkeypatch) + state = {} + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" + for at, load, memory, cache in [(130, 0.58, 18.5, 139.4), (160, 0.53, 18.2, 138.9)]: + now[0] = at + host.update(loadRatio=load, availableMemoryBytes=int(memory * 1024**3), + cacheFreeBytes=int(cache * 1024**3), swapSampleAt=at) + reason, _ = controller._fresh_mutation_gate("backend", ["backend"], state) + assert reason is None + + +@pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"]) +def test_unsafe_sample_restarts_the_entire_stability_window(controller_module, monkeypatch, change): + module = controller_module + controller, now, host, capacity = _host_gate_sample(module, monkeypatch) + state = {} + controller._fresh_mutation_gate("backend", ["backend"], state) + now[0] = 130 + host["swapSampleAt"] = 130 + original = copy.deepcopy(capacity) + if change == "load": + host["loadRatio"] = 0.71 + elif change == "memory": + host["availableMemoryBytes"] = 14 * 1024**3 + elif change == "cache": + host["cacheFreeBytes"] = 29 * 1024**3 + elif change == "swap": + host["swapCritical"] = True + elif change == "stale": + host["swapSampleAt"] = 1 + elif change == "future": + host["swapSampleAt"] = 131 + elif change == "pressure": + capacity["decision"]["pressure"] = "pressure" + else: + monkeypatch.setattr(controller, "_host_evidence", lambda: ("blocked", "host_guard_paused", capacity)) + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is not None + capacity.clear() + capacity.update(original) + host = capacity["host"] + now[0] = 160 + host["swapSampleAt"] = 160 + monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, capacity)) + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" + now[0] = 220 + host["swapSampleAt"] = 220 + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is None + + +def test_missing_observation_interval_cannot_count_as_stable_headroom(controller_module, monkeypatch): + module = controller_module + controller, now, host, _ = _host_gate_sample(module, monkeypatch) + state = {} + controller._fresh_mutation_gate("backend", ["backend"], state) + now[0] = 200 + host["swapSampleAt"] = 200 + assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" + + +@pytest.mark.parametrize("profile,claims", [("frontend", ["frontend"]), ("backend", ["backend", "checks"])]) +def test_profile_or_reservation_change_requires_new_stability_window(controller_module, monkeypatch, profile, claims): + module = controller_module + controller, now, host, _ = _host_gate_sample(module, monkeypatch) + state = {} + controller._fresh_mutation_gate("backend", ["backend"], state) + now[0] = 160 + host["swapSampleAt"] = 160 + assert controller._fresh_mutation_gate(profile, claims, state)[0] == "runner_admission_host_headroom_not_stable" + + def _public_observation(module, now: float): rows = {} containers = {} @@ -257,6 +340,173 @@ def _lease_fixture(module, *, enabled=("backend", "frontend"), busy=(), queue=() return controller, state, observation, inventory_rows, containers +def _retired_security_inventory(module): + rows = [ + {"name": profile.display_name, "id": 101 + index, "status": "offline", + "busy": False, "disabled": True, + "labels": [profile.runner_label, *profile.legacy_labels]} + for index, profile in enumerate(module.RUNNER_PROFILES.values()) + ] + rows.append({ + "name": "itworx-security-runner", "id": 9, "status": "offline", + "busy": False, "disabled": True, + "labels": ["linux-validation", "docker-validation", "ubuntu-latest", + "ubuntu-24.04", "ubuntu-22.04", "mcp-validation", "unraid"], + }) + return rows + + +def test_exact_retired_v1_security_labels_do_not_block_new_profile_inventory(controller_module): + module = controller_module + rows, ids = module.RunnerAdmissionController._normalized_runner_rows( + _retired_security_inventory(module) + ) + claims, profile_ids, _ = module.RunnerAdmissionController._active_claims(rows, ids, {}) + assert claims == [] + assert set(profile_ids) == set(module.PROFILE_IDS) + assert rows["itworx-security-runner"]["disabled"] is True + + +@pytest.mark.parametrize("change", ["enabled", "busy", "missing", "foreign", "replaced_id"]) +def test_retired_v1_security_variant_refuses_live_or_changed_registration(controller_module, change): + module = controller_module + inventory = _retired_security_inventory(module) + old = inventory[-1] + if change == "enabled": + old["disabled"] = False + elif change == "busy": + old["busy"] = True + elif change == "missing": + old["labels"].remove("mcp-validation") + elif change == "replaced_id": + old["id"] = 99 + else: + old["labels"].append("foreign-capability") + with pytest.raises(module.RunnerAdmissionError, match="legacy_runner_labels_invalid"): + module.RunnerAdmissionController._normalized_runner_rows(inventory) + + +def test_retired_v1_security_active_job_still_refuses_admission(controller_module): + module = controller_module + rows, ids = module.RunnerAdmissionController._normalized_runner_rows( + _retired_security_inventory(module) + ) + with pytest.raises(module.RunnerAdmissionError, match="legacy_claim_unmapped"): + module.RunnerAdmissionController._active_claims(rows, ids, {9: [{"id": "foreign-active"}]}) + + +def test_original_security_label_contract_is_preserved(controller_module): + module = controller_module + inventory = _retired_security_inventory(module) + inventory[-1]["labels"] = sorted(module.LEGACY_REQUIRED_LABELS["itworx-security-runner"]) + rows, _ = module.RunnerAdmissionController._normalized_runner_rows(inventory) + assert rows["itworx-security-runner"]["labels"] == set(inventory[-1]["labels"]) + + +def _migrated_tower_inventory(module): + return [*_retired_security_inventory(module), + {"name": "itworx-unraid-deploy", "id": 13, "status": "online", "busy": False, + "disabled": False, "labels": ["unraid-deploy"]}, + {"name": "itworx-windows11-native", "id": 14, "status": "online", "busy": False, + "disabled": False, "labels": ["windows-native"]}] + + +def test_migrated_tower_preserves_idle_native_registrations_without_family_claims(controller_module): + module = controller_module + rows, ids = module.RunnerAdmissionController._normalized_runner_rows(_migrated_tower_inventory(module)) + claims, profiles, assigned = module.RunnerAdmissionController._active_claims(rows, ids, {}) + assert claims == [] + assert set(profiles) == set(module.PROFILE_IDS) + assert 13 not in assigned and 14 not in assigned + + +@pytest.mark.parametrize("index", [-2, -1]) +@pytest.mark.parametrize("change", ["id", "labels", "name"]) +def test_preserved_native_identity_or_capability_drift_refuses(controller_module, index, change): + module = controller_module + inventory = _migrated_tower_inventory(module) + if change == "id": + inventory[index]["id"] = 500 + elif change == "labels": + inventory[index]["labels"].append("linux-validation") + else: + inventory[index]["name"] = "foreign-native" + with pytest.raises(module.RunnerAdmissionError, match="preserved_native_identity_invalid"): + module.RunnerAdmissionController._normalized_runner_rows(inventory) + + +@pytest.mark.parametrize("index", [-2, -1]) +@pytest.mark.parametrize("busy", [True, False]) +def test_active_preserved_native_blocks_family_admission_until_it_drains(controller_module, index, busy): + module = controller_module + inventory = _migrated_tower_inventory(module) + row = inventory[index] + row["busy"] = busy + rows, ids = module.RunnerAdmissionController._normalized_runner_rows(inventory) + active = {} if busy else {row["id"]: [{"id": "native-active"}]} + with pytest.raises(module.RunnerAdmissionError, match="preserved_native_active"): + module.RunnerAdmissionController._active_claims(rows, ids, active) + + +def test_foreign_enabled_registration_still_refuses_with_preserved_natives(controller_module): + module = controller_module + inventory = _migrated_tower_inventory(module) + inventory.append({"name": "foreign", "id": 500, "status": "online", "busy": False, + "disabled": False, "labels": ["linux-validation"]}) + rows, ids = module.RunnerAdmissionController._normalized_runner_rows(inventory) + with pytest.raises(module.RunnerAdmissionError, match="enabled_registration_unmapped"): + module.RunnerAdmissionController._active_claims(rows, ids, {}) + + +def test_absent_retired_container_is_confirmed_by_successful_exact_inventory(controller_module, monkeypatch): + module = controller_module + calls = [] + def run(args, **_kwargs): + calls.append(args) + if args[1] == "inspect": + return SimpleNamespace(returncode=1, stdout="[]\n", stderr="opaque failure") + return SimpleNamespace(returncode=0, stdout="", stderr="") + monkeypatch.setattr(module.subprocess, "run", run) + assert module.RunnerAdmissionController()._container("itworx-security-runner") is None + assert all(args[1] != "inspect" for args in calls) + + +@pytest.mark.parametrize("stdout,exit_code", [ + ("", 1), ("foreign", 0), ("a" * 64 + " wrong-name\n", 0), + (("a" * 64 + " itworx-security-runner\n") * 2, 0), +]) +def test_retired_container_inventory_failure_or_ambiguity_refuses(controller_module, monkeypatch, stdout, exit_code): + module = controller_module + monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: + SimpleNamespace(returncode=exit_code, stdout=stdout, stderr="opaque failure")) + with pytest.raises(module.RunnerAdmissionError): + module.RunnerAdmissionController()._container("itworx-security-runner") + + +def test_present_retired_container_inspect_failure_is_not_absence(controller_module, monkeypatch): + module = controller_module + def run(args, **_kwargs): + if args[1] == "inspect": + return SimpleNamespace(returncode=1, stdout="[]\n", stderr="opaque failure") + return SimpleNamespace(returncode=0, stdout="a" * 64 + " itworx-security-runner\n", stderr="") + monkeypatch.setattr(module.subprocess, "run", run) + with pytest.raises(module.RunnerAdmissionError, match="container_inspect_failed"): + module.RunnerAdmissionController()._container("itworx-security-runner") + + +def test_present_retired_container_replacement_refuses(controller_module, monkeypatch): + module = controller_module + def run(args, **_kwargs): + if args[1] == "inspect": + row = {"Id": "b" * 64, "Name": "/itworx-security-runner", "State": {"Running": False}, + "Config": {"Labels": {"com.docker.compose.project": module.COMPOSE_PROJECT}}} + return SimpleNamespace(returncode=0, stdout=json.dumps([row]), stderr="") + return SimpleNamespace(returncode=0, stdout="a" * 64 + " itworx-security-runner\n", stderr="") + monkeypatch.setattr(module.subprocess, "run", run) + with pytest.raises(module.RunnerAdmissionError, match="container_identity_invalid"): + module.RunnerAdmissionController()._container("itworx-security-runner") + + def _raw_job(module, job, *, runner_id=None, active=False): row = { "id": job["id"], @@ -514,7 +764,7 @@ def test_runner_inventory_maps_exact_gitea_display_names_to_fixed_containers( "disabled": True, "labels": [profile.runner_label, *profile.legacy_labels], } - for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=10) + for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=100) ] by_name, by_id = module.RunnerAdmissionController._normalized_runner_rows(rows) diff --git a/backend/tests/test_runner_build_acceptance_workflow.py b/backend/tests/test_runner_build_acceptance_workflow.py new file mode 100644 index 0000000..e7fc10d --- /dev/null +++ b/backend/tests/test_runner_build_acceptance_workflow.py @@ -0,0 +1,222 @@ +"""Exercise the real workflow's receipt publisher without Docker or network.""" +from __future__ import annotations + +import copy +import json +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".gitea/workflows/runner-build-acceptance.yml" +REVISION = "a" * 40 +TREE = "b" * 40 +MARKER = "ITWORX_CI_JOB_DOCKER_PROBE_RESULT=" + + +@pytest.fixture +def workflow(): + return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + + +def test_only_manual_fixed_build_profile_and_pinned_actions(workflow): + # PyYAML's YAML 1.1 parser treats the Actions `on` key as boolean true. + events = workflow.get("on", workflow.get(True)) + assert set(events) == {"workflow_dispatch"} + assert set(events["workflow_dispatch"]["inputs"]) == {"source_sha"} + assert workflow["permissions"] == {"contents": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is False + assert set(workflow["jobs"]) == {"build-runner-acceptance"} + job = workflow["jobs"]["build-runner-acceptance"] + assert job["runs-on"] == "itworx-ci-build" + assert "services" not in job and "container" not in job + checkout, probe, upload = job["steps"] + assert checkout["uses"] == "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + assert checkout["with"]["persist-credentials"] is False + assert checkout["with"]["github-server-url"] == "https://gitea.example.com" + assert checkout["with"]["ref"] == "${{ gitea.sha }}" + assert "secrets." not in WORKFLOW.read_text(encoding="utf-8") + assert probe["shell"] == "bash" + assert upload["uses"] == "ChristopherHX/gitea-upload-artifact@62ac910c5d3dfa85c7cb2df15afe2e342b2407c2" + assert upload.get("if") is None # Actions default success; never always(). + assert upload["with"]["path"] == "${{ runner.temp }}/runner-build-acceptance/receipt.json" + assert upload["with"]["if-no-files-found"] == "error" + assert upload["with"]["overwrite"] is False + + +@pytest.fixture +def publisher(workflow, monkeypatch, tmp_path): + script = workflow["jobs"]["build-runner-acceptance"]["steps"][1]["run"] + script = script.split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + environment = { + "EXPECTED_SOURCE_SHA": REVISION, + "DISPATCH_SHA": REVISION, + "DISPATCH_REF": "refs/heads/main", + "DISPATCH_EVENT": "workflow_dispatch", + "RUN_REPOSITORY": "Jens/Unraid-AppOps-Gateway", + "RUN_ID": "22840", + "RUN_NUMBER": "1261", + "RUN_ATTEMPT": "1", + "RUN_JOB": "build-runner-acceptance", + "RUNNER_TEMP": str(tmp_path), + } + probe = { + "schema": "itworx.ci.job-docker-probe/v1", "status": "passed", + "sourceRevision": REVISION, "sourceTree": TREE, + "engineVersion": "29.8.2", "engineRootless": True, + "engineStorageDriver": "overlay2", "engineCgroupDriver": "none", + "engineSocket": "unix:///run/itworx-job-docker/docker.sock", + "engineSocketUid": 1000, "composeVersion": "5.5.1", + "buildRun": True, "composeHealthy": True, "composeDnsResolved": True, + "archiveNormalized": True, "archiveLoaded": True, "holdSeconds": 30, + "imageLabels": { + "org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway", + "org.opencontainers.image.revision": REVISION, + "tech.itworx.source-tree": TREE, "tech.itworx.ci.profile": "build", + }, + "imageId": "sha256:" + "c" * 64, + "archiveSha256": "sha256:" + "d" * 64, + } + state = {"calls": [], "probe": probe, "exit": 0, "dirty": "", "tree": TREE} + + def run(args, **kwargs): + state["calls"].append(args) + assert kwargs["stdin"] == subprocess.DEVNULL + if args[0] == "git": + assert kwargs["stderr"] == subprocess.DEVNULL + command = args[1:] + values = { + ("rev-parse", "HEAD"): REVISION, + ("cat-file", "-t", REVISION): "commit", + ("rev-parse", "HEAD^{tree}"): state["tree"], + ("cat-file", "-t", TREE): "tree", + ("rev-parse", REVISION + "^{tree}"): TREE, + ("status", "--porcelain", "--untracked-files=no"): state["dirty"], + } + return subprocess.CompletedProcess(args, 0, values[tuple(command)] + "\n") + assert args == ["/usr/local/bin/itworx-ci-job-docker-probe", REVISION, + TREE, "--hold-seconds", "30"] + assert kwargs["stderr"] == subprocess.DEVNULL + assert kwargs["timeout"] == 1000 + if state.get("timeout"): + raise subprocess.TimeoutExpired(args, 1000, stderr="PRIVATE_TEST_SECRET") + output = state.get("raw", MARKER + json.dumps(state["probe"]) + "\n") + kwargs["stdout"].write(output.encode()) + return subprocess.CompletedProcess(args, state["exit"]) + + def execute(): + with monkeypatch.context() as patch: + for key, value in environment.items(): + patch.setenv(key, value) + patch.setattr(subprocess, "run", run) + # Execute repository-owned workflow code; every subprocess is intercepted. + previous_umask = os.umask(0o077) + try: + exec(compile(script, str(WORKFLOW), "exec"), {}) # noqa: S102 + finally: + os.umask(previous_umask) + + state["environment"] = environment + state["execute"] = execute + state["receipt"] = tmp_path / "runner-build-acceptance/receipt.json" + return state + + +def test_success_publishes_only_finite_post_cleanup_receipt(publisher): + publisher["execute"]() + receipt = json.loads(publisher["receipt"].read_text()) + assert receipt["cleanupVerified"] is True + assert receipt["probeExitCode"] == 0 + assert receipt["probe"] == publisher["probe"] + assert receipt["gitea"] == { + "repository": "Jens/Unraid-AppOps-Gateway", + "workflow": "runner-build-acceptance.yml", "jobKey": "build-runner-acceptance", + "runId": 22840, "runNumber": 1261, "runAttempt": 1, + "nativeCorrelationRequired": True, + } + + +def test_missing_attempt_is_honest_null_not_invented_one(publisher): + publisher["environment"]["RUN_ATTEMPT"] = "" + publisher["execute"]() + run = json.loads(publisher["receipt"].read_text())["gitea"] + assert run["runAttempt"] is None + assert run["nativeCorrelationRequired"] is True + + +@pytest.mark.parametrize("failed_probe", [False, True]) +def test_publisher_fixture_restores_process_umask(publisher, failed_probe, monkeypatch): + mask = [0o022] + + def umask(value): + previous, mask[0] = mask[0], value + return previous + + monkeypatch.setattr(os, "umask", umask) + publisher["exit"] = 70 if failed_probe else 0 + if failed_probe: + with pytest.raises(SystemExit): + publisher["execute"]() + else: + publisher["execute"]() + assert mask[0] == 0o022 + + +@pytest.mark.parametrize(("key", "value"), [ + ("EXPECTED_SOURCE_SHA", "$(echo PRIVATE_TEST_SECRET)"), + ("EXPECTED_SOURCE_SHA", "e" * 40), + ("DISPATCH_SHA", "e" * 40), + ("DISPATCH_REF", "refs/heads/other"), + ("DISPATCH_EVENT", "push"), + ("RUN_REPOSITORY", "Jens/Other"), + ("RUN_ID", ""), ("RUN_ID", "0"), + ("RUN_ATTEMPT", "garbage"), + ("RUN_JOB", "other"), +]) +def test_invalid_identity_refuses_before_probe(publisher, key, value): + publisher["environment"][key] = value + with pytest.raises(SystemExit, match="no success receipt published"): + publisher["execute"]() + assert all(args[0] == "git" for args in publisher["calls"]) + assert not publisher["receipt"].exists() + + +@pytest.mark.parametrize(("key", "value"), [("dirty", " M tracked.txt"), ("tree", "invalid")]) +def test_bad_checkout_tree_refuses_before_probe(publisher, key, value): + publisher[key] = value + with pytest.raises(SystemExit, match="no success receipt published"): + publisher["execute"]() + assert all(args[0] == "git" for args in publisher["calls"]) + assert not publisher["receipt"].exists() + + +@pytest.mark.parametrize("mode", ["cleanup_failure", "hold_failure", "timeout", "secret_output"]) +def test_marker_does_not_authorize_success_before_exit_zero(publisher, mode): + if mode in {"cleanup_failure", "hold_failure"}: + publisher["exit"] = 70 + elif mode == "timeout": + publisher["timeout"] = True + else: + publisher["raw"] = "PRIVATE_TEST_SECRET\n" + MARKER + json.dumps(publisher["probe"]) + with pytest.raises(SystemExit) as failure: + publisher["execute"]() + assert "PRIVATE_TEST_SECRET" not in str(failure.value) + assert not publisher["receipt"].exists() + + +@pytest.mark.parametrize(("key", "value"), [ + ("engineRootless", False), ("engineRootless", 1), + ("archiveLoaded", False), ("composeHealthy", False), + ("sourceRevision", "e" * 40), ("sourceTree", "e" * 40), + ("imageId", "invalid"), ("unexpected", "PRIVATE_TEST_SECRET"), +]) +def test_false_or_foreign_probe_receipt_is_never_published(publisher, key, value): + publisher["probe"] = copy.deepcopy(publisher["probe"]) + publisher["probe"][key] = value + with pytest.raises(SystemExit) as failure: + publisher["execute"]() + assert "PRIVATE_TEST_SECRET" not in str(failure.value) + assert not publisher["receipt"].exists() diff --git a/backend/tests/test_runner_fleet_specialize.py b/backend/tests/test_runner_fleet_specialize.py index a43e8e5..68de036 100644 --- a/backend/tests/test_runner_fleet_specialize.py +++ b/backend/tests/test_runner_fleet_specialize.py @@ -4308,16 +4308,18 @@ def test_legacy_queue_gate_rejects_job_duplicated_across_status_snapshots(specia specialize._check_legacy_queued_jobs() -def test_image_workflow_builds_from_protected_appops_main_with_gitea_checkout_url(): +def test_image_workflow_builds_from_protected_appops_main_with_private_https_checkout_url(): workflow = (ROOT / ".gitea" / "workflows" / "ci-task-images.yml").read_text(encoding="utf-8") build = (ROOT / "scripts" / "build_ci_task_images.sh").read_text(encoding="utf-8") frontend = (ROOT / "ops" / "runner-task-images" / "frontend" / "Dockerfile").read_text(encoding="utf-8") - assert "github-server-url: http://gitea-runner-internal:3000" in workflow - assert "runs-on: linux-validation" in workflow - assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output"' in workflow - assert "runs-on: itworx-ci-build" not in workflow - assert "--job-local" not in workflow + assert "github-server-url: https://gitea.example.com" in workflow + for name in ("managed-validation.yml", "runner-build-acceptance.yml", "ci-task-images.yml"): + content = (ROOT / ".gitea" / "workflows" / name).read_text(encoding="utf-8") + assert "gitea-runner-internal" not in content + assert "runs-on: itworx-ci-build" in workflow + assert 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output" --job-local' in workflow + assert "runs-on: linux-validation" not in workflow assert "jens/unraid-appops-gateway" in build assert "Jens/Unraid-AppOps-Gateway" in build assert "corepack prepare pnpm@11.15.0 --activate" in frontend @@ -4375,6 +4377,9 @@ def test_gitea_pretty_registration_names_map_only_to_fixed_container_identity( def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): rows, containers, dind, journal, model = _active_recovery_fixture(specialize) + dind.update({"Id": "a" * 64, "Image": specialize._DIND_IMAGE_ID}) + for profile in specialize.RUNNER_PROFILES.values(): + containers[profile.container_name]["State"]["Pid"] = 0 events = [] class Volume: @@ -4425,6 +4430,10 @@ def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): ) monkeypatch.setattr(specialize, "_wait_healthy", lambda *_args, **_kwargs: events.append("healthy")) monkeypatch.setattr(specialize, "verify_profile_task_images", lambda _profiles: True) + monkeypatch.setattr(specialize, "_bind_active_dind_family", + lambda *_args: events.append("owned-populated-family")) + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "") + monkeypatch.setattr(specialize, "_run", lambda args, **_kwargs: events.append(("docker", args)) or "") monkeypatch.setattr( specialize, "_resource_proof", @@ -4433,7 +4442,8 @@ def _stub_active_recovery(specialize, monkeypatch, *, family_exists=True): def inspect(name): if name == specialize.DIND_NAME: - dind["State"] = {"Running": True, "Health": {"Status": "healthy"}} + dind["State"] = {"Running": True, "Pid": 77, + "StartedAt": "2026-10-12T00:00:00Z", "Health": {"Status": "healthy"}} return dind return None @@ -4461,11 +4471,270 @@ def test_active_recovery_restarts_only_fixed_dind_after_two_fresh_gates( assert compose_bytes == b"captured-compose" assert args == ("up", "-d", "--no-deps", "runner-dind") assert kwargs["timeout"] <= 30 - assert events.count("fresh-host-gate") == 2 + assert events.count("fresh-host-gate") == 3 assert events.index("fresh-host-gate") < events.index("healthy") -def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_dind( +def test_active_recovery_survives_disappearing_empty_family(specialize, monkeypatch): + events, _rows, _containers, _dind = _stub_active_recovery( + specialize, monkeypatch, family_exists=False + ) + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: { + "exists": False, + "safeToStartDind": False, + "safeToStartDindReason": "resource_cgroup_family_missing", + "rootSubtreeControllers": ["cpu", "memory", "pids"], + }) + monkeypatch.setattr(specialize, "_bind_active_dind_family", + lambda *_args: events.append("owned-populated-family"), raising=False) + + result = specialize.ensure_active_fleet_ready() + + assert result == {"ready": True, "reason": None} + assert "cgroup-create" not in events + assert events.index("healthy") < events.index("owned-populated-family") + + +@pytest.mark.parametrize("failure", ["health", "inspect-after-binding", "binding", "images", "envelope", "guard"]) +def test_active_recovery_stops_only_its_new_empty_start_on_failed_acceptance( + specialize, monkeypatch, failure, +): + events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch) + if failure == "health": + def fail_health(*_args, **_kwargs): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_dind_recovery_unverified") + monkeypatch.setattr(specialize, "_wait_healthy", fail_health) + elif failure == "inspect-after-binding": + original_inspect = specialize._inspect + count = [0] + def fail_inspect(name): + count[0] += 1 + if count[0] == 2: + raise RuntimeError("sentinel-secret") + return original_inspect(name) + monkeypatch.setattr(specialize, "_inspect", fail_inspect) + elif failure == "binding": + def fail_binding(*_args): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable") + monkeypatch.setattr(specialize, "_bind_active_dind_family", fail_binding) + elif failure == "images": + monkeypatch.setattr(specialize, "verify_profile_task_images", lambda *_args: False) + elif failure == "envelope": + def fail_proof(*_args): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_resource_envelope_unready") + monkeypatch.setattr(specialize, "_resource_proof", fail_proof) + else: + count = [0] + def fail_guard(): + count[0] += 1 + if count[0] == 3: + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_host_guard_paused") + monkeypatch.setattr(specialize, "_fresh_recovery_host_gate", fail_guard) + + result = specialize.ensure_active_fleet_ready() + + assert result["ready"] is False + assert "sentinel-secret" not in str(result) + assert ("docker", ["docker", "stop", "--time", "30", "a" * 64]) in events + + +@pytest.mark.parametrize("drift", ["id", "pid", "started", "job", "profile", "profile-pid", "registration", "busy", "enabled", "volume"]) +def test_active_recovery_rollback_refuses_changed_or_foreign_state(specialize, monkeypatch, drift): + events, rows, containers, dind = _stub_active_recovery(specialize, monkeypatch) + inspected = specialize._inspect(specialize.DIND_NAME) + binding = specialize._active_dind_binding(inspected, {}) + marker, journal, model, compose = specialize._active_release_bundle() + identity = specialize._active_recovery_identity(marker, journal, compose) + if drift == "id": + dind["Id"] = "b" * 64 + elif drift in {"pid", "started"}: + key, value = ("Pid", 88) if drift == "pid" else ("StartedAt", "2026-10-12T00:01:00Z") + def changed_inspect(_name): + changed = copy.deepcopy(dind) + changed["State"][key] = value + return changed + monkeypatch.setattr(specialize, "_inspect", changed_inspect) + elif drift == "job": + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job") + elif drift == "profile": + containers[specialize.RUNNER_PROFILES["backend"].container_name]["State"]["Running"] = True + elif drift == "profile-pid": + containers[specialize.RUNNER_PROFILES["backend"].container_name]["State"]["Pid"] = 88 + elif drift == "registration": + rows[0]["id"] += 1000 + elif drift == "busy": + rows[0]["busy"] = True + elif drift == "enabled": + rows[0]["disabled"] = False + else: + def changed_bundle(): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_active_release_untrusted") + monkeypatch.setattr(specialize, "_active_release_bundle", changed_bundle) + try: + specialize._stop_new_active_dind(binding, model, identity) + except specialize.RunnerFleetSpecializeError: + pass + assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events) + + +@pytest.mark.parametrize("case", ["missing-original", "replaced", "same-cid-restarted", "uninspectable", "inspect-once", "nonempty"]) +def test_active_recovery_failed_start_never_infers_cleanup_ownership(specialize, monkeypatch, case): + events, _rows, containers, _dind = _stub_active_recovery(specialize, monkeypatch) + original_inspect = specialize._inspect + if case == "missing-original": + containers.pop(specialize.DIND_NAME) + elif case == "replaced": + def replaced(name): + item = copy.deepcopy(original_inspect(name)) + item["Id"] = "b" * 64 + return item + monkeypatch.setattr(specialize, "_inspect", replaced) + elif case == "uninspectable": + def fail_inspect(_name): + raise RuntimeError("sentinel-secret") + monkeypatch.setattr(specialize, "_inspect", fail_inspect) + elif case == "inspect-once": + count = [0] + def fail_once(name): + count[0] += 1 + if count[0] == 1: + raise RuntimeError("sentinel-secret") + return original_inspect(name) + monkeypatch.setattr(specialize, "_inspect", fail_once) + elif case == "same-cid-restarted": + def restarted(name): + item = copy.deepcopy(original_inspect(name)) + item["State"].update({"Pid": 88, "StartedAt": "2026-10-12T00:01:00Z"}) + return item + monkeypatch.setattr(specialize, "_inspect", restarted) + else: + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job") + def fail_start(*_args, **_kwargs): + raise RuntimeError("sentinel-secret") + if case != "inspect-once": + monkeypatch.setattr(specialize, "_run_compose", fail_start) + + result = specialize.ensure_active_fleet_ready() + + assert result["ready"] is False + assert "sentinel-secret" not in str(result) + assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events) + + +def test_active_binder_uses_exact_owned_population_without_migration_journal_writes(specialize, monkeypatch): + binder = specialize._bind_active_dind_family + events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch) + # Exercise the real binder rather than the high-level lifecycle stub. + item = specialize._inspect(specialize.DIND_NAME) + calls = [] + monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\n77\n78\n") + monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", lambda **kwargs: calls.append(kwargs)) + monkeypatch.setattr(specialize, "_write_journal", lambda *_args: pytest.fail("migration evidence must remain intact")) + marker, journal, model, compose = specialize._active_release_bundle() + binder(item, model, specialize._active_recovery_identity(marker, journal, compose)) + + assert calls == [{"expected_container_id": "a" * 64, "expected_host_pid": 77, + "expected_process_ids": [77, 78], "allow_owned_partial": True}] + assert "cgroup-create" not in events + + +@pytest.mark.parametrize("case", ["job", "job-after", "duplicate", "zero", "missing-init", "unknown-pid", "foreign-family", "replaced", "replaced-before", "restarted-before"]) +def test_active_binder_rejects_untrusted_or_nonempty_population(specialize, monkeypatch, case): + binder = specialize._bind_active_dind_family + _events, _rows, _containers, dind = _stub_active_recovery(specialize, monkeypatch) + item = copy.deepcopy(specialize._inspect(specialize.DIND_NAME)) + if case == "replaced-before": + dind["Id"] = "b" * 64 + elif case == "restarted-before": + original_inspect = specialize._inspect + def restarted(name): + changed = copy.deepcopy(original_inspect(name)) + changed["State"].update({"Pid": 88, "StartedAt": "2026-10-12T00:01:00Z"}) + return changed + monkeypatch.setattr(specialize, "_inspect", restarted) + text = {"duplicate": "PID\n77\n77\n", "zero": "PID\n77\n0\n", + "missing-init": "PID\n78\n", "unknown-pid": "PID\nsentinel-secret\n"}.get(case, "PID\n77\n78\n") + monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: text) + if case == "job": + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job") + calls = [] + def bind(**kwargs): + calls.append(kwargs) + if case == "foreign-family": + raise RuntimeError("sentinel-secret") + if case == "replaced": + dind["Id"] = "b" * 64 + elif case == "job-after": + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job") + monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", bind) + marker, journal, model, compose = specialize._active_release_bundle() + with pytest.raises(specialize.RunnerFleetSpecializeError) as error: + binder(item, model, specialize._active_recovery_identity(marker, journal, compose)) + assert "sentinel-secret" not in str(error.value) + if case not in {"foreign-family", "replaced", "job-after"}: + assert calls == [] + + +def test_active_recovery_can_finish_existing_empty_bounded_start_without_stopping_it(specialize, monkeypatch): + events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch) + specialize._inspect(specialize.DIND_NAME) + ready = iter([{"ready": False}, {"ready": True}]) + monkeypatch.setattr(specialize, "verified_topology", lambda: next(ready)) + assert specialize.ensure_active_fleet_ready() == {"ready": True, "reason": None} + assert "owned-populated-family" in events + assert not any(isinstance(item, tuple) and item[0] in {"compose", "docker"} for item in events) + + +@pytest.mark.parametrize("drift", ["sourceScriptSha256", "imageSourceRevision", "imageSourceTree", "imageManifestSha256", "compose", "registration"]) +def test_active_binder_and_cleanup_refuse_valid_same_model_release_drift(specialize, monkeypatch, drift): + binder = specialize._bind_active_dind_family + events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch) + item = specialize._inspect(specialize.DIND_NAME) + binding = specialize._active_dind_binding(item, {}) + marker, journal, model, compose = specialize._active_release_bundle() + identity = specialize._active_recovery_identity(marker, journal, compose) + changed_marker, changed_journal, changed_compose = copy.deepcopy(marker), copy.deepcopy(journal), compose + if drift == "compose": + changed_compose += b" " + elif drift == "registration": + changed_journal["newRegistrationIds"]["backend"] += 1000 + else: + changed_marker[drift] = "f" * (40 if drift in {"imageSourceRevision", "imageSourceTree"} else 64) + monkeypatch.setattr(specialize, "_active_release_bundle", lambda: (changed_marker, changed_journal, model, changed_compose)) + monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "PID\n77\n78\n") + monkeypatch.setattr(specialize, "ensure_owned_dind_cgroup_family", lambda **_kwargs: pytest.fail("changed source cannot bind")) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="active_release_changed"): + binder(item, model, identity) + monkeypatch.setattr(specialize, "_run", lambda args, **_kwargs: events.append(("docker", args)) or "") + with pytest.raises(specialize.RunnerFleetSpecializeError, match="active_release_changed"): + specialize._stop_new_active_dind(binding, model, identity) + assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events) + + +@pytest.mark.parametrize("failure", ["binding", "proof"]) +def test_active_recovery_preserves_preexisting_running_daemon_on_failure(specialize, monkeypatch, failure): + events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch) + specialize._inspect(specialize.DIND_NAME) + monkeypatch.setattr(specialize, "verified_topology", lambda: {"ready": False}) + def fail_bind(*_args): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_cgroup_family_unavailable") + if failure == "binding": + monkeypatch.setattr(specialize, "_bind_active_dind_family", fail_bind) + assert specialize.ensure_active_fleet_ready()["ready"] is False + assert not any(isinstance(item, tuple) and item[0] in {"compose", "docker"} for item in events) + + +def test_active_recovery_does_not_stop_new_daemon_with_work_appearing_after_binding(specialize, monkeypatch): + events, _rows, _containers, _dind = _stub_active_recovery(specialize, monkeypatch) + def failed_binding(*_args): + monkeypatch.setattr(specialize, "_run_dind_docker", lambda *_args, **_kwargs: "foreign-job") + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_dind_recovery_unverified") + monkeypatch.setattr(specialize, "_bind_active_dind_family", failed_binding) + assert specialize.ensure_active_fleet_ready()["ready"] is False + assert not any(isinstance(item, tuple) and item[0] == "docker" for item in events) + + +def test_active_recovery_makes_no_cgroup_or_start_write_if_host_guard_pauses_before_dind( specialize, monkeypatch ): events, _rows, _containers, _dind = _stub_active_recovery( @@ -4495,9 +4764,8 @@ def test_active_recovery_rolls_back_empty_cgroup_if_host_guard_pauses_before_din "ready": False, "reason": "runner_fleet_specialization_host_guard_paused", } - assert rollback == [ - {"created": True, "initial_controllers": ["cpu", "memory", "pids"]} - ] + assert rollback == [] + assert "cgroup-create" not in events assert not any(isinstance(item, tuple) and item[0] == "compose" for item in events) diff --git a/backend/tests/test_runner_task_image_producer.py b/backend/tests/test_runner_task_image_producer.py index 1734e67..0e0107d 100644 --- a/backend/tests/test_runner_task_image_producer.py +++ b/backend/tests/test_runner_task_image_producer.py @@ -11,7 +11,7 @@ import tarfile from pathlib import Path import pytest - +import yaml ROOT = Path(__file__).resolve().parents[2] PROFILES = ("checks", "backend", "frontend", "security", "browser", "build") @@ -30,6 +30,39 @@ BASE_LABELS = { } +def test_post_activation_producer_uses_fixed_job_local_build_route_and_preserves_artifact_contract(): + workflow = yaml.safe_load((ROOT / ".gitea/workflows/ci-task-images.yml").read_text(encoding="utf-8")) + events = workflow.get("on", workflow.get(True)) + assert events == {"push": {"branches": ["main"]}} + assert workflow["permissions"] == {"contents": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is False + assert set(workflow["jobs"]) == {"build"} + job = workflow["jobs"]["build"] + assert job["runs-on"] == "itworx-ci-build" + assert job["timeout-minutes"] == 90 + assert "services" not in job and "container" not in job + checkout, build, upload = job["steps"] + assert checkout["uses"] == "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + assert checkout["with"] == { + "fetch-depth": 1, "persist-credentials": False, + "github-server-url": "https://gitea.example.com", + } + assert build["run"].strip() == ( + 'set -euo pipefail\n' + 'output="${RUNNER_TEMP}/ci-task-images-${GITHUB_SHA}"\n' + 'bash scripts/build_ci_task_images.sh "$GITHUB_SHA" "$GITHUB_RUN_ID" "$output" --job-local' + ) + assert "env" not in build + assert upload["uses"] == "ChristopherHX/gitea-upload-artifact@62ac910c5d3dfa85c7cb2df15afe2e342b2407c2" + assert upload.get("if") is None + assert upload["with"] == { + "name": "ci-task-images-${{ github.sha }}", + "path": "${{ runner.temp }}/ci-task-images-${{ github.sha }}/*", + "if-no-files-found": "error", "retention-days": 90, + "compression-level": 6, "overwrite": False, + } + + def _run_producer(tmp_path: Path, mutation: str = ""): repo = tmp_path / "repository" repo.mkdir() @@ -121,6 +154,7 @@ def _run_producer(tmp_path: Path, mutation: str = ""): capture_output=True, text=True, timeout=30, + check=False, ) return result, output, revision, tree