from __future__ import annotations import importlib.util import hashlib import json import shlex import shutil import subprocess from pathlib import Path from typing import Any import pytest ROOT = Path(__file__).resolve().parents[2] UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py" ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs" UNSET = object() SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh" PRIMARY = "itworx-security-runner" SECONDARY = "itworx-validation-runner-2" LEGACY = "itworx-unraid-deploy" WINDOWS_NATIVE = "itworx-windows11-native" OTHER_FIXED_RUNNERS = ( "itworx-validation-runner-3", "itworx-docker-runner", "itworx-browser-runner", "itworx-validation-runner-burst-1", ) @pytest.fixture def upgrade_module(): spec = importlib.util.spec_from_file_location( "appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _runner( runner_id: int, name: str, *, busy: object = False, disabled: object = False, ) -> dict[str, object]: return { "id": runner_id, "name": name, "busy": busy, "disabled": disabled, } def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]: runners = [ _runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY, disabled=True), ] if include_legacy: runners.append(_runner(3, LEGACY, disabled=True)) return runners class ServiceStub: def __init__( self, runners: list[dict[str, object]] | None = None, jobs: object = UNSET, *, inventory_error: Exception | None = None, request_error: Exception | None = None, ) -> None: self.runners = list(runners if runners is not None else _safe_runners()) self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs self.inventory_error = inventory_error self.request_error = request_error self.requests: list[tuple[str, dict[str, object]]] = [] def _gitea_runner_inventory(self) -> list[dict[str, object]]: if self.inventory_error is not None: raise self.inventory_error return self.runners def _gitea_request( self, path: str, params: dict[str, object] | None = None ) -> object: self.requests.append((path, dict(params or {}))) if self.request_error is not None: raise self.request_error return self.jobs def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]: return module.require_cutover_quiescence(service) def _assert_rejected(module: Any, service: ServiceStub) -> None: with pytest.raises(RuntimeError): _require_quiescent(module, service) def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled( upgrade_module, ): service = ServiceStub( jobs={ "jobs": [ {"id": 101, "runner_id": 1, "status": "in_progress"}, ], "total_count": 1, } ) _assert_rejected(upgrade_module, service) assert len(service.requests) == 1 assert service.requests[0][0] == ACTIVE_JOBS_PATH assert service.requests[0][1].get("status") == "in_progress" assert service.requests[0][1].get("limit") == 100 @pytest.mark.parametrize( "runners", [ [_runner(2, SECONDARY, disabled=True)], [_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)], [ _runner(1, PRIMARY, busy=True, disabled=True), _runner(2, SECONDARY, disabled=True), ], ], ids=("primary-missing", "primary-not-fenced", "primary-busy"), ) def test_primary_runner_must_be_unique_present_disabled_and_idle( upgrade_module, runners ): _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize( "runners", [ [_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)], [ _runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY, busy=True, disabled=True), ], ], ids=("secondary-not-fenced", "secondary-busy"), ) def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners): _assert_rejected(upgrade_module, ServiceStub(runners=runners)) def test_duplicate_primary_registration_is_rejected(upgrade_module): runners = _safe_runners() runners.append(_runner(11, PRIMARY, disabled=True)) _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize("bad_value", [None, "false", 0, 1]) def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value): busy_runners = _safe_runners() busy_runners[0]["busy"] = bad_value _assert_rejected(upgrade_module, ServiceStub(runners=busy_runners)) disabled_runners = _safe_runners() disabled_runners[0]["disabled"] = bad_value _assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners)) @pytest.mark.parametrize( "legacy", [ _runner(3, LEGACY, disabled=False), _runner(3, LEGACY, busy=True, disabled=True), ], ids=("legacy-enabled", "legacy-busy"), ) def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy): runners = _safe_runners() runners.append(legacy) _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS) def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name): runners = _safe_runners() runners.append(_runner(10, name, busy=True, disabled=False)) _assert_rejected(upgrade_module, ServiceStub(runners=runners)) def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module): runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")] _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize( "jobs", [ None, [], {}, {"jobs": []}, {"jobs": [], "total_count": 1}, {"jobs": [], "total_count": True}, {"jobs": [], "total_count": "0"}, {"jobs": None, "total_count": 0}, {"jobs": {}, "total_count": 0}, {"jobs": [None], "total_count": 1}, {"jobs": ["not-a-job"], "total_count": 1}, {"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1}, {"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1}, ], ids=( "response-null", "response-list", "jobs-field-missing", "jobs-count-missing", "jobs-count-mismatch", "jobs-count-bool", "jobs-count-not-integer", "jobs-null", "jobs-not-list", "job-null", "job-not-object", "job-status-missing", "job-status-mismatch", ), ) def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs): _assert_rejected(upgrade_module, ServiceStub(jobs=jobs)) def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module): jobs = { "jobs": [ {"id": index + 1, "runner_id": 90, "status": "in_progress"} for index in range(100) ], "total_count": 100, } _assert_rejected( upgrade_module, ServiceStub( runners=[ *_safe_runners(), _runner(90, "known-unrelated-runner", disabled=True), ], jobs=jobs, ), ) @pytest.mark.parametrize( "runner_id", [None, True, False, 0, -1, "1", 999], ids=("missing", "true", "false", "zero", "negative", "string", "unknown"), ) def test_active_job_with_invalid_or_unknown_runner_id_fails_closed( upgrade_module, runner_id ): jobs = { "jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}], "total_count": 1, } _assert_rejected(upgrade_module, ServiceStub(jobs=jobs)) def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false( upgrade_module, ): runners = [ *_safe_runners(), _runner(10, "itworx-docker-runner", busy=False, disabled=True), ] service = ServiceStub( runners=runners, jobs={ "jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}], "total_count": 1, }, ) _assert_rejected(upgrade_module, service) @pytest.mark.parametrize("where", ["inventory", "jobs"]) def test_gitea_api_errors_fail_closed(upgrade_module, where): service = ServiceStub( inventory_error=OSError("inventory unavailable") if where == "inventory" else None, request_error=OSError("jobs unavailable") if where == "jobs" else None, ) expected_message = ( "inventory unavailable" if where == "inventory" else "jobs unavailable" ) with pytest.raises(OSError, match=expected_message): _require_quiescent(upgrade_module, service) def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module): service = ServiceStub( runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")], jobs={ "jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}], "total_count": 1, }, ) _assert_rejected(upgrade_module, service) def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch): def unexpected_shell(*_args, **_kwargs): raise AssertionError("cutover guard must not execute Docker or shell commands") monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell) unrelated = _runner(90, "known-unrelated-runner", disabled=True) service = ServiceStub( runners=[*_safe_runners(include_legacy=True), unrelated], jobs={"jobs": [], "total_count": 0}, ) result = _require_quiescent(upgrade_module, service) assert result == { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY, LEGACY], "activeFleetJobs": 0, } assert service.requests == [ ( ACTIVE_JOBS_PATH, { "status": "in_progress", "page": 1, "limit": 100, "sort": "id", "order": "asc", }, ) ] def _bash_function(name: str) -> str: script = SCRIPT_PATH.read_text(encoding="utf-8") start = script.index(f"{name}() {{") end = script.index("\n}", start) + 2 return script[start:end] def _labels_digest(labels: list[object]) -> str: canonical = json.dumps( labels, sort_keys=True, separators=(",", ":"), ensure_ascii=False ).encode("utf-8") return hashlib.sha256(canonical).hexdigest() @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("rows", "total_count", "expected_status", "expected_snapshot"), [ ( [ { "id": 41, "name": WINDOWS_NATIVE, "busy": False, "disabled": False, "labels": ["windows-validation"], }, { "id": 42, "name": "itworx-security-runner", "busy": False, "disabled": True, "labels": [], }, ], 2, 0, f"41:false:{_labels_digest(['windows-validation'])}", ), ( [ { "id": 42, "name": "itworx-security-runner", "busy": False, "disabled": True, "labels": [], } ], 1, 3, "", ), ( [ { "id": 41, "name": WINDOWS_NATIVE, "busy": False, "disabled": False, "labels": [], } ], 2, 1, "", ), ( [ { "id": 41, "name": WINDOWS_NATIVE, "busy": False, "disabled": False, "labels": [], }, { "id": 41, "name": "other-runner", "busy": False, "disabled": False, "labels": [], }, ], 2, 1, "", ), ( [ { "id": 41, "name": WINDOWS_NATIVE, "busy": False, "disabled": False, "labels": [], }, { "id": 42, "name": WINDOWS_NATIVE, "busy": False, "disabled": True, "labels": [], }, ], 2, 1, "", ), ( [ { "id": index + 1, "name": WINDOWS_NATIVE if index == 0 else f"other-{index}", "busy": False, "disabled": False, "labels": [], } for index in range(100) ], 100, 1, "", ), ( [ { "id": 41, "name": WINDOWS_NATIVE, "busy": "false", "disabled": False, "labels": [], } ], 1, 1, "", ), ], ids=( "known-registration", "optional-registration-absent", "incomplete-inventory", "duplicate-id", "duplicate-name", "bounded-inventory-limit", "malformed-inventory-row", ), ) def test_windows_snapshot_requires_a_complete_unique_inventory( tmp_path, rows, total_count, expected_status, expected_snapshot ): token_file = tmp_path / "token" token_file.write_text("synthetic-token", encoding="utf-8") inventory = json.dumps({"total_count": total_count, "runners": rows}) body = "\n".join( [ f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}", f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}", f"MOCK_INVENTORY={shlex.quote(inventory)}", 'curl() { printf "%s" "$MOCK_INVENTORY"; }', _bash_function("windows_native_snapshot"), "windows_native_snapshot", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert result.returncode == expected_status, result.stderr.decode(errors="replace") assert result.stdout.decode(errors="replace").strip() == expected_snapshot @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") def test_windows_snapshot_fails_closed_on_inventory_api_error(tmp_path): token_file = tmp_path / "token" token_file.write_text("synthetic-token", encoding="utf-8") body = "\n".join( [ f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}", f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}", 'curl() { return 22; }', _bash_function("windows_native_snapshot"), "windows_native_snapshot", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert result.returncode == 1 assert result.stdout == b"" @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("prior_disabled", "expected_patches"), [("false", ["patch:41:true", "patch:41:false"]), ("true", [])], ids=("restore-prior-enabled", "preserve-prior-paused"), ) def test_windows_admission_capture_and_restore_preserve_prior_state( prior_disabled, expected_patches ): labels = _labels_digest(["windows-validation"]) body = "\n".join( [ f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}", 'windows_native_saved=0; windows_native_id=""; windows_native_prior_disabled=""; windows_native_labels_digest=""', 'mock_id=41; mock_disabled="' + prior_disabled + '"; mock_labels="' + labels + '"', 'windows_native_snapshot() { printf "%s:%s:%s\\n" "$mock_id" "$mock_disabled" "$mock_labels"; }', 'set_runner_disabled_by_id() { printf "patch:%s:%s\\n" "$1" "$2"; mock_disabled="$2"; }', _bash_function("capture_windows_native_admission"), _bash_function("restore_windows_native_admission"), "capture_windows_native_admission", '[[ "$windows_native_saved" == 1 ]] || exit 91', '[[ "$mock_disabled" == true ]] || exit 92', "restore_windows_native_admission", 'printf "final:%s:saved:%s\\n" "$mock_disabled" "$windows_native_saved"', ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert result.returncode == 0, result.stderr.decode(errors="replace") output = result.stdout.decode(errors="replace").splitlines() assert output[:-1] == expected_patches assert output[-1] == f"final:{prior_disabled}:saved:0" @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( "snapshot", [ "99:true:{digest}", "41:true:{other_digest}", ], ids=("registration-replaced", "labels-drifted"), ) def test_windows_admission_restore_rejects_identity_or_label_drift(snapshot): expected_digest = _labels_digest(["windows-validation"]) other_digest = _labels_digest(["changed-label"]) rendered = snapshot.format(digest=expected_digest, other_digest=other_digest) body = "\n".join( [ 'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false', f'windows_native_labels_digest="{expected_digest}"', f'MOCK_SNAPSHOT="{rendered}"', 'windows_native_snapshot() { printf "%s\\n" "$MOCK_SNAPSHOT"; }', 'set_runner_disabled_by_id() { echo unexpected-patch; }', _bash_function("restore_windows_native_admission"), 'if restore_windows_native_admission; then exit 93; else status=$?; fi', 'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"', ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert result.returncode == 0, result.stderr.decode(errors="replace") assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1" @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") def test_windows_admission_restore_failure_keeps_recovery_intent(): labels = _labels_digest(["windows-validation"]) body = "\n".join( [ 'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false', f'windows_native_labels_digest="{labels}"', f'mock_snapshot="41:true:{labels}"', 'windows_native_snapshot() { printf "%s\\n" "$mock_snapshot"; }', 'set_runner_disabled_by_id() { return 1; }', _bash_function("restore_windows_native_admission"), 'if restore_windows_native_admission; then exit 94; else status=$?; fi', 'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"', ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert result.returncode == 0, result.stderr.decode(errors="replace") assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1" @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("proof", "executor_status", "allowed"), [ ("", 0, False), ("not-json", 0, False), ( json.dumps( { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY], "activeFleetJobs": 1, } ), 0, False, ), ( json.dumps( { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY], "activeFleetJobs": 0, } ), 1, False, ), ( json.dumps( { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY], "activeFleetJobs": 0, } ), 0, True, ), ], ids=( "old-executor-empty", "malformed-proof", "active-job", "executor-error", "quiescent", ), ) def test_cutover_shell_requires_real_successful_quiescence_proof( proof, executor_status, allowed ): # The first python3 invocation stands in for the installed executor; the second # invokes the actual JSON parser inside check_cutover. body = "\n".join( [ "ROOT=/synthetic", f"MOCK_PROOF={shlex.quote(proof)}", f"MOCK_EXECUTOR_STATUS={executor_status}", "python3() {", ' if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then', ' printf "%s" "$MOCK_PROOF"', ' return "$MOCK_EXECUTOR_STATUS"', " fi", ' command python3 "$@"', "}", _bash_function("check_cutover"), "check_cutover", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("inventory", "docker_status", "allowed"), [("", 1, False), ("running-container-id", 0, False), ("", 0, True)], ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"), ) def test_dind_idle_shell_requires_successful_empty_docker_inventory( inventory, docker_status, allowed ): body = "\n".join( [ "DIND=synthetic-dind", f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}", f"MOCK_DOCKER_STATUS={docker_status}", 'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }', _bash_function("check_dind_idle"), "check_dind_idle", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") def _restore_exit_harness( *, cutover_started: int, dispatch_started: int, exit_status: int, fail_first_cp: bool = False, fail_windows_patch: bool = False, ): labels = _labels_digest(["windows-validation"]) body = "\n".join( [ "set -Eeuo pipefail", "backup=/synthetic-backup", "COMPOSE_FILE=/synthetic-compose", "PRIMARY_CONFIG=/synthetic-primary", "SECONDARY_CONFIG=/synthetic-secondary", "ROOT=/synthetic-root", "RUNNER_GUARD_SCRIPT=/synthetic-runner-guard", "HOST_GUARD_SCRIPT=/synthetic-host-guard", "guard_changed=0", "host_guard_changed=0", f"cutover_started={cutover_started}", f"dispatch_started={dispatch_started}", 'new_runners=("new-container")', 'registered=("new-registration")', 'fenced=("itworx-security-runner:false")', "created_paths=()", "windows_native_saved=1", "windows_native_id=41", "windows_native_prior_disabled=false", f'windows_native_labels_digest="{labels}"', "mock_windows_disabled=true", f"fail_first_cp={int(fail_first_cp)}", f"fail_windows_patch={int(fail_windows_patch)}", "cp_calls=0", "exec 3>&1", 'cp() { printf "cp:%s\\n" "$*" >&3; ((cp_calls += 1)); if ((fail_first_cp && cp_calls == 1)); then return 1; fi; }', 'docker() { printf "docker:%s\\n" "$*" >&3; }', 'delete_runner() { printf "delete:%s\\n" "$*" >&3; }', 'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2" >&3; }', 'set_runner_disabled_by_id() { printf "windows-patch:%s:%s\\n" "$1" "$2" >&3; if ((fail_windows_patch)); then return 1; fi; mock_windows_disabled="$2"; }', 'windows_native_snapshot() { printf "windows-snapshot\\n" >&3; printf "41:%s:%s\\n" "$mock_windows_disabled" "$windows_native_labels_digest"; }', 'stop_capacity_daemon() { printf "stop-daemon\\n" >&3; }', 'rm() { printf "rm:%s\\n" "$*" >&3; }', _bash_function("restore_managed_failure"), _bash_function("restore_windows_native_admission"), _bash_function("restore"), "trap restore EXIT", f"exit {exit_status}", ] ) return subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("cutover_started", "dispatch_started"), [(0, 0), (1, 0), (1, 1)], ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"), ) def test_exit_cleanup_restores_windows_after_managed_recovery( cutover_started, dispatch_started ): result = _restore_exit_harness( cutover_started=cutover_started, dispatch_started=dispatch_started, exit_status=1, ) output = result.stdout.decode(errors="replace").splitlines() errors = result.stderr.decode(errors="replace") assert result.returncode == 1, errors windows_patch_index = output.index("windows-patch:41:false") assert not any( line.startswith("docker:") for line in output[windows_patch_index + 1 :] ) if dispatch_started: assert output == [ "windows-snapshot", "windows-patch:41:false", "windows-snapshot", ] assert "runner_fleet_recovery_needed" in errors else: assert any(line.startswith("cp:") for line in output) assert "delete:new-registration" in output assert "restore-fence:itworx-security-runner:false" in output assert ("stop-daemon" in output) is bool(cutover_started) compose_indexes = [ index for index, line in enumerate(output) if line.startswith("docker:compose ") ] assert bool(compose_indexes) is bool(cutover_started) if compose_indexes: assert compose_indexes[-1] < windows_patch_index @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") def test_copy_failure_still_restores_windows_after_managed_cleanup_aborts(): result = _restore_exit_harness( cutover_started=1, dispatch_started=0, exit_status=1, fail_first_cp=True, ) output = result.stdout.decode(errors="replace").splitlines() errors = result.stderr.decode(errors="replace") assert result.returncode == 75 assert output == [ "cp:-a -- /synthetic-backup/docker-compose.yml /synthetic-compose", "windows-snapshot", "windows-patch:41:false", "windows-snapshot", ] assert "managed rollback did not complete" in errors assert not any(line.startswith("docker:") for line in output) @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") def test_exit_cleanup_restores_windows_on_success(): result = _restore_exit_harness( cutover_started=0, dispatch_started=1, exit_status=0, ) assert result.returncode == 0, result.stderr.decode(errors="replace") assert result.stdout.decode(errors="replace").splitlines() == [ "windows-snapshot", "windows-patch:41:false", "windows-snapshot", ] script = SCRIPT_PATH.read_text(encoding="utf-8") dispatch_index = script.index("dispatch_started=1") restore_index = script.index("if ! restore_windows_native_admission; then", dispatch_index) ready_index = script.index("printf 'ci_runner_fleet_v2_ready", restore_index) trap_removed_index = script.index("trap - EXIT", ready_index) assert restore_index < ready_index < trap_removed_index assert "docker " not in script[restore_index:trap_removed_index] @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") def test_post_dispatch_windows_restore_failure_marks_recovery_needed(): result = _restore_exit_harness( cutover_started=1, dispatch_started=1, exit_status=1, fail_windows_patch=True, ) assert result.returncode == 75 assert ( "runner_fleet_recovery_needed: Windows-native admission state could not be restored" in result.stderr.decode(errors="replace") )