from __future__ import annotations import importlib.util import json import shlex import shutil import subprocess from pathlib import Path from typing import Any import pytest ROOT = Path(__file__).resolve().parents[2] UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py" ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs" UNSET = object() SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh" PRIMARY = "itworx-security-runner" SECONDARY = "itworx-validation-runner-2" LEGACY = "itworx-unraid-deploy" OTHER_FIXED_RUNNERS = ( "itworx-validation-runner-3", "itworx-docker-runner", "itworx-browser-runner", "itworx-validation-runner-burst-1", ) @pytest.fixture def upgrade_module(): spec = importlib.util.spec_from_file_location( "appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _runner( runner_id: int, name: str, *, busy: object = False, disabled: object = False, ) -> dict[str, object]: return { "id": runner_id, "name": name, "busy": busy, "disabled": disabled, } def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]: runners = [ _runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY, disabled=True), ] if include_legacy: runners.append(_runner(3, LEGACY, disabled=True)) return runners class ServiceStub: def __init__( self, runners: list[dict[str, object]] | None = None, jobs: object = UNSET, *, inventory_error: Exception | None = None, request_error: Exception | None = None, ) -> None: self.runners = list(runners if runners is not None else _safe_runners()) self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs self.inventory_error = inventory_error self.request_error = request_error self.requests: list[tuple[str, dict[str, object]]] = [] def _gitea_runner_inventory(self) -> list[dict[str, object]]: if self.inventory_error is not None: raise self.inventory_error return self.runners def _gitea_request( self, path: str, params: dict[str, object] | None = None ) -> object: self.requests.append((path, dict(params or {}))) if self.request_error is not None: raise self.request_error return self.jobs def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]: return module.require_cutover_quiescence(service) def _assert_rejected(module: Any, service: ServiceStub) -> None: with pytest.raises(RuntimeError): _require_quiescent(module, service) def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled( upgrade_module, ): service = ServiceStub( jobs={ "jobs": [ {"id": 101, "runner_id": 1, "status": "in_progress"}, ], "total_count": 1, } ) _assert_rejected(upgrade_module, service) assert len(service.requests) == 1 assert service.requests[0][0] == ACTIVE_JOBS_PATH assert service.requests[0][1].get("status") == "in_progress" assert service.requests[0][1].get("limit") == 100 @pytest.mark.parametrize( "runners", [ [_runner(2, SECONDARY, disabled=True)], [_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)], [ _runner(1, PRIMARY, busy=True, disabled=True), _runner(2, SECONDARY, disabled=True), ], ], ids=("primary-missing", "primary-not-fenced", "primary-busy"), ) def test_primary_runner_must_be_unique_present_disabled_and_idle( upgrade_module, runners ): _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize( "runners", [ [_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)], [ _runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY, busy=True, disabled=True), ], ], ids=("secondary-not-fenced", "secondary-busy"), ) def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners): _assert_rejected(upgrade_module, ServiceStub(runners=runners)) def test_duplicate_primary_registration_is_rejected(upgrade_module): runners = _safe_runners() runners.append(_runner(11, PRIMARY, disabled=True)) _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize("bad_value", [None, "false", 0, 1]) def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value): busy_runners = _safe_runners() busy_runners[0]["busy"] = bad_value _assert_rejected(upgrade_module, ServiceStub(runners=busy_runners)) disabled_runners = _safe_runners() disabled_runners[0]["disabled"] = bad_value _assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners)) @pytest.mark.parametrize( "legacy", [ _runner(3, LEGACY, disabled=False), _runner(3, LEGACY, busy=True, disabled=True), ], ids=("legacy-enabled", "legacy-busy"), ) def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy): runners = _safe_runners() runners.append(legacy) _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS) def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name): runners = _safe_runners() runners.append(_runner(10, name, busy=True, disabled=False)) _assert_rejected(upgrade_module, ServiceStub(runners=runners)) def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module): runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")] _assert_rejected(upgrade_module, ServiceStub(runners=runners)) @pytest.mark.parametrize( "jobs", [ None, [], {}, {"jobs": []}, {"jobs": [], "total_count": 1}, {"jobs": [], "total_count": True}, {"jobs": [], "total_count": "0"}, {"jobs": None, "total_count": 0}, {"jobs": {}, "total_count": 0}, {"jobs": [None], "total_count": 1}, {"jobs": ["not-a-job"], "total_count": 1}, {"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1}, {"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1}, ], ids=( "response-null", "response-list", "jobs-field-missing", "jobs-count-missing", "jobs-count-mismatch", "jobs-count-bool", "jobs-count-not-integer", "jobs-null", "jobs-not-list", "job-null", "job-not-object", "job-status-missing", "job-status-mismatch", ), ) def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs): _assert_rejected(upgrade_module, ServiceStub(jobs=jobs)) def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module): jobs = { "jobs": [ {"id": index + 1, "runner_id": 90, "status": "in_progress"} for index in range(100) ], "total_count": 100, } _assert_rejected( upgrade_module, ServiceStub( runners=[ *_safe_runners(), _runner(90, "known-unrelated-runner", disabled=True), ], jobs=jobs, ), ) @pytest.mark.parametrize( "runner_id", [None, True, False, 0, -1, "1", 999], ids=("missing", "true", "false", "zero", "negative", "string", "unknown"), ) def test_active_job_with_invalid_or_unknown_runner_id_fails_closed( upgrade_module, runner_id ): jobs = { "jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}], "total_count": 1, } _assert_rejected(upgrade_module, ServiceStub(jobs=jobs)) def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false( upgrade_module, ): runners = [ *_safe_runners(), _runner(10, "itworx-docker-runner", busy=False, disabled=True), ] service = ServiceStub( runners=runners, jobs={ "jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}], "total_count": 1, }, ) _assert_rejected(upgrade_module, service) @pytest.mark.parametrize("where", ["inventory", "jobs"]) def test_gitea_api_errors_fail_closed(upgrade_module, where): service = ServiceStub( inventory_error=OSError("inventory unavailable") if where == "inventory" else None, request_error=OSError("jobs unavailable") if where == "jobs" else None, ) expected_message = ( "inventory unavailable" if where == "inventory" else "jobs unavailable" ) with pytest.raises(OSError, match=expected_message): _require_quiescent(upgrade_module, service) def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module): service = ServiceStub( runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")], jobs={ "jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}], "total_count": 1, }, ) _assert_rejected(upgrade_module, service) def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch): def unexpected_shell(*_args, **_kwargs): raise AssertionError("cutover guard must not execute Docker or shell commands") monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell) unrelated = _runner(90, "known-unrelated-runner", disabled=True) service = ServiceStub( runners=[*_safe_runners(include_legacy=True), unrelated], jobs={"jobs": [], "total_count": 0}, ) result = _require_quiescent(upgrade_module, service) assert result == { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY, LEGACY], "activeFleetJobs": 0, } assert service.requests == [ ( ACTIVE_JOBS_PATH, { "status": "in_progress", "page": 1, "limit": 100, "sort": "id", "order": "asc", }, ) ] def _bash_function(name: str) -> str: script = SCRIPT_PATH.read_text(encoding="utf-8") start = script.index(f"{name}() {{") end = script.index("\n}", start) + 2 return script[start:end] @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("proof", "executor_status", "allowed"), [ ("", 0, False), ("not-json", 0, False), ( json.dumps( { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY], "activeFleetJobs": 1, } ), 0, False, ), ( json.dumps( { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY], "activeFleetJobs": 0, } ), 1, False, ), ( json.dumps( { "status": "quiescent", "fencedRunners": [PRIMARY, SECONDARY], "activeFleetJobs": 0, } ), 0, True, ), ], ids=( "old-executor-empty", "malformed-proof", "active-job", "executor-error", "quiescent", ), ) def test_cutover_shell_requires_real_successful_quiescence_proof( proof, executor_status, allowed ): # The first python3 invocation stands in for the installed executor; the second # invokes the actual JSON parser inside check_cutover. body = "\n".join( [ "ROOT=/synthetic", f"MOCK_PROOF={shlex.quote(proof)}", f"MOCK_EXECUTOR_STATUS={executor_status}", "python3() {", ' if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then', ' printf "%s" "$MOCK_PROOF"', ' return "$MOCK_EXECUTOR_STATUS"', " fi", ' command python3 "$@"', "}", _bash_function("check_cutover"), "check_cutover", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("inventory", "docker_status", "allowed"), [("", 1, False), ("running-container-id", 0, False), ("", 0, True)], ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"), ) def test_dind_idle_shell_requires_successful_empty_docker_inventory( inventory, docker_status, allowed ): body = "\n".join( [ "DIND=synthetic-dind", f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}", f"MOCK_DOCKER_STATUS={docker_status}", 'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }', _bash_function("check_dind_idle"), "check_dind_idle", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace") @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable") @pytest.mark.parametrize( ("cutover_started", "dispatch_started"), [(0, 0), (1, 0), (1, 1)], ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"), ) def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started): body = "\n".join( [ "backup=/synthetic-backup", "COMPOSE_FILE=/synthetic-compose", "PRIMARY_CONFIG=/synthetic-primary", "SECONDARY_CONFIG=/synthetic-secondary", "ROOT=/synthetic-root", "guard_changed=0", "host_guard_changed=0", f"cutover_started={cutover_started}", f"dispatch_started={dispatch_started}", 'new_runners=("new-container")', 'registered=("new-registration")', 'fenced=("itworx-security-runner:false")', "created_paths=()", "exec 3>&1", 'cp() { printf "cp:%s\\n" "$*"; }', 'docker() { printf "docker:%s\\n" "$*" >&3; }', 'delete_runner() { printf "delete:%s\\n" "$*"; }', 'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }', 'stop_capacity_daemon() { printf "stop-daemon\\n"; }', 'rm() { printf "rm:%s\\n" "$*"; }', _bash_function("restore"), "false", "restore", ] ) result = subprocess.run( ["bash"], input=body.encode(), capture_output=True, check=False ) output = result.stdout.decode(errors="replace") errors = result.stderr.decode(errors="replace") assert result.returncode == 1, errors if dispatch_started: assert output == "" assert "runner_fleet_recovery_needed" in errors else: assert "cp:" in output assert "delete:new-registration" in output assert "restore-fence:itworx-security-runner:false" in output assert ("stop-daemon" in output) is bool(cutover_started) assert ("docker:compose " in output) is bool(cutover_started)