Unraid-AppOps-Gateway-Public/backend/tests/test_runner_fleet_cutover_safety.py

from __future__ import annotations

import importlib.util
import json
import shlex
import shutil
import subprocess
from pathlib import Path
from typing import Any

import pytest

ROOT = Path(__file__).resolve().parents[2]
UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py"
ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs"
UNSET = object()
SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh"

PRIMARY = "itworx-security-runner"
SECONDARY = "itworx-validation-runner-2"
LEGACY = "itworx-unraid-deploy"
OTHER_FIXED_RUNNERS = (
    "itworx-validation-runner-3",
    "itworx-docker-runner",
    "itworx-browser-runner",
    "itworx-validation-runner-burst-1",
)


@pytest.fixture
def upgrade_module():
    spec = importlib.util.spec_from_file_location(
        "appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH
    )
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _runner(
    runner_id: int,
    name: str,
    *,
    busy: object = False,
    disabled: object = False,
) -> dict[str, object]:
    return {
        "id": runner_id,
        "name": name,
        "busy": busy,
        "disabled": disabled,
    }


def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]:
    runners = [
        _runner(1, PRIMARY, disabled=True),
        _runner(2, SECONDARY, disabled=True),
    ]
    if include_legacy:
        runners.append(_runner(3, LEGACY, disabled=True))
    return runners


class ServiceStub:
    def __init__(
        self,
        runners: list[dict[str, object]] | None = None,
        jobs: object = UNSET,
        *,
        inventory_error: Exception | None = None,
        request_error: Exception | None = None,
    ) -> None:
        self.runners = list(runners if runners is not None else _safe_runners())
        self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs
        self.inventory_error = inventory_error
        self.request_error = request_error
        self.requests: list[tuple[str, dict[str, object]]] = []

    def _gitea_runner_inventory(self) -> list[dict[str, object]]:
        if self.inventory_error is not None:
            raise self.inventory_error
        return self.runners

    def _gitea_request(
        self, path: str, params: dict[str, object] | None = None
    ) -> object:
        self.requests.append((path, dict(params or {})))
        if self.request_error is not None:
            raise self.request_error
        return self.jobs


def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]:
    return module.require_cutover_quiescence(service)


def _assert_rejected(module: Any, service: ServiceStub) -> None:
    with pytest.raises(RuntimeError):
        _require_quiescent(module, service)


def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled(
    upgrade_module,
):
    service = ServiceStub(
        jobs={
            "jobs": [
                {"id": 101, "runner_id": 1, "status": "in_progress"},
            ],
            "total_count": 1,
        }
    )

    _assert_rejected(upgrade_module, service)

    assert len(service.requests) == 1
    assert service.requests[0][0] == ACTIVE_JOBS_PATH
    assert service.requests[0][1].get("status") == "in_progress"
    assert service.requests[0][1].get("limit") == 100


@pytest.mark.parametrize(
    "runners",
    [
        [_runner(2, SECONDARY, disabled=True)],
        [_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)],
        [
            _runner(1, PRIMARY, busy=True, disabled=True),
            _runner(2, SECONDARY, disabled=True),
        ],
    ],
    ids=("primary-missing", "primary-not-fenced", "primary-busy"),
)
def test_primary_runner_must_be_unique_present_disabled_and_idle(
    upgrade_module, runners
):
    _assert_rejected(upgrade_module, ServiceStub(runners=runners))


@pytest.mark.parametrize(
    "runners",
    [
        [_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)],
        [
            _runner(1, PRIMARY, disabled=True),
            _runner(2, SECONDARY, busy=True, disabled=True),
        ],
    ],
    ids=("secondary-not-fenced", "secondary-busy"),
)
def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners):
    _assert_rejected(upgrade_module, ServiceStub(runners=runners))


def test_duplicate_primary_registration_is_rejected(upgrade_module):
    runners = _safe_runners()
    runners.append(_runner(11, PRIMARY, disabled=True))

    _assert_rejected(upgrade_module, ServiceStub(runners=runners))


@pytest.mark.parametrize("bad_value", [None, "false", 0, 1])
def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value):
    busy_runners = _safe_runners()
    busy_runners[0]["busy"] = bad_value
    _assert_rejected(upgrade_module, ServiceStub(runners=busy_runners))

    disabled_runners = _safe_runners()
    disabled_runners[0]["disabled"] = bad_value
    _assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners))


@pytest.mark.parametrize(
    "legacy",
    [
        _runner(3, LEGACY, disabled=False),
        _runner(3, LEGACY, busy=True, disabled=True),
    ],
    ids=("legacy-enabled", "legacy-busy"),
)
def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy):
    runners = _safe_runners()
    runners.append(legacy)

    _assert_rejected(upgrade_module, ServiceStub(runners=runners))


@pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS)
def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name):
    runners = _safe_runners()
    runners.append(_runner(10, name, busy=True, disabled=False))

    _assert_rejected(upgrade_module, ServiceStub(runners=runners))


def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module):
    runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")]
    _assert_rejected(upgrade_module, ServiceStub(runners=runners))


@pytest.mark.parametrize(
    "jobs",
    [
        None,
        [],
        {},
        {"jobs": []},
        {"jobs": [], "total_count": 1},
        {"jobs": [], "total_count": True},
        {"jobs": [], "total_count": "0"},
        {"jobs": None, "total_count": 0},
        {"jobs": {}, "total_count": 0},
        {"jobs": [None], "total_count": 1},
        {"jobs": ["not-a-job"], "total_count": 1},
        {"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1},
        {"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1},
    ],
    ids=(
        "response-null",
        "response-list",
        "jobs-field-missing",
        "jobs-count-missing",
        "jobs-count-mismatch",
        "jobs-count-bool",
        "jobs-count-not-integer",
        "jobs-null",
        "jobs-not-list",
        "job-null",
        "job-not-object",
        "job-status-missing",
        "job-status-mismatch",
    ),
)
def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs):
    _assert_rejected(upgrade_module, ServiceStub(jobs=jobs))


def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module):
    jobs = {
        "jobs": [
            {"id": index + 1, "runner_id": 90, "status": "in_progress"}
            for index in range(100)
        ],
        "total_count": 100,
    }

    _assert_rejected(
        upgrade_module,
        ServiceStub(
            runners=[
                *_safe_runners(),
                _runner(90, "known-unrelated-runner", disabled=True),
            ],
            jobs=jobs,
        ),
    )


@pytest.mark.parametrize(
    "runner_id",
    [None, True, False, 0, -1, "1", 999],
    ids=("missing", "true", "false", "zero", "negative", "string", "unknown"),
)
def test_active_job_with_invalid_or_unknown_runner_id_fails_closed(
    upgrade_module, runner_id
):
    jobs = {
        "jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}],
        "total_count": 1,
    }

    _assert_rejected(upgrade_module, ServiceStub(jobs=jobs))


def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false(
    upgrade_module,
):
    runners = [
        *_safe_runners(),
        _runner(10, "itworx-docker-runner", busy=False, disabled=True),
    ]
    service = ServiceStub(
        runners=runners,
        jobs={
            "jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}],
            "total_count": 1,
        },
    )

    _assert_rejected(upgrade_module, service)


@pytest.mark.parametrize("where", ["inventory", "jobs"])
def test_gitea_api_errors_fail_closed(upgrade_module, where):
    service = ServiceStub(
        inventory_error=OSError("inventory unavailable")
        if where == "inventory"
        else None,
        request_error=OSError("jobs unavailable") if where == "jobs" else None,
    )

    expected_message = (
        "inventory unavailable" if where == "inventory" else "jobs unavailable"
    )
    with pytest.raises(OSError, match=expected_message):
        _require_quiescent(upgrade_module, service)


def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module):
    service = ServiceStub(
        runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")],
        jobs={
            "jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}],
            "total_count": 1,
        },
    )
    _assert_rejected(upgrade_module, service)


def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch):
    def unexpected_shell(*_args, **_kwargs):
        raise AssertionError("cutover guard must not execute Docker or shell commands")

    monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell)
    unrelated = _runner(90, "known-unrelated-runner", disabled=True)
    service = ServiceStub(
        runners=[*_safe_runners(include_legacy=True), unrelated],
        jobs={"jobs": [], "total_count": 0},
    )

    result = _require_quiescent(upgrade_module, service)

    assert result == {
        "status": "quiescent",
        "fencedRunners": [PRIMARY, SECONDARY, LEGACY],
        "activeFleetJobs": 0,
    }
    assert service.requests == [
        (
            ACTIVE_JOBS_PATH,
            {
                "status": "in_progress",
                "page": 1,
                "limit": 100,
                "sort": "id",
                "order": "asc",
            },
        )
    ]


def _bash_function(name: str) -> str:
    script = SCRIPT_PATH.read_text(encoding="utf-8")
    start = script.index(f"{name}() {{")
    end = script.index("\n}", start) + 2
    return script[start:end]


@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
    ("proof", "executor_status", "allowed"),
    [
        ("", 0, False),
        ("not-json", 0, False),
        (
            json.dumps(
                {
                    "status": "quiescent",
                    "fencedRunners": [PRIMARY, SECONDARY],
                    "activeFleetJobs": 1,
                }
            ),
            0,
            False,
        ),
        (
            json.dumps(
                {
                    "status": "quiescent",
                    "fencedRunners": [PRIMARY, SECONDARY],
                    "activeFleetJobs": 0,
                }
            ),
            1,
            False,
        ),
        (
            json.dumps(
                {
                    "status": "quiescent",
                    "fencedRunners": [PRIMARY, SECONDARY],
                    "activeFleetJobs": 0,
                }
            ),
            0,
            True,
        ),
    ],
    ids=(
        "old-executor-empty",
        "malformed-proof",
        "active-job",
        "executor-error",
        "quiescent",
    ),
)
def test_cutover_shell_requires_real_successful_quiescence_proof(
    proof, executor_status, allowed
):
    # The first python3 invocation stands in for the installed executor; the second
    # invokes the actual JSON parser inside check_cutover.
    body = "\n".join(
        [
            "ROOT=/synthetic",
            f"MOCK_PROOF={shlex.quote(proof)}",
            f"MOCK_EXECUTOR_STATUS={executor_status}",
            "python3() {",
            '  if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then',
            '    printf "%s" "$MOCK_PROOF"',
            '    return "$MOCK_EXECUTOR_STATUS"',
            "  fi",
            '  command python3 "$@"',
            "}",
            _bash_function("check_cutover"),
            "check_cutover",
        ]
    )
    result = subprocess.run(
        ["bash"], input=body.encode(), capture_output=True, check=False
    )
    assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")


@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
    ("inventory", "docker_status", "allowed"),
    [("", 1, False), ("running-container-id", 0, False), ("", 0, True)],
    ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"),
)
def test_dind_idle_shell_requires_successful_empty_docker_inventory(
    inventory, docker_status, allowed
):
    body = "\n".join(
        [
            "DIND=synthetic-dind",
            f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}",
            f"MOCK_DOCKER_STATUS={docker_status}",
            'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }',
            _bash_function("check_dind_idle"),
            "check_dind_idle",
        ]
    )
    result = subprocess.run(
        ["bash"], input=body.encode(), capture_output=True, check=False
    )
    assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")


@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
@pytest.mark.parametrize(
    ("cutover_started", "dispatch_started"),
    [(0, 0), (1, 0), (1, 1)],
    ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
)
def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started):
    body = "\n".join(
        [
            "backup=/synthetic-backup",
            "COMPOSE_FILE=/synthetic-compose",
            "PRIMARY_CONFIG=/synthetic-primary",
            "SECONDARY_CONFIG=/synthetic-secondary",
            "ROOT=/synthetic-root",
            "guard_changed=0",
            "host_guard_changed=0",
            f"cutover_started={cutover_started}",
            f"dispatch_started={dispatch_started}",
            'new_runners=("new-container")',
            'registered=("new-registration")',
            'fenced=("itworx-security-runner:false")',
            "created_paths=()",
            "exec 3>&1",
            'cp() { printf "cp:%s\\n" "$*"; }',
            'docker() { printf "docker:%s\\n" "$*" >&3; }',
            'delete_runner() { printf "delete:%s\\n" "$*"; }',
            'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }',
            'stop_capacity_daemon() { printf "stop-daemon\\n"; }',
            'rm() { printf "rm:%s\\n" "$*"; }',
            _bash_function("restore"),
            "false",
            "restore",
        ]
    )
    result = subprocess.run(
        ["bash"], input=body.encode(), capture_output=True, check=False
    )
    output = result.stdout.decode(errors="replace")
    errors = result.stderr.decode(errors="replace")
    assert result.returncode == 1, errors
    if dispatch_started:
        assert output == ""
        assert "runner_fleet_recovery_needed" in errors
    else:
        assert "cp:" in output
        assert "delete:new-registration" in output
        assert "restore-fence:itworx-security-runner:false" in output
        assert ("stop-daemon" in output) is bool(cutover_started)
        assert ("docker:compose " in output) is bool(cutover_started)