893 lines
29 KiB
Python
893 lines
29 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import hashlib
|
|
import json
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py"
|
|
ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs"
|
|
UNSET = object()
|
|
SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh"
|
|
|
|
PRIMARY = "itworx-security-runner"
|
|
SECONDARY = "itworx-validation-runner-2"
|
|
LEGACY = "itworx-unraid-deploy"
|
|
WINDOWS_NATIVE = "itworx-windows11-native"
|
|
OTHER_FIXED_RUNNERS = (
|
|
"itworx-validation-runner-3",
|
|
"itworx-docker-runner",
|
|
"itworx-browser-runner",
|
|
"itworx-validation-runner-burst-1",
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def upgrade_module():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _runner(
|
|
runner_id: int,
|
|
name: str,
|
|
*,
|
|
busy: object = False,
|
|
disabled: object = False,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"id": runner_id,
|
|
"name": name,
|
|
"busy": busy,
|
|
"disabled": disabled,
|
|
}
|
|
|
|
|
|
def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]:
|
|
runners = [
|
|
_runner(1, PRIMARY, disabled=True),
|
|
_runner(2, SECONDARY, disabled=True),
|
|
]
|
|
if include_legacy:
|
|
runners.append(_runner(3, LEGACY, disabled=True))
|
|
return runners
|
|
|
|
|
|
class ServiceStub:
|
|
def __init__(
|
|
self,
|
|
runners: list[dict[str, object]] | None = None,
|
|
jobs: object = UNSET,
|
|
*,
|
|
inventory_error: Exception | None = None,
|
|
request_error: Exception | None = None,
|
|
) -> None:
|
|
self.runners = list(runners if runners is not None else _safe_runners())
|
|
self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs
|
|
self.inventory_error = inventory_error
|
|
self.request_error = request_error
|
|
self.requests: list[tuple[str, dict[str, object]]] = []
|
|
|
|
def _gitea_runner_inventory(self) -> list[dict[str, object]]:
|
|
if self.inventory_error is not None:
|
|
raise self.inventory_error
|
|
return self.runners
|
|
|
|
def _gitea_request(
|
|
self, path: str, params: dict[str, object] | None = None
|
|
) -> object:
|
|
self.requests.append((path, dict(params or {})))
|
|
if self.request_error is not None:
|
|
raise self.request_error
|
|
return self.jobs
|
|
|
|
|
|
def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]:
|
|
return module.require_cutover_quiescence(service)
|
|
|
|
|
|
def _assert_rejected(module: Any, service: ServiceStub) -> None:
|
|
with pytest.raises(RuntimeError):
|
|
_require_quiescent(module, service)
|
|
|
|
|
|
def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled(
|
|
upgrade_module,
|
|
):
|
|
service = ServiceStub(
|
|
jobs={
|
|
"jobs": [
|
|
{"id": 101, "runner_id": 1, "status": "in_progress"},
|
|
],
|
|
"total_count": 1,
|
|
}
|
|
)
|
|
|
|
_assert_rejected(upgrade_module, service)
|
|
|
|
assert len(service.requests) == 1
|
|
assert service.requests[0][0] == ACTIVE_JOBS_PATH
|
|
assert service.requests[0][1].get("status") == "in_progress"
|
|
assert service.requests[0][1].get("limit") == 100
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"runners",
|
|
[
|
|
[_runner(2, SECONDARY, disabled=True)],
|
|
[_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)],
|
|
[
|
|
_runner(1, PRIMARY, busy=True, disabled=True),
|
|
_runner(2, SECONDARY, disabled=True),
|
|
],
|
|
],
|
|
ids=("primary-missing", "primary-not-fenced", "primary-busy"),
|
|
)
|
|
def test_primary_runner_must_be_unique_present_disabled_and_idle(
|
|
upgrade_module, runners
|
|
):
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"runners",
|
|
[
|
|
[_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)],
|
|
[
|
|
_runner(1, PRIMARY, disabled=True),
|
|
_runner(2, SECONDARY, busy=True, disabled=True),
|
|
],
|
|
],
|
|
ids=("secondary-not-fenced", "secondary-busy"),
|
|
)
|
|
def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners):
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
|
|
|
|
|
def test_duplicate_primary_registration_is_rejected(upgrade_module):
|
|
runners = _safe_runners()
|
|
runners.append(_runner(11, PRIMARY, disabled=True))
|
|
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
|
|
|
|
|
@pytest.mark.parametrize("bad_value", [None, "false", 0, 1])
|
|
def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value):
|
|
busy_runners = _safe_runners()
|
|
busy_runners[0]["busy"] = bad_value
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=busy_runners))
|
|
|
|
disabled_runners = _safe_runners()
|
|
disabled_runners[0]["disabled"] = bad_value
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"legacy",
|
|
[
|
|
_runner(3, LEGACY, disabled=False),
|
|
_runner(3, LEGACY, busy=True, disabled=True),
|
|
],
|
|
ids=("legacy-enabled", "legacy-busy"),
|
|
)
|
|
def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy):
|
|
runners = _safe_runners()
|
|
runners.append(legacy)
|
|
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
|
|
|
|
|
@pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS)
|
|
def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name):
|
|
runners = _safe_runners()
|
|
runners.append(_runner(10, name, busy=True, disabled=False))
|
|
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
|
|
|
|
|
def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module):
|
|
runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")]
|
|
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"jobs",
|
|
[
|
|
None,
|
|
[],
|
|
{},
|
|
{"jobs": []},
|
|
{"jobs": [], "total_count": 1},
|
|
{"jobs": [], "total_count": True},
|
|
{"jobs": [], "total_count": "0"},
|
|
{"jobs": None, "total_count": 0},
|
|
{"jobs": {}, "total_count": 0},
|
|
{"jobs": [None], "total_count": 1},
|
|
{"jobs": ["not-a-job"], "total_count": 1},
|
|
{"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1},
|
|
{"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1},
|
|
],
|
|
ids=(
|
|
"response-null",
|
|
"response-list",
|
|
"jobs-field-missing",
|
|
"jobs-count-missing",
|
|
"jobs-count-mismatch",
|
|
"jobs-count-bool",
|
|
"jobs-count-not-integer",
|
|
"jobs-null",
|
|
"jobs-not-list",
|
|
"job-null",
|
|
"job-not-object",
|
|
"job-status-missing",
|
|
"job-status-mismatch",
|
|
),
|
|
)
|
|
def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs):
|
|
_assert_rejected(upgrade_module, ServiceStub(jobs=jobs))
|
|
|
|
|
|
def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module):
|
|
jobs = {
|
|
"jobs": [
|
|
{"id": index + 1, "runner_id": 90, "status": "in_progress"}
|
|
for index in range(100)
|
|
],
|
|
"total_count": 100,
|
|
}
|
|
|
|
_assert_rejected(
|
|
upgrade_module,
|
|
ServiceStub(
|
|
runners=[
|
|
*_safe_runners(),
|
|
_runner(90, "known-unrelated-runner", disabled=True),
|
|
],
|
|
jobs=jobs,
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"runner_id",
|
|
[None, True, False, 0, -1, "1", 999],
|
|
ids=("missing", "true", "false", "zero", "negative", "string", "unknown"),
|
|
)
|
|
def test_active_job_with_invalid_or_unknown_runner_id_fails_closed(
|
|
upgrade_module, runner_id
|
|
):
|
|
jobs = {
|
|
"jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}],
|
|
"total_count": 1,
|
|
}
|
|
|
|
_assert_rejected(upgrade_module, ServiceStub(jobs=jobs))
|
|
|
|
|
|
def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false(
|
|
upgrade_module,
|
|
):
|
|
runners = [
|
|
*_safe_runners(),
|
|
_runner(10, "itworx-docker-runner", busy=False, disabled=True),
|
|
]
|
|
service = ServiceStub(
|
|
runners=runners,
|
|
jobs={
|
|
"jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}],
|
|
"total_count": 1,
|
|
},
|
|
)
|
|
|
|
_assert_rejected(upgrade_module, service)
|
|
|
|
|
|
@pytest.mark.parametrize("where", ["inventory", "jobs"])
|
|
def test_gitea_api_errors_fail_closed(upgrade_module, where):
|
|
service = ServiceStub(
|
|
inventory_error=OSError("inventory unavailable")
|
|
if where == "inventory"
|
|
else None,
|
|
request_error=OSError("jobs unavailable") if where == "jobs" else None,
|
|
)
|
|
|
|
expected_message = (
|
|
"inventory unavailable" if where == "inventory" else "jobs unavailable"
|
|
)
|
|
with pytest.raises(OSError, match=expected_message):
|
|
_require_quiescent(upgrade_module, service)
|
|
|
|
|
|
def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module):
|
|
service = ServiceStub(
|
|
runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")],
|
|
jobs={
|
|
"jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}],
|
|
"total_count": 1,
|
|
},
|
|
)
|
|
_assert_rejected(upgrade_module, service)
|
|
|
|
|
|
def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch):
|
|
def unexpected_shell(*_args, **_kwargs):
|
|
raise AssertionError("cutover guard must not execute Docker or shell commands")
|
|
|
|
monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell)
|
|
unrelated = _runner(90, "known-unrelated-runner", disabled=True)
|
|
service = ServiceStub(
|
|
runners=[*_safe_runners(include_legacy=True), unrelated],
|
|
jobs={"jobs": [], "total_count": 0},
|
|
)
|
|
|
|
result = _require_quiescent(upgrade_module, service)
|
|
|
|
assert result == {
|
|
"status": "quiescent",
|
|
"fencedRunners": [PRIMARY, SECONDARY, LEGACY],
|
|
"activeFleetJobs": 0,
|
|
}
|
|
assert service.requests == [
|
|
(
|
|
ACTIVE_JOBS_PATH,
|
|
{
|
|
"status": "in_progress",
|
|
"page": 1,
|
|
"limit": 100,
|
|
"sort": "id",
|
|
"order": "asc",
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def _bash_function(name: str) -> str:
|
|
script = SCRIPT_PATH.read_text(encoding="utf-8")
|
|
start = script.index(f"{name}() {{")
|
|
end = script.index("\n}", start) + 2
|
|
return script[start:end]
|
|
|
|
|
|
def _labels_digest(labels: list[object]) -> str:
|
|
canonical = json.dumps(
|
|
labels, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
|
).encode("utf-8")
|
|
return hashlib.sha256(canonical).hexdigest()
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
@pytest.mark.parametrize(
|
|
("rows", "total_count", "expected_status", "expected_snapshot"),
|
|
[
|
|
(
|
|
[
|
|
{
|
|
"id": 41,
|
|
"name": WINDOWS_NATIVE,
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["windows-validation"],
|
|
},
|
|
{
|
|
"id": 42,
|
|
"name": "itworx-security-runner",
|
|
"busy": False,
|
|
"disabled": True,
|
|
"labels": [],
|
|
},
|
|
],
|
|
2,
|
|
0,
|
|
f"41:false:{_labels_digest(['windows-validation'])}",
|
|
),
|
|
(
|
|
[
|
|
{
|
|
"id": 42,
|
|
"name": "itworx-security-runner",
|
|
"busy": False,
|
|
"disabled": True,
|
|
"labels": [],
|
|
}
|
|
],
|
|
1,
|
|
3,
|
|
"",
|
|
),
|
|
(
|
|
[
|
|
{
|
|
"id": 41,
|
|
"name": WINDOWS_NATIVE,
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": [],
|
|
}
|
|
],
|
|
2,
|
|
1,
|
|
"",
|
|
),
|
|
(
|
|
[
|
|
{
|
|
"id": 41,
|
|
"name": WINDOWS_NATIVE,
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": [],
|
|
},
|
|
{
|
|
"id": 41,
|
|
"name": "other-runner",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": [],
|
|
},
|
|
],
|
|
2,
|
|
1,
|
|
"",
|
|
),
|
|
(
|
|
[
|
|
{
|
|
"id": 41,
|
|
"name": WINDOWS_NATIVE,
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": [],
|
|
},
|
|
{
|
|
"id": 42,
|
|
"name": WINDOWS_NATIVE,
|
|
"busy": False,
|
|
"disabled": True,
|
|
"labels": [],
|
|
},
|
|
],
|
|
2,
|
|
1,
|
|
"",
|
|
),
|
|
(
|
|
[
|
|
{
|
|
"id": index + 1,
|
|
"name": WINDOWS_NATIVE if index == 0 else f"other-{index}",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": [],
|
|
}
|
|
for index in range(100)
|
|
],
|
|
100,
|
|
1,
|
|
"",
|
|
),
|
|
(
|
|
[
|
|
{
|
|
"id": 41,
|
|
"name": WINDOWS_NATIVE,
|
|
"busy": "false",
|
|
"disabled": False,
|
|
"labels": [],
|
|
}
|
|
],
|
|
1,
|
|
1,
|
|
"",
|
|
),
|
|
],
|
|
ids=(
|
|
"known-registration",
|
|
"optional-registration-absent",
|
|
"incomplete-inventory",
|
|
"duplicate-id",
|
|
"duplicate-name",
|
|
"bounded-inventory-limit",
|
|
"malformed-inventory-row",
|
|
),
|
|
)
|
|
def test_windows_snapshot_requires_a_complete_unique_inventory(
|
|
tmp_path, rows, total_count, expected_status, expected_snapshot
|
|
):
|
|
token_file = tmp_path / "token"
|
|
token_file.write_text("synthetic-token", encoding="utf-8")
|
|
inventory = json.dumps({"total_count": total_count, "runners": rows})
|
|
body = "\n".join(
|
|
[
|
|
f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}",
|
|
f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
|
|
f"MOCK_INVENTORY={shlex.quote(inventory)}",
|
|
'curl() { printf "%s" "$MOCK_INVENTORY"; }',
|
|
_bash_function("windows_native_snapshot"),
|
|
"windows_native_snapshot",
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert result.returncode == expected_status, result.stderr.decode(errors="replace")
|
|
assert result.stdout.decode(errors="replace").strip() == expected_snapshot
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
def test_windows_snapshot_fails_closed_on_inventory_api_error(tmp_path):
|
|
token_file = tmp_path / "token"
|
|
token_file.write_text("synthetic-token", encoding="utf-8")
|
|
body = "\n".join(
|
|
[
|
|
f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}",
|
|
f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
|
|
'curl() { return 22; }',
|
|
_bash_function("windows_native_snapshot"),
|
|
"windows_native_snapshot",
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert result.returncode == 1
|
|
assert result.stdout == b""
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
@pytest.mark.parametrize(
|
|
("prior_disabled", "expected_patches"),
|
|
[("false", ["patch:41:true", "patch:41:false"]), ("true", [])],
|
|
ids=("restore-prior-enabled", "preserve-prior-paused"),
|
|
)
|
|
def test_windows_admission_capture_and_restore_preserve_prior_state(
|
|
prior_disabled, expected_patches
|
|
):
|
|
labels = _labels_digest(["windows-validation"])
|
|
body = "\n".join(
|
|
[
|
|
f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
|
|
'windows_native_saved=0; windows_native_id=""; windows_native_prior_disabled=""; windows_native_labels_digest=""',
|
|
'mock_id=41; mock_disabled="' + prior_disabled + '"; mock_labels="' + labels + '"',
|
|
'windows_native_snapshot() { printf "%s:%s:%s\\n" "$mock_id" "$mock_disabled" "$mock_labels"; }',
|
|
'set_runner_disabled_by_id() { printf "patch:%s:%s\\n" "$1" "$2"; mock_disabled="$2"; }',
|
|
_bash_function("capture_windows_native_admission"),
|
|
_bash_function("restore_windows_native_admission"),
|
|
"capture_windows_native_admission",
|
|
'[[ "$windows_native_saved" == 1 ]] || exit 91',
|
|
'[[ "$mock_disabled" == true ]] || exit 92',
|
|
"restore_windows_native_admission",
|
|
'printf "final:%s:saved:%s\\n" "$mock_disabled" "$windows_native_saved"',
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert result.returncode == 0, result.stderr.decode(errors="replace")
|
|
output = result.stdout.decode(errors="replace").splitlines()
|
|
assert output[:-1] == expected_patches
|
|
assert output[-1] == f"final:{prior_disabled}:saved:0"
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
@pytest.mark.parametrize(
|
|
"snapshot",
|
|
[
|
|
"99:true:{digest}",
|
|
"41:true:{other_digest}",
|
|
],
|
|
ids=("registration-replaced", "labels-drifted"),
|
|
)
|
|
def test_windows_admission_restore_rejects_identity_or_label_drift(snapshot):
|
|
expected_digest = _labels_digest(["windows-validation"])
|
|
other_digest = _labels_digest(["changed-label"])
|
|
rendered = snapshot.format(digest=expected_digest, other_digest=other_digest)
|
|
body = "\n".join(
|
|
[
|
|
'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false',
|
|
f'windows_native_labels_digest="{expected_digest}"',
|
|
f'MOCK_SNAPSHOT="{rendered}"',
|
|
'windows_native_snapshot() { printf "%s\\n" "$MOCK_SNAPSHOT"; }',
|
|
'set_runner_disabled_by_id() { echo unexpected-patch; }',
|
|
_bash_function("restore_windows_native_admission"),
|
|
'if restore_windows_native_admission; then exit 93; else status=$?; fi',
|
|
'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"',
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert result.returncode == 0, result.stderr.decode(errors="replace")
|
|
assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1"
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
def test_windows_admission_restore_failure_keeps_recovery_intent():
|
|
labels = _labels_digest(["windows-validation"])
|
|
body = "\n".join(
|
|
[
|
|
'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false',
|
|
f'windows_native_labels_digest="{labels}"',
|
|
f'mock_snapshot="41:true:{labels}"',
|
|
'windows_native_snapshot() { printf "%s\\n" "$mock_snapshot"; }',
|
|
'set_runner_disabled_by_id() { return 1; }',
|
|
_bash_function("restore_windows_native_admission"),
|
|
'if restore_windows_native_admission; then exit 94; else status=$?; fi',
|
|
'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"',
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert result.returncode == 0, result.stderr.decode(errors="replace")
|
|
assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1"
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
@pytest.mark.parametrize(
|
|
("proof", "executor_status", "allowed"),
|
|
[
|
|
("", 0, False),
|
|
("not-json", 0, False),
|
|
(
|
|
json.dumps(
|
|
{
|
|
"status": "quiescent",
|
|
"fencedRunners": [PRIMARY, SECONDARY],
|
|
"activeFleetJobs": 1,
|
|
}
|
|
),
|
|
0,
|
|
False,
|
|
),
|
|
(
|
|
json.dumps(
|
|
{
|
|
"status": "quiescent",
|
|
"fencedRunners": [PRIMARY, SECONDARY],
|
|
"activeFleetJobs": 0,
|
|
}
|
|
),
|
|
1,
|
|
False,
|
|
),
|
|
(
|
|
json.dumps(
|
|
{
|
|
"status": "quiescent",
|
|
"fencedRunners": [PRIMARY, SECONDARY],
|
|
"activeFleetJobs": 0,
|
|
}
|
|
),
|
|
0,
|
|
True,
|
|
),
|
|
],
|
|
ids=(
|
|
"old-executor-empty",
|
|
"malformed-proof",
|
|
"active-job",
|
|
"executor-error",
|
|
"quiescent",
|
|
),
|
|
)
|
|
def test_cutover_shell_requires_real_successful_quiescence_proof(
|
|
proof, executor_status, allowed
|
|
):
|
|
# The first python3 invocation stands in for the installed executor; the second
|
|
# invokes the actual JSON parser inside check_cutover.
|
|
body = "\n".join(
|
|
[
|
|
"ROOT=/synthetic",
|
|
f"MOCK_PROOF={shlex.quote(proof)}",
|
|
f"MOCK_EXECUTOR_STATUS={executor_status}",
|
|
"python3() {",
|
|
' if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then',
|
|
' printf "%s" "$MOCK_PROOF"',
|
|
' return "$MOCK_EXECUTOR_STATUS"',
|
|
" fi",
|
|
' command python3 "$@"',
|
|
"}",
|
|
_bash_function("check_cutover"),
|
|
"check_cutover",
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
@pytest.mark.parametrize(
|
|
("inventory", "docker_status", "allowed"),
|
|
[("", 1, False), ("running-container-id", 0, False), ("", 0, True)],
|
|
ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"),
|
|
)
|
|
def test_dind_idle_shell_requires_successful_empty_docker_inventory(
|
|
inventory, docker_status, allowed
|
|
):
|
|
body = "\n".join(
|
|
[
|
|
"DIND=synthetic-dind",
|
|
f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}",
|
|
f"MOCK_DOCKER_STATUS={docker_status}",
|
|
'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }',
|
|
_bash_function("check_dind_idle"),
|
|
"check_dind_idle",
|
|
]
|
|
)
|
|
result = subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
|
|
|
|
|
|
def _restore_exit_harness(
|
|
*,
|
|
cutover_started: int,
|
|
dispatch_started: int,
|
|
exit_status: int,
|
|
fail_first_cp: bool = False,
|
|
fail_windows_patch: bool = False,
|
|
):
|
|
labels = _labels_digest(["windows-validation"])
|
|
body = "\n".join(
|
|
[
|
|
"set -Eeuo pipefail",
|
|
"backup=/synthetic-backup",
|
|
"COMPOSE_FILE=/synthetic-compose",
|
|
"PRIMARY_CONFIG=/synthetic-primary",
|
|
"SECONDARY_CONFIG=/synthetic-secondary",
|
|
"ROOT=/synthetic-root",
|
|
"RUNNER_GUARD_SCRIPT=/synthetic-runner-guard",
|
|
"HOST_GUARD_SCRIPT=/synthetic-host-guard",
|
|
"guard_changed=0",
|
|
"host_guard_changed=0",
|
|
f"cutover_started={cutover_started}",
|
|
f"dispatch_started={dispatch_started}",
|
|
'new_runners=("new-container")',
|
|
'registered=("new-registration")',
|
|
'fenced=("itworx-security-runner:false")',
|
|
"created_paths=()",
|
|
"windows_native_saved=1",
|
|
"windows_native_id=41",
|
|
"windows_native_prior_disabled=false",
|
|
f'windows_native_labels_digest="{labels}"',
|
|
"mock_windows_disabled=true",
|
|
f"fail_first_cp={int(fail_first_cp)}",
|
|
f"fail_windows_patch={int(fail_windows_patch)}",
|
|
"cp_calls=0",
|
|
"exec 3>&1",
|
|
'cp() { printf "cp:%s\\n" "$*" >&3; ((cp_calls += 1)); if ((fail_first_cp && cp_calls == 1)); then return 1; fi; }',
|
|
'docker() { printf "docker:%s\\n" "$*" >&3; }',
|
|
'delete_runner() { printf "delete:%s\\n" "$*" >&3; }',
|
|
'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2" >&3; }',
|
|
'set_runner_disabled_by_id() { printf "windows-patch:%s:%s\\n" "$1" "$2" >&3; if ((fail_windows_patch)); then return 1; fi; mock_windows_disabled="$2"; }',
|
|
'windows_native_snapshot() { printf "windows-snapshot\\n" >&3; printf "41:%s:%s\\n" "$mock_windows_disabled" "$windows_native_labels_digest"; }',
|
|
'stop_capacity_daemon() { printf "stop-daemon\\n" >&3; }',
|
|
'rm() { printf "rm:%s\\n" "$*" >&3; }',
|
|
_bash_function("restore_managed_failure"),
|
|
_bash_function("restore_windows_native_admission"),
|
|
_bash_function("restore"),
|
|
"trap restore EXIT",
|
|
f"exit {exit_status}",
|
|
]
|
|
)
|
|
return subprocess.run(
|
|
["bash"], input=body.encode(), capture_output=True, check=False
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
@pytest.mark.parametrize(
|
|
("cutover_started", "dispatch_started"),
|
|
[(0, 0), (1, 0), (1, 1)],
|
|
ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
|
|
)
|
|
def test_exit_cleanup_restores_windows_after_managed_recovery(
|
|
cutover_started, dispatch_started
|
|
):
|
|
result = _restore_exit_harness(
|
|
cutover_started=cutover_started,
|
|
dispatch_started=dispatch_started,
|
|
exit_status=1,
|
|
)
|
|
output = result.stdout.decode(errors="replace").splitlines()
|
|
errors = result.stderr.decode(errors="replace")
|
|
assert result.returncode == 1, errors
|
|
windows_patch_index = output.index("windows-patch:41:false")
|
|
assert not any(
|
|
line.startswith("docker:") for line in output[windows_patch_index + 1 :]
|
|
)
|
|
if dispatch_started:
|
|
assert output == [
|
|
"windows-snapshot",
|
|
"windows-patch:41:false",
|
|
"windows-snapshot",
|
|
]
|
|
assert "runner_fleet_recovery_needed" in errors
|
|
else:
|
|
assert any(line.startswith("cp:") for line in output)
|
|
assert "delete:new-registration" in output
|
|
assert "restore-fence:itworx-security-runner:false" in output
|
|
assert ("stop-daemon" in output) is bool(cutover_started)
|
|
compose_indexes = [
|
|
index
|
|
for index, line in enumerate(output)
|
|
if line.startswith("docker:compose ")
|
|
]
|
|
assert bool(compose_indexes) is bool(cutover_started)
|
|
if compose_indexes:
|
|
assert compose_indexes[-1] < windows_patch_index
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
def test_copy_failure_still_restores_windows_after_managed_cleanup_aborts():
|
|
result = _restore_exit_harness(
|
|
cutover_started=1,
|
|
dispatch_started=0,
|
|
exit_status=1,
|
|
fail_first_cp=True,
|
|
)
|
|
output = result.stdout.decode(errors="replace").splitlines()
|
|
errors = result.stderr.decode(errors="replace")
|
|
assert result.returncode == 75
|
|
assert output == [
|
|
"cp:-a -- /synthetic-backup/docker-compose.yml /synthetic-compose",
|
|
"windows-snapshot",
|
|
"windows-patch:41:false",
|
|
"windows-snapshot",
|
|
]
|
|
assert "managed rollback did not complete" in errors
|
|
assert not any(line.startswith("docker:") for line in output)
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
def test_exit_cleanup_restores_windows_on_success():
|
|
result = _restore_exit_harness(
|
|
cutover_started=0,
|
|
dispatch_started=1,
|
|
exit_status=0,
|
|
)
|
|
assert result.returncode == 0, result.stderr.decode(errors="replace")
|
|
assert result.stdout.decode(errors="replace").splitlines() == [
|
|
"windows-snapshot",
|
|
"windows-patch:41:false",
|
|
"windows-snapshot",
|
|
]
|
|
|
|
script = SCRIPT_PATH.read_text(encoding="utf-8")
|
|
dispatch_index = script.index("dispatch_started=1")
|
|
restore_index = script.index("if ! restore_windows_native_admission; then", dispatch_index)
|
|
ready_index = script.index("printf 'ci_runner_fleet_v2_ready", restore_index)
|
|
trap_removed_index = script.index("trap - EXIT", ready_index)
|
|
assert restore_index < ready_index < trap_removed_index
|
|
assert "docker " not in script[restore_index:trap_removed_index]
|
|
|
|
|
|
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
|
def test_post_dispatch_windows_restore_failure_marks_recovery_needed():
|
|
result = _restore_exit_harness(
|
|
cutover_started=1,
|
|
dispatch_started=1,
|
|
exit_status=1,
|
|
fail_windows_patch=True,
|
|
)
|
|
assert result.returncode == 75
|
|
assert (
|
|
"runner_fleet_recovery_needed: Windows-native admission state could not be restored"
|
|
in result.stderr.decode(errors="replace")
|
|
)
|