Publish curated Unraid-AppOps-Gateway source from 3c5fb0c6d1f4e693e4a0fa50e7151509b4450786
This commit is contained in:
Public Source Publisher committed
1 parent
3c7997bf4b
commit
2de6e5c85e
5 files changed
+786
-6
No files matched your search
@@ -0,0 +1,509 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py"
|
||||
ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs"
|
||||
UNSET = object()
|
||||
SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh"
|
||||
|
||||
PRIMARY = "itworx-security-runner"
|
||||
SECONDARY = "itworx-validation-runner-2"
|
||||
LEGACY = "itworx-unraid-deploy"
|
||||
OTHER_FIXED_RUNNERS = (
|
||||
"itworx-validation-runner-3",
|
||||
"itworx-docker-runner",
|
||||
"itworx-browser-runner",
|
||||
"itworx-validation-runner-burst-1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upgrade_module():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _runner(
|
||||
runner_id: int,
|
||||
name: str,
|
||||
*,
|
||||
busy: object = False,
|
||||
disabled: object = False,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"id": runner_id,
|
||||
"name": name,
|
||||
"busy": busy,
|
||||
"disabled": disabled,
|
||||
}
|
||||
|
||||
|
||||
def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]:
|
||||
runners = [
|
||||
_runner(1, PRIMARY, disabled=True),
|
||||
_runner(2, SECONDARY, disabled=True),
|
||||
]
|
||||
if include_legacy:
|
||||
runners.append(_runner(3, LEGACY, disabled=True))
|
||||
return runners
|
||||
|
||||
|
||||
class ServiceStub:
|
||||
def __init__(
|
||||
self,
|
||||
runners: list[dict[str, object]] | None = None,
|
||||
jobs: object = UNSET,
|
||||
*,
|
||||
inventory_error: Exception | None = None,
|
||||
request_error: Exception | None = None,
|
||||
) -> None:
|
||||
self.runners = list(runners if runners is not None else _safe_runners())
|
||||
self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs
|
||||
self.inventory_error = inventory_error
|
||||
self.request_error = request_error
|
||||
self.requests: list[tuple[str, dict[str, object]]] = []
|
||||
|
||||
def _gitea_runner_inventory(self) -> list[dict[str, object]]:
|
||||
if self.inventory_error is not None:
|
||||
raise self.inventory_error
|
||||
return self.runners
|
||||
|
||||
def _gitea_request(
|
||||
self, path: str, params: dict[str, object] | None = None
|
||||
) -> object:
|
||||
self.requests.append((path, dict(params or {})))
|
||||
if self.request_error is not None:
|
||||
raise self.request_error
|
||||
return self.jobs
|
||||
|
||||
|
||||
def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]:
|
||||
return module.require_cutover_quiescence(service)
|
||||
|
||||
|
||||
def _assert_rejected(module: Any, service: ServiceStub) -> None:
|
||||
with pytest.raises(RuntimeError):
|
||||
_require_quiescent(module, service)
|
||||
|
||||
|
||||
def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled(
|
||||
upgrade_module,
|
||||
):
|
||||
service = ServiceStub(
|
||||
jobs={
|
||||
"jobs": [
|
||||
{"id": 101, "runner_id": 1, "status": "in_progress"},
|
||||
],
|
||||
"total_count": 1,
|
||||
}
|
||||
)
|
||||
|
||||
_assert_rejected(upgrade_module, service)
|
||||
|
||||
assert len(service.requests) == 1
|
||||
assert service.requests[0][0] == ACTIVE_JOBS_PATH
|
||||
assert service.requests[0][1].get("status") == "in_progress"
|
||||
assert service.requests[0][1].get("limit") == 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"runners",
|
||||
[
|
||||
[_runner(2, SECONDARY, disabled=True)],
|
||||
[_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)],
|
||||
[
|
||||
_runner(1, PRIMARY, busy=True, disabled=True),
|
||||
_runner(2, SECONDARY, disabled=True),
|
||||
],
|
||||
],
|
||||
ids=("primary-missing", "primary-not-fenced", "primary-busy"),
|
||||
)
|
||||
def test_primary_runner_must_be_unique_present_disabled_and_idle(
|
||||
upgrade_module, runners
|
||||
):
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"runners",
|
||||
[
|
||||
[_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)],
|
||||
[
|
||||
_runner(1, PRIMARY, disabled=True),
|
||||
_runner(2, SECONDARY, busy=True, disabled=True),
|
||||
],
|
||||
],
|
||||
ids=("secondary-not-fenced", "secondary-busy"),
|
||||
)
|
||||
def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners):
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
||||
|
||||
|
||||
def test_duplicate_primary_registration_is_rejected(upgrade_module):
|
||||
runners = _safe_runners()
|
||||
runners.append(_runner(11, PRIMARY, disabled=True))
|
||||
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_value", [None, "false", 0, 1])
|
||||
def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value):
|
||||
busy_runners = _safe_runners()
|
||||
busy_runners[0]["busy"] = bad_value
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=busy_runners))
|
||||
|
||||
disabled_runners = _safe_runners()
|
||||
disabled_runners[0]["disabled"] = bad_value
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"legacy",
|
||||
[
|
||||
_runner(3, LEGACY, disabled=False),
|
||||
_runner(3, LEGACY, busy=True, disabled=True),
|
||||
],
|
||||
ids=("legacy-enabled", "legacy-busy"),
|
||||
)
|
||||
def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy):
|
||||
runners = _safe_runners()
|
||||
runners.append(legacy)
|
||||
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS)
|
||||
def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name):
|
||||
runners = _safe_runners()
|
||||
runners.append(_runner(10, name, busy=True, disabled=False))
|
||||
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
||||
|
||||
|
||||
def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module):
|
||||
runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")]
|
||||
_assert_rejected(upgrade_module, ServiceStub(runners=runners))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"jobs",
|
||||
[
|
||||
None,
|
||||
[],
|
||||
{},
|
||||
{"jobs": []},
|
||||
{"jobs": [], "total_count": 1},
|
||||
{"jobs": [], "total_count": True},
|
||||
{"jobs": [], "total_count": "0"},
|
||||
{"jobs": None, "total_count": 0},
|
||||
{"jobs": {}, "total_count": 0},
|
||||
{"jobs": [None], "total_count": 1},
|
||||
{"jobs": ["not-a-job"], "total_count": 1},
|
||||
{"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1},
|
||||
{"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1},
|
||||
],
|
||||
ids=(
|
||||
"response-null",
|
||||
"response-list",
|
||||
"jobs-field-missing",
|
||||
"jobs-count-missing",
|
||||
"jobs-count-mismatch",
|
||||
"jobs-count-bool",
|
||||
"jobs-count-not-integer",
|
||||
"jobs-null",
|
||||
"jobs-not-list",
|
||||
"job-null",
|
||||
"job-not-object",
|
||||
"job-status-missing",
|
||||
"job-status-mismatch",
|
||||
),
|
||||
)
|
||||
def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs):
|
||||
_assert_rejected(upgrade_module, ServiceStub(jobs=jobs))
|
||||
|
||||
|
||||
def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module):
|
||||
jobs = {
|
||||
"jobs": [
|
||||
{"id": index + 1, "runner_id": 90, "status": "in_progress"}
|
||||
for index in range(100)
|
||||
],
|
||||
"total_count": 100,
|
||||
}
|
||||
|
||||
_assert_rejected(
|
||||
upgrade_module,
|
||||
ServiceStub(
|
||||
runners=[
|
||||
*_safe_runners(),
|
||||
_runner(90, "known-unrelated-runner", disabled=True),
|
||||
],
|
||||
jobs=jobs,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"runner_id",
|
||||
[None, True, False, 0, -1, "1", 999],
|
||||
ids=("missing", "true", "false", "zero", "negative", "string", "unknown"),
|
||||
)
|
||||
def test_active_job_with_invalid_or_unknown_runner_id_fails_closed(
|
||||
upgrade_module, runner_id
|
||||
):
|
||||
jobs = {
|
||||
"jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}],
|
||||
"total_count": 1,
|
||||
}
|
||||
|
||||
_assert_rejected(upgrade_module, ServiceStub(jobs=jobs))
|
||||
|
||||
|
||||
def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false(
|
||||
upgrade_module,
|
||||
):
|
||||
runners = [
|
||||
*_safe_runners(),
|
||||
_runner(10, "itworx-docker-runner", busy=False, disabled=True),
|
||||
]
|
||||
service = ServiceStub(
|
||||
runners=runners,
|
||||
jobs={
|
||||
"jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}],
|
||||
"total_count": 1,
|
||||
},
|
||||
)
|
||||
|
||||
_assert_rejected(upgrade_module, service)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("where", ["inventory", "jobs"])
|
||||
def test_gitea_api_errors_fail_closed(upgrade_module, where):
|
||||
service = ServiceStub(
|
||||
inventory_error=OSError("inventory unavailable")
|
||||
if where == "inventory"
|
||||
else None,
|
||||
request_error=OSError("jobs unavailable") if where == "jobs" else None,
|
||||
)
|
||||
|
||||
expected_message = (
|
||||
"inventory unavailable" if where == "inventory" else "jobs unavailable"
|
||||
)
|
||||
with pytest.raises(OSError, match=expected_message):
|
||||
_require_quiescent(upgrade_module, service)
|
||||
|
||||
|
||||
def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module):
|
||||
service = ServiceStub(
|
||||
runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")],
|
||||
jobs={
|
||||
"jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}],
|
||||
"total_count": 1,
|
||||
},
|
||||
)
|
||||
_assert_rejected(upgrade_module, service)
|
||||
|
||||
|
||||
def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch):
|
||||
def unexpected_shell(*_args, **_kwargs):
|
||||
raise AssertionError("cutover guard must not execute Docker or shell commands")
|
||||
|
||||
monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell)
|
||||
unrelated = _runner(90, "known-unrelated-runner", disabled=True)
|
||||
service = ServiceStub(
|
||||
runners=[*_safe_runners(include_legacy=True), unrelated],
|
||||
jobs={"jobs": [], "total_count": 0},
|
||||
)
|
||||
|
||||
result = _require_quiescent(upgrade_module, service)
|
||||
|
||||
assert result == {
|
||||
"status": "quiescent",
|
||||
"fencedRunners": [PRIMARY, SECONDARY, LEGACY],
|
||||
"activeFleetJobs": 0,
|
||||
}
|
||||
assert service.requests == [
|
||||
(
|
||||
ACTIVE_JOBS_PATH,
|
||||
{
|
||||
"status": "in_progress",
|
||||
"page": 1,
|
||||
"limit": 100,
|
||||
"sort": "id",
|
||||
"order": "asc",
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _bash_function(name: str) -> str:
|
||||
script = SCRIPT_PATH.read_text(encoding="utf-8")
|
||||
start = script.index(f"{name}() {{")
|
||||
end = script.index("\n}", start) + 2
|
||||
return script[start:end]
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
||||
@pytest.mark.parametrize(
|
||||
("proof", "executor_status", "allowed"),
|
||||
[
|
||||
("", 0, False),
|
||||
("not-json", 0, False),
|
||||
(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "quiescent",
|
||||
"fencedRunners": [PRIMARY, SECONDARY],
|
||||
"activeFleetJobs": 1,
|
||||
}
|
||||
),
|
||||
0,
|
||||
False,
|
||||
),
|
||||
(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "quiescent",
|
||||
"fencedRunners": [PRIMARY, SECONDARY],
|
||||
"activeFleetJobs": 0,
|
||||
}
|
||||
),
|
||||
1,
|
||||
False,
|
||||
),
|
||||
(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "quiescent",
|
||||
"fencedRunners": [PRIMARY, SECONDARY],
|
||||
"activeFleetJobs": 0,
|
||||
}
|
||||
),
|
||||
0,
|
||||
True,
|
||||
),
|
||||
],
|
||||
ids=(
|
||||
"old-executor-empty",
|
||||
"malformed-proof",
|
||||
"active-job",
|
||||
"executor-error",
|
||||
"quiescent",
|
||||
),
|
||||
)
|
||||
def test_cutover_shell_requires_real_successful_quiescence_proof(
|
||||
proof, executor_status, allowed
|
||||
):
|
||||
# The first python3 invocation stands in for the installed executor; the second
|
||||
# invokes the actual JSON parser inside check_cutover.
|
||||
body = "\n".join(
|
||||
[
|
||||
"ROOT=/synthetic",
|
||||
f"MOCK_PROOF={shlex.quote(proof)}",
|
||||
f"MOCK_EXECUTOR_STATUS={executor_status}",
|
||||
"python3() {",
|
||||
' if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then',
|
||||
' printf "%s" "$MOCK_PROOF"',
|
||||
' return "$MOCK_EXECUTOR_STATUS"',
|
||||
" fi",
|
||||
' command python3 "$@"',
|
||||
"}",
|
||||
_bash_function("check_cutover"),
|
||||
"check_cutover",
|
||||
]
|
||||
)
|
||||
result = subprocess.run(
|
||||
["bash"], input=body.encode(), capture_output=True, check=False
|
||||
)
|
||||
assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
||||
@pytest.mark.parametrize(
|
||||
("inventory", "docker_status", "allowed"),
|
||||
[("", 1, False), ("running-container-id", 0, False), ("", 0, True)],
|
||||
ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"),
|
||||
)
|
||||
def test_dind_idle_shell_requires_successful_empty_docker_inventory(
|
||||
inventory, docker_status, allowed
|
||||
):
|
||||
body = "\n".join(
|
||||
[
|
||||
"DIND=synthetic-dind",
|
||||
f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}",
|
||||
f"MOCK_DOCKER_STATUS={docker_status}",
|
||||
'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }',
|
||||
_bash_function("check_dind_idle"),
|
||||
"check_dind_idle",
|
||||
]
|
||||
)
|
||||
result = subprocess.run(
|
||||
["bash"], input=body.encode(), capture_output=True, check=False
|
||||
)
|
||||
assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
|
||||
@pytest.mark.parametrize(
|
||||
("cutover_started", "dispatch_started"),
|
||||
[(0, 0), (1, 0), (1, 1)],
|
||||
ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
|
||||
)
|
||||
def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started):
|
||||
body = "\n".join(
|
||||
[
|
||||
"backup=/synthetic-backup",
|
||||
"COMPOSE_FILE=/synthetic-compose",
|
||||
"PRIMARY_CONFIG=/synthetic-primary",
|
||||
"SECONDARY_CONFIG=/synthetic-secondary",
|
||||
"ROOT=/synthetic-root",
|
||||
"guard_changed=0",
|
||||
"host_guard_changed=0",
|
||||
f"cutover_started={cutover_started}",
|
||||
f"dispatch_started={dispatch_started}",
|
||||
'new_runners=("new-container")',
|
||||
'registered=("new-registration")',
|
||||
'fenced=("itworx-security-runner:false")',
|
||||
"created_paths=()",
|
||||
"exec 3>&1",
|
||||
'cp() { printf "cp:%s\\n" "$*"; }',
|
||||
'docker() { printf "docker:%s\\n" "$*" >&3; }',
|
||||
'delete_runner() { printf "delete:%s\\n" "$*"; }',
|
||||
'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }',
|
||||
'stop_capacity_daemon() { printf "stop-daemon\\n"; }',
|
||||
'rm() { printf "rm:%s\\n" "$*"; }',
|
||||
_bash_function("restore"),
|
||||
"false",
|
||||
"restore",
|
||||
]
|
||||
)
|
||||
result = subprocess.run(
|
||||
["bash"], input=body.encode(), capture_output=True, check=False
|
||||
)
|
||||
output = result.stdout.decode(errors="replace")
|
||||
errors = result.stderr.decode(errors="replace")
|
||||
assert result.returncode == 1, errors
|
||||
if dispatch_started:
|
||||
assert output == ""
|
||||
assert "runner_fleet_recovery_needed" in errors
|
||||
else:
|
||||
assert "cp:" in output
|
||||
assert "delete:new-registration" in output
|
||||
assert "restore-fence:itworx-security-runner:false" in output
|
||||
assert ("stop-daemon" in output) is bool(cutover_started)
|
||||
assert ("docker:compose " in output) is bool(cutover_started)
|
||||
Reference in new issue
Block a user