1048 lines
34 KiB
Python
1048 lines
34 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
|
|
|
|
|
|
@pytest.fixture
|
|
def fleet_module():
|
|
spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _healthy_snapshot():
|
|
return {
|
|
"status": "healthy",
|
|
"networks": {"cleanupCandidates": []},
|
|
"storage": {"status": "healthy"},
|
|
"topology": {
|
|
"dind": {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"runners": [
|
|
{
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
},
|
|
{
|
|
"name": "itworx-security-runner",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
},
|
|
]
|
|
},
|
|
}
|
|
|
|
|
|
|
|
|
|
def test_container_projection_includes_restart_policy(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
payload = [
|
|
{
|
|
"Config": {"Labels": {"com.docker.compose.project": "gitea-deploy-runners"}},
|
|
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped", "MaximumRetryCount": 0}},
|
|
"State": {
|
|
"Running": True,
|
|
"StartedAt": "2026-09-21T16:00:00Z",
|
|
"OOMKilled": False,
|
|
"Health": {"Status": "healthy"},
|
|
},
|
|
"RestartCount": 2,
|
|
}
|
|
]
|
|
monkeypatch.setattr(service, "_run", lambda _args, timeout=20: __import__("json").dumps(payload))
|
|
|
|
result = service._container("gitea-runner-dind")
|
|
|
|
assert result["restartPolicy"] == "unless-stopped"
|
|
assert result["running"] is True
|
|
assert result["health"] == "healthy"
|
|
assert result["restarts"] == 2
|
|
|
|
|
|
def test_snapshot_keeps_exited_dind_actionable_without_querying_dind_networks(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
states = {
|
|
"gitea-runner-dind": {
|
|
"name": "gitea-runner-dind",
|
|
"running": False,
|
|
"health": "none",
|
|
"restarts": 0,
|
|
"startedAt": "",
|
|
"oomKilled": False,
|
|
"restartPolicy": "no",
|
|
},
|
|
"itworx-validation-runner-2": {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"itworx-security-runner": {
|
|
"name": "itworx-security-runner",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
}
|
|
monkeypatch.setattr(service, "_container", lambda name: states[name])
|
|
monkeypatch.setattr(service, "_optional_container", lambda _name: None)
|
|
monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_network_inventory",
|
|
lambda: pytest.fail("network inventory must not query an exited DinD daemon"),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"cleanup_candidates",
|
|
lambda: pytest.fail("cleanup must not query an exited DinD daemon"),
|
|
)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
|
|
result = service.snapshot()
|
|
|
|
assert result["status"] == "critical"
|
|
assert result["topology"]["dind"]["running"] is False
|
|
assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []}
|
|
assert result["policy"]["hostDockerMutationTargets"][0] == "gitea-runner-dind"
|
|
|
|
|
|
def test_snapshot_does_not_query_dind_api_during_health_start_grace(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
states = {
|
|
"gitea-runner-dind": {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "starting",
|
|
"restarts": 0,
|
|
"startedAt": datetime.fromtimestamp(now - 10, UTC).isoformat(),
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"itworx-validation-runner-2": {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(),
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"itworx-security-runner": {
|
|
"name": "itworx-security-runner",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(),
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
}
|
|
monkeypatch.setattr(service, "_container", lambda name: states[name])
|
|
monkeypatch.setattr(service, "_optional_container", lambda _name: None)
|
|
monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_network_inventory",
|
|
lambda: pytest.fail("DinD API must not be queried during health start grace"),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"cleanup_candidates",
|
|
lambda: pytest.fail("cleanup must not query DinD during health start grace"),
|
|
)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
|
|
result = service.snapshot()
|
|
|
|
assert result["status"] == "healthy"
|
|
assert result["topology"]["dind"]["health"] == "starting"
|
|
assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []}
|
|
|
|
|
|
def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"] = {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "starting",
|
|
"startedAt": "2026-09-21T16:00:00Z",
|
|
"restartPolicy": "unless-stopped",
|
|
}
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert events == []
|
|
assert result["dindRecovery"] == {
|
|
"status": "starting",
|
|
"target": "gitea-runner-dind",
|
|
}
|
|
assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"}
|
|
|
|
|
|
def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["topology"]["dind"] = {
|
|
"name": "gitea-runner-dind",
|
|
"running": False,
|
|
"health": "none",
|
|
"startedAt": "",
|
|
"restartPolicy": "no",
|
|
}
|
|
commands: list[tuple[list[str], int]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda target: target == "gitea-runner-dind")
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30),
|
|
(["docker", "start", "gitea-runner-dind"], 30),
|
|
]
|
|
assert result["dindRecovery"]["status"] == "requested"
|
|
assert result["schedulerRecovery"]["status"] == "deferred_for_dind_recovery"
|
|
assert any(
|
|
event.get("action") == "start"
|
|
and event.get("target") == "gitea-runner-dind"
|
|
and event.get("reason") == "known_ci_dind_exited"
|
|
for event in events
|
|
)
|
|
|
|
|
|
def test_automatic_cycle_never_restarts_running_unhealthy_dind(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["topology"]["dind"] = {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "unhealthy",
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"restartPolicy": "unless-stopped",
|
|
}
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["dindRecovery"] == {
|
|
"status": "deferred",
|
|
"target": "gitea-runner-dind",
|
|
}
|
|
assert any(
|
|
event.get("action") == "restart.deferred"
|
|
and event.get("target") == "gitea-runner-dind"
|
|
for event in events
|
|
)
|
|
|
|
|
|
def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
recent = datetime.fromtimestamp(now - 60, UTC).isoformat()
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_events",
|
|
lambda: [
|
|
{"at": recent, "action": "start", "target": "gitea-runner-dind"},
|
|
{"at": recent, "action": "restart", "target": "gitea-runner-dind"},
|
|
{"at": recent, "action": "start", "target": "gitea-runner-dind"},
|
|
],
|
|
)
|
|
|
|
assert service._recovery_available("gitea-runner-dind") is False
|
|
|
|
|
|
def test_scheduler_candidate_uses_live_gitea_queue_and_fixed_label_target(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["linux-validation", "ubuntu-latest"],
|
|
}
|
|
]
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() == {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
}
|
|
|
|
|
|
def test_scheduler_candidate_never_targets_busy_runner(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": False,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_scheduler_candidate_never_restarts_runner_with_in_progress_job(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["linux-validation"],
|
|
}],
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {"jobs": [{"id": 22931, "created_at": created_at, "labels": ["linux-validation"]}]}
|
|
if params.get("status") == "in_progress":
|
|
return {"jobs": [{"id": 22932, "runner_id": 17, "status": "in_progress"}]}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_scheduler_candidate_never_hides_disabled_busy_runner(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
if params.get("disabled") == "false":
|
|
return {"total_count": 0, "runners": []}
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": True,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_scheduler_candidate_reads_later_runner_inventory_pages(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
runner_calls: list[dict[str, object]] = []
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
runner_calls.append(params)
|
|
if params.get("page") == 2:
|
|
return {
|
|
"total_count": 2,
|
|
"runners": [
|
|
{
|
|
"id": 18,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": True,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
],
|
|
}
|
|
return {
|
|
"total_count": 2,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "unrelated-runner",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["unrelated"],
|
|
}
|
|
],
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
assert [call.get("page") for call in runner_calls] == [1, 2]
|
|
|
|
|
|
def test_scheduler_candidate_fails_closed_on_malformed_runner_inventory(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
|
|
def fake_gitea(path, params=None):
|
|
if path.endswith("/runners"):
|
|
return {"total_count": 1}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
with pytest.raises(
|
|
fleet_module.RunnerFleetError,
|
|
match="runner_fleet_gitea_invalid_runner_inventory",
|
|
):
|
|
service._scheduler_recovery_candidate()
|
|
|
|
|
|
@pytest.mark.parametrize("unsafe_first", [True, False])
|
|
def test_scheduler_candidate_vetoes_any_unsafe_duplicate_fixed_runner(
|
|
fleet_module, monkeypatch, unsafe_first
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
unsafe = {
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": True,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
idle = {
|
|
"id": 18,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
runners = [unsafe, idle] if unsafe_first else [idle, unsafe]
|
|
return {"total_count": 2, "runners": runners}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_automatic_cycle_restarts_only_exact_known_idle_runner_for_stale_scheduler(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[tuple[list[str], int]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_record", lambda **event: events.append(event) or True
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45)
|
|
]
|
|
assert any(
|
|
event.get("action") == "restart"
|
|
and event.get("target") == "itworx-validation-runner-2"
|
|
and event.get("reason") == "stale_scheduler_assignment"
|
|
for event in events
|
|
)
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "requested",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
}
|
|
|
|
|
|
def test_scheduler_restart_persists_intent_before_invoking_docker(
|
|
fleet_module, monkeypatch, tmp_path
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
|
|
def observe_durable_intent(args, timeout=20):
|
|
assert args == [
|
|
"docker",
|
|
"restart",
|
|
"--timeout",
|
|
"30",
|
|
"itworx-validation-runner-2",
|
|
]
|
|
assert timeout == 45
|
|
persisted = service._events()
|
|
assert persisted[-1] == {
|
|
"at": persisted[-1]["at"],
|
|
"action": "restart",
|
|
"target": "itworx-validation-runner-2",
|
|
"reason": "stale_scheduler_assignment",
|
|
"result": "intent",
|
|
"ageSeconds": 900,
|
|
}
|
|
return ""
|
|
|
|
monkeypatch.setattr(service, "_run", observe_durable_intent)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["schedulerRecovery"]["status"] == "requested"
|
|
|
|
|
|
def test_scheduler_recovery_refuses_restart_when_intent_cannot_be_persisted(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_safe_state_root", lambda: False)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "failed",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "restart_intent_not_persisted",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_refuses_restart_when_intent_write_raises_oserror(
|
|
fleet_module, monkeypatch, tmp_path
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
fleet_module.os,
|
|
"open",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
PermissionError("state storage is unwritable")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "failed",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "restart_intent_not_persisted",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_keeps_intent_when_outcome_write_fails(
|
|
fleet_module, monkeypatch, tmp_path
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
event_open_count = 0
|
|
real_open = fleet_module.os.open
|
|
monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(fleet_module, "RECOVERY_BUDGET", 1)
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
|
|
def fail_outcome_open(path, flags, mode=0o777):
|
|
nonlocal event_open_count
|
|
if Path(path).name.startswith(".events."):
|
|
event_open_count += 1
|
|
if event_open_count == 2:
|
|
raise PermissionError("outcome storage is unwritable")
|
|
return real_open(path, flags, mode)
|
|
|
|
monkeypatch.setattr(fleet_module.os, "open", fail_outcome_open)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"]
|
|
]
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "requested",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
}
|
|
assert [event["action"] for event in service._events()] == ["restart"]
|
|
assert service._recovery_available("itworx-validation-runner-2") is False
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("running", "health"),
|
|
[
|
|
(True, "starting"),
|
|
(True, "none"),
|
|
(True, "unhealthy"),
|
|
(False, "healthy"),
|
|
],
|
|
)
|
|
def test_scheduler_recovery_rechecks_strict_runner_health_before_restart(
|
|
fleet_module, monkeypatch, running, health
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": running,
|
|
"health": health,
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **_event: True)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "ineligible",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "docker_health_not_healthy",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_rejects_duplicate_fresh_inspect_results(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[tuple[list[str], int]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
|
|
def duplicate_inspect(args, timeout=20):
|
|
commands.append((args, timeout))
|
|
return "[{}, {}]"
|
|
|
|
monkeypatch.setattr(service, "_run", duplicate_inspect)
|
|
monkeypatch.setattr(service, "_record", lambda **_event: True)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "inspect", "itworx-validation-runner-2"], 20)
|
|
]
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "ineligible",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "docker_health_not_healthy",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_rejects_runner_without_healthy_docker_state(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["runners"][0]["health"] = "unhealthy"
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "ineligible",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "docker_health_not_healthy",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_refuses_unknown_target(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "Gitea",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"]["status"] == "rejected"
|
|
|
|
|
|
def test_scheduler_recovery_obeys_existing_recovery_budget(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: False)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"]["status"] == "budget_exhausted"
|
|
|
|
|
|
def test_scheduler_recovery_does_nothing_without_stale_candidate(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {"status": "not_needed"}
|