1517 lines
49 KiB
Python
1517 lines
49 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
import types
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
|
|
|
|
|
|
@pytest.fixture
|
|
def fleet_module(monkeypatch):
|
|
if sys.platform == "win32" and "fcntl" not in sys.modules:
|
|
fcntl = types.ModuleType("fcntl")
|
|
fcntl.LOCK_EX = 2
|
|
fcntl.LOCK_UN = 8
|
|
fcntl.LOCK_NB = 4
|
|
fcntl.flock = lambda *_args, **_kwargs: None
|
|
monkeypatch.setitem(sys.modules, "fcntl", fcntl)
|
|
spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
module._host_guard_state_unpatched = module._host_guard_state
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"runner_fleet_specialize",
|
|
types.SimpleNamespace(specialization_state=lambda: "legacy"),
|
|
)
|
|
monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", ""))
|
|
monkeypatch.setattr(
|
|
module.RunnerCapacityController,
|
|
"snapshot",
|
|
lambda _self: _healthy_capacity_snapshot(),
|
|
)
|
|
return module
|
|
|
|
|
|
def _healthy_capacity_snapshot():
|
|
return {
|
|
"status": "ok",
|
|
"decision": {"pressure": "headroom"},
|
|
"host": {
|
|
"swapCritical": False,
|
|
"loadRatio": 0.1,
|
|
"availableMemoryBytes": 12 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapSampleAt": datetime.now(UTC).timestamp(),
|
|
},
|
|
}
|
|
|
|
|
|
def _healthy_snapshot():
|
|
runners = [
|
|
{
|
|
"name": name,
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
}
|
|
for name in (
|
|
"itworx-security-runner",
|
|
"itworx-validation-runner-2",
|
|
)
|
|
]
|
|
return {
|
|
"status": "healthy",
|
|
"generatedAt": "2026-10-01T10:00:00Z",
|
|
"capacity": _healthy_capacity_snapshot(),
|
|
"networks": {"count": 0, "capacity": None, "cleanupCandidates": []},
|
|
"storage": {
|
|
"status": "healthy",
|
|
"freeBytes": 30 * 1024**3,
|
|
"totalBytes": 40 * 1024**3,
|
|
},
|
|
"scheduler": {
|
|
"status": "healthy",
|
|
"candidate": None,
|
|
"stallThresholdSeconds": 600,
|
|
},
|
|
"topology": {
|
|
"dind": {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"runners": runners,
|
|
"generalSlots": 2,
|
|
"runnerCount": len(runners),
|
|
"specializedRunners": [],
|
|
"burstRunner": None,
|
|
},
|
|
}
|
|
|
|
|
|
|
|
|
|
def test_container_projection_includes_restart_policy(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
payload = [
|
|
{
|
|
"Config": {"Labels": {"com.docker.compose.project": "gitea-deploy-runners"}},
|
|
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped", "MaximumRetryCount": 0}},
|
|
"State": {
|
|
"Running": True,
|
|
"StartedAt": "2026-09-21T16:00:00Z",
|
|
"OOMKilled": False,
|
|
"Health": {"Status": "healthy"},
|
|
},
|
|
"RestartCount": 2,
|
|
}
|
|
]
|
|
monkeypatch.setattr(service, "_run", lambda _args, timeout=20: __import__("json").dumps(payload))
|
|
|
|
result = service._container("gitea-runner-dind")
|
|
|
|
assert result["restartPolicy"] == "unless-stopped"
|
|
assert result["running"] is True
|
|
assert result["health"] == "healthy"
|
|
assert result["restarts"] == 2
|
|
|
|
|
|
def test_snapshot_keeps_exited_dind_actionable_without_querying_dind_networks(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
states = {
|
|
"gitea-runner-dind": {
|
|
"name": "gitea-runner-dind",
|
|
"running": False,
|
|
"health": "none",
|
|
"restarts": 0,
|
|
"startedAt": "",
|
|
"oomKilled": False,
|
|
"restartPolicy": "no",
|
|
},
|
|
"itworx-validation-runner-2": {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"itworx-security-runner": {
|
|
"name": "itworx-security-runner",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
}
|
|
monkeypatch.setattr(service, "_container", lambda name: states[name])
|
|
monkeypatch.setattr(service, "_optional_container", lambda _name: None)
|
|
monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_network_inventory",
|
|
lambda: pytest.fail("network inventory must not query an exited DinD daemon"),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"cleanup_candidates",
|
|
lambda: pytest.fail("cleanup must not query an exited DinD daemon"),
|
|
)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
|
|
result = service.snapshot()
|
|
|
|
assert result["status"] == "critical"
|
|
assert result["topology"]["dind"]["running"] is False
|
|
assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []}
|
|
assert result["policy"]["hostDockerMutationTargets"][0] == "gitea-runner-dind"
|
|
|
|
|
|
def test_snapshot_does_not_query_dind_api_during_health_start_grace(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
states = {
|
|
"gitea-runner-dind": {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "starting",
|
|
"restarts": 0,
|
|
"startedAt": datetime.fromtimestamp(now - 10, UTC).isoformat(),
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"itworx-validation-runner-2": {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(),
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
"itworx-security-runner": {
|
|
"name": "itworx-security-runner",
|
|
"running": True,
|
|
"health": "healthy",
|
|
"restarts": 0,
|
|
"startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(),
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
}
|
|
monkeypatch.setattr(service, "_container", lambda name: states[name])
|
|
monkeypatch.setattr(service, "_optional_container", lambda _name: None)
|
|
monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_network_inventory",
|
|
lambda: pytest.fail("DinD API must not be queried during health start grace"),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"cleanup_candidates",
|
|
lambda: pytest.fail("cleanup must not query DinD during health start grace"),
|
|
)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
|
|
result = service.snapshot()
|
|
|
|
assert result["status"] == "healthy"
|
|
assert result["topology"]["dind"]["health"] == "starting"
|
|
assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []}
|
|
|
|
|
|
def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"] = {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "starting",
|
|
"startedAt": "2026-09-21T16:00:00Z",
|
|
"restartPolicy": "unless-stopped",
|
|
}
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert events == []
|
|
assert result["dindRecovery"] == {
|
|
"status": "starting",
|
|
"target": "gitea-runner-dind",
|
|
}
|
|
assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"}
|
|
|
|
|
|
def test_automatic_cycle_recovers_critical_exited_dind_with_safe_host_capacity(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["topology"]["dind"].update(
|
|
{
|
|
"running": False,
|
|
"health": "none",
|
|
"startedAt": "",
|
|
"restartPolicy": "no",
|
|
}
|
|
)
|
|
commands: list[tuple[list[str], int]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda target: target == "gitea-runner-dind")
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30),
|
|
(["docker", "start", "gitea-runner-dind"], 30),
|
|
]
|
|
assert [event["action"] for event in events] == [
|
|
"restart-policy.update",
|
|
"start",
|
|
]
|
|
assert result["status"] == "ok"
|
|
assert result["dindRecovery"] == {
|
|
"status": "requested",
|
|
"restartPolicy": "requested",
|
|
"target": "gitea-runner-dind",
|
|
}
|
|
|
|
|
|
def test_automatic_cycle_repairs_fixed_dind_policy_when_healthy_and_guard_clear(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
|
commands: list[tuple[list[str], int]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30)
|
|
]
|
|
assert result["dindRecovery"] == {
|
|
"status": "not_needed",
|
|
"restartPolicy": "requested",
|
|
}
|
|
assert len(events) == 1
|
|
assert events[0]["action"] == "restart-policy.update"
|
|
|
|
|
|
def test_automatic_cycle_never_restarts_running_unhealthy_dind(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "healthy"
|
|
snapshot["topology"]["dind"] = {
|
|
"name": "gitea-runner-dind",
|
|
"running": True,
|
|
"health": "unhealthy",
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"restartPolicy": "unless-stopped",
|
|
}
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["dindRecovery"] == {
|
|
"status": "deferred",
|
|
"target": "gitea-runner-dind",
|
|
}
|
|
assert any(
|
|
event.get("action") == "restart.deferred"
|
|
and event.get("target") == "gitea-runner-dind"
|
|
for event in events
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("guard_state", "guard_reason"),
|
|
[
|
|
(("paused", "host_resource_guard_paused"), "host_resource_guard_paused"),
|
|
(("unavailable", "host_guard_state_untrusted"), "host_guard_state_untrusted"),
|
|
],
|
|
)
|
|
def test_automatic_cycle_never_mutates_when_host_guard_is_paused_or_untrusted(
|
|
fleet_module, monkeypatch, guard_state, guard_reason
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"].update(
|
|
{"running": False, "health": "none", "restartPolicy": "no"}
|
|
)
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: guard_state)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == guard_reason
|
|
assert commands == []
|
|
assert events == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"action",
|
|
[
|
|
"restart-policy-update",
|
|
"dind-start",
|
|
"scheduler-restart",
|
|
"network-remove",
|
|
"cache-prune",
|
|
"runner-start",
|
|
],
|
|
)
|
|
def test_guard_pause_appearing_after_cycle_entry_blocks_each_mutation(
|
|
fleet_module, monkeypatch, action
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
dind = snapshot["topology"]["dind"]
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
if action == "restart-policy-update":
|
|
dind["restartPolicy"] = "no"
|
|
elif action == "dind-start":
|
|
dind.update({"running": False, "health": "none"})
|
|
elif action == "scheduler-restart":
|
|
snapshot["topology"]["runners"] = [
|
|
{
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
}
|
|
]
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22,
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda name: {
|
|
"name": name,
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
elif action == "network-remove":
|
|
snapshot["networks"]["cleanupCandidates"] = [
|
|
{
|
|
"id": "a" * 64,
|
|
"name": "GITEA-ACTIONS-TASK-22-WORKFLOW-ci-JOB-test-network",
|
|
"ageSeconds": 3600,
|
|
"reason": "empty_gitea_actions_task_network_in_dedicated_dind",
|
|
}
|
|
]
|
|
elif action == "cache-prune":
|
|
snapshot["storage"]["status"] = "critical"
|
|
elif action == "runner-start":
|
|
snapshot["topology"]["runners"] = [
|
|
{
|
|
"name": "itworx-validation-runner-2",
|
|
"running": False,
|
|
"health": "unhealthy",
|
|
}
|
|
]
|
|
|
|
guard_states = iter(
|
|
[
|
|
("clear", ""),
|
|
("paused", "host_resource_guard_paused"),
|
|
]
|
|
)
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: next(guard_states))
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == "host_resource_guard_paused"
|
|
assert commands == []
|
|
assert events == []
|
|
|
|
|
|
def test_critical_current_pressure_after_cycle_entry_blocks_mutation(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
|
current_samples = iter(
|
|
[
|
|
{
|
|
"status": "ok",
|
|
"decision": {"pressure": "critical"},
|
|
"host": {"swapCritical": True},
|
|
}
|
|
]
|
|
)
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
|
monkeypatch.setattr(
|
|
fleet_module.RunnerCapacityController,
|
|
"snapshot",
|
|
lambda _self: next(current_samples),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append(args) or "",
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == "runner_capacity_pressure_critical"
|
|
assert commands == []
|
|
|
|
|
|
def test_stale_capacity_sample_blocks_automatic_mutation(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
|
stale = _healthy_capacity_snapshot()
|
|
stale["host"]["swapSampleAt"] = (
|
|
datetime.now(UTC).timestamp()
|
|
- fleet_module.CAPACITY_SAMPLE_MAX_AGE_SECONDS
|
|
- 1
|
|
)
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
|
monkeypatch.setattr(
|
|
fleet_module.RunnerCapacityController,
|
|
"snapshot",
|
|
lambda _self: stale,
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append(args) or "",
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == "runner_capacity_telemetry_unavailable"
|
|
assert commands == []
|
|
|
|
|
|
@pytest.mark.parametrize("capacity", [{"status": "unavailable"}, {"status": "not_configured"}])
|
|
def test_unavailable_or_missing_current_pressure_blocks_mutation(
|
|
fleet_module, monkeypatch, capacity
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
|
monkeypatch.setattr(
|
|
fleet_module.RunnerCapacityController,
|
|
"snapshot",
|
|
lambda _self: capacity,
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append(args) or "",
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == "runner_capacity_telemetry_unavailable"
|
|
assert commands == []
|
|
|
|
|
|
def test_automatic_cycle_rejects_incomplete_critical_snapshot(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["topology"]["dind"].update(
|
|
{"running": False, "health": "none", "restartPolicy": "no"}
|
|
)
|
|
snapshot["topology"].pop("runnerCount")
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda *_args, **_kwargs: pytest.fail("critical fleet must not mutate")
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_record", lambda **_event: pytest.fail("critical fleet must not audit a mutation")
|
|
)
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == "runner_fleet_snapshot_untrusted"
|
|
assert result["dindRecovery"]["status"] == "deferred"
|
|
|
|
|
|
def test_automatic_cycle_recovers_stalled_scheduler_in_critical_snapshot(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["scheduler"] = {
|
|
"status": "stalled",
|
|
"candidate": {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
"stallThresholdSeconds": 600,
|
|
}
|
|
commands: list[tuple[list[str], int]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: snapshot["scheduler"]["candidate"],
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda name: {
|
|
"name": name,
|
|
"running": True,
|
|
"health": "healthy",
|
|
"startedAt": "2026-09-16T18:00:00Z",
|
|
"restarts": 0,
|
|
"oomKilled": False,
|
|
"restartPolicy": "unless-stopped",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event) or True)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45)
|
|
]
|
|
assert result["status"] == "ok"
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "requested",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
}
|
|
assert [event["action"] for event in events] == ["restart", "restart.result"]
|
|
|
|
|
|
def test_automatic_cycle_allows_critical_storage_prune_with_safe_host_capacity(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["storage"] = {
|
|
"status": "critical",
|
|
"freeBytes": 8 * 1024**3,
|
|
"totalBytes": 40 * 1024**3,
|
|
}
|
|
commands: list[list[str]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
["image", "prune", "--force"],
|
|
["builder", "prune", "--force", "--filter", "until=168h"],
|
|
]
|
|
assert result["status"] == "ok"
|
|
assert events == [
|
|
{
|
|
"action": "cache.prune",
|
|
"target": "gitea-runner-dind",
|
|
"reason": "ci_storage_critical",
|
|
"result": "completed",
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("capacity", "reason"),
|
|
[
|
|
({"status": "ok", "decision": {"pressure": "critical"}, "host": {"swapCritical": False}},
|
|
"runner_capacity_pressure_critical"),
|
|
({"status": "unavailable"}, "runner_capacity_telemetry_unavailable"),
|
|
({"status": "ok", "decision": {}, "host": {}}, "runner_capacity_telemetry_untrusted"),
|
|
],
|
|
)
|
|
def test_automatic_cycle_blocks_critical_or_untrusted_capacity_snapshot(
|
|
fleet_module, monkeypatch, capacity, reason
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
snapshot["status"] = "critical"
|
|
snapshot["capacity"] = capacity
|
|
snapshot["topology"]["dind"].update(
|
|
{"running": False, "health": "none", "restartPolicy": "no"}
|
|
)
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda *_args, **_kwargs: pytest.fail("unsafe capacity must not mutate")
|
|
)
|
|
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["status"] == "deferred"
|
|
assert result["reason"] == reason
|
|
|
|
|
|
def test_host_guard_probe_fails_closed_when_path_is_not_posix_root(monkeypatch, fleet_module):
|
|
monkeypatch.setattr(fleet_module.os, "name", "nt")
|
|
|
|
assert fleet_module._host_guard_state_unpatched() == (
|
|
"unavailable",
|
|
"host_guard_state_untrusted",
|
|
)
|
|
|
|
|
|
def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
recent = datetime.fromtimestamp(now - 60, UTC).isoformat()
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_events",
|
|
lambda: [
|
|
{"at": recent, "action": "start", "target": "gitea-runner-dind"},
|
|
{"at": recent, "action": "restart", "target": "gitea-runner-dind"},
|
|
{"at": recent, "action": "start", "target": "gitea-runner-dind"},
|
|
],
|
|
)
|
|
|
|
assert service._recovery_available("gitea-runner-dind") is False
|
|
|
|
|
|
def test_scheduler_candidate_uses_live_gitea_queue_and_fixed_label_target(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["linux-validation", "ubuntu-latest"],
|
|
}
|
|
]
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() == {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
}
|
|
|
|
|
|
def test_scheduler_candidate_never_targets_busy_runner(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": False,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_scheduler_candidate_never_restarts_runner_with_in_progress_job(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["linux-validation"],
|
|
}],
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {"jobs": [{"id": 22931, "created_at": created_at, "labels": ["linux-validation"]}]}
|
|
if params.get("status") == "in_progress":
|
|
return {"jobs": [{"id": 22932, "runner_id": 17, "status": "in_progress"}]}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_scheduler_candidate_never_hides_disabled_busy_runner(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
if params.get("disabled") == "false":
|
|
return {"total_count": 0, "runners": []}
|
|
return {
|
|
"total_count": 1,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": True,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_scheduler_candidate_reads_later_runner_inventory_pages(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
runner_calls: list[dict[str, object]] = []
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
runner_calls.append(params)
|
|
if params.get("page") == 2:
|
|
return {
|
|
"total_count": 2,
|
|
"runners": [
|
|
{
|
|
"id": 18,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": True,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
],
|
|
}
|
|
return {
|
|
"total_count": 2,
|
|
"runners": [
|
|
{
|
|
"id": 17,
|
|
"name": "unrelated-runner",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["unrelated"],
|
|
}
|
|
],
|
|
}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
assert [call.get("page") for call in runner_calls] == [1, 2]
|
|
|
|
|
|
def test_scheduler_candidate_fails_closed_on_malformed_runner_inventory(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
|
|
def fake_gitea(path, params=None):
|
|
if path.endswith("/runners"):
|
|
return {"total_count": 1}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
with pytest.raises(
|
|
fleet_module.RunnerFleetError,
|
|
match="runner_fleet_gitea_invalid_runner_inventory",
|
|
):
|
|
service._scheduler_recovery_candidate()
|
|
|
|
|
|
@pytest.mark.parametrize("unsafe_first", [True, False])
|
|
def test_scheduler_candidate_vetoes_any_unsafe_duplicate_fixed_runner(
|
|
fleet_module, monkeypatch, unsafe_first
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
now = 1_800_000_000
|
|
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
|
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
|
unsafe = {
|
|
"id": 17,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": True,
|
|
"disabled": True,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
idle = {
|
|
"id": 18,
|
|
"name": "itworx-validation-runner-2",
|
|
"status": "online",
|
|
"busy": False,
|
|
"disabled": False,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
|
|
def fake_gitea(path, params=None):
|
|
params = params or {}
|
|
if path.endswith("/runners"):
|
|
runners = [unsafe, idle] if unsafe_first else [idle, unsafe]
|
|
return {"total_count": 2, "runners": runners}
|
|
if params.get("status") == "queued":
|
|
return {
|
|
"jobs": [
|
|
{
|
|
"id": 22931,
|
|
"created_at": created_at,
|
|
"labels": ["linux-validation"],
|
|
}
|
|
]
|
|
}
|
|
return {"jobs": []}
|
|
|
|
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
|
|
|
assert service._scheduler_recovery_candidate() is None
|
|
|
|
|
|
def test_automatic_cycle_restarts_only_exact_known_idle_runner_for_stale_scheduler(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[tuple[list[str], int]] = []
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_record", lambda **event: events.append(event) or True
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45)
|
|
]
|
|
assert any(
|
|
event.get("action") == "restart"
|
|
and event.get("target") == "itworx-validation-runner-2"
|
|
and event.get("reason") == "stale_scheduler_assignment"
|
|
for event in events
|
|
)
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "requested",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
}
|
|
|
|
|
|
def test_scheduler_restart_persists_intent_before_invoking_docker(
|
|
fleet_module, monkeypatch, tmp_path
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
|
|
def observe_durable_intent(args, timeout=20):
|
|
assert args == [
|
|
"docker",
|
|
"restart",
|
|
"--timeout",
|
|
"30",
|
|
"itworx-validation-runner-2",
|
|
]
|
|
assert timeout == 45
|
|
persisted = service._events()
|
|
assert persisted[-1] == {
|
|
"at": persisted[-1]["at"],
|
|
"action": "restart",
|
|
"target": "itworx-validation-runner-2",
|
|
"reason": "stale_scheduler_assignment",
|
|
"result": "intent",
|
|
"ageSeconds": 900,
|
|
}
|
|
return ""
|
|
|
|
monkeypatch.setattr(service, "_run", observe_durable_intent)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert result["schedulerRecovery"]["status"] == "requested"
|
|
|
|
|
|
def test_scheduler_recovery_refuses_restart_when_intent_cannot_be_persisted(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_safe_state_root", lambda: False)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "failed",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "restart_intent_not_persisted",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_refuses_restart_when_intent_write_raises_oserror(
|
|
fleet_module, monkeypatch, tmp_path
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
fleet_module.os,
|
|
"open",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
PermissionError("state storage is unwritable")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "failed",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "restart_intent_not_persisted",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_keeps_intent_when_outcome_write_fails(
|
|
fleet_module, monkeypatch, tmp_path
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
event_open_count = 0
|
|
real_open = fleet_module.os.open
|
|
monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(fleet_module, "RECOVERY_BUDGET", 1)
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": True,
|
|
"health": "healthy",
|
|
},
|
|
)
|
|
|
|
def fail_outcome_open(path, flags, mode=0o777):
|
|
nonlocal event_open_count
|
|
if Path(path).name.startswith(".events."):
|
|
event_open_count += 1
|
|
if event_open_count == 2:
|
|
raise PermissionError("outcome storage is unwritable")
|
|
return real_open(path, flags, mode)
|
|
|
|
monkeypatch.setattr(fleet_module.os, "open", fail_outcome_open)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"]
|
|
]
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "requested",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
}
|
|
assert [event["action"] for event in service._events()] == ["restart"]
|
|
assert service._recovery_available("itworx-validation-runner-2") is False
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("running", "health"),
|
|
[
|
|
(True, "starting"),
|
|
(True, "none"),
|
|
(True, "unhealthy"),
|
|
(False, "healthy"),
|
|
],
|
|
)
|
|
def test_scheduler_recovery_rechecks_strict_runner_health_before_restart(
|
|
fleet_module, monkeypatch, running, health
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_container",
|
|
lambda _target: {
|
|
"name": "itworx-validation-runner-2",
|
|
"running": running,
|
|
"health": health,
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **_event: True)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "ineligible",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "docker_health_not_healthy",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_rejects_duplicate_fresh_inspect_results(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[tuple[list[str], int]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
|
|
def duplicate_inspect(args, timeout=20):
|
|
commands.append((args, timeout))
|
|
return "[{}, {}]"
|
|
|
|
monkeypatch.setattr(service, "_run", duplicate_inspect)
|
|
monkeypatch.setattr(service, "_record", lambda **_event: True)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == [
|
|
(["docker", "inspect", "itworx-validation-runner-2"], 20)
|
|
]
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "ineligible",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "docker_health_not_healthy",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_rejects_runner_without_healthy_docker_state(
|
|
fleet_module, monkeypatch
|
|
):
|
|
service = fleet_module.RunnerFleetService()
|
|
snapshot = _healthy_snapshot()
|
|
next(
|
|
runner
|
|
for runner in snapshot["topology"]["runners"]
|
|
if runner["name"] == "itworx-validation-runner-2"
|
|
)["health"] = "unhealthy"
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(
|
|
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
|
)
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {
|
|
"status": "ineligible",
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"reason": "docker_health_not_healthy",
|
|
}
|
|
|
|
|
|
def test_scheduler_recovery_refuses_unknown_target(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "Gitea",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"]["status"] == "rejected"
|
|
|
|
|
|
def test_scheduler_recovery_obeys_existing_recovery_budget(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_scheduler_recovery_candidate",
|
|
lambda: {
|
|
"target": "itworx-validation-runner-2",
|
|
"jobId": 22931,
|
|
"label": "linux-validation",
|
|
"ageSeconds": 900,
|
|
"reason": "stale_scheduler_assignment",
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_recovery_available", lambda _target: False)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"]["status"] == "budget_exhausted"
|
|
|
|
|
|
def test_scheduler_recovery_does_nothing_without_stale_candidate(fleet_module, monkeypatch):
|
|
service = fleet_module.RunnerFleetService()
|
|
commands: list[list[str]] = []
|
|
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
|
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
|
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
|
monkeypatch.setattr(service, "_record", lambda **_event: None)
|
|
|
|
result = service.automatic_cycle()
|
|
|
|
assert commands == []
|
|
assert result["schedulerRecovery"] == {"status": "not_needed"}
|