Publish curated Unraid-AppOps-Gateway source from a506550ef83ef2d81651e37c98a89f7ee330c730
This commit is contained in:
Public Source Publisher committed
1 parent
2facb8de34
commit
767e66921e
68 files changed
+15924
-313
No files matched your search
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -12,41 +14,93 @@ FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fleet_module():
|
||||
def fleet_module(monkeypatch):
|
||||
if sys.platform == "win32" and "fcntl" not in sys.modules:
|
||||
fcntl = types.ModuleType("fcntl")
|
||||
fcntl.LOCK_EX = 2
|
||||
fcntl.LOCK_UN = 8
|
||||
fcntl.LOCK_NB = 4
|
||||
fcntl.flock = lambda *_args, **_kwargs: None
|
||||
monkeypatch.setitem(sys.modules, "fcntl", fcntl)
|
||||
spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
module._host_guard_state_unpatched = module._host_guard_state
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"runner_fleet_specialize",
|
||||
types.SimpleNamespace(specialization_state=lambda: "legacy"),
|
||||
)
|
||||
monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", ""))
|
||||
monkeypatch.setattr(
|
||||
module.RunnerCapacityController,
|
||||
"snapshot",
|
||||
lambda _self: _healthy_capacity_snapshot(),
|
||||
)
|
||||
return module
|
||||
|
||||
|
||||
def _healthy_capacity_snapshot():
|
||||
return {
|
||||
"status": "ok",
|
||||
"decision": {"pressure": "headroom"},
|
||||
"host": {
|
||||
"swapCritical": False,
|
||||
"loadRatio": 0.1,
|
||||
"availableMemoryBytes": 12 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapSampleAt": datetime.now(UTC).timestamp(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _healthy_snapshot():
|
||||
runners = [
|
||||
{
|
||||
"name": name,
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"restarts": 0,
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
"oomKilled": False,
|
||||
"restartPolicy": "unless-stopped",
|
||||
}
|
||||
for name in (
|
||||
"itworx-security-runner",
|
||||
"itworx-validation-runner-2",
|
||||
)
|
||||
]
|
||||
return {
|
||||
"status": "healthy",
|
||||
"networks": {"cleanupCandidates": []},
|
||||
"storage": {"status": "healthy"},
|
||||
"generatedAt": "2026-10-01T10:00:00Z",
|
||||
"capacity": _healthy_capacity_snapshot(),
|
||||
"networks": {"count": 0, "capacity": None, "cleanupCandidates": []},
|
||||
"storage": {
|
||||
"status": "healthy",
|
||||
"freeBytes": 30 * 1024**3,
|
||||
"totalBytes": 40 * 1024**3,
|
||||
},
|
||||
"scheduler": {
|
||||
"status": "healthy",
|
||||
"candidate": None,
|
||||
"stallThresholdSeconds": 600,
|
||||
},
|
||||
"topology": {
|
||||
"dind": {
|
||||
"name": "gitea-runner-dind",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"restarts": 0,
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
"oomKilled": False,
|
||||
"restartPolicy": "unless-stopped",
|
||||
},
|
||||
"runners": [
|
||||
{
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
},
|
||||
{
|
||||
"name": "itworx-security-runner",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
},
|
||||
]
|
||||
"runners": runners,
|
||||
"generalSlots": 2,
|
||||
"runnerCount": len(runners),
|
||||
"specializedRunners": [],
|
||||
"burstRunner": None,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -220,19 +274,20 @@ def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting(
|
||||
assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"}
|
||||
|
||||
|
||||
def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind(
|
||||
def test_automatic_cycle_recovers_critical_exited_dind_with_safe_host_capacity(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["status"] = "critical"
|
||||
snapshot["topology"]["dind"] = {
|
||||
"name": "gitea-runner-dind",
|
||||
"running": False,
|
||||
"health": "none",
|
||||
"startedAt": "",
|
||||
"restartPolicy": "no",
|
||||
}
|
||||
snapshot["topology"]["dind"].update(
|
||||
{
|
||||
"running": False,
|
||||
"health": "none",
|
||||
"startedAt": "",
|
||||
"restartPolicy": "no",
|
||||
}
|
||||
)
|
||||
commands: list[tuple[list[str], int]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
@@ -250,14 +305,45 @@ def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind(
|
||||
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30),
|
||||
(["docker", "start", "gitea-runner-dind"], 30),
|
||||
]
|
||||
assert result["dindRecovery"]["status"] == "requested"
|
||||
assert result["schedulerRecovery"]["status"] == "deferred_for_dind_recovery"
|
||||
assert any(
|
||||
event.get("action") == "start"
|
||||
and event.get("target") == "gitea-runner-dind"
|
||||
and event.get("reason") == "known_ci_dind_exited"
|
||||
for event in events
|
||||
assert [event["action"] for event in events] == [
|
||||
"restart-policy.update",
|
||||
"start",
|
||||
]
|
||||
assert result["status"] == "ok"
|
||||
assert result["dindRecovery"] == {
|
||||
"status": "requested",
|
||||
"restartPolicy": "requested",
|
||||
"target": "gitea-runner-dind",
|
||||
}
|
||||
|
||||
|
||||
def test_automatic_cycle_repairs_fixed_dind_policy_when_healthy_and_guard_clear(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
||||
commands: list[tuple[list[str], int]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
||||
)
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert commands == [
|
||||
(["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30)
|
||||
]
|
||||
assert result["dindRecovery"] == {
|
||||
"status": "not_needed",
|
||||
"restartPolicy": "requested",
|
||||
}
|
||||
assert len(events) == 1
|
||||
assert events[0]["action"] == "restart-policy.update"
|
||||
|
||||
|
||||
def test_automatic_cycle_never_restarts_running_unhealthy_dind(
|
||||
@@ -265,7 +351,7 @@ def test_automatic_cycle_never_restarts_running_unhealthy_dind(
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["status"] = "critical"
|
||||
snapshot["status"] = "healthy"
|
||||
snapshot["topology"]["dind"] = {
|
||||
"name": "gitea-runner-dind",
|
||||
"running": True,
|
||||
@@ -293,6 +379,385 @@ def test_automatic_cycle_never_restarts_running_unhealthy_dind(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("guard_state", "guard_reason"),
|
||||
[
|
||||
(("paused", "host_resource_guard_paused"), "host_resource_guard_paused"),
|
||||
(("unavailable", "host_guard_state_untrusted"), "host_guard_state_untrusted"),
|
||||
],
|
||||
)
|
||||
def test_automatic_cycle_never_mutates_when_host_guard_is_paused_or_untrusted(
|
||||
fleet_module, monkeypatch, guard_state, guard_reason
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["topology"]["dind"].update(
|
||||
{"running": False, "health": "none", "restartPolicy": "no"}
|
||||
)
|
||||
commands: list[list[str]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
||||
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: guard_state)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == guard_reason
|
||||
assert commands == []
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"action",
|
||||
[
|
||||
"restart-policy-update",
|
||||
"dind-start",
|
||||
"scheduler-restart",
|
||||
"network-remove",
|
||||
"cache-prune",
|
||||
"runner-start",
|
||||
],
|
||||
)
|
||||
def test_guard_pause_appearing_after_cycle_entry_blocks_each_mutation(
|
||||
fleet_module, monkeypatch, action
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
dind = snapshot["topology"]["dind"]
|
||||
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
||||
if action == "restart-policy-update":
|
||||
dind["restartPolicy"] = "no"
|
||||
elif action == "dind-start":
|
||||
dind.update({"running": False, "health": "none"})
|
||||
elif action == "scheduler-restart":
|
||||
snapshot["topology"]["runners"] = [
|
||||
{
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_scheduler_recovery_candidate",
|
||||
lambda: {
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22,
|
||||
"ageSeconds": 900,
|
||||
"reason": "stale_scheduler_assignment",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_container",
|
||||
lambda name: {
|
||||
"name": name,
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
},
|
||||
)
|
||||
elif action == "network-remove":
|
||||
snapshot["networks"]["cleanupCandidates"] = [
|
||||
{
|
||||
"id": "a" * 64,
|
||||
"name": "GITEA-ACTIONS-TASK-22-WORKFLOW-ci-JOB-test-network",
|
||||
"ageSeconds": 3600,
|
||||
"reason": "empty_gitea_actions_task_network_in_dedicated_dind",
|
||||
}
|
||||
]
|
||||
elif action == "cache-prune":
|
||||
snapshot["storage"]["status"] = "critical"
|
||||
elif action == "runner-start":
|
||||
snapshot["topology"]["runners"] = [
|
||||
{
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": False,
|
||||
"health": "unhealthy",
|
||||
}
|
||||
]
|
||||
|
||||
guard_states = iter(
|
||||
[
|
||||
("clear", ""),
|
||||
("paused", "host_resource_guard_paused"),
|
||||
]
|
||||
)
|
||||
commands: list[list[str]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: next(guard_states))
|
||||
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
||||
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
||||
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == "host_resource_guard_paused"
|
||||
assert commands == []
|
||||
assert events == []
|
||||
|
||||
|
||||
def test_critical_current_pressure_after_cycle_entry_blocks_mutation(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
||||
current_samples = iter(
|
||||
[
|
||||
{
|
||||
"status": "ok",
|
||||
"decision": {"pressure": "critical"},
|
||||
"host": {"swapCritical": True},
|
||||
}
|
||||
]
|
||||
)
|
||||
commands: list[list[str]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
||||
monkeypatch.setattr(
|
||||
fleet_module.RunnerCapacityController,
|
||||
"snapshot",
|
||||
lambda _self: next(current_samples),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20: commands.append(args) or "",
|
||||
)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == "runner_capacity_pressure_critical"
|
||||
assert commands == []
|
||||
|
||||
|
||||
def test_stale_capacity_sample_blocks_automatic_mutation(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
||||
stale = _healthy_capacity_snapshot()
|
||||
stale["host"]["swapSampleAt"] = (
|
||||
datetime.now(UTC).timestamp()
|
||||
- fleet_module.CAPACITY_SAMPLE_MAX_AGE_SECONDS
|
||||
- 1
|
||||
)
|
||||
commands: list[list[str]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
||||
monkeypatch.setattr(
|
||||
fleet_module.RunnerCapacityController,
|
||||
"snapshot",
|
||||
lambda _self: stale,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20: commands.append(args) or "",
|
||||
)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == "runner_capacity_telemetry_unavailable"
|
||||
assert commands == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [{"status": "unavailable"}, {"status": "not_configured"}])
|
||||
def test_unavailable_or_missing_current_pressure_blocks_mutation(
|
||||
fleet_module, monkeypatch, capacity
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["topology"]["dind"]["restartPolicy"] = "no"
|
||||
commands: list[list[str]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
||||
monkeypatch.setattr(
|
||||
fleet_module.RunnerCapacityController,
|
||||
"snapshot",
|
||||
lambda _self: capacity,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20: commands.append(args) or "",
|
||||
)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == "runner_capacity_telemetry_unavailable"
|
||||
assert commands == []
|
||||
|
||||
|
||||
def test_automatic_cycle_rejects_incomplete_critical_snapshot(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["status"] = "critical"
|
||||
snapshot["topology"]["dind"].update(
|
||||
{"running": False, "health": "none", "restartPolicy": "no"}
|
||||
)
|
||||
snapshot["topology"].pop("runnerCount")
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(
|
||||
service, "_run", lambda *_args, **_kwargs: pytest.fail("critical fleet must not mutate")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_record", lambda **_event: pytest.fail("critical fleet must not audit a mutation")
|
||||
)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == "runner_fleet_snapshot_untrusted"
|
||||
assert result["dindRecovery"]["status"] == "deferred"
|
||||
|
||||
|
||||
def test_automatic_cycle_recovers_stalled_scheduler_in_critical_snapshot(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["status"] = "critical"
|
||||
snapshot["scheduler"] = {
|
||||
"status": "stalled",
|
||||
"candidate": {
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22931,
|
||||
"label": "linux-validation",
|
||||
"ageSeconds": 900,
|
||||
"reason": "stale_scheduler_assignment",
|
||||
},
|
||||
"stallThresholdSeconds": 600,
|
||||
}
|
||||
commands: list[tuple[list[str], int]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_scheduler_recovery_candidate",
|
||||
lambda: snapshot["scheduler"]["candidate"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_container",
|
||||
lambda name: {
|
||||
"name": name,
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
"restarts": 0,
|
||||
"oomKilled": False,
|
||||
"restartPolicy": "unless-stopped",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20: commands.append((args, timeout)) or "",
|
||||
)
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event) or True)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert commands == [
|
||||
(["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45)
|
||||
]
|
||||
assert result["status"] == "ok"
|
||||
assert result["schedulerRecovery"] == {
|
||||
"status": "requested",
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22931,
|
||||
}
|
||||
assert [event["action"] for event in events] == ["restart", "restart.result"]
|
||||
|
||||
|
||||
def test_automatic_cycle_allows_critical_storage_prune_with_safe_host_capacity(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["status"] = "critical"
|
||||
snapshot["storage"] = {
|
||||
"status": "critical",
|
||||
"freeBytes": 8 * 1024**3,
|
||||
"totalBytes": 40 * 1024**3,
|
||||
}
|
||||
commands: list[list[str]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None)
|
||||
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "")
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert commands == [
|
||||
["image", "prune", "--force"],
|
||||
["builder", "prune", "--force", "--filter", "until=168h"],
|
||||
]
|
||||
assert result["status"] == "ok"
|
||||
assert events == [
|
||||
{
|
||||
"action": "cache.prune",
|
||||
"target": "gitea-runner-dind",
|
||||
"reason": "ci_storage_critical",
|
||||
"result": "completed",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("capacity", "reason"),
|
||||
[
|
||||
({"status": "ok", "decision": {"pressure": "critical"}, "host": {"swapCritical": False}},
|
||||
"runner_capacity_pressure_critical"),
|
||||
({"status": "unavailable"}, "runner_capacity_telemetry_unavailable"),
|
||||
({"status": "ok", "decision": {}, "host": {}}, "runner_capacity_telemetry_untrusted"),
|
||||
],
|
||||
)
|
||||
def test_automatic_cycle_blocks_critical_or_untrusted_capacity_snapshot(
|
||||
fleet_module, monkeypatch, capacity, reason
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["status"] = "critical"
|
||||
snapshot["capacity"] = capacity
|
||||
snapshot["topology"]["dind"].update(
|
||||
{"running": False, "health": "none", "restartPolicy": "no"}
|
||||
)
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(
|
||||
service, "_run", lambda *_args, **_kwargs: pytest.fail("unsafe capacity must not mutate")
|
||||
)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", ""))
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == reason
|
||||
|
||||
|
||||
def test_host_guard_probe_fails_closed_when_path_is_not_posix_root(monkeypatch, fleet_module):
|
||||
monkeypatch.setattr(fleet_module.os, "name", "nt")
|
||||
|
||||
assert fleet_module._host_guard_state_unpatched() == (
|
||||
"unavailable",
|
||||
"host_guard_state_untrusted",
|
||||
)
|
||||
|
||||
|
||||
def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
@@ -952,7 +1417,11 @@ def test_scheduler_recovery_rejects_runner_without_healthy_docker_state(
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = _healthy_snapshot()
|
||||
snapshot["topology"]["runners"][0]["health"] = "unhealthy"
|
||||
next(
|
||||
runner
|
||||
for runner in snapshot["topology"]["runners"]
|
||||
if runner["name"] == "itworx-validation-runner-2"
|
||||
)["health"] = "unhealthy"
|
||||
commands: list[list[str]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(
|
||||
|
||||
Reference in new issue
Block a user