496 lines
18 KiB
Python
496 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker"
|
|
CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py"
|
|
|
|
|
|
@pytest.fixture
|
|
def capacity_module(monkeypatch):
|
|
if sys.platform == "win32":
|
|
# The daemon lock is not exercised by these in-process cycle tests.
|
|
monkeypatch.setitem(sys.modules, "fcntl", types.ModuleType("fcntl"))
|
|
monkeypatch.syspath_prepend(str(CONTROLLER_ROOT))
|
|
spec = importlib.util.spec_from_file_location(
|
|
"appops_runner_capacity_controller_test", CONTROLLER_PATH
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _wire_read_only_host(service, module, monkeypatch, samples):
|
|
counters = iter(samples)
|
|
monkeypatch.setattr(service, "_strict_swap_pages", lambda: next(counters))
|
|
monkeypatch.setattr(module.os, "cpu_count", lambda: 20)
|
|
monkeypatch.setattr(module.os, "getloadavg", lambda: (4.0, 3.0, 2.0), raising=False)
|
|
monkeypatch.setattr(module.os, "sysconf", lambda _name: 4096, raising=False)
|
|
monkeypatch.setattr(service, "_mem_available_bytes", lambda: 16 * 1024**3)
|
|
monkeypatch.setattr(service, "_cache_free_bytes", lambda: 32 * 1024**3)
|
|
|
|
def forbidden(*_args, **_kwargs):
|
|
pytest.fail("read-only host sampling touched burst inventory or a mutation")
|
|
|
|
for name in ("_burst_container", "_runner_registration", "_gitea_request",
|
|
"_run", "_read_state", "_write_state", "_record"):
|
|
monkeypatch.setattr(service, name, forbidden)
|
|
|
|
|
|
def test_host_snapshot_measures_swap_without_optional_burst(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [(50, 70)])
|
|
prior = {"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}
|
|
preserved = dict(prior)
|
|
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("baseline already available"))
|
|
|
|
result = service.host_snapshot(prior, now=100.0)
|
|
|
|
assert result["status"] == "observed"
|
|
assert result["host"]["loadRatio"] == 0.2
|
|
assert result["host"]["swapInKbps"] == 200.0
|
|
assert result["host"]["swapOutKbps"] == 280.0
|
|
assert result["host"]["swapCritical"] is False
|
|
assert prior == preserved
|
|
|
|
|
|
@pytest.mark.parametrize("prior", [None, {"swapSampleAt": 0}, {
|
|
"swapSampleAt": 1.0, "swapInPages": 5, "swapOutPages": 5,
|
|
}, {"swapSampleAt": float("nan"), "swapInPages": 5, "swapOutPages": 5}])
|
|
def test_host_snapshot_takes_two_real_samples_for_missing_or_stale_baseline(
|
|
capacity_module, monkeypatch, prior
|
|
):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (266, 532)])
|
|
elapsed = iter([4.0, 5.0])
|
|
sleeps = []
|
|
monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed))
|
|
monkeypatch.setattr(capacity_module.time, "sleep", sleeps.append)
|
|
|
|
result = service.host_snapshot(prior, now=100.0)
|
|
|
|
assert result["status"] == "observed"
|
|
assert result["host"]["swapInKbps"] == 1024.0
|
|
assert result["host"]["swapOutKbps"] == 2048.0
|
|
assert result["host"]["swapSampleAt"] == 101.0
|
|
assert sleeps == [1.0]
|
|
|
|
|
|
def test_host_snapshot_reports_unavailable_if_swap_counter_resets(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (9, 20)])
|
|
elapsed = iter([4.0, 5.0])
|
|
monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed))
|
|
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: None)
|
|
|
|
result = service.host_snapshot(now=100.0)
|
|
|
|
assert result["status"] == "unavailable"
|
|
assert result["reason"] == "runner_capacity_swap_baseline_unavailable"
|
|
assert result["host"] is None
|
|
|
|
|
|
def test_host_snapshot_rejects_reset_against_recent_protected_baseline(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [(9, 20), (10, 21)])
|
|
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("recent counter reset must fail closed"))
|
|
|
|
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 10, "swapOutPages": 20}, now=100.0)
|
|
|
|
assert result["status"] == "unavailable"
|
|
assert result["reason"] == "runner_capacity_swap_baseline_unavailable"
|
|
assert result["host"] is None
|
|
|
|
|
|
def test_host_snapshot_handles_absent_load_tuple(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)])
|
|
monkeypatch.setattr(capacity_module.os, "getloadavg", lambda: ())
|
|
|
|
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0)
|
|
|
|
assert result["status"] == "unavailable"
|
|
assert result["host"] is None
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["", "pswpin 1\n", "pswpin -1\npswpout 2\n",
|
|
"pswpin 1\npswpin 2\npswpout 3\n", "pswpin invalid\npswpout 2\n", "x" * 65537])
|
|
def test_strict_swap_reader_rejects_missing_or_invalid_counters(capacity_module, monkeypatch, raw):
|
|
monkeypatch.setattr(capacity_module, "Path", lambda _path: types.SimpleNamespace(
|
|
open=lambda **_kwargs: io.StringIO(raw)
|
|
))
|
|
with pytest.raises(capacity_module.RunnerCapacityError, match="swap_telemetry_unavailable"):
|
|
capacity_module.RunnerCapacityController._strict_swap_pages()
|
|
|
|
|
|
def test_host_snapshot_does_not_turn_missing_swap_into_zero(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [])
|
|
|
|
def unavailable():
|
|
raise capacity_module.RunnerCapacityError("runner_capacity_swap_telemetry_unavailable")
|
|
|
|
monkeypatch.setattr(service, "_strict_swap_pages", unavailable)
|
|
result = service.host_snapshot(now=100.0)
|
|
assert result["status"] == "unavailable"
|
|
assert result["host"] is None
|
|
|
|
|
|
@pytest.mark.parametrize("metric", ["cpu_count", "getloadavg", "sysconf", "memory", "cache"])
|
|
def test_host_snapshot_rejects_invalid_global_metrics(capacity_module, monkeypatch, metric):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)])
|
|
if metric == "getloadavg":
|
|
monkeypatch.setattr(capacity_module.os, metric, lambda: (float("nan"), 0, 0))
|
|
elif metric in {"cpu_count", "sysconf"}:
|
|
monkeypatch.setattr(capacity_module.os, metric, lambda *_args: 0)
|
|
else:
|
|
monkeypatch.setattr(service, "_mem_available_bytes" if metric == "memory" else "_cache_free_bytes", lambda: True)
|
|
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0)
|
|
assert result["status"] == "unavailable"
|
|
assert result["host"] is None
|
|
|
|
|
|
def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True):
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_burst_container",
|
|
lambda: {
|
|
"name": "itworx-validation-runner-burst-1",
|
|
"running": running,
|
|
"health": "healthy" if running else "none",
|
|
"startedAt": "",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_runner_registration",
|
|
lambda: {
|
|
"id": 44,
|
|
"status": "online" if running else "offline",
|
|
"busy": busy,
|
|
"disabled": disabled,
|
|
"labels": ["linux-validation", "ubuntu-latest"],
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_read_state", lambda: {})
|
|
monkeypatch.setattr(service, "_write_state", lambda _state: None)
|
|
monkeypatch.setattr(service, "_record", lambda _event: None)
|
|
monkeypatch.setattr(service, "_host_guard_state", lambda: ("clear", ""))
|
|
|
|
|
|
def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=False, disabled=True)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_queue",
|
|
lambda: {"depth": 2, "oldestSeconds": 20},
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_host",
|
|
lambda _previous, now: {
|
|
"cpuCount": 20,
|
|
"loadOne": 5.0,
|
|
"loadRatio": 0.25,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
},
|
|
)
|
|
commands = []
|
|
disabled = []
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20, check=True: commands.append(args) or "",
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_set_disabled",
|
|
lambda runner_id, value: disabled.append((runner_id, value)),
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["action"] == "start"
|
|
assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]]
|
|
assert disabled == [(44, False)]
|
|
|
|
|
|
def test_cycle_drains_busy_burst_under_host_pressure(capacity_module, monkeypatch):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=True, busy=True, disabled=False)
|
|
monkeypatch.setattr(service, "_queue", lambda: {"depth": 3, "oldestSeconds": 90})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_host",
|
|
lambda _previous, now: {
|
|
"cpuCount": 20,
|
|
"loadOne": 19.0,
|
|
"loadRatio": 0.95,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
},
|
|
)
|
|
disabled = []
|
|
commands = []
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_set_disabled",
|
|
lambda runner_id, value: disabled.append((runner_id, value)),
|
|
)
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20, check=True: commands.append(args) or "",
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["action"] == "drain"
|
|
assert disabled == [(44, True)]
|
|
assert commands == []
|
|
|
|
|
|
def test_cycle_stops_only_disabled_idle_runner_without_active_assignment(
|
|
capacity_module, monkeypatch
|
|
):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=True, busy=False, disabled=True)
|
|
monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1})
|
|
monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_host",
|
|
lambda _previous, now: {
|
|
"cpuCount": 20,
|
|
"loadOne": 5.0,
|
|
"loadRatio": 0.25,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: False)
|
|
commands = []
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20, check=True: commands.append(args) or "",
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["action"] == "stop"
|
|
assert commands == [
|
|
["docker", "stop", "--time", "30", "itworx-validation-runner-burst-1"]
|
|
]
|
|
|
|
|
|
def test_cycle_never_stops_when_gitea_still_has_active_assignment(
|
|
capacity_module, monkeypatch
|
|
):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=True, busy=False, disabled=True)
|
|
monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1})
|
|
monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_host",
|
|
lambda _previous, now: {
|
|
"cpuCount": 20,
|
|
"loadOne": 5.0,
|
|
"loadRatio": 0.25,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: True)
|
|
commands = []
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_run",
|
|
lambda args, timeout=20, check=True: commands.append(args) or "",
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["result"] == "stop_deferred_not_proven_idle"
|
|
assert commands == []
|
|
|
|
|
|
def test_paused_host_guard_blocks_burst_start_before_docker_or_gitea_write(
|
|
capacity_module, monkeypatch
|
|
):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=False, disabled=True)
|
|
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_host",
|
|
lambda _previous, now: {
|
|
"cpuCount": 20,
|
|
"loadOne": 2.0,
|
|
"loadRatio": 0.1,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
},
|
|
)
|
|
monkeypatch.setattr(service, "_host_guard_state", lambda: ("paused", "guard"))
|
|
commands = []
|
|
disabled = []
|
|
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_set_disabled",
|
|
lambda runner_id, value: disabled.append((runner_id, value)),
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["action"] == "start"
|
|
assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused"
|
|
assert commands == []
|
|
assert disabled == []
|
|
|
|
|
|
def test_guard_is_rechecked_after_docker_start_before_gitea_enable(
|
|
capacity_module, monkeypatch
|
|
):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=False, disabled=True)
|
|
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_host",
|
|
lambda _previous, now: {
|
|
"cpuCount": 20,
|
|
"loadOne": 2.0,
|
|
"loadRatio": 0.1,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
},
|
|
)
|
|
guard_states = iter([("clear", ""), ("paused", "guard")])
|
|
monkeypatch.setattr(service, "_host_guard_state", lambda: next(guard_states))
|
|
commands = []
|
|
disabled = []
|
|
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_set_disabled",
|
|
lambda runner_id, value: disabled.append((runner_id, value)),
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused"
|
|
assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]]
|
|
assert disabled == []
|
|
|
|
|
|
def test_fresh_pressure_is_rechecked_before_burst_start(
|
|
capacity_module, monkeypatch
|
|
):
|
|
service = capacity_module.RunnerCapacityController()
|
|
_wire_common(service, monkeypatch, running=False, disabled=True)
|
|
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
|
|
samples = iter(
|
|
[
|
|
{
|
|
"loadRatio": 0.1,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapCritical": False,
|
|
},
|
|
{
|
|
"loadRatio": 0.95,
|
|
"availableMemoryBytes": 20 * 1024**3,
|
|
"cacheFreeBytes": 100 * 1024**3,
|
|
"swapCritical": False,
|
|
},
|
|
]
|
|
)
|
|
|
|
def sample(_previous, now):
|
|
values = next(samples)
|
|
return {
|
|
"cpuCount": 20,
|
|
"loadOne": values["loadRatio"] * 20,
|
|
**values,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapSampleAt": now,
|
|
"swapInPages": 0,
|
|
"swapOutPages": 0,
|
|
}
|
|
|
|
monkeypatch.setattr(service, "_host", sample)
|
|
commands = []
|
|
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
|
|
disabled = []
|
|
monkeypatch.setattr(
|
|
service,
|
|
"_set_disabled",
|
|
lambda runner_id, value: disabled.append((runner_id, value)),
|
|
)
|
|
|
|
result = service.cycle()
|
|
|
|
assert result["decision"]["result"] == "failed:runner_capacity_pressure_critical"
|
|
assert commands == []
|
|
assert disabled == []
|