from __future__ import annotations import importlib.util import io import sys import types from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker" CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py" @pytest.fixture def capacity_module(monkeypatch): if sys.platform == "win32": # The daemon lock is not exercised by these in-process cycle tests. monkeypatch.setitem(sys.modules, "fcntl", types.ModuleType("fcntl")) monkeypatch.syspath_prepend(str(CONTROLLER_ROOT)) spec = importlib.util.spec_from_file_location( "appops_runner_capacity_controller_test", CONTROLLER_PATH ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _wire_read_only_host(service, module, monkeypatch, samples): counters = iter(samples) monkeypatch.setattr(service, "_strict_swap_pages", lambda: next(counters)) monkeypatch.setattr(module.os, "cpu_count", lambda: 20) monkeypatch.setattr(module.os, "getloadavg", lambda: (4.0, 3.0, 2.0), raising=False) monkeypatch.setattr(module.os, "sysconf", lambda _name: 4096, raising=False) monkeypatch.setattr(service, "_mem_available_bytes", lambda: 16 * 1024**3) monkeypatch.setattr(service, "_cache_free_bytes", lambda: 32 * 1024**3) def forbidden(*_args, **_kwargs): pytest.fail("read-only host sampling touched burst inventory or a mutation") for name in ("_burst_container", "_runner_registration", "_gitea_request", "_run", "_read_state", "_write_state", "_record"): monkeypatch.setattr(service, name, forbidden) def test_host_snapshot_measures_swap_without_optional_burst(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, [(50, 70)]) prior = {"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0} preserved = dict(prior) monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("baseline already available")) result = service.host_snapshot(prior, now=100.0) assert result["status"] == "observed" assert result["host"]["loadRatio"] == 0.2 assert result["host"]["swapInKbps"] == 200.0 assert result["host"]["swapOutKbps"] == 280.0 assert result["host"]["swapCritical"] is False assert prior == preserved @pytest.mark.parametrize("prior", [None, {"swapSampleAt": 0}, { "swapSampleAt": 1.0, "swapInPages": 5, "swapOutPages": 5, }, {"swapSampleAt": float("nan"), "swapInPages": 5, "swapOutPages": 5}]) def test_host_snapshot_takes_two_real_samples_for_missing_or_stale_baseline( capacity_module, monkeypatch, prior ): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (266, 532)]) elapsed = iter([4.0, 5.0]) sleeps = [] monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed)) monkeypatch.setattr(capacity_module.time, "sleep", sleeps.append) result = service.host_snapshot(prior, now=100.0) assert result["status"] == "observed" assert result["host"]["swapInKbps"] == 1024.0 assert result["host"]["swapOutKbps"] == 2048.0 assert result["host"]["swapSampleAt"] == 101.0 assert sleeps == [1.0] def test_host_snapshot_reports_unavailable_if_swap_counter_resets(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (9, 20)]) elapsed = iter([4.0, 5.0]) monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed)) monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: None) result = service.host_snapshot(now=100.0) assert result["status"] == "unavailable" assert result["reason"] == "runner_capacity_swap_baseline_unavailable" assert result["host"] is None def test_host_snapshot_rejects_reset_against_recent_protected_baseline(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, [(9, 20), (10, 21)]) monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("recent counter reset must fail closed")) result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 10, "swapOutPages": 20}, now=100.0) assert result["status"] == "unavailable" assert result["reason"] == "runner_capacity_swap_baseline_unavailable" assert result["host"] is None def test_host_snapshot_handles_absent_load_tuple(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)]) monkeypatch.setattr(capacity_module.os, "getloadavg", lambda: ()) result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0) assert result["status"] == "unavailable" assert result["host"] is None @pytest.mark.parametrize("raw", ["", "pswpin 1\n", "pswpin -1\npswpout 2\n", "pswpin 1\npswpin 2\npswpout 3\n", "pswpin invalid\npswpout 2\n", "x" * 65537]) def test_strict_swap_reader_rejects_missing_or_invalid_counters(capacity_module, monkeypatch, raw): monkeypatch.setattr(capacity_module, "Path", lambda _path: types.SimpleNamespace( open=lambda **_kwargs: io.StringIO(raw) )) with pytest.raises(capacity_module.RunnerCapacityError, match="swap_telemetry_unavailable"): capacity_module.RunnerCapacityController._strict_swap_pages() def test_host_snapshot_does_not_turn_missing_swap_into_zero(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, []) def unavailable(): raise capacity_module.RunnerCapacityError("runner_capacity_swap_telemetry_unavailable") monkeypatch.setattr(service, "_strict_swap_pages", unavailable) result = service.host_snapshot(now=100.0) assert result["status"] == "unavailable" assert result["host"] is None @pytest.mark.parametrize("metric", ["cpu_count", "getloadavg", "sysconf", "memory", "cache"]) def test_host_snapshot_rejects_invalid_global_metrics(capacity_module, monkeypatch, metric): service = capacity_module.RunnerCapacityController() _wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)]) if metric == "getloadavg": monkeypatch.setattr(capacity_module.os, metric, lambda: (float("nan"), 0, 0)) elif metric in {"cpu_count", "sysconf"}: monkeypatch.setattr(capacity_module.os, metric, lambda *_args: 0) else: monkeypatch.setattr(service, "_mem_available_bytes" if metric == "memory" else "_cache_free_bytes", lambda: True) result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0) assert result["status"] == "unavailable" assert result["host"] is None def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True): monkeypatch.setattr( service, "_burst_container", lambda: { "name": "itworx-validation-runner-burst-1", "running": running, "health": "healthy" if running else "none", "startedAt": "", }, ) monkeypatch.setattr( service, "_runner_registration", lambda: { "id": 44, "status": "online" if running else "offline", "busy": busy, "disabled": disabled, "labels": ["linux-validation", "ubuntu-latest"], }, ) monkeypatch.setattr(service, "_read_state", lambda: {}) monkeypatch.setattr(service, "_write_state", lambda _state: None) monkeypatch.setattr(service, "_record", lambda _event: None) monkeypatch.setattr(service, "_host_guard_state", lambda: ("clear", "")) def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=False, disabled=True) monkeypatch.setattr( service, "_queue", lambda: {"depth": 2, "oldestSeconds": 20}, ) monkeypatch.setattr( service, "_host", lambda _previous, now: { "cpuCount": 20, "loadOne": 5.0, "loadRatio": 0.25, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, }, ) commands = [] disabled = [] monkeypatch.setattr( service, "_run", lambda args, timeout=20, check=True: commands.append(args) or "", ) monkeypatch.setattr( service, "_set_disabled", lambda runner_id, value: disabled.append((runner_id, value)), ) result = service.cycle() assert result["decision"]["action"] == "start" assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]] assert disabled == [(44, False)] def test_cycle_drains_busy_burst_under_host_pressure(capacity_module, monkeypatch): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=True, busy=True, disabled=False) monkeypatch.setattr(service, "_queue", lambda: {"depth": 3, "oldestSeconds": 90}) monkeypatch.setattr( service, "_host", lambda _previous, now: { "cpuCount": 20, "loadOne": 19.0, "loadRatio": 0.95, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, }, ) disabled = [] commands = [] monkeypatch.setattr( service, "_set_disabled", lambda runner_id, value: disabled.append((runner_id, value)), ) monkeypatch.setattr( service, "_run", lambda args, timeout=20, check=True: commands.append(args) or "", ) result = service.cycle() assert result["decision"]["action"] == "drain" assert disabled == [(44, True)] assert commands == [] def test_cycle_stops_only_disabled_idle_runner_without_active_assignment( capacity_module, monkeypatch ): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=True, busy=False, disabled=True) monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1}) monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None}) monkeypatch.setattr( service, "_host", lambda _previous, now: { "cpuCount": 20, "loadOne": 5.0, "loadRatio": 0.25, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, }, ) monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: False) commands = [] monkeypatch.setattr( service, "_run", lambda args, timeout=20, check=True: commands.append(args) or "", ) result = service.cycle() assert result["decision"]["action"] == "stop" assert commands == [ ["docker", "stop", "--time", "30", "itworx-validation-runner-burst-1"] ] def test_cycle_never_stops_when_gitea_still_has_active_assignment( capacity_module, monkeypatch ): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=True, busy=False, disabled=True) monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1}) monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None}) monkeypatch.setattr( service, "_host", lambda _previous, now: { "cpuCount": 20, "loadOne": 5.0, "loadRatio": 0.25, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, }, ) monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: True) commands = [] monkeypatch.setattr( service, "_run", lambda args, timeout=20, check=True: commands.append(args) or "", ) result = service.cycle() assert result["decision"]["result"] == "stop_deferred_not_proven_idle" assert commands == [] def test_paused_host_guard_blocks_burst_start_before_docker_or_gitea_write( capacity_module, monkeypatch ): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=False, disabled=True) monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60}) monkeypatch.setattr( service, "_host", lambda _previous, now: { "cpuCount": 20, "loadOne": 2.0, "loadRatio": 0.1, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, }, ) monkeypatch.setattr(service, "_host_guard_state", lambda: ("paused", "guard")) commands = [] disabled = [] monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "") monkeypatch.setattr( service, "_set_disabled", lambda runner_id, value: disabled.append((runner_id, value)), ) result = service.cycle() assert result["decision"]["action"] == "start" assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused" assert commands == [] assert disabled == [] def test_guard_is_rechecked_after_docker_start_before_gitea_enable( capacity_module, monkeypatch ): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=False, disabled=True) monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60}) monkeypatch.setattr( service, "_host", lambda _previous, now: { "cpuCount": 20, "loadOne": 2.0, "loadRatio": 0.1, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, }, ) guard_states = iter([("clear", ""), ("paused", "guard")]) monkeypatch.setattr(service, "_host_guard_state", lambda: next(guard_states)) commands = [] disabled = [] monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "") monkeypatch.setattr( service, "_set_disabled", lambda runner_id, value: disabled.append((runner_id, value)), ) result = service.cycle() assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused" assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]] assert disabled == [] def test_fresh_pressure_is_rechecked_before_burst_start( capacity_module, monkeypatch ): service = capacity_module.RunnerCapacityController() _wire_common(service, monkeypatch, running=False, disabled=True) monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60}) samples = iter( [ { "loadRatio": 0.1, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapCritical": False, }, { "loadRatio": 0.95, "availableMemoryBytes": 20 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapCritical": False, }, ] ) def sample(_previous, now): values = next(samples) return { "cpuCount": 20, "loadOne": values["loadRatio"] * 20, **values, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapSampleAt": now, "swapInPages": 0, "swapOutPages": 0, } monkeypatch.setattr(service, "_host", sample) commands = [] monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "") disabled = [] monkeypatch.setattr( service, "_set_disabled", lambda runner_id, value: disabled.append((runner_id, value)), ) result = service.cycle() assert result["decision"]["result"] == "failed:runner_capacity_pressure_critical" assert commands == [] assert disabled == []