Publish curated Unraid-AppOps-Gateway source from a506550ef83ef2d81651e37c98a89f7ee330c730
This commit is contained in:
Public Source Publisher committed
1 parent
2facb8de34
commit
767e66921e
68 files changed
+15924
-313
No files matched your search
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -14,6 +16,9 @@ CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py"
|
||||
|
||||
@pytest.fixture
|
||||
def capacity_module(monkeypatch):
|
||||
if sys.platform == "win32":
|
||||
# The daemon lock is not exercised by these in-process cycle tests.
|
||||
monkeypatch.setitem(sys.modules, "fcntl", types.ModuleType("fcntl"))
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER_ROOT))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_runner_capacity_controller_test", CONTROLLER_PATH
|
||||
@@ -25,6 +30,137 @@ def capacity_module(monkeypatch):
|
||||
return module
|
||||
|
||||
|
||||
def _wire_read_only_host(service, module, monkeypatch, samples):
|
||||
counters = iter(samples)
|
||||
monkeypatch.setattr(service, "_strict_swap_pages", lambda: next(counters))
|
||||
monkeypatch.setattr(module.os, "cpu_count", lambda: 20)
|
||||
monkeypatch.setattr(module.os, "getloadavg", lambda: (4.0, 3.0, 2.0), raising=False)
|
||||
monkeypatch.setattr(module.os, "sysconf", lambda _name: 4096, raising=False)
|
||||
monkeypatch.setattr(service, "_mem_available_bytes", lambda: 16 * 1024**3)
|
||||
monkeypatch.setattr(service, "_cache_free_bytes", lambda: 32 * 1024**3)
|
||||
|
||||
def forbidden(*_args, **_kwargs):
|
||||
pytest.fail("read-only host sampling touched burst inventory or a mutation")
|
||||
|
||||
for name in ("_burst_container", "_runner_registration", "_gitea_request",
|
||||
"_run", "_read_state", "_write_state", "_record"):
|
||||
monkeypatch.setattr(service, name, forbidden)
|
||||
|
||||
|
||||
def test_host_snapshot_measures_swap_without_optional_burst(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [(50, 70)])
|
||||
prior = {"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}
|
||||
preserved = dict(prior)
|
||||
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("baseline already available"))
|
||||
|
||||
result = service.host_snapshot(prior, now=100.0)
|
||||
|
||||
assert result["status"] == "observed"
|
||||
assert result["host"]["loadRatio"] == 0.2
|
||||
assert result["host"]["swapInKbps"] == 200.0
|
||||
assert result["host"]["swapOutKbps"] == 280.0
|
||||
assert result["host"]["swapCritical"] is False
|
||||
assert prior == preserved
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prior", [None, {"swapSampleAt": 0}, {
|
||||
"swapSampleAt": 1.0, "swapInPages": 5, "swapOutPages": 5,
|
||||
}, {"swapSampleAt": float("nan"), "swapInPages": 5, "swapOutPages": 5}])
|
||||
def test_host_snapshot_takes_two_real_samples_for_missing_or_stale_baseline(
|
||||
capacity_module, monkeypatch, prior
|
||||
):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (266, 532)])
|
||||
elapsed = iter([4.0, 5.0])
|
||||
sleeps = []
|
||||
monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed))
|
||||
monkeypatch.setattr(capacity_module.time, "sleep", sleeps.append)
|
||||
|
||||
result = service.host_snapshot(prior, now=100.0)
|
||||
|
||||
assert result["status"] == "observed"
|
||||
assert result["host"]["swapInKbps"] == 1024.0
|
||||
assert result["host"]["swapOutKbps"] == 2048.0
|
||||
assert result["host"]["swapSampleAt"] == 101.0
|
||||
assert sleeps == [1.0]
|
||||
|
||||
|
||||
def test_host_snapshot_reports_unavailable_if_swap_counter_resets(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20), (9, 20)])
|
||||
elapsed = iter([4.0, 5.0])
|
||||
monkeypatch.setattr(capacity_module.time, "monotonic", lambda: next(elapsed))
|
||||
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: None)
|
||||
|
||||
result = service.host_snapshot(now=100.0)
|
||||
|
||||
assert result["status"] == "unavailable"
|
||||
assert result["reason"] == "runner_capacity_swap_baseline_unavailable"
|
||||
assert result["host"] is None
|
||||
|
||||
|
||||
def test_host_snapshot_rejects_reset_against_recent_protected_baseline(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [(9, 20), (10, 21)])
|
||||
monkeypatch.setattr(capacity_module.time, "sleep", lambda _seconds: pytest.fail("recent counter reset must fail closed"))
|
||||
|
||||
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 10, "swapOutPages": 20}, now=100.0)
|
||||
|
||||
assert result["status"] == "unavailable"
|
||||
assert result["reason"] == "runner_capacity_swap_baseline_unavailable"
|
||||
assert result["host"] is None
|
||||
|
||||
|
||||
def test_host_snapshot_handles_absent_load_tuple(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)])
|
||||
monkeypatch.setattr(capacity_module.os, "getloadavg", lambda: ())
|
||||
|
||||
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0)
|
||||
|
||||
assert result["status"] == "unavailable"
|
||||
assert result["host"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["", "pswpin 1\n", "pswpin -1\npswpout 2\n",
|
||||
"pswpin 1\npswpin 2\npswpout 3\n", "pswpin invalid\npswpout 2\n", "x" * 65537])
|
||||
def test_strict_swap_reader_rejects_missing_or_invalid_counters(capacity_module, monkeypatch, raw):
|
||||
monkeypatch.setattr(capacity_module, "Path", lambda _path: types.SimpleNamespace(
|
||||
open=lambda **_kwargs: io.StringIO(raw)
|
||||
))
|
||||
with pytest.raises(capacity_module.RunnerCapacityError, match="swap_telemetry_unavailable"):
|
||||
capacity_module.RunnerCapacityController._strict_swap_pages()
|
||||
|
||||
|
||||
def test_host_snapshot_does_not_turn_missing_swap_into_zero(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [])
|
||||
|
||||
def unavailable():
|
||||
raise capacity_module.RunnerCapacityError("runner_capacity_swap_telemetry_unavailable")
|
||||
|
||||
monkeypatch.setattr(service, "_strict_swap_pages", unavailable)
|
||||
result = service.host_snapshot(now=100.0)
|
||||
assert result["status"] == "unavailable"
|
||||
assert result["host"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metric", ["cpu_count", "getloadavg", "sysconf", "memory", "cache"])
|
||||
def test_host_snapshot_rejects_invalid_global_metrics(capacity_module, monkeypatch, metric):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_read_only_host(service, capacity_module, monkeypatch, [(10, 20)])
|
||||
if metric == "getloadavg":
|
||||
monkeypatch.setattr(capacity_module.os, metric, lambda: (float("nan"), 0, 0))
|
||||
elif metric in {"cpu_count", "sysconf"}:
|
||||
monkeypatch.setattr(capacity_module.os, metric, lambda *_args: 0)
|
||||
else:
|
||||
monkeypatch.setattr(service, "_mem_available_bytes" if metric == "memory" else "_cache_free_bytes", lambda: True)
|
||||
result = service.host_snapshot({"swapSampleAt": 99.0, "swapInPages": 0, "swapOutPages": 0}, now=100.0)
|
||||
assert result["status"] == "unavailable"
|
||||
assert result["host"] is None
|
||||
|
||||
|
||||
def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True):
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
@@ -50,6 +186,7 @@ def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=Tr
|
||||
monkeypatch.setattr(service, "_read_state", lambda: {})
|
||||
monkeypatch.setattr(service, "_write_state", lambda _state: None)
|
||||
monkeypatch.setattr(service, "_record", lambda _event: None)
|
||||
monkeypatch.setattr(service, "_host_guard_state", lambda: ("clear", ""))
|
||||
|
||||
|
||||
def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch):
|
||||
@@ -218,3 +355,141 @@ def test_cycle_never_stops_when_gitea_still_has_active_assignment(
|
||||
|
||||
assert result["decision"]["result"] == "stop_deferred_not_proven_idle"
|
||||
assert commands == []
|
||||
|
||||
|
||||
def test_paused_host_guard_blocks_burst_start_before_docker_or_gitea_write(
|
||||
capacity_module, monkeypatch
|
||||
):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=False, disabled=True)
|
||||
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_host",
|
||||
lambda _previous, now: {
|
||||
"cpuCount": 20,
|
||||
"loadOne": 2.0,
|
||||
"loadRatio": 0.1,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapCritical": False,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(service, "_host_guard_state", lambda: ("paused", "guard"))
|
||||
commands = []
|
||||
disabled = []
|
||||
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_set_disabled",
|
||||
lambda runner_id, value: disabled.append((runner_id, value)),
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["action"] == "start"
|
||||
assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused"
|
||||
assert commands == []
|
||||
assert disabled == []
|
||||
|
||||
|
||||
def test_guard_is_rechecked_after_docker_start_before_gitea_enable(
|
||||
capacity_module, monkeypatch
|
||||
):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=False, disabled=True)
|
||||
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_host",
|
||||
lambda _previous, now: {
|
||||
"cpuCount": 20,
|
||||
"loadOne": 2.0,
|
||||
"loadRatio": 0.1,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapCritical": False,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
},
|
||||
)
|
||||
guard_states = iter([("clear", ""), ("paused", "guard")])
|
||||
monkeypatch.setattr(service, "_host_guard_state", lambda: next(guard_states))
|
||||
commands = []
|
||||
disabled = []
|
||||
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_set_disabled",
|
||||
lambda runner_id, value: disabled.append((runner_id, value)),
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["result"] == "failed:runner_capacity_host_guard_paused"
|
||||
assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]]
|
||||
assert disabled == []
|
||||
|
||||
|
||||
def test_fresh_pressure_is_rechecked_before_burst_start(
|
||||
capacity_module, monkeypatch
|
||||
):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=False, disabled=True)
|
||||
monkeypatch.setattr(service, "_queue", lambda: {"depth": 2, "oldestSeconds": 60})
|
||||
samples = iter(
|
||||
[
|
||||
{
|
||||
"loadRatio": 0.1,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapCritical": False,
|
||||
},
|
||||
{
|
||||
"loadRatio": 0.95,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapCritical": False,
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
def sample(_previous, now):
|
||||
values = next(samples)
|
||||
return {
|
||||
"cpuCount": 20,
|
||||
"loadOne": values["loadRatio"] * 20,
|
||||
**values,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(service, "_host", sample)
|
||||
commands = []
|
||||
monkeypatch.setattr(service, "_run", lambda args, **_kwargs: commands.append(args) or "")
|
||||
disabled = []
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_set_disabled",
|
||||
lambda runner_id, value: disabled.append((runner_id, value)),
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["result"] == "failed:runner_capacity_pressure_critical"
|
||||
assert commands == []
|
||||
assert disabled == []
|
||||
Reference in new issue
Block a user