Unraid-AppOps-Gateway-Public/backend/tests/test_runner_capacity_controller.py

from __future__ import annotations

import importlib.util
import sys
from pathlib import Path

import pytest


ROOT = Path(__file__).resolve().parents[2]
CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker"
CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py"


@pytest.fixture
def capacity_module(monkeypatch):
    monkeypatch.syspath_prepend(str(CONTROLLER_ROOT))
    spec = importlib.util.spec_from_file_location(
        "appops_runner_capacity_controller_test", CONTROLLER_PATH
    )
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = module
    spec.loader.exec_module(module)
    return module


def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True):
    monkeypatch.setattr(
        service,
        "_burst_container",
        lambda: {
            "name": "itworx-validation-runner-burst-1",
            "running": running,
            "health": "healthy" if running else "none",
            "startedAt": "",
        },
    )
    monkeypatch.setattr(
        service,
        "_runner_registration",
        lambda: {
            "id": 44,
            "status": "online" if running else "offline",
            "busy": busy,
            "disabled": disabled,
            "labels": ["linux-validation", "ubuntu-latest"],
        },
    )
    monkeypatch.setattr(service, "_read_state", lambda: {})
    monkeypatch.setattr(service, "_write_state", lambda _state: None)
    monkeypatch.setattr(service, "_record", lambda _event: None)


def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch):
    service = capacity_module.RunnerCapacityController()
    _wire_common(service, monkeypatch, running=False, disabled=True)
    monkeypatch.setattr(
        service,
        "_queue",
        lambda: {"depth": 2, "oldestSeconds": 20},
    )
    monkeypatch.setattr(
        service,
        "_host",
        lambda _previous, now: {
            "cpuCount": 20,
            "loadOne": 5.0,
            "loadRatio": 0.25,
            "availableMemoryBytes": 20 * 1024**3,
            "cacheFreeBytes": 100 * 1024**3,
            "swapInKbps": 0.0,
            "swapOutKbps": 0.0,
            "swapPressureSamples": 0,
            "swapCritical": False,
            "swapSampleAt": now,
            "swapInPages": 0,
            "swapOutPages": 0,
        },
    )
    commands = []
    disabled = []
    monkeypatch.setattr(
        service,
        "_run",
        lambda args, timeout=20, check=True: commands.append(args) or "",
    )
    monkeypatch.setattr(
        service,
        "_set_disabled",
        lambda runner_id, value: disabled.append((runner_id, value)),
    )

    result = service.cycle()

    assert result["decision"]["action"] == "start"
    assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]]
    assert disabled == [(44, False)]


def test_cycle_drains_busy_burst_under_host_pressure(capacity_module, monkeypatch):
    service = capacity_module.RunnerCapacityController()
    _wire_common(service, monkeypatch, running=True, busy=True, disabled=False)
    monkeypatch.setattr(service, "_queue", lambda: {"depth": 3, "oldestSeconds": 90})
    monkeypatch.setattr(
        service,
        "_host",
        lambda _previous, now: {
            "cpuCount": 20,
            "loadOne": 19.0,
            "loadRatio": 0.95,
            "availableMemoryBytes": 20 * 1024**3,
            "cacheFreeBytes": 100 * 1024**3,
            "swapInKbps": 0.0,
            "swapOutKbps": 0.0,
            "swapPressureSamples": 0,
            "swapCritical": False,
            "swapSampleAt": now,
            "swapInPages": 0,
            "swapOutPages": 0,
        },
    )
    disabled = []
    commands = []
    monkeypatch.setattr(
        service,
        "_set_disabled",
        lambda runner_id, value: disabled.append((runner_id, value)),
    )
    monkeypatch.setattr(
        service,
        "_run",
        lambda args, timeout=20, check=True: commands.append(args) or "",
    )

    result = service.cycle()

    assert result["decision"]["action"] == "drain"
    assert disabled == [(44, True)]
    assert commands == []


def test_cycle_stops_only_disabled_idle_runner_without_active_assignment(
    capacity_module, monkeypatch
):
    service = capacity_module.RunnerCapacityController()
    _wire_common(service, monkeypatch, running=True, busy=False, disabled=True)
    monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1})
    monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None})
    monkeypatch.setattr(
        service,
        "_host",
        lambda _previous, now: {
            "cpuCount": 20,
            "loadOne": 5.0,
            "loadRatio": 0.25,
            "availableMemoryBytes": 20 * 1024**3,
            "cacheFreeBytes": 100 * 1024**3,
            "swapInKbps": 0.0,
            "swapOutKbps": 0.0,
            "swapPressureSamples": 0,
            "swapCritical": False,
            "swapSampleAt": now,
            "swapInPages": 0,
            "swapOutPages": 0,
        },
    )
    monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: False)
    commands = []
    monkeypatch.setattr(
        service,
        "_run",
        lambda args, timeout=20, check=True: commands.append(args) or "",
    )

    result = service.cycle()

    assert result["decision"]["action"] == "stop"
    assert commands == [
        ["docker", "stop", "--time", "30", "itworx-validation-runner-burst-1"]
    ]


def test_cycle_never_stops_when_gitea_still_has_active_assignment(
    capacity_module, monkeypatch
):
    service = capacity_module.RunnerCapacityController()
    _wire_common(service, monkeypatch, running=True, busy=False, disabled=True)
    monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1})
    monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None})
    monkeypatch.setattr(
        service,
        "_host",
        lambda _previous, now: {
            "cpuCount": 20,
            "loadOne": 5.0,
            "loadRatio": 0.25,
            "availableMemoryBytes": 20 * 1024**3,
            "cacheFreeBytes": 100 * 1024**3,
            "swapInKbps": 0.0,
            "swapOutKbps": 0.0,
            "swapPressureSamples": 0,
            "swapCritical": False,
            "swapSampleAt": now,
            "swapInPages": 0,
            "swapOutPages": 0,
        },
    )
    monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: True)
    commands = []
    monkeypatch.setattr(
        service,
        "_run",
        lambda args, timeout=20, check=True: commands.append(args) or "",
    )

    result = service.cycle()

    assert result["decision"]["result"] == "stop_deferred_not_proven_idle"
    assert commands == []