1641 lines
56 KiB
Python
1641 lines
56 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
import time
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from backend.app.services.runner_statistics import sanitize_statistics
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker"
|
|
|
|
|
|
def _load(name: str, path: Path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def controller_module(monkeypatch):
|
|
profiles = _load(
|
|
"appops_runner_profiles_for_admission_controller_test",
|
|
CONTROLLER_ROOT / "runner_profiles.py",
|
|
)
|
|
monkeypatch.setitem(sys.modules, "runner_profiles", profiles)
|
|
policy = _load(
|
|
"appops_runner_admission_policy_for_controller_test",
|
|
CONTROLLER_ROOT / "runner_admission_policy.py",
|
|
)
|
|
monkeypatch.setitem(sys.modules, "runner_admission_policy", policy)
|
|
return _load(
|
|
"appops_runner_admission_controller_test",
|
|
CONTROLLER_ROOT / "runner_admission_controller.py",
|
|
)
|
|
|
|
|
|
def _public_observation(module, now: float):
|
|
rows = {}
|
|
containers = {}
|
|
profile_runner_ids = {}
|
|
runner_profile_ids = {}
|
|
active = {}
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
runner_id = 100 + index
|
|
rows[profile.container_name] = {
|
|
"id": runner_id,
|
|
"name": profile.container_name,
|
|
"status": "offline",
|
|
"busy": False,
|
|
"disabled": True,
|
|
"labels": {profile.runner_label, *profile.legacy_labels},
|
|
}
|
|
containers[profile.container_name] = {"running": False, "health": "none"}
|
|
profile_runner_ids[profile.id] = runner_id
|
|
runner_profile_ids[runner_id] = profile.id
|
|
active[runner_id] = []
|
|
return {
|
|
"specialization": "active",
|
|
"at": now,
|
|
"runnerRows": rows,
|
|
"profileRunnerIds": profile_runner_ids,
|
|
"runnerProfileIds": runner_profile_ids,
|
|
"containers": containers,
|
|
"claims": [],
|
|
"queue": [
|
|
{
|
|
"id": "private-job-id",
|
|
"labels": ["linux-validation"],
|
|
"queued_at": now - 44,
|
|
}
|
|
],
|
|
"activeJobs": active,
|
|
"resourceProof": {
|
|
"ready": True,
|
|
"effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI,
|
|
"effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB,
|
|
"activeClaims": 0,
|
|
"heavyClaims": 0,
|
|
"runningRunnerContainers": 0,
|
|
"cgroups": {},
|
|
"digest": "a" * 64,
|
|
},
|
|
"resourceReason": None,
|
|
"hostReason": None,
|
|
"capacity": {
|
|
"host": {
|
|
"availableMemoryBytes": 12 * 1024**3,
|
|
"cacheFreeBytes": 40 * 1024**3,
|
|
"loadRatio": 0.25,
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _install_gitea_observation(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
*,
|
|
queued_rows=(),
|
|
pending_rows=(),
|
|
active_rows=(),
|
|
):
|
|
runner_rows = {}
|
|
by_id = {}
|
|
profile_runner_ids = {}
|
|
runner_profile_ids = {}
|
|
containers = {}
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
runner_id = 100 + index
|
|
row = {
|
|
"id": runner_id,
|
|
"name": profile.container_name,
|
|
"status": "offline",
|
|
"busy": False,
|
|
"disabled": True,
|
|
"labels": {profile.runner_label, *profile.legacy_labels},
|
|
}
|
|
runner_rows[profile.container_name] = row
|
|
by_id[runner_id] = row
|
|
profile_runner_ids[profile.id] = runner_id
|
|
runner_profile_ids[runner_id] = profile.id
|
|
containers[profile.container_name] = {
|
|
"id": f"{index:064x}",
|
|
"running": False,
|
|
"health": "none",
|
|
}
|
|
containers[module.DIND_NAME] = {
|
|
"id": "f" * 64,
|
|
"running": True,
|
|
"health": "healthy",
|
|
}
|
|
proof = {
|
|
"ready": True,
|
|
"effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI,
|
|
"effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB,
|
|
"activeClaims": 0,
|
|
"heavyClaims": 0,
|
|
"runningRunnerContainers": 0,
|
|
"cgroups": {},
|
|
"digest": "e" * 64,
|
|
}
|
|
|
|
def job_rows(status):
|
|
if status == "queued":
|
|
return list(queued_rows), True
|
|
if status == "pending":
|
|
return list(pending_rows), True
|
|
if status == "in_progress":
|
|
return list(active_rows), True
|
|
return [], True
|
|
|
|
monkeypatch.setattr(controller, "_specialization_state", lambda: "active")
|
|
monkeypatch.setattr(controller, "_runner_inventory", lambda: [])
|
|
monkeypatch.setattr(controller, "_normalized_runner_rows", lambda _rows: (runner_rows, by_id))
|
|
monkeypatch.setattr(controller, "_job_rows", job_rows)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_active_claims",
|
|
lambda *_args: ([], profile_runner_ids, runner_profile_ids),
|
|
)
|
|
monkeypatch.setattr(controller, "_container", lambda name: containers.get(name))
|
|
monkeypatch.setattr(controller, "_verify_profile_task_images", lambda *_args: True)
|
|
monkeypatch.setattr(controller, "_resource_attestation", lambda *_args: proof)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_host_evidence",
|
|
lambda: (
|
|
"clear",
|
|
None,
|
|
{
|
|
"host": {
|
|
"availableMemoryBytes": 8 * 1024**3,
|
|
"cacheFreeBytes": 20 * 1024**3,
|
|
"loadRatio": 0.25,
|
|
}
|
|
},
|
|
),
|
|
)
|
|
|
|
|
|
def _lease_fixture(module, *, enabled=("backend", "frontend"), busy=(), queue=()):
|
|
controller = module.RunnerAdmissionController(clock=lambda: time.time())
|
|
state = module._initial_state()
|
|
rows = {}
|
|
containers = {}
|
|
profile_runner_ids = {}
|
|
active_jobs = {}
|
|
inventory_rows = []
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
runner_id = 100 + index
|
|
is_enabled = profile.id in enabled
|
|
is_busy = profile.id in busy
|
|
labels = [profile.runner_label, *profile.legacy_labels]
|
|
rows[profile.container_name] = {
|
|
"id": runner_id,
|
|
"name": profile.container_name,
|
|
"status": "online" if is_enabled else "offline",
|
|
"busy": is_busy,
|
|
"disabled": not is_enabled,
|
|
"labels": set(labels),
|
|
}
|
|
containers[profile.container_name] = {
|
|
"id": f"{index:064x}",
|
|
"running": is_enabled,
|
|
"health": "healthy" if is_enabled else "none",
|
|
}
|
|
profile_runner_ids[profile.id] = runner_id
|
|
state["registrationIds"][profile.id] = runner_id
|
|
state["profiles"][profile.id].update(
|
|
observed=True,
|
|
enabledByController=is_enabled,
|
|
operatorDisabled=False,
|
|
operatorEnabled=False,
|
|
)
|
|
inventory_rows.append(
|
|
{
|
|
"name": profile.display_name,
|
|
"id": runner_id,
|
|
"status": "online" if is_enabled else "offline",
|
|
"busy": is_busy,
|
|
"disabled": not is_enabled,
|
|
"labels": labels,
|
|
}
|
|
)
|
|
if is_busy:
|
|
active_jobs[runner_id] = [
|
|
{
|
|
"id": f"active-{profile.id}",
|
|
"labels": labels,
|
|
"startedAt": module._utc_iso(),
|
|
"attempt": 1,
|
|
}
|
|
]
|
|
by_id = {row["id"]: row for row in rows.values()}
|
|
claims, _profile_ids, _runner_profiles = module.RunnerAdmissionController._active_claims(
|
|
rows, by_id, active_jobs
|
|
)
|
|
observation = {
|
|
"runnerRows": rows,
|
|
"containers": containers,
|
|
"activeJobs": active_jobs,
|
|
"profileRunnerIds": profile_runner_ids,
|
|
"claims": claims,
|
|
"queue": list(queue),
|
|
}
|
|
return controller, state, observation, inventory_rows, containers
|
|
|
|
|
|
def _raw_job(module, job, *, runner_id=None, active=False):
|
|
row = {
|
|
"id": job["id"],
|
|
"labels": list(job["labels"]),
|
|
"attempt": 1,
|
|
}
|
|
if active:
|
|
row.update(
|
|
runner_id=runner_id,
|
|
started_at=module._utc_iso(),
|
|
status="in_progress",
|
|
)
|
|
else:
|
|
row["created_at"] = module._utc_iso(float(job["queued_at"]))
|
|
return row
|
|
|
|
|
|
def _install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory_rows,
|
|
containers,
|
|
*,
|
|
queue_rows=(),
|
|
active_rows=(),
|
|
saved=None,
|
|
):
|
|
import copy
|
|
|
|
def save(state):
|
|
if saved is not None:
|
|
saved.append(copy.deepcopy(state))
|
|
|
|
monkeypatch.setattr(controller, "_save_state", save)
|
|
|
|
def job_rows(status):
|
|
if status == "queued" or status == "pending":
|
|
return list(queue_rows), True
|
|
if status == "in_progress":
|
|
return list(active_rows), True
|
|
return [], True
|
|
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_job_rows",
|
|
job_rows,
|
|
)
|
|
monkeypatch.setattr(controller, "_runner_inventory", lambda: [dict(row) for row in inventory_rows])
|
|
monkeypatch.setattr(controller, "_container", lambda name: dict(containers[name]))
|
|
monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("clear", ""))
|
|
|
|
def stop_container(args, **_kwargs):
|
|
assert args[:2] == ["docker", "stop"]
|
|
containers[args[2]]["running"] = False
|
|
containers[args[2]]["health"] = "none"
|
|
return ""
|
|
|
|
monkeypatch.setattr(controller, "_docker_run", stop_container)
|
|
|
|
|
|
def _complete_state(module, now: float):
|
|
state = module._initial_state()
|
|
state["terminalHistoryComplete"] = True
|
|
state["terminalHistoryReason"] = None
|
|
state["historySamples"] = [
|
|
{
|
|
"at": module._utc_iso(now - 30),
|
|
"activeJobs": 0,
|
|
"queuedJobs": 2,
|
|
"cpuUsed": 1.25,
|
|
"memoryUsedBytes": 2048,
|
|
},
|
|
{
|
|
"at": module._utc_iso(now),
|
|
"activeJobs": 0,
|
|
"queuedJobs": 1,
|
|
"cpuUsed": 1.5,
|
|
"memoryUsedBytes": 4096,
|
|
},
|
|
]
|
|
state["terminalHistory"] = [
|
|
{
|
|
"jobId": str(index),
|
|
"attempt": 1,
|
|
"profileId": "checks",
|
|
"result": "success",
|
|
"createdAt": module._utc_iso(now - 300 + index * 30),
|
|
"startedAt": module._utc_iso(now - 290 + index * 30),
|
|
"finishedAt": module._utc_iso(now - 250 + index * 30),
|
|
"waitSeconds": index * 10,
|
|
"durationSeconds": index * 20,
|
|
}
|
|
for index in range(1, 4)
|
|
]
|
|
return state
|
|
|
|
|
|
def test_public_projection_matches_fixed_v1_and_uses_only_observed_statistics(
|
|
controller_module,
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller = module.RunnerAdmissionController(clock=lambda: now)
|
|
payload = controller._public_snapshot(
|
|
_public_observation(module, now), _complete_state(module, now)
|
|
)
|
|
|
|
assert set(payload) == {
|
|
"schemaVersion",
|
|
"status",
|
|
"generatedAt",
|
|
"scope",
|
|
"budget",
|
|
"host",
|
|
"profiles",
|
|
"window",
|
|
"history",
|
|
}
|
|
assert payload["scope"] == "ci-fleet"
|
|
assert payload["status"] == "healthy"
|
|
assert payload["budget"] == {
|
|
"cpuLimit": 8,
|
|
"memoryLimitBytes": 16 * 1024**3,
|
|
"activeJobLimit": 2,
|
|
"heavyJobLimit": 1,
|
|
"reservedCpu": 1,
|
|
"reservedMemoryBytes": 2 * 1024**3,
|
|
"activeJobs": 0,
|
|
"heavyJobs": 0,
|
|
}
|
|
assert payload["host"]["cpuBusyPercent"] == 18.75
|
|
assert payload["profiles"][0]["cpuLimit"] is None
|
|
assert payload["profiles"][0]["memoryLimitBytes"] is None
|
|
backend = next(row for row in payload["profiles"] if row["id"] == "backend")
|
|
frontend = next(row for row in payload["profiles"] if row["id"] == "frontend")
|
|
assert backend["queueDepth"] == frontend["queueDepth"] == 1
|
|
checks = payload["profiles"][0]
|
|
assert checks["completedJobs"] == 3
|
|
assert checks["p50WaitSeconds"] == 20
|
|
assert checks["p95WaitSeconds"] == 30
|
|
assert checks["p50DurationSeconds"] == 40
|
|
assert checks["p95DurationSeconds"] == 60
|
|
assert checks["lastDurationSeconds"] == 60
|
|
assert payload["window"]["coverage"] == "bounded"
|
|
assert "private-job-id" not in json.dumps(payload["history"])
|
|
assert all(set(row) == {"at", "activeJobs", "queuedJobs", "cpuUsed", "memoryUsedBytes"}
|
|
for row in payload["history"])
|
|
assert sanitize_statistics(payload)["status"] == "healthy"
|
|
|
|
|
|
def test_base_projection_does_not_claim_unattested_capacity(controller_module):
|
|
payload = controller_module.RunnerAdmissionController._base_snapshot(
|
|
"not_configured", "runner_admission_not_configured", time.time()
|
|
)
|
|
assert payload["budget"] == {
|
|
"cpuLimit": None,
|
|
"memoryLimitBytes": None,
|
|
"activeJobLimit": None,
|
|
"heavyJobLimit": None,
|
|
"reservedCpu": None,
|
|
"reservedMemoryBytes": None,
|
|
"activeJobs": None,
|
|
"heavyJobs": None,
|
|
}
|
|
assert all(row["cpuLimit"] is None and row["memoryLimitBytes"] is None
|
|
for row in payload["profiles"])
|
|
assert payload["window"]["coverage"] == "unavailable"
|
|
|
|
|
|
def test_capacity_projection_uses_independent_host_sampler_without_burst_state(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
calls = []
|
|
sample = {
|
|
"cpuCount": 4,
|
|
"loadOne": 1.0,
|
|
"loadRatio": 0.25,
|
|
"availableMemoryBytes": 16 * 1024**3,
|
|
"cacheFreeBytes": 40 * 1024**3,
|
|
"swapInKbps": 0.0,
|
|
"swapOutKbps": 0.0,
|
|
"swapPressureSamples": 0,
|
|
"swapCritical": False,
|
|
"swapSampleAt": time.time(),
|
|
"swapInPages": 100,
|
|
"swapOutPages": 200,
|
|
}
|
|
|
|
class HostSampler:
|
|
def host_snapshot(self, previous=None):
|
|
calls.append(previous)
|
|
return {"status": "observed", "generatedAt": "2026-10-01T00:00:00Z", "host": sample}
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"runner_capacity_controller",
|
|
SimpleNamespace(RunnerCapacityController=HostSampler),
|
|
)
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"runner_capacity_policy",
|
|
SimpleNamespace(
|
|
START_MAX_LOAD_RATIO=0.70,
|
|
START_MIN_AVAILABLE_MEMORY_BYTES=12 * 1024**3,
|
|
START_MIN_CACHE_FREE_BYTES=30 * 1024**3,
|
|
STOP_LOAD_RATIO=0.90,
|
|
STOP_MIN_AVAILABLE_MEMORY_BYTES=8 * 1024**3,
|
|
STOP_MIN_CACHE_FREE_BYTES=15 * 1024**3,
|
|
),
|
|
)
|
|
controller = module.RunnerAdmissionController()
|
|
|
|
first = controller._capacity_snapshot()
|
|
second = controller._capacity_snapshot()
|
|
|
|
assert calls[0] is None
|
|
assert calls[1] == sample
|
|
assert first["status"] == second["status"] == "ok"
|
|
assert first["decision"] == {"pressure": "headroom"}
|
|
assert second["host"]["swapOutPages"] == 200
|
|
|
|
|
|
def test_capacity_projection_fails_closed_on_unavailable_host_sample(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
|
|
class HostSampler:
|
|
def host_snapshot(self, previous=None):
|
|
return {"status": "unavailable", "host": None}
|
|
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"runner_capacity_controller",
|
|
SimpleNamespace(RunnerCapacityController=HostSampler),
|
|
)
|
|
controller = module.RunnerAdmissionController()
|
|
|
|
with pytest.raises(module.RunnerAdmissionError, match="host_telemetry_unavailable"):
|
|
controller._capacity_snapshot()
|
|
|
|
|
|
def test_runner_inventory_maps_exact_gitea_display_names_to_fixed_containers(
|
|
controller_module,
|
|
):
|
|
module = controller_module
|
|
rows = [
|
|
{
|
|
"name": profile.display_name,
|
|
"id": index,
|
|
"status": "offline",
|
|
"busy": False,
|
|
"disabled": True,
|
|
"labels": [profile.runner_label, *profile.legacy_labels],
|
|
}
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=10)
|
|
]
|
|
|
|
by_name, by_id = module.RunnerAdmissionController._normalized_runner_rows(rows)
|
|
|
|
for profile in module.RUNNER_PROFILES.values():
|
|
observed = by_name[profile.container_name]
|
|
assert observed["displayName"] == profile.display_name
|
|
assert observed["id"] in by_id
|
|
|
|
canonical_alias = [dict(rows[0], name=module.RUNNER_PROFILES["checks"].container_name)]
|
|
with pytest.raises(
|
|
module.RunnerAdmissionError,
|
|
match="runner_admission_registration_name_invalid",
|
|
):
|
|
module.RunnerAdmissionController._normalized_runner_rows(canonical_alias)
|
|
|
|
duplicate_alias = [
|
|
rows[0],
|
|
dict(rows[1], name=module.RUNNER_PROFILES["checks"].display_name),
|
|
]
|
|
with pytest.raises(
|
|
module.RunnerAdmissionError,
|
|
match="runner_admission_runner_inventory_duplicate",
|
|
):
|
|
module.RunnerAdmissionController._normalized_runner_rows(duplicate_alias)
|
|
|
|
|
|
def test_snapshot_reads_only_atomic_projection_and_downshifts_stale_health(
|
|
controller_module, monkeypatch, tmp_path
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller = module.RunnerAdmissionController(clock=lambda: now)
|
|
projection = module.RunnerAdmissionController._base_snapshot(
|
|
"healthy", "unused", now - 120
|
|
)
|
|
monkeypatch.setattr(module, "STATE_ROOT", tmp_path)
|
|
monkeypatch.setattr(module, "PUBLIC_STATISTICS_FILE", tmp_path / "statistics.json")
|
|
monkeypatch.setattr(module, "_safe_directory", lambda *_args, **_kwargs: True)
|
|
monkeypatch.setattr(
|
|
module.RunnerAdmissionController,
|
|
"_safe_private_file",
|
|
staticmethod(lambda *_args, **_kwargs: True),
|
|
)
|
|
module.PUBLIC_STATISTICS_FILE.write_text(json.dumps(projection), encoding="utf-8")
|
|
monkeypatch.setattr(controller, "_observe", lambda **_kwargs: pytest.fail("snapshot polled"))
|
|
monkeypatch.setattr(controller, "_request", lambda *_args, **_kwargs: pytest.fail("snapshot called Gitea"))
|
|
monkeypatch.setattr(controller, "_docker_run", lambda *_args, **_kwargs: pytest.fail("snapshot called Docker"))
|
|
|
|
result = controller.snapshot()
|
|
|
|
assert result["status"] == "degraded"
|
|
assert result["generatedAt"] == projection["generatedAt"]
|
|
assert result["budget"]["cpuLimit"] is None
|
|
|
|
|
|
def test_projection_rejects_naive_future_and_out_of_order_history(controller_module):
|
|
module = controller_module
|
|
now = time.time()
|
|
assert module._parse_time("2026-10-01T12:00:00") is None
|
|
projection = module.RunnerAdmissionController._base_snapshot(
|
|
"healthy", "unused", now
|
|
)
|
|
projection["history"] = [
|
|
{
|
|
"at": module._utc_iso(now),
|
|
"activeJobs": 0,
|
|
"queuedJobs": 0,
|
|
"cpuUsed": None,
|
|
"memoryUsedBytes": None,
|
|
},
|
|
{
|
|
"at": module._utc_iso(now - 30),
|
|
"activeJobs": 0,
|
|
"queuedJobs": 0,
|
|
"cpuUsed": None,
|
|
"memoryUsedBytes": None,
|
|
},
|
|
]
|
|
assert not module.RunnerAdmissionController._valid_projection(projection)
|
|
|
|
|
|
def test_resource_attestation_uses_fixed_outer_names_and_mapping_contract(
|
|
controller_module,
|
|
):
|
|
module = controller_module
|
|
requests = []
|
|
proof = {
|
|
"ready": True,
|
|
"effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI,
|
|
"effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB,
|
|
"activeClaims": 1,
|
|
"heavyClaims": 0,
|
|
"runningRunnerContainers": 1,
|
|
"cgroups": {"familyPath": "/itworx-ci", "boundedGroups": 7},
|
|
"digest": "b" * 64,
|
|
}
|
|
controller = module.RunnerAdmissionController(
|
|
resource_attestor=lambda **kwargs: requests.append(kwargs) or proof
|
|
)
|
|
empty_legacy_inventory = {
|
|
name: None for name in module.ALL_LEGACY_RUNNER_NAMES
|
|
}
|
|
|
|
result = controller._resource_attestation(["checks"], empty_legacy_inventory)
|
|
|
|
assert result == proof
|
|
assert requests[0]["active_profiles"] == ("checks",)
|
|
assert requests[0]["expected_host_budget"] == {"cpuMilli": 8000, "memoryMiB": 16384}
|
|
assert requests[0]["expected_outer_containers"] == (
|
|
*module.PROFILE_RUNNER_NAMES,
|
|
module.DIND_NAME,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("verified", [True, False])
|
|
def test_observation_requires_source_bound_profile_task_images(
|
|
controller_module, monkeypatch, verified
|
|
):
|
|
module = controller_module
|
|
calls = []
|
|
fake_specializer = SimpleNamespace(
|
|
verify_profile_task_images=lambda profile_ids: calls.append(tuple(profile_ids))
|
|
or verified
|
|
)
|
|
monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer)
|
|
attestation_requests = []
|
|
proof = {
|
|
"ready": True,
|
|
"effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI,
|
|
"effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB,
|
|
"activeClaims": 0,
|
|
"heavyClaims": 0,
|
|
"runningRunnerContainers": 0,
|
|
"cgroups": {},
|
|
"digest": "f" * 64,
|
|
}
|
|
controller = module.RunnerAdmissionController(
|
|
resource_attestor=lambda **kwargs: attestation_requests.append(kwargs) or proof
|
|
)
|
|
rows = {}
|
|
by_id = {}
|
|
profile_runner_ids = {}
|
|
runner_profile_ids = {}
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
row = {
|
|
"id": index,
|
|
"name": profile.container_name,
|
|
"disabled": True,
|
|
"busy": False,
|
|
"labels": {profile.runner_label, *profile.legacy_labels},
|
|
}
|
|
rows[profile.container_name] = row
|
|
by_id[index] = row
|
|
profile_runner_ids[profile.id] = index
|
|
runner_profile_ids[index] = profile.id
|
|
|
|
monkeypatch.setattr(controller, "_specialization_state", lambda: "active")
|
|
monkeypatch.setattr(controller, "_runner_inventory", lambda: [])
|
|
monkeypatch.setattr(controller, "_normalized_runner_rows", lambda _rows: (rows, by_id))
|
|
monkeypatch.setattr(controller, "_job_rows", lambda _status: ([], True))
|
|
monkeypatch.setattr(controller, "_normalized_jobs", lambda *_args: ([], {}))
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_active_claims",
|
|
lambda *_args: ([], profile_runner_ids, runner_profile_ids),
|
|
)
|
|
monkeypatch.setattr(controller, "_container", lambda name: {"name": name, "running": False, "health": "none"})
|
|
monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, {}))
|
|
|
|
observation = controller._observe()
|
|
|
|
assert calls == [tuple(module.PROFILE_IDS)]
|
|
assert observation["taskImagesVerified"] is verified
|
|
if verified:
|
|
assert observation["resourceProof"] == proof
|
|
assert observation["resourceReason"] is None
|
|
assert len(attestation_requests) == 1
|
|
else:
|
|
assert observation["resourceProof"] is None
|
|
assert observation["resourceReason"] == "runner_admission_task_image_unverified"
|
|
assert attestation_requests == []
|
|
|
|
|
|
def test_dependency_pending_job_does_not_count_or_admit_as_runnable(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller = module.RunnerAdmissionController(clock=lambda: now)
|
|
pending_build = {
|
|
"id": 101,
|
|
"labels": [module.RUNNER_PROFILES["build"].runner_label],
|
|
"attempt": 1,
|
|
"status": "requested",
|
|
"created_at": module._utc_iso(now - 600),
|
|
}
|
|
queued_checks = {
|
|
"id": 102,
|
|
"labels": [module.RUNNER_PROFILES["checks"].runner_label],
|
|
"attempt": 1,
|
|
"created_at": module._utc_iso(now - 30),
|
|
}
|
|
_install_gitea_observation(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
queued_rows=[queued_checks],
|
|
pending_rows=[pending_build],
|
|
)
|
|
|
|
observation = controller._observe()
|
|
state = module._initial_state()
|
|
assert controller._record_aggregate_sample(state, observation)
|
|
projection = controller._public_snapshot(observation, state)
|
|
choice = module.select_admission(
|
|
observation["queue"],
|
|
available_profiles=controller._enabled_candidate_profiles(observation, state),
|
|
active_claims=[],
|
|
cursor=0,
|
|
)
|
|
|
|
assert [job["id"] for job in observation["queue"]] == ["102"]
|
|
assert next(row for row in projection["profiles"] if row["id"] == "checks")["queueDepth"] == 1
|
|
build = next(row for row in projection["profiles"] if row["id"] == "build")
|
|
assert build["queueDepth"] == 0
|
|
assert build["oldestQueuedSeconds"] is None
|
|
assert projection["history"][-1]["queuedJobs"] == 1
|
|
assert (choice.profile_id, choice.job_id) == ("checks", "102")
|
|
|
|
|
|
def test_observation_still_rejects_malformed_pending_inventory(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController()
|
|
_install_gitea_observation(monkeypatch, controller, module)
|
|
original_job_rows = controller._job_rows
|
|
|
|
def malformed_pending_rows(status):
|
|
if status == "pending":
|
|
return ([{"id": 103, "labels": [], "created_at": module._utc_iso()}], True)
|
|
return original_job_rows(status)
|
|
|
|
monkeypatch.setattr(controller, "_job_rows", malformed_pending_rows)
|
|
|
|
with pytest.raises(
|
|
module.RunnerAdmissionError,
|
|
match="runner_admission_job_inventory_invalid",
|
|
):
|
|
controller._observe()
|
|
|
|
|
|
def test_pending_only_dependency_job_does_not_start_a_runner(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller = module.RunnerAdmissionController(clock=lambda: now)
|
|
pending_build = {
|
|
"id": 101,
|
|
"labels": [module.RUNNER_PROFILES["build"].runner_label],
|
|
"attempt": 1,
|
|
"status": "requested",
|
|
"created_at": module._utc_iso(now - 600),
|
|
}
|
|
_install_gitea_observation(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
pending_rows=[pending_build],
|
|
)
|
|
state = module._initial_state()
|
|
state["terminalHistoryUpdatedAt"] = now
|
|
starts = []
|
|
enables = []
|
|
projections = []
|
|
monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext())
|
|
monkeypatch.setattr(controller, "_read_state", lambda: state)
|
|
monkeypatch.setattr(controller, "_save_state", lambda _state: None)
|
|
monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False)
|
|
monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False)
|
|
monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False)
|
|
monkeypatch.setattr(controller, "_write_projection", projections.append)
|
|
monkeypatch.setattr(controller, "_docker_run", lambda args, **_kwargs: starts.append(args))
|
|
monkeypatch.setattr(controller, "_set_disabled", lambda *args: enables.append(args))
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_continue_pending",
|
|
lambda *_args, **_kwargs: pytest.fail("no runner startup intent should be created"),
|
|
)
|
|
|
|
result = controller.cycle()
|
|
|
|
assert result["status"] == "runner_admission_no_compatible_queue"
|
|
assert starts == []
|
|
assert enables == []
|
|
assert state["pending"] is None
|
|
assert next(row for row in projections[0]["profiles"] if row["id"] == "build")["queueDepth"] == 0
|
|
|
|
|
|
def test_aggregate_history_uses_counter_deltas_and_real_memory(controller_module):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller = module.RunnerAdmissionController(clock=lambda: now)
|
|
state = module._initial_state()
|
|
state["lastFamilyCpuUsageUsec"] = 100_000_000
|
|
state["lastFamilyCpuSampleAt"] = now - 20
|
|
observation = {
|
|
"activeJobs": {21: [{"id": "active-1"}]},
|
|
"queue": [{"id": "queued-1"}, {"id": "queued-2"}],
|
|
"resourceProof": {
|
|
"familyCpuUsageUsec": 110_000_000,
|
|
"familyMemoryCurrentBytes": 512 * 1024**2,
|
|
},
|
|
}
|
|
|
|
assert controller._record_aggregate_sample(state, observation)
|
|
assert state["historySamples"][-1] == {
|
|
"at": module._utc_iso(now),
|
|
"activeJobs": 1,
|
|
"queuedJobs": 2,
|
|
"cpuUsed": 0.5,
|
|
"memoryUsedBytes": 512 * 1024**2,
|
|
}
|
|
|
|
state["lastFamilyCpuUsageUsec"] = 200_000_000
|
|
state["lastFamilyCpuSampleAt"] = now - 20
|
|
state["historySamples"][-1]["at"] = module._utc_iso(now - 30)
|
|
observation["resourceProof"]["familyCpuUsageUsec"] = 100_000_000
|
|
now += 31
|
|
controller._clock = lambda: now
|
|
assert controller._record_aggregate_sample(state, observation)
|
|
assert state["historySamples"][-1]["cpuUsed"] is None
|
|
|
|
|
|
def test_controller_owned_drain_rechecks_operator_and_allows_guard_pause(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module, enabled=("checks",)
|
|
)
|
|
entry = state["profiles"]["checks"]
|
|
entry.update(
|
|
idleSince=now - module.IDLE_DRAIN_SECONDS - 1,
|
|
)
|
|
profile = module.RUNNER_PROFILES["checks"]
|
|
runner_id = observation["profileRunnerIds"]["checks"]
|
|
disabled = []
|
|
container_running = {"value": True}
|
|
row = next(item for item in inventory if item["id"] == runner_id)
|
|
monkeypatch.setattr(controller, "_save_state", lambda _state: None)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_job_rows",
|
|
lambda _status: ([], True),
|
|
)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_runner_inventory",
|
|
lambda: [dict(item) for item in inventory],
|
|
)
|
|
monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("paused", "guard"))
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_set_disabled",
|
|
lambda actual_id, value: (
|
|
disabled.append((actual_id, value)),
|
|
row.update(disabled=value),
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_container",
|
|
lambda name: {
|
|
"id": containers[name]["id"],
|
|
"running": container_running["value"],
|
|
"health": "healthy" if container_running["value"] else "none",
|
|
},
|
|
)
|
|
def stop(args, **_kwargs):
|
|
assert args == ["docker", "stop", profile.container_name]
|
|
container_running["value"] = False
|
|
return ""
|
|
|
|
monkeypatch.setattr(controller, "_docker_run", stop)
|
|
|
|
action = controller._drain_one_idle_profile(state, observation)
|
|
|
|
assert action == "runner_admission_drained_idle_profile"
|
|
assert disabled == [(runner_id, True)]
|
|
assert entry["enabledByController"] is False
|
|
assert entry["drainOwned"] is False
|
|
assert container_running["value"] is False
|
|
|
|
|
|
def test_drained_runner_requires_a_full_exact_container_id(controller_module):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController()
|
|
state = module._initial_state()
|
|
state["profiles"]["checks"]["drainOwned"] = True
|
|
|
|
assert controller._stop_drained_profile(
|
|
state,
|
|
profile_id="checks",
|
|
runner_id=101,
|
|
expected_container_id="",
|
|
) == "runner_admission_container_identity_invalid"
|
|
|
|
|
|
def test_drain_does_not_stop_when_operator_reopens_registration_after_disable(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module, enabled=("checks",)
|
|
)
|
|
entry = state["profiles"]["checks"]
|
|
entry.update(
|
|
idleSince=now - module.IDLE_DRAIN_SECONDS - 1,
|
|
)
|
|
runner_id = observation["profileRunnerIds"]["checks"]
|
|
row = next(item for item in inventory if item["id"] == runner_id)
|
|
calls = 0
|
|
patches = []
|
|
|
|
monkeypatch.setattr(controller, "_save_state", lambda _state: None)
|
|
monkeypatch.setattr(controller, "_job_rows", lambda _status: ([], True))
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_runner_inventory",
|
|
lambda: [dict(item) for item in inventory],
|
|
)
|
|
monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("clear", ""))
|
|
|
|
def operator_reopens(actual_id, _value):
|
|
patches.append((actual_id, True))
|
|
row.update(disabled=False)
|
|
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_set_disabled",
|
|
operator_reopens,
|
|
)
|
|
monkeypatch.setattr(controller, "_container", lambda name: dict(containers[name]))
|
|
|
|
def docker(args, **_kwargs):
|
|
nonlocal calls
|
|
calls += 1
|
|
return ""
|
|
|
|
monkeypatch.setattr(controller, "_docker_run", docker)
|
|
|
|
action = controller._drain_one_idle_profile(state, observation)
|
|
|
|
assert action == "runner_admission_drain_stop_deferred"
|
|
assert calls == 0
|
|
assert entry["drainOwned"] is True
|
|
assert entry["enabledByController"] is False
|
|
|
|
fresh_rows, _ = controller._normalized_runner_rows(inventory)
|
|
observation["runnerRows"] = fresh_rows
|
|
assert controller._reconcile_ownership(state, observation)
|
|
assert entry["drainOwned"] is False
|
|
assert entry["operatorEnabled"] is True
|
|
assert controller._drain_one_idle_profile(state, observation) is None
|
|
assert patches == [(runner_id, True)]
|
|
|
|
|
|
def test_busy_controller_owned_runner_gets_a_durable_disable_but_job_is_not_stopped(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module, enabled=("backend",), busy=("backend",)
|
|
)
|
|
profile = module.RUNNER_PROFILES["backend"]
|
|
runner_id = observation["profileRunnerIds"]["backend"]
|
|
active = _raw_job(
|
|
module,
|
|
{"id": "active-backend", "labels": [profile.runner_label]},
|
|
runner_id=runner_id,
|
|
active=True,
|
|
)
|
|
saved = []
|
|
patches = []
|
|
stops = []
|
|
_install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory,
|
|
containers,
|
|
active_rows=[active],
|
|
saved=saved,
|
|
)
|
|
|
|
def disable(actual_id, disabled):
|
|
patches.append((actual_id, disabled))
|
|
next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled
|
|
|
|
monkeypatch.setattr(controller, "_set_disabled", disable)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_docker_run",
|
|
lambda args, **_kwargs: stops.append(args),
|
|
)
|
|
|
|
action = controller._drain_one_idle_profile(state, observation)
|
|
|
|
assert action == "runner_admission_drain_stop_deferred"
|
|
assert patches == [(runner_id, True)]
|
|
assert stops == []
|
|
assert saved[-1]["profiles"]["backend"]["drainOwned"] is True
|
|
assert state["profiles"]["backend"]["drainOwned"] is True
|
|
assert state["profiles"]["backend"]["enabledByController"] is False
|
|
fresh_rows, fresh_by_id = controller._normalized_runner_rows(inventory)
|
|
claims, _, _ = controller._active_claims(
|
|
fresh_rows, fresh_by_id, {runner_id: [{"id": "active-backend"}]}
|
|
)
|
|
assert claims == ["backend"]
|
|
|
|
|
|
def test_cycle_fences_a_busy_owned_lease_even_without_a_resource_proof(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module, enabled=("backend",), busy=("backend",)
|
|
)
|
|
state["terminalHistoryUpdatedAt"] = module._utc_iso(now)
|
|
observation.update(specialization="active", at=now)
|
|
saved = []
|
|
patches = []
|
|
active = _raw_job(
|
|
module,
|
|
{
|
|
"id": "active-backend",
|
|
"labels": [module.RUNNER_PROFILES["backend"].runner_label],
|
|
},
|
|
runner_id=observation["profileRunnerIds"]["backend"],
|
|
active=True,
|
|
)
|
|
_install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory,
|
|
containers,
|
|
active_rows=[active],
|
|
saved=saved,
|
|
)
|
|
monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext())
|
|
monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation)
|
|
monkeypatch.setattr(controller, "_read_state", lambda: state)
|
|
monkeypatch.setattr(controller, "_terminal_history", lambda: ([], True, None))
|
|
monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False)
|
|
monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False)
|
|
monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": "healthy"})
|
|
monkeypatch.setattr(controller, "_write_projection", lambda _projection: None)
|
|
|
|
def disable(actual_id, disabled):
|
|
patches.append((actual_id, disabled))
|
|
next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled
|
|
|
|
monkeypatch.setattr(controller, "_set_disabled", disable)
|
|
|
|
result = controller.cycle()
|
|
|
|
assert result["status"] == "runner_admission_drain_stop_deferred"
|
|
assert patches == [(observation["profileRunnerIds"]["backend"], True)]
|
|
assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True
|
|
assert saved[-1]["profiles"]["backend"]["drainOwned"] is True
|
|
|
|
|
|
def test_oldest_other_profile_eventually_replaces_sustained_backend_frontend_claims(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
older_build = {
|
|
"id": "older-build",
|
|
"labels": [module.RUNNER_PROFILES["build"].runner_label],
|
|
"queued_at": now - 20,
|
|
}
|
|
newer_backend = {
|
|
"id": "newer-backend",
|
|
"labels": ["linux-validation"],
|
|
"queued_at": now - 10,
|
|
}
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module,
|
|
enabled=("backend", "frontend"),
|
|
queue=[older_build, newer_backend],
|
|
)
|
|
for profile_id in ("backend", "frontend"):
|
|
state["profiles"][profile_id]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1
|
|
raw_queue = [_raw_job(module, older_build), _raw_job(module, newer_backend)]
|
|
patches = []
|
|
_install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory,
|
|
containers,
|
|
queue_rows=raw_queue,
|
|
)
|
|
|
|
def disable(actual_id, disabled):
|
|
patches.append((actual_id, disabled))
|
|
next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled
|
|
|
|
monkeypatch.setattr(controller, "_set_disabled", disable)
|
|
|
|
first = controller._drain_one_idle_profile(state, observation)
|
|
assert first == "runner_admission_drained_idle_profile"
|
|
assert patches == [(observation["profileRunnerIds"]["backend"], True)]
|
|
assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is False
|
|
|
|
# The next poll can yield the other idle broad-label lease as the older
|
|
# heavy job remains ahead of a newer backend job.
|
|
rows, by_id = controller._normalized_runner_rows(inventory)
|
|
observation["runnerRows"] = rows
|
|
observation["claims"] = controller._active_claims(rows, by_id, {})[0]
|
|
second = controller._drain_one_idle_profile(state, observation)
|
|
assert second == "runner_admission_drained_idle_profile"
|
|
assert patches == [
|
|
(observation["profileRunnerIds"]["backend"], True),
|
|
(observation["profileRunnerIds"]["frontend"], True),
|
|
]
|
|
assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is False
|
|
|
|
rows, by_id = controller._normalized_runner_rows(inventory)
|
|
claims, _, _ = controller._active_claims(rows, by_id, {})
|
|
choice = module.select_admission(
|
|
[older_build, newer_backend],
|
|
available_profiles=controller._enabled_candidate_profiles(
|
|
{
|
|
"runnerRows": rows,
|
|
"containers": containers,
|
|
"activeJobs": {},
|
|
"profileRunnerIds": observation["profileRunnerIds"],
|
|
},
|
|
state,
|
|
),
|
|
active_claims=claims,
|
|
cursor=state["cursor"],
|
|
)
|
|
assert (choice.profile_id, choice.job_id) == ("build", "older-build")
|
|
|
|
|
|
def test_prepatch_drain_intent_survives_restart_and_retries_with_complete_inventory(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module, enabled=("backend",), busy=("backend",)
|
|
)
|
|
runner_id = observation["profileRunnerIds"]["backend"]
|
|
active = _raw_job(
|
|
module,
|
|
{"id": "active-backend", "labels": ["itworx-ci-backend"]},
|
|
runner_id=runner_id,
|
|
active=True,
|
|
)
|
|
saved = []
|
|
patches = []
|
|
_install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory,
|
|
containers,
|
|
active_rows=[active],
|
|
saved=saved,
|
|
)
|
|
|
|
def incomplete(status):
|
|
if status == "pending":
|
|
raise module.RunnerAdmissionError("runner_admission_pending_inventory_incomplete")
|
|
return ([active] if status == "in_progress" else [], True)
|
|
|
|
monkeypatch.setattr(controller, "_job_rows", incomplete)
|
|
assert controller._drain_one_idle_profile(state, observation) == (
|
|
"runner_admission_pending_inventory_incomplete"
|
|
)
|
|
assert saved[-1]["profiles"]["backend"]["drainOwned"] is True
|
|
assert patches == []
|
|
|
|
recovered = copy.deepcopy(saved[-1])
|
|
assert controller._reconcile_ownership(recovered, observation) is False
|
|
assert recovered["profiles"]["backend"]["drainOwned"] is True
|
|
assert recovered["profiles"]["backend"]["operatorEnabled"] is False
|
|
|
|
def complete(status):
|
|
if status in {"queued", "pending"}:
|
|
return [], True
|
|
return [active], True
|
|
|
|
monkeypatch.setattr(controller, "_job_rows", complete)
|
|
|
|
def disable(actual_id, disabled):
|
|
patches.append((actual_id, disabled))
|
|
next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled
|
|
|
|
monkeypatch.setattr(controller, "_set_disabled", disable)
|
|
assert controller._drain_one_idle_profile(recovered, observation) == (
|
|
"runner_admission_drain_stop_deferred"
|
|
)
|
|
assert patches == [(runner_id, True)]
|
|
assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True
|
|
|
|
|
|
def test_stale_idle_drain_intent_is_cleared_when_fresh_queue_no_longer_yields(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
backend_job = {
|
|
"id": "newer-backend",
|
|
"labels": ["linux-validation"],
|
|
"queued_at": now - 10,
|
|
}
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module,
|
|
enabled=("backend",),
|
|
busy=(),
|
|
queue=[backend_job],
|
|
)
|
|
entry = state["profiles"]["backend"]
|
|
entry.update(drainOwned=True, enabledByController=True, idleSince=None)
|
|
saved = []
|
|
patches = []
|
|
_install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory,
|
|
containers,
|
|
queue_rows=[_raw_job(module, backend_job)],
|
|
saved=saved,
|
|
)
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_set_disabled",
|
|
lambda actual_id, disabled: patches.append((actual_id, disabled)),
|
|
)
|
|
recovered = copy.deepcopy(state)
|
|
|
|
assert controller._reconcile_ownership(recovered, observation) is False
|
|
result = controller._drain_one_idle_profile(recovered, observation)
|
|
|
|
assert result == "runner_admission_drain_deferred_queue"
|
|
assert patches == []
|
|
assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True
|
|
assert recovered["profiles"]["backend"]["drainOwned"] is False
|
|
assert saved[-1]["profiles"]["backend"]["drainOwned"] is False
|
|
|
|
|
|
def test_queue_identity_is_rechecked_before_yielding_and_busy_peer_is_never_stopped(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
now = time.time()
|
|
build = {
|
|
"id": "build",
|
|
"labels": [module.RUNNER_PROFILES["build"].runner_label],
|
|
"queued_at": now - 20,
|
|
}
|
|
backend = {
|
|
"id": "backend",
|
|
"labels": ["linux-validation"],
|
|
"queued_at": now - 10,
|
|
}
|
|
controller, state, observation, inventory, containers = _lease_fixture(
|
|
module,
|
|
enabled=("backend", "frontend"),
|
|
busy=(),
|
|
queue=[build, backend],
|
|
)
|
|
state["profiles"]["backend"]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1
|
|
raw_build = _raw_job(module, build)
|
|
fresh_backend = {**backend, "queued_at": now - 30}
|
|
fresh_build = {**build, "queued_at": now - 20}
|
|
raw_backend = _raw_job(module, fresh_backend)
|
|
raw_build = _raw_job(module, fresh_build)
|
|
patches = []
|
|
stops = []
|
|
fresh_queue = [raw_backend, raw_build]
|
|
fresh_active = []
|
|
_install_lease_io(
|
|
monkeypatch,
|
|
controller,
|
|
module,
|
|
inventory,
|
|
containers,
|
|
# Fresh inventory reverses the queue identity order; the controller
|
|
# must not act on the stale observation's earlier build.
|
|
queue_rows=fresh_queue,
|
|
active_rows=fresh_active,
|
|
)
|
|
def disable(actual_id, disabled):
|
|
patches.append((actual_id, disabled))
|
|
next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled
|
|
|
|
monkeypatch.setattr(controller, "_set_disabled", disable)
|
|
|
|
def stop(args, **_kwargs):
|
|
stops.append(args)
|
|
containers[args[2]]["running"] = False
|
|
containers[args[2]]["health"] = "none"
|
|
return ""
|
|
|
|
monkeypatch.setattr(controller, "_docker_run", stop)
|
|
|
|
# Fresh backend demand is older, and Backend is still serving it. A drain
|
|
# must not proceed just because a stale snapshot preferred Build.
|
|
result = controller._drain_one_idle_profile(state, observation)
|
|
|
|
assert result == "runner_admission_drain_deferred_queue"
|
|
assert patches == []
|
|
assert stops == []
|
|
assert state["profiles"]["backend"]["drainOwned"] is False
|
|
|
|
# Once the fresh queue again places Build first, the busy Frontend lease
|
|
# gets its FetchTask fenced first, but its active process stays running.
|
|
raw_build["created_at"] = module._utc_iso(now - 40)
|
|
raw_backend["created_at"] = module._utc_iso(now - 30)
|
|
fresh_queue[:] = [raw_build, raw_backend]
|
|
frontend_id = observation["profileRunnerIds"]["frontend"]
|
|
frontend_row = next(row for row in inventory if row["id"] == frontend_id)
|
|
frontend_row["busy"] = True
|
|
frontend_active = _raw_job(
|
|
module,
|
|
{"id": "active-frontend", "labels": ["itworx-ci-frontend"]},
|
|
runner_id=frontend_id,
|
|
active=True,
|
|
)
|
|
fresh_active.append(frontend_active)
|
|
observation["runnerRows"][module.RUNNER_PROFILES["frontend"].container_name][
|
|
"busy"
|
|
] = True
|
|
observation["activeJobs"][frontend_id] = [{"id": "active-frontend"}]
|
|
observation["claims"] = ["backend", "frontend"]
|
|
first = controller._drain_one_idle_profile(state, observation)
|
|
assert first == "runner_admission_drain_stop_deferred"
|
|
assert patches == [(frontend_id, True)]
|
|
assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is True
|
|
|
|
rows, by_id = controller._normalized_runner_rows(inventory)
|
|
observation["runnerRows"] = rows
|
|
observation["claims"] = controller._active_claims(
|
|
rows, by_id, observation["activeJobs"]
|
|
)[0]
|
|
second = controller._drain_one_idle_profile(state, observation)
|
|
assert second == "runner_admission_drained_idle_profile"
|
|
assert patches == [
|
|
(frontend_id, True),
|
|
(observation["profileRunnerIds"]["backend"], True),
|
|
]
|
|
assert stops == [["docker", "stop", module.RUNNER_PROFILES["backend"].container_name]]
|
|
assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is True
|
|
|
|
|
|
def test_bootstrap_receipt_seeds_only_stopped_disabled_fixed_baseline(
|
|
controller_module, monkeypatch, tmp_path
|
|
):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController()
|
|
state = module._initial_state()
|
|
monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json")
|
|
saved = []
|
|
consumed = []
|
|
|
|
rows = {}
|
|
containers = {}
|
|
registrations = []
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
labels = [profile.runner_label, *profile.legacy_labels]
|
|
rows[profile.container_name] = {
|
|
"id": 100 + index,
|
|
"displayName": profile.display_name,
|
|
"disabled": True,
|
|
"busy": False,
|
|
"labels": set(labels),
|
|
}
|
|
containers[profile.container_name] = {
|
|
"id": f"{index:064x}",
|
|
"running": False,
|
|
"health": "none",
|
|
}
|
|
registrations.append(
|
|
{
|
|
"profileId": profile.id,
|
|
"displayName": profile.display_name,
|
|
"containerName": profile.container_name,
|
|
"id": 100 + index,
|
|
"labels": labels,
|
|
}
|
|
)
|
|
receipt = {
|
|
"status": "pending",
|
|
"verifiedRunners": [
|
|
profile.container_name for profile in module.RUNNER_PROFILES.values()
|
|
],
|
|
"registrations": registrations,
|
|
}
|
|
receipt_sha = "d" * 64
|
|
fake_specializer = SimpleNamespace(
|
|
read_admission_bootstrap_receipt=lambda: (receipt, receipt_sha),
|
|
consume_admission_bootstrap_receipt=lambda digest: (
|
|
consumed.append((digest, state["revision"])) or True
|
|
),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer)
|
|
monkeypatch.setattr(controller, "_save_state", lambda value: (
|
|
saved.append(value["revision"]),
|
|
value.update(revision=value["revision"] + 1),
|
|
))
|
|
|
|
seeded = controller._seed_or_consume_bootstrap_receipt(
|
|
state,
|
|
{
|
|
"runnerRows": rows,
|
|
"containers": containers,
|
|
"resourceProof": {"ready": True, "runningRunnerContainers": 0},
|
|
"claims": [],
|
|
},
|
|
)
|
|
|
|
assert seeded is True
|
|
assert saved == [0]
|
|
assert consumed == [(receipt_sha, 1)]
|
|
assert state["bootstrapReceiptSha256"] == receipt_sha
|
|
assert all(entry["observed"] for entry in state["profiles"].values())
|
|
assert all(not entry["operatorDisabled"] for entry in state["profiles"].values())
|
|
|
|
|
|
def test_bootstrap_receipt_rejects_any_running_initial_runner(
|
|
controller_module, monkeypatch, tmp_path
|
|
):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController()
|
|
state = module._initial_state()
|
|
monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json")
|
|
rows = {}
|
|
containers = {}
|
|
registrations = []
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
labels = [profile.runner_label, *profile.legacy_labels]
|
|
rows[profile.container_name] = {
|
|
"id": 100 + index,
|
|
"displayName": profile.display_name,
|
|
"disabled": True,
|
|
"busy": False,
|
|
"labels": set(labels),
|
|
}
|
|
containers[profile.container_name] = {
|
|
"id": f"{index:064x}",
|
|
"running": index == 1,
|
|
"health": "healthy" if index == 1 else "none",
|
|
}
|
|
registrations.append(
|
|
{
|
|
"profileId": profile.id,
|
|
"displayName": profile.display_name,
|
|
"containerName": profile.container_name,
|
|
"id": 100 + index,
|
|
"labels": labels,
|
|
}
|
|
)
|
|
fake_specializer = SimpleNamespace(
|
|
read_admission_bootstrap_receipt=lambda: (
|
|
{
|
|
"status": "pending",
|
|
"verifiedRunners": [
|
|
profile.container_name for profile in module.RUNNER_PROFILES.values()
|
|
],
|
|
"registrations": registrations,
|
|
},
|
|
"e" * 64,
|
|
),
|
|
consume_admission_bootstrap_receipt=lambda _digest: pytest.fail(
|
|
"receipt consumed before live baseline verification"
|
|
),
|
|
)
|
|
monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer)
|
|
saved = []
|
|
monkeypatch.setattr(controller, "_save_state", lambda value: saved.append(value))
|
|
|
|
with pytest.raises(module.RunnerAdmissionError):
|
|
controller._seed_or_consume_bootstrap_receipt(
|
|
state,
|
|
{
|
|
"runnerRows": rows,
|
|
"containers": containers,
|
|
"resourceProof": {"ready": True, "runningRunnerContainers": 1},
|
|
"claims": [],
|
|
},
|
|
)
|
|
assert saved == []
|
|
|
|
|
|
def test_unattested_more_than_two_running_runner_outers_blocks_claims(
|
|
controller_module,
|
|
):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController(
|
|
resource_attestor=lambda **_kwargs: {
|
|
"ready": True,
|
|
"effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI,
|
|
"effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB,
|
|
"activeClaims": 0,
|
|
"heavyClaims": 0,
|
|
"runningRunnerContainers": 3,
|
|
"cgroups": {},
|
|
"digest": "c" * 64,
|
|
}
|
|
)
|
|
|
|
with pytest.raises(
|
|
module.RunnerAdmissionError,
|
|
match="runner_admission_running_outer_budget_exceeded",
|
|
):
|
|
controller._resource_attestation(
|
|
[],
|
|
{name: None for name in module.ALL_LEGACY_RUNNER_NAMES},
|
|
)
|
|
|
|
|
|
def test_operator_open_then_disable_is_latched_from_bootstrap_baseline(
|
|
controller_module,
|
|
):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController()
|
|
state = module._initial_state()
|
|
entry = state["profiles"]["checks"]
|
|
entry.update(observed=True, operatorDisabled=False, operatorEnabled=False)
|
|
profile = module.RUNNER_PROFILES["checks"]
|
|
rows = {}
|
|
for index, item in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
rows[item.container_name] = {
|
|
"id": 200 + index,
|
|
"disabled": True,
|
|
"busy": False,
|
|
}
|
|
state["registrationIds"][item.id] = 200 + index
|
|
rows[profile.container_name]["disabled"] = False
|
|
observation = {
|
|
"runnerRows": rows
|
|
}
|
|
|
|
assert controller._reconcile_ownership(state, observation)
|
|
assert entry["operatorEnabled"] is True
|
|
assert entry["enabledByController"] is False
|
|
rows[profile.container_name]["disabled"] = True
|
|
|
|
assert controller._reconcile_ownership(state, observation)
|
|
assert entry["operatorDisabled"] is True
|
|
assert entry["operatorEnabled"] is False
|
|
|
|
|
|
def test_registration_identity_change_fails_closed(controller_module):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController()
|
|
state = module._initial_state()
|
|
rows = {}
|
|
for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1):
|
|
expected_id = 300 + index
|
|
state["registrationIds"][profile.id] = expected_id
|
|
rows[profile.container_name] = {
|
|
"id": expected_id,
|
|
"disabled": True,
|
|
"busy": False,
|
|
}
|
|
rows[module.RUNNER_PROFILES["checks"].container_name]["id"] += 1
|
|
|
|
with pytest.raises(
|
|
module.RunnerAdmissionError,
|
|
match="runner_admission_registration_identity_changed",
|
|
):
|
|
controller._reconcile_ownership(state, {"runnerRows": rows})
|
|
|
|
|
|
def test_pending_runner_start_does_not_enable_after_guard_pauses(
|
|
controller_module, monkeypatch
|
|
):
|
|
module = controller_module
|
|
controller = module.RunnerAdmissionController(clock=lambda: time.time())
|
|
state = module._initial_state()
|
|
profile = module.RUNNER_PROFILES["checks"]
|
|
runner_id = 101
|
|
job_id = "job-42"
|
|
state["pending"] = {"profileId": "checks", "runnerId": runner_id, "jobId": job_id}
|
|
initial = {
|
|
"runnerRows": {profile.container_name: {"id": runner_id, "disabled": True, "busy": False}},
|
|
"queue": [{"id": job_id, "labels": [profile.runner_label], "queued_at": time.time()}],
|
|
"claims": ["checks"],
|
|
"containers": {profile.container_name: {"running": False, "health": "none"}},
|
|
"resourceProof": {"ready": True},
|
|
}
|
|
after_start = copy.deepcopy(initial)
|
|
after_start["containers"][profile.container_name] = {"running": True, "health": "healthy"}
|
|
observations = iter([after_start, after_start])
|
|
gates = iter([(None, {"host": {"swapSampleAt": time.time()}}),
|
|
("runner_admission_host_guard_paused", None)])
|
|
starts = []
|
|
enables = []
|
|
monkeypatch.setattr(controller, "_save_state", lambda _state: None)
|
|
monkeypatch.setattr(controller, "_observe", lambda **_kwargs: next(observations))
|
|
monkeypatch.setattr(controller, "_fresh_mutation_gate", lambda *_args, **_kwargs: next(gates))
|
|
monkeypatch.setattr(controller, "_docker_run", lambda args, **_kwargs: starts.append(args))
|
|
monkeypatch.setattr(controller, "_set_disabled", lambda *args: enables.append(args))
|
|
|
|
result = controller._continue_pending(state, initial, reserved_profile_id="checks")
|
|
|
|
assert result == "runner_admission_host_guard_paused"
|
|
assert starts == [["docker", "start", profile.container_name]]
|
|
assert enables == []
|