Publish curated Unraid-AppOps-Gateway source from bc001a6df859f7c9836fa145afb739d1ceebc524
This commit is contained in:
Public Source Publisher committed
1 parent
631b846397
commit
ab3fa553f0
15 files changed
+3687
-227
No files matched your search
@@ -5,6 +5,7 @@ import sys
|
||||
import types
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from threading import Event, Thread, current_thread
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -107,6 +108,45 @@ def _healthy_snapshot():
|
||||
|
||||
|
||||
|
||||
def _jobs_page(jobs=()):
|
||||
rows = list(jobs)
|
||||
return {"total_count": len(rows), "jobs": rows}
|
||||
|
||||
|
||||
def _scheduler_api(state, error_type):
|
||||
def fake_gitea(path, params=None):
|
||||
params = params or {}
|
||||
if path.endswith("/runners"):
|
||||
runners = state.get(
|
||||
"runners",
|
||||
[
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
],
|
||||
)
|
||||
return {"total_count": len(runners), "runners": runners}
|
||||
status = params.get("status")
|
||||
if status == state.get("error_status"):
|
||||
raise error_type("runner_fleet_gitea_test_error")
|
||||
rows = state.get(
|
||||
{"queued": "queued", "pending": "pending", "in_progress": "active"}.get(
|
||||
status, ""
|
||||
),
|
||||
[],
|
||||
)
|
||||
if status == state.get("incomplete_status"):
|
||||
return {"total_count": len(rows) + 1, "jobs": rows}
|
||||
return _jobs_page(rows)
|
||||
|
||||
return fake_gitea
|
||||
|
||||
|
||||
def test_container_projection_includes_restart_policy(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
payload = [
|
||||
@@ -780,9 +820,11 @@ def test_scheduler_candidate_uses_live_gitea_queue_and_fixed_label_target(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
monotonic = [1_000.0]
|
||||
created_at = datetime.fromtimestamp(1_800_000_000 - 7_200, UTC).isoformat()
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
params = params or {}
|
||||
@@ -801,33 +843,533 @@ def test_scheduler_candidate_uses_live_gitea_queue_and_fixed_label_target(
|
||||
]
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"jobs": [
|
||||
return _jobs_page(
|
||||
[
|
||||
{
|
||||
"id": 22931,
|
||||
"status": "queued",
|
||||
"run_id": 3001,
|
||||
"attempt": 1,
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"jobs": []}
|
||||
)
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
assert service._scheduler_recovery_candidate() == {
|
||||
# An old created_at is not evidence of an old runnable/claimable interval.
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
candidate = None
|
||||
for _ in range(fleet_module.SCHEDULER_STALL_SECONDS // 60):
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
|
||||
assert candidate == {
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22931,
|
||||
"label": "linux-validation",
|
||||
"ageSeconds": 900,
|
||||
"runnerId": 17,
|
||||
"labels": ("linux-validation",),
|
||||
"reason": "stale_scheduler_assignment",
|
||||
"runIdentity": (("run_id", 3001), ("attempt", 1)),
|
||||
"ageSeconds": fleet_module.SCHEDULER_STALL_SECONDS,
|
||||
}
|
||||
|
||||
|
||||
def test_pending_age_is_not_carried_into_newly_queued_claimability(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [1_000.0]
|
||||
created_at = datetime.fromtimestamp(1_800_000_000 - 7_200, UTC).isoformat()
|
||||
pending_job = {
|
||||
"id": 22931,
|
||||
"status": "requested",
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
queued_job = {**pending_job, "status": "queued"}
|
||||
state = {"pending": [pending_job], "queued": [], "active": []}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
state["pending"] = []
|
||||
state["queued"] = [queued_job]
|
||||
monotonic[0] += fleet_module.SCHEDULER_STALL_SECONDS
|
||||
# The first queued observation starts the timer even though created_at is old.
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
for _ in range(fleet_module.SCHEDULER_STALL_SECONDS // 60):
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
|
||||
assert candidate is not None
|
||||
assert candidate["jobId"] == 22931
|
||||
assert candidate["ageSeconds"] == fleet_module.SCHEDULER_STALL_SECONDS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("uncertainty", ["error", "truncated"])
|
||||
def test_scheduler_claimability_resets_after_unknown_or_incomplete_observation(
|
||||
fleet_module, monkeypatch, uncertainty
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [2_000.0]
|
||||
old_job = {
|
||||
"id": 22931,
|
||||
"status": "queued",
|
||||
"created_at": datetime.fromtimestamp(1_800_000_000 - 7_200, UTC).isoformat(),
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
state = {"queued": [old_job], "pending": [], "active": []}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
monotonic[0] += 60
|
||||
if uncertainty == "error":
|
||||
state["error_status"] = "pending"
|
||||
else:
|
||||
state["incomplete_status"] = "queued"
|
||||
with pytest.raises(fleet_module.RunnerFleetError):
|
||||
service._scheduler_recovery_candidate()
|
||||
|
||||
state.pop("error_status", None)
|
||||
state.pop("incomplete_status", None)
|
||||
monotonic[0] += 60
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
for _ in range((fleet_module.SCHEDULER_STALL_SECONDS // 60) - 2):
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
# The pre-error timer would have matured at this instant if unknown time
|
||||
# had been bridged instead of starting over at the first fresh sample.
|
||||
assert candidate is None
|
||||
for _ in range(2):
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
assert candidate is not None
|
||||
assert candidate["ageSeconds"] == fleet_module.SCHEDULER_STALL_SECONDS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("loss", ["busy", "active"])
|
||||
def test_scheduler_claimability_resets_when_target_becomes_unavailable(
|
||||
fleet_module, monkeypatch, loss
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [3_000.0]
|
||||
job = {
|
||||
"id": 22931,
|
||||
"status": "queued",
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
state = {"queued": [job], "pending": [], "active": []}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
monotonic[0] += 60
|
||||
if loss == "busy":
|
||||
state["runners"] = [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": True,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
else:
|
||||
state["active"] = [
|
||||
{"id": 22932, "status": "in_progress", "runner_id": 17}
|
||||
]
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
state.pop("runners", None)
|
||||
state["active"] = []
|
||||
monotonic[0] += 60
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
for _ in range((fleet_module.SCHEDULER_STALL_SECONDS // 60) - 1):
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
assert candidate is None
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
assert candidate is not None
|
||||
assert candidate["ageSeconds"] == fleet_module.SCHEDULER_STALL_SECONDS
|
||||
|
||||
|
||||
def test_scheduler_claimability_gap_restarts_the_observation_window(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [4_000.0]
|
||||
state = {
|
||||
"queued": [{"id": 22931, "status": "queued", "labels": ["linux-validation"]}],
|
||||
"pending": [],
|
||||
"active": [],
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
monotonic[0] += fleet_module.SCHEDULER_OBSERVATION_MAX_GAP_SECONDS + 1
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
monotonic[0] += fleet_module.SCHEDULER_STALL_SECONDS
|
||||
# A second long gap resets again instead of treating elapsed wall time as observed.
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
|
||||
def test_older_eligible_sample_cannot_repopulate_after_newer_busy_sample(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [1_000.0]
|
||||
queued_job = {
|
||||
"id": 22931,
|
||||
"status": "queued",
|
||||
"run_id": 3001,
|
||||
"attempt": 1,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
idle_state = {"queued": [queued_job], "pending": [], "active": []}
|
||||
live_state = {"queued": [queued_job], "pending": [], "active": []}
|
||||
stale_request = _scheduler_api(idle_state, fleet_module.RunnerFleetError)
|
||||
live_request = _scheduler_api(live_state, fleet_module.RunnerFleetError)
|
||||
older_sample_entered = Event()
|
||||
release_older_sample = Event()
|
||||
older_result: dict[str, object] = {}
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
if current_thread().name == "older-idle-sample":
|
||||
if not older_sample_entered.is_set():
|
||||
older_sample_entered.set()
|
||||
assert release_older_sample.wait(timeout=5)
|
||||
return stale_request(path, params)
|
||||
return live_request(path, params)
|
||||
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
monotonic[0] = 1_010.0
|
||||
older_sample = Thread(
|
||||
name="older-idle-sample",
|
||||
target=lambda: older_result.setdefault(
|
||||
"candidate", service._scheduler_recovery_candidate()
|
||||
),
|
||||
)
|
||||
older_sample.start()
|
||||
assert older_sample_entered.wait(timeout=5)
|
||||
|
||||
monotonic[0] = 1_020.0
|
||||
live_state["runners"] = [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": True,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
monotonic[0] = 1_030.0
|
||||
release_older_sample.set()
|
||||
older_sample.join(timeout=5)
|
||||
assert not older_sample.is_alive()
|
||||
assert older_result["candidate"] is None
|
||||
|
||||
live_state.pop("runners")
|
||||
monotonic[0] = 1_031.0
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
for _ in range(9):
|
||||
monotonic[0] += 60
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
|
||||
# The rejected stale sample must not have started the ten-minute dwell.
|
||||
assert monotonic[0] == 1_571.0
|
||||
assert candidate is None
|
||||
monotonic[0] += 59
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
monotonic[0] += 1
|
||||
candidate = service._scheduler_recovery_candidate()
|
||||
assert candidate is not None
|
||||
assert candidate["ageSeconds"] == fleet_module.SCHEDULER_STALL_SECONDS
|
||||
|
||||
|
||||
def test_scheduler_claimability_is_process_local_and_lost_on_restart(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [5_000.0]
|
||||
state = {
|
||||
"queued": [{"id": 22931, "status": "queued", "labels": ["linux-validation"]}],
|
||||
"pending": [],
|
||||
"active": [],
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
fleet_module.RunnerFleetService._clear_scheduler_claimability()
|
||||
monotonic[0] += fleet_module.SCHEDULER_STALL_SECONDS
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("identity_change", ["origin", "job", "run", "labels", "runner"])
|
||||
def test_scheduler_claimability_restarts_when_job_or_runner_identity_changes(
|
||||
fleet_module, monkeypatch, identity_change
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [6_000.0]
|
||||
job = {
|
||||
"id": 22931,
|
||||
"run_id": 3001,
|
||||
"attempt": 1,
|
||||
"status": "queued",
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
state = {"queued": [job], "pending": [], "active": []}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
for _ in range((fleet_module.SCHEDULER_STALL_SECONDS // 60) - 1):
|
||||
monotonic[0] += 60
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
if identity_change == "origin":
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "GITEA_API_BASE", "http://other-gitea:3000"
|
||||
)
|
||||
elif identity_change == "job":
|
||||
job["id"] = 22932
|
||||
elif identity_change == "run":
|
||||
job["run_id"] = 3002
|
||||
elif identity_change == "labels":
|
||||
job["labels"] = ["ubuntu-latest"]
|
||||
state["runners"] = [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation", "ubuntu-latest"],
|
||||
}
|
||||
]
|
||||
else:
|
||||
state["runners"] = [
|
||||
{
|
||||
"id": 18,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
monotonic[0] += 60
|
||||
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
|
||||
def test_scheduler_candidate_pages_jobs_at_gitea_fifty_row_limit(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
queued = [
|
||||
{"id": job_id, "status": "queued", "labels": ["unmapped-label"]}
|
||||
for job_id in range(1, 51)
|
||||
]
|
||||
queued.append(
|
||||
{"id": 51, "status": "queued", "labels": ["linux-validation"]}
|
||||
)
|
||||
job_calls: list[dict[str, object]] = []
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
params = params or {}
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
],
|
||||
}
|
||||
status = params["status"]
|
||||
if status == "queued":
|
||||
job_calls.append(params)
|
||||
start = (int(params["page"]) - 1) * 50
|
||||
return {"total_count": 51, "jobs": queued[start : start + 50]}
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
assert [(call["page"], call["limit"]) for call in job_calls] == [
|
||||
(1, fleet_module.GITEA_JOB_PAGE_LIMIT),
|
||||
(2, fleet_module.GITEA_JOB_PAGE_LIMIT),
|
||||
]
|
||||
|
||||
|
||||
def test_scheduler_candidate_rejects_job_inventory_over_one_thousand_rows(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
if path.endswith("/runners"):
|
||||
return {"total_count": 0, "runners": []}
|
||||
return {"total_count": 1_001, "jobs": []}
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
with pytest.raises(
|
||||
fleet_module.RunnerFleetError,
|
||||
match="runner_fleet_gitea_invalid_job_inventory",
|
||||
):
|
||||
service._scheduler_recovery_candidate()
|
||||
|
||||
|
||||
def test_scheduler_candidate_fails_closed_when_job_appears_in_two_statuses(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
monotonic = [7_000.0]
|
||||
job = {"id": 22931, "labels": ["linux-validation"]}
|
||||
state = {
|
||||
"queued": [{**job, "status": "queued"}],
|
||||
"pending": [{**job, "status": "requested"}],
|
||||
"active": [],
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
fleet_module, "_scheduler_monotonic_now", lambda: monotonic[0]
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
fleet_module.RunnerFleetError,
|
||||
match="runner_fleet_gitea_job_status_changed_during_read",
|
||||
):
|
||||
service._scheduler_recovery_candidate()
|
||||
state["pending"] = []
|
||||
monotonic[0] += fleet_module.SCHEDULER_STALL_SECONDS
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
|
||||
def test_scheduler_candidate_does_not_classify_offline_target_as_claimable(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
state = {
|
||||
"queued": [
|
||||
{
|
||||
"id": 22931,
|
||||
"status": "queued",
|
||||
"created_at": datetime.fromtimestamp(
|
||||
1_800_000_000 - 7_200, UTC
|
||||
).isoformat(),
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
],
|
||||
"pending": [],
|
||||
"active": [],
|
||||
"runners": [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "offline",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
],
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("label_source", "expected_error"),
|
||||
[
|
||||
("job", "runner_fleet_gitea_invalid_queued_job"),
|
||||
("runner", "runner_fleet_gitea_invalid_runner_inventory"),
|
||||
],
|
||||
)
|
||||
def test_scheduler_candidate_rejects_labels_that_would_change_when_trimmed(
|
||||
fleet_module, monkeypatch, label_source, expected_error
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
state = {
|
||||
"queued": [
|
||||
{"id": 22931, "status": "queued", "labels": ["linux-validation"]}
|
||||
],
|
||||
"pending": [],
|
||||
"active": [],
|
||||
}
|
||||
if label_source == "job":
|
||||
state["queued"][0]["labels"] = [" linux-validation"]
|
||||
else:
|
||||
state["runners"] = [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
"status": "online",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation "],
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
service, "_gitea_request", _scheduler_api(state, fleet_module.RunnerFleetError)
|
||||
)
|
||||
|
||||
with pytest.raises(fleet_module.RunnerFleetError, match=expected_error):
|
||||
service._scheduler_recovery_candidate()
|
||||
|
||||
|
||||
def test_scheduler_candidate_never_targets_busy_runner(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
params = params or {}
|
||||
@@ -846,16 +1388,10 @@ def test_scheduler_candidate_never_targets_busy_runner(fleet_module, monkeypatch
|
||||
]
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"jobs": [
|
||||
{
|
||||
"id": 22931,
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"jobs": []}
|
||||
return _jobs_page(
|
||||
[{"id": 22931, "status": "queued", "labels": ["linux-validation"]}]
|
||||
)
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
@@ -866,9 +1402,6 @@ def test_scheduler_candidate_never_restarts_runner_with_in_progress_job(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
params = params or {}
|
||||
@@ -885,10 +1418,14 @@ def test_scheduler_candidate_never_restarts_runner_with_in_progress_job(
|
||||
}],
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {"jobs": [{"id": 22931, "created_at": created_at, "labels": ["linux-validation"]}]}
|
||||
return _jobs_page(
|
||||
[{"id": 22931, "status": "queued", "labels": ["linux-validation"]}]
|
||||
)
|
||||
if params.get("status") == "in_progress":
|
||||
return {"jobs": [{"id": 22932, "runner_id": 17, "status": "in_progress"}]}
|
||||
return {"jobs": []}
|
||||
return _jobs_page(
|
||||
[{"id": 22932, "runner_id": 17, "status": "in_progress"}]
|
||||
)
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
@@ -897,9 +1434,6 @@ def test_scheduler_candidate_never_restarts_runner_with_in_progress_job(
|
||||
|
||||
def test_scheduler_candidate_never_hides_disabled_busy_runner(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
params = params or {}
|
||||
@@ -920,16 +1454,10 @@ def test_scheduler_candidate_never_hides_disabled_busy_runner(fleet_module, monk
|
||||
]
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"jobs": [
|
||||
{
|
||||
"id": 22931,
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"jobs": []}
|
||||
return _jobs_page(
|
||||
[{"id": 22931, "status": "queued", "labels": ["linux-validation"]}]
|
||||
)
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
@@ -940,9 +1468,6 @@ def test_scheduler_candidate_reads_later_runner_inventory_pages(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
runner_calls: list[dict[str, object]] = []
|
||||
|
||||
def fake_gitea(path, params=None):
|
||||
@@ -977,16 +1502,10 @@ def test_scheduler_candidate_reads_later_runner_inventory_pages(
|
||||
],
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"jobs": [
|
||||
{
|
||||
"id": 22931,
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"jobs": []}
|
||||
return _jobs_page(
|
||||
[{"id": 22931, "status": "queued", "labels": ["linux-validation"]}]
|
||||
)
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
@@ -1002,7 +1521,7 @@ def test_scheduler_candidate_fails_closed_on_malformed_runner_inventory(
|
||||
def fake_gitea(path, params=None):
|
||||
if path.endswith("/runners"):
|
||||
return {"total_count": 1}
|
||||
return {"jobs": []}
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
@@ -1018,9 +1537,6 @@ def test_scheduler_candidate_vetoes_any_unsafe_duplicate_fixed_runner(
|
||||
fleet_module, monkeypatch, unsafe_first
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
created_at = datetime.fromtimestamp(now - 900, UTC).isoformat()
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
unsafe = {
|
||||
"id": 17,
|
||||
"name": "itworx-validation-runner-2",
|
||||
@@ -1044,22 +1560,149 @@ def test_scheduler_candidate_vetoes_any_unsafe_duplicate_fixed_runner(
|
||||
runners = [unsafe, idle] if unsafe_first else [idle, unsafe]
|
||||
return {"total_count": 2, "runners": runners}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"jobs": [
|
||||
{
|
||||
"id": 22931,
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
]
|
||||
}
|
||||
return {"jobs": []}
|
||||
return _jobs_page(
|
||||
[{"id": 22931, "status": "queued", "labels": ["linux-validation"]}]
|
||||
)
|
||||
return _jobs_page()
|
||||
|
||||
monkeypatch.setattr(service, "_gitea_request", fake_gitea)
|
||||
|
||||
assert service._scheduler_recovery_candidate() is None
|
||||
|
||||
|
||||
def test_automatic_cycle_rechecks_the_same_scheduler_candidate_before_restart(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
commands: list[list[str]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
candidate = {
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22931,
|
||||
"label": "linux-validation",
|
||||
"runnerId": 17,
|
||||
"labels": ("linux-validation",),
|
||||
"runIdentity": (("run_id", 3001), ("attempt", 1)),
|
||||
"ageSeconds": 600,
|
||||
"reason": "stale_scheduler_assignment",
|
||||
}
|
||||
changed = {**candidate, "runIdentity": (("run_id", 3002), ("attempt", 1))}
|
||||
candidates = iter([candidate, changed])
|
||||
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
||||
monkeypatch.setattr(
|
||||
service, "_scheduler_recovery_candidate", lambda: next(candidates)
|
||||
)
|
||||
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_container",
|
||||
lambda _target: {
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
||||
)
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event) or True)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert commands == []
|
||||
assert result["schedulerRecovery"] == {
|
||||
"status": "ineligible",
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22931,
|
||||
"reason": "scheduler_claimability_changed",
|
||||
}
|
||||
assert any(
|
||||
event.get("action") == "restart.result"
|
||||
and event.get("result") == "cancelled"
|
||||
for event in events
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("unsafe_source", "expected_reason"),
|
||||
[
|
||||
("host_guard", "host_resource_guard_paused"),
|
||||
("pressure", "runner_capacity_pressure_critical"),
|
||||
],
|
||||
)
|
||||
def test_automatic_cycle_rechecks_host_safety_after_final_scheduler_read(
|
||||
fleet_module, monkeypatch, unsafe_source, expected_reason
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
commands: list[list[str]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
safety_changed = [False]
|
||||
candidate = {
|
||||
"target": "itworx-validation-runner-2",
|
||||
"jobId": 22931,
|
||||
"label": "linux-validation",
|
||||
"runnerId": 17,
|
||||
"labels": ("linux-validation",),
|
||||
"runIdentity": (("run_id", 3001), ("attempt", 1)),
|
||||
"ageSeconds": 600,
|
||||
"reason": "stale_scheduler_assignment",
|
||||
}
|
||||
scheduler_reads = [0]
|
||||
|
||||
def read_candidate():
|
||||
scheduler_reads[0] += 1
|
||||
if scheduler_reads[0] == 2:
|
||||
safety_changed[0] = True
|
||||
return candidate
|
||||
|
||||
def read_host_guard():
|
||||
if safety_changed[0] and unsafe_source == "host_guard":
|
||||
return "paused", "host_resource_guard_paused"
|
||||
return "clear", ""
|
||||
|
||||
def read_capacity(_self):
|
||||
capacity = _healthy_capacity_snapshot()
|
||||
if safety_changed[0] and unsafe_source == "pressure":
|
||||
capacity["decision"]["pressure"] = "critical"
|
||||
return capacity
|
||||
|
||||
monkeypatch.setattr(service, "snapshot", _healthy_snapshot)
|
||||
monkeypatch.setattr(service, "_scheduler_recovery_candidate", read_candidate)
|
||||
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_container",
|
||||
lambda _target: {
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(fleet_module, "_host_guard_state", read_host_guard)
|
||||
monkeypatch.setattr(
|
||||
fleet_module.RunnerCapacityController, "snapshot", read_capacity
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_run", lambda args, timeout=20: commands.append(args) or ""
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service, "_record", lambda **event: events.append(event) or True
|
||||
)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["status"] == "deferred"
|
||||
assert result["reason"] == expected_reason
|
||||
assert commands == []
|
||||
# The durable intent remains in the budget even though the final fresh
|
||||
# safety check cancelled the Docker mutation.
|
||||
assert any(
|
||||
event.get("action") == "restart" and event.get("result") == "intent"
|
||||
for event in events
|
||||
)
|
||||
assert not any(event.get("action") == "restart.result" for event in events)
|
||||
|
||||
|
||||
def test_automatic_cycle_restarts_only_exact_known_idle_runner_for_stale_scheduler(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
|
||||
Reference in new issue
Block a user