Publish curated Unraid-AppOps-Gateway source from a194da2defbd709ea58bd4c9503966dbd37b11f2
This commit is contained in:
Public Source Publisher committed
1 parent
2c5788da5d
commit
c5224725b7
11 files changed
+626
-18
No files matched your search
@@ -316,6 +316,69 @@ def test_safe_live_metric_fluctuations_do_not_restart_stable_headroom(controller
|
||||
assert reason is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"])
|
||||
def test_cached_post_start_sample_preserves_stability_without_authorizing_enable(
|
||||
controller_module, monkeypatch, change
|
||||
):
|
||||
controller, now, host, capacity = _host_gate_sample(controller_module, monkeypatch)
|
||||
state = {}
|
||||
controller._fresh_mutation_gate("backend", ["backend"], state)
|
||||
now[0] = 160
|
||||
host["swapSampleAt"] = 160
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is None
|
||||
established = copy.deepcopy(state["hostStability"])
|
||||
now[0] = 161
|
||||
assert controller._fresh_mutation_gate(
|
||||
"backend", ["backend"], state, minimum_sample_at=160
|
||||
)[0] == "runner_admission_host_telemetry_not_refreshed"
|
||||
assert state["hostStability"] == established
|
||||
|
||||
now[0] = 162
|
||||
host["swapSampleAt"] = 162
|
||||
assert controller._fresh_mutation_gate(
|
||||
"backend", ["backend"], state, minimum_sample_at=160
|
||||
)[0] is None
|
||||
if change == "load":
|
||||
host["loadRatio"] = 0.71
|
||||
elif change == "memory":
|
||||
host["availableMemoryBytes"] = 15 * 1024**3 - 1
|
||||
elif change == "cache":
|
||||
host["cacheFreeBytes"] = 0
|
||||
elif change == "swap":
|
||||
host["swapCritical"] = True
|
||||
elif change == "stale":
|
||||
now[0] = 253
|
||||
elif change == "future":
|
||||
host["swapSampleAt"] = 163
|
||||
elif change == "pressure":
|
||||
capacity["decision"]["pressure"] = "critical"
|
||||
elif change == "guard":
|
||||
monkeypatch.setattr(controller, "_host_evidence", lambda: ("blocked", "runner_capacity_pressure_critical", capacity))
|
||||
assert controller._fresh_mutation_gate(
|
||||
"backend", ["backend"], state, minimum_sample_at=162
|
||||
)[0] is not None
|
||||
assert state["hostStability"] is None
|
||||
|
||||
|
||||
def test_cached_post_start_wait_never_shortens_required_stability(controller_module, monkeypatch):
|
||||
controller, now, host, _ = _host_gate_sample(controller_module, monkeypatch)
|
||||
state = {}
|
||||
controller._fresh_mutation_gate("backend", ["backend"], state)
|
||||
now[0] = 130
|
||||
host["swapSampleAt"] = 130
|
||||
assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable"
|
||||
now[0] = 131
|
||||
assert controller._fresh_mutation_gate(
|
||||
"backend", ["backend"], state, minimum_sample_at=130
|
||||
)[0] == "runner_admission_host_telemetry_not_refreshed"
|
||||
now[0] = 140
|
||||
host["swapSampleAt"] = 140
|
||||
assert controller._fresh_mutation_gate(
|
||||
"backend", ["backend"], state, minimum_sample_at=130
|
||||
)[0] == "runner_admission_host_headroom_not_stable"
|
||||
assert state["hostStability"]["since"] == 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"])
|
||||
def test_unsafe_sample_restarts_the_entire_stability_window(controller_module, monkeypatch, change):
|
||||
module = controller_module
|
||||
@@ -1539,6 +1602,142 @@ def test_busy_controller_owned_runner_gets_a_durable_disable_but_job_is_not_stop
|
||||
assert claims == ["backend"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("busy_flag,active_inventory", [(True, False), (False, True), (True, True)])
|
||||
def test_busy_disabled_drain_does_not_starve_other_profile(
|
||||
controller_module, monkeypatch, busy_flag, active_inventory
|
||||
):
|
||||
module = controller_module
|
||||
waiting = {"id": "waiting-backend", "labels": ["itworx-ci-backend"], "queued_at": time.time() - 30}
|
||||
controller, state, observation, _inventory, containers = _lease_fixture(
|
||||
module, enabled=("frontend",), busy=("frontend",), queue=(waiting,)
|
||||
)
|
||||
profile = module.RUNNER_PROFILES["frontend"]
|
||||
runner_id = observation["profileRunnerIds"]["frontend"]
|
||||
row = observation["runnerRows"][profile.container_name]
|
||||
row.update(disabled=True, busy=busy_flag)
|
||||
state["profiles"]["frontend"].update(enabledByController=False, drainOwned=True)
|
||||
if not active_inventory:
|
||||
observation["activeJobs"][runner_id] = []
|
||||
stops = []
|
||||
monkeypatch.setattr(controller, "_stop_drained_profile", lambda *args, **kwargs: stops.append(kwargs) or "runner_admission_drain_stop_deferred")
|
||||
|
||||
assert controller._drain_one_idle_profile(state, observation) is None
|
||||
assert stops == []
|
||||
assert containers[profile.container_name]["running"] is True
|
||||
assert state["profiles"]["frontend"]["drainOwned"] is True
|
||||
assert observation["claims"] == ["frontend"]
|
||||
choice = module.select_admission(
|
||||
observation["queue"], available_profiles=["backend"], active_claims=observation["claims"]
|
||||
)
|
||||
assert choice.profile_id == "backend"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocked", [None, "resource", "host", "capacity"])
|
||||
@pytest.mark.parametrize("projection_status", ["healthy", "degraded"])
|
||||
def test_cycle_can_select_second_slot_during_busy_drain_but_keeps_guards(
|
||||
controller_module, monkeypatch, blocked, projection_status
|
||||
):
|
||||
module = controller_module
|
||||
now = time.time()
|
||||
waiting = {"id": "waiting-backend", "labels": ["itworx-ci-backend"], "queued_at": now - 30}
|
||||
controller, state, observation, _inventory, _containers = _lease_fixture(
|
||||
module, enabled=("frontend",), busy=("frontend",), queue=(waiting,)
|
||||
)
|
||||
profile = module.RUNNER_PROFILES["frontend"]
|
||||
observation["runnerRows"][profile.container_name]["disabled"] = True
|
||||
state["profiles"]["frontend"].update(enabledByController=False, drainOwned=True)
|
||||
state["terminalHistoryUpdatedAt"] = now
|
||||
observation.update(specialization="active", at=now, resourceProof={"ready": True}, hostReason=None)
|
||||
if blocked == "resource":
|
||||
observation.update(resourceProof=None, resourceReason="runner_admission_resource_attestation_unready")
|
||||
elif blocked == "host":
|
||||
observation["hostReason"] = "runner_capacity_pressure_critical"
|
||||
elif blocked == "capacity":
|
||||
observation["claims"].append("build")
|
||||
selected = []
|
||||
monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext())
|
||||
monkeypatch.setattr(controller, "_read_state", lambda: state)
|
||||
monkeypatch.setattr(controller, "_save_state", lambda _state: None)
|
||||
monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation)
|
||||
monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_record_idle_state", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": projection_status, "reason": None})
|
||||
monkeypatch.setattr(controller, "_write_projection", lambda _projection: None)
|
||||
monkeypatch.setattr(controller, "_stop_drained_profile", lambda *_args, **_kwargs: pytest.fail("busy job must be retained"))
|
||||
monkeypatch.setattr(controller, "_continue_pending", lambda *_args, **kwargs: selected.append(kwargs["reserved_profile_id"]) or "test_selected")
|
||||
|
||||
result = controller.cycle()
|
||||
|
||||
assert selected == (["backend"] if blocked is None else [])
|
||||
assert observation["claims"] == (["frontend", "build"] if blocked == "capacity" else ["frontend"])
|
||||
assert state["profiles"]["frontend"]["drainOwned"] is True
|
||||
if blocked is not None:
|
||||
assert state["pending"] is None
|
||||
assert result["status"] != "test_selected"
|
||||
assert result["statistics"]["reason"] == result["status"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["reserved", "started"])
|
||||
@pytest.mark.parametrize("blocked", [None, "operator", "fresh_queue", "guard"])
|
||||
def test_pending_start_does_not_starve_guarded_idle_lease_cleanup(
|
||||
controller_module, monkeypatch, stage, blocked
|
||||
):
|
||||
module = controller_module
|
||||
now = time.time()
|
||||
waiting = {"id": "queued-backend", "labels": [module.RUNNER_PROFILES["backend"].runner_label], "queued_at": now - 600}
|
||||
controller, state, observation, inventory, containers = _lease_fixture(
|
||||
module, enabled=("checks",), queue=(waiting,)
|
||||
)
|
||||
checks = module.RUNNER_PROFILES["checks"]
|
||||
checks_id = observation["profileRunnerIds"]["checks"]
|
||||
pending = {"profileId": "backend", "runnerId": observation["profileRunnerIds"]["backend"], "jobId": waiting["id"], "createdAt": module._utc_iso(now - 200), "stage": stage}
|
||||
state.update(pending=pending, terminalHistoryUpdatedAt=module._utc_iso(now))
|
||||
state["profiles"]["checks"]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1
|
||||
observation.update(specialization="active", at=now, resourceProof={"ready": True}, claims=["checks", "backend"])
|
||||
if stage == "started":
|
||||
containers[module.RUNNER_PROFILES["backend"].container_name].update(running=True, health="healthy")
|
||||
if blocked == "operator":
|
||||
state["profiles"]["checks"]["operatorEnabled"] = True
|
||||
queue_rows = [_raw_job(module, waiting)]
|
||||
if blocked == "fresh_queue":
|
||||
queued_check = {"id": "queued-checks", "labels": [checks.runner_label], "queued_at": now - 10}
|
||||
observation["queue"].append(queued_check)
|
||||
queue_rows.append(_raw_job(module, queued_check))
|
||||
_install_lease_io(monkeypatch, controller, module, inventory, containers, queue_rows=queue_rows)
|
||||
if blocked == "guard":
|
||||
monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("blocked", "guard"))
|
||||
patches, continued = [], []
|
||||
def disable(runner_id, value):
|
||||
patches.append((runner_id, value))
|
||||
next(row for row in inventory if row["id"] == runner_id)["disabled"] = value
|
||||
monkeypatch.setattr(controller, "_set_disabled", disable)
|
||||
monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext())
|
||||
monkeypatch.setattr(controller, "_read_state", lambda: state)
|
||||
monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation)
|
||||
monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False)
|
||||
monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": "healthy"})
|
||||
monkeypatch.setattr(controller, "_write_projection", lambda *_args: None)
|
||||
monkeypatch.setattr(controller, "_continue_pending", lambda *_args, **_kwargs: continued.append(True) or "runner_admission_host_headroom_insufficient")
|
||||
result = controller.cycle()
|
||||
assert state["pending"] is pending
|
||||
if blocked is None:
|
||||
assert result["status"] == "runner_admission_drained_idle_profile"
|
||||
assert patches == [(checks_id, True)] and continued == []
|
||||
assert containers[checks.container_name]["running"] is False
|
||||
else:
|
||||
assert patches == [] and containers[checks.container_name]["running"] is True
|
||||
expected = {
|
||||
"operator": "runner_admission_host_headroom_insufficient",
|
||||
"fresh_queue": "runner_admission_drain_deferred_queue",
|
||||
"guard": "runner_admission_drain_state_changed",
|
||||
}
|
||||
assert result["status"] == expected[blocked]
|
||||
|
||||
|
||||
def test_cycle_fences_a_busy_owned_lease_even_without_a_resource_proof(
|
||||
controller_module, monkeypatch
|
||||
):
|
||||
|
||||
Reference in new issue
Block a user