from __future__ import annotations import importlib.util from datetime import UTC, datetime from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py" @pytest.fixture def fleet_module(): spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def _healthy_snapshot(): return { "status": "healthy", "networks": {"cleanupCandidates": []}, "storage": {"status": "healthy"}, "topology": { "dind": { "name": "gitea-runner-dind", "running": True, "health": "healthy", "startedAt": "2026-09-16T18:00:00Z", "restartPolicy": "unless-stopped", }, "runners": [ { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", "startedAt": "2026-09-16T18:00:00Z", }, { "name": "itworx-security-runner", "running": True, "health": "healthy", "startedAt": "2026-09-16T18:00:00Z", }, ] }, } def test_container_projection_includes_restart_policy(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() payload = [ { "Config": {"Labels": {"com.docker.compose.project": "gitea-deploy-runners"}}, "HostConfig": {"RestartPolicy": {"Name": "unless-stopped", "MaximumRetryCount": 0}}, "State": { "Running": True, "StartedAt": "2026-09-21T16:00:00Z", "OOMKilled": False, "Health": {"Status": "healthy"}, }, "RestartCount": 2, } ] monkeypatch.setattr(service, "_run", lambda _args, timeout=20: __import__("json").dumps(payload)) result = service._container("gitea-runner-dind") assert result["restartPolicy"] == "unless-stopped" assert result["running"] is True assert result["health"] == "healthy" assert result["restarts"] == 2 def test_snapshot_keeps_exited_dind_actionable_without_querying_dind_networks( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() states = { "gitea-runner-dind": { "name": "gitea-runner-dind", "running": False, "health": "none", "restarts": 0, "startedAt": "", "oomKilled": False, "restartPolicy": "no", }, "itworx-validation-runner-2": { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", "oomKilled": False, "restartPolicy": "unless-stopped", }, "itworx-security-runner": { "name": "itworx-security-runner", "running": True, "health": "healthy", "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", "oomKilled": False, "restartPolicy": "unless-stopped", }, } monkeypatch.setattr(service, "_container", lambda name: states[name]) monkeypatch.setattr(service, "_optional_container", lambda _name: None) monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000}) monkeypatch.setattr( service, "_network_inventory", lambda: pytest.fail("network inventory must not query an exited DinD daemon"), ) monkeypatch.setattr( service, "cleanup_candidates", lambda: pytest.fail("cleanup must not query an exited DinD daemon"), ) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) result = service.snapshot() assert result["status"] == "critical" assert result["topology"]["dind"]["running"] is False assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []} assert result["policy"]["hostDockerMutationTargets"][0] == "gitea-runner-dind" def test_snapshot_does_not_query_dind_api_during_health_start_grace( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 monkeypatch.setattr(fleet_module.time, "time", lambda: now) states = { "gitea-runner-dind": { "name": "gitea-runner-dind", "running": True, "health": "starting", "restarts": 0, "startedAt": datetime.fromtimestamp(now - 10, UTC).isoformat(), "oomKilled": False, "restartPolicy": "unless-stopped", }, "itworx-validation-runner-2": { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", "restarts": 0, "startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(), "oomKilled": False, "restartPolicy": "unless-stopped", }, "itworx-security-runner": { "name": "itworx-security-runner", "running": True, "health": "healthy", "restarts": 0, "startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(), "oomKilled": False, "restartPolicy": "unless-stopped", }, } monkeypatch.setattr(service, "_container", lambda name: states[name]) monkeypatch.setattr(service, "_optional_container", lambda _name: None) monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000}) monkeypatch.setattr( service, "_network_inventory", lambda: pytest.fail("DinD API must not be queried during health start grace"), ) monkeypatch.setattr( service, "cleanup_candidates", lambda: pytest.fail("cleanup must not query DinD during health start grace"), ) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) result = service.snapshot() assert result["status"] == "healthy" assert result["topology"]["dind"]["health"] == "starting" assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []} def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"] = { "name": "gitea-runner-dind", "running": True, "health": "starting", "startedAt": "2026-09-21T16:00:00Z", "restartPolicy": "unless-stopped", } commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [] assert events == [] assert result["dindRecovery"] == { "status": "starting", "target": "gitea-runner-dind", } assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"} def test_automatic_cycle_sets_self_restart_and_starts_only_fixed_exited_dind( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["topology"]["dind"] = { "name": "gitea-runner-dind", "running": False, "health": "none", "startedAt": "", "restartPolicy": "no", } commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_recovery_available", lambda target: target == "gitea-runner-dind") monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append((args, timeout)) or "", ) monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [ (["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30), (["docker", "start", "gitea-runner-dind"], 30), ] assert result["dindRecovery"]["status"] == "requested" assert result["schedulerRecovery"]["status"] == "deferred_for_dind_recovery" assert any( event.get("action") == "start" and event.get("target") == "gitea-runner-dind" and event.get("reason") == "known_ci_dind_exited" for event in events ) def test_automatic_cycle_never_restarts_running_unhealthy_dind( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["topology"]["dind"] = { "name": "gitea-runner-dind", "running": True, "health": "unhealthy", "startedAt": "2026-09-16T18:00:00Z", "restartPolicy": "unless-stopped", } commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [] assert result["dindRecovery"] == { "status": "deferred", "target": "gitea-runner-dind", } assert any( event.get("action") == "restart.deferred" and event.get("target") == "gitea-runner-dind" for event in events ) def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 monkeypatch.setattr(fleet_module.time, "time", lambda: now) recent = datetime.fromtimestamp(now - 60, UTC).isoformat() monkeypatch.setattr( service, "_events", lambda: [ {"at": recent, "action": "start", "target": "gitea-runner-dind"}, {"at": recent, "action": "restart", "target": "gitea-runner-dind"}, {"at": recent, "action": "start", "target": "gitea-runner-dind"}, ], ) assert service._recovery_available("gitea-runner-dind") is False def test_scheduler_candidate_uses_live_gitea_queue_and_fixed_label_target( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): return { "total_count": 1, "runners": [ { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation", "ubuntu-latest"], } ] } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() == { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", } def test_scheduler_candidate_never_targets_busy_runner(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): return { "total_count": 1, "runners": [ { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": False, "labels": ["linux-validation"], } ] } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_scheduler_candidate_never_restarts_runner_with_in_progress_job( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): return { "total_count": 1, "runners": [{ "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation"], }], } if params.get("status") == "queued": return {"jobs": [{"id": 22931, "created_at": created_at, "labels": ["linux-validation"]}]} if params.get("status") == "in_progress": return {"jobs": [{"id": 22932, "runner_id": 17, "status": "in_progress"}]} return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_scheduler_candidate_never_hides_disabled_busy_runner(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): if params.get("disabled") == "false": return {"total_count": 0, "runners": []} return { "total_count": 1, "runners": [ { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": True, "labels": ["linux-validation"], } ] } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_scheduler_candidate_reads_later_runner_inventory_pages( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) runner_calls: list[dict[str, object]] = [] def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): runner_calls.append(params) if params.get("page") == 2: return { "total_count": 2, "runners": [ { "id": 18, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": True, "labels": ["linux-validation"], } ], } return { "total_count": 2, "runners": [ { "id": 17, "name": "unrelated-runner", "status": "online", "busy": False, "disabled": False, "labels": ["unrelated"], } ], } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None assert [call.get("page") for call in runner_calls] == [1, 2] def test_scheduler_candidate_fails_closed_on_malformed_runner_inventory( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() def fake_gitea(path, params=None): if path.endswith("/runners"): return {"total_count": 1} return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) with pytest.raises( fleet_module.RunnerFleetError, match="runner_fleet_gitea_invalid_runner_inventory", ): service._scheduler_recovery_candidate() @pytest.mark.parametrize("unsafe_first", [True, False]) def test_scheduler_candidate_vetoes_any_unsafe_duplicate_fixed_runner( fleet_module, monkeypatch, unsafe_first ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) unsafe = { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": True, "labels": ["linux-validation"], } idle = { "id": 18, "name": "itworx-validation-runner-2", "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation"], } def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): runners = [unsafe, idle] if unsafe_first else [idle, unsafe] return {"total_count": 2, "runners": runners} if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_automatic_cycle_restarts_only_exact_known_idle_runner_for_stale_scheduler( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append((args, timeout)) or "", ) monkeypatch.setattr( service, "_record", lambda **event: events.append(event) or True ) result = service.automatic_cycle() assert commands == [ (["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45) ] assert any( event.get("action") == "restart" and event.get("target") == "itworx-validation-runner-2" and event.get("reason") == "stale_scheduler_assignment" for event in events ) assert result["schedulerRecovery"] == { "status": "requested", "target": "itworx-validation-runner-2", "jobId": 22931, } def test_scheduler_restart_persists_intent_before_invoking_docker( fleet_module, monkeypatch, tmp_path ): service = fleet_module.RunnerFleetService() monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path) monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) def observe_durable_intent(args, timeout=20): assert args == [ "docker", "restart", "--timeout", "30", "itworx-validation-runner-2", ] assert timeout == 45 persisted = service._events() assert persisted[-1] == { "at": persisted[-1]["at"], "action": "restart", "target": "itworx-validation-runner-2", "reason": "stale_scheduler_assignment", "result": "intent", "ageSeconds": 900, } return "" monkeypatch.setattr(service, "_run", observe_durable_intent) result = service.automatic_cycle() assert result["schedulerRecovery"]["status"] == "requested" def test_scheduler_recovery_refuses_restart_when_intent_cannot_be_persisted( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) monkeypatch.setattr(service, "_safe_state_root", lambda: False) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "failed", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "restart_intent_not_persisted", } def test_scheduler_recovery_refuses_restart_when_intent_write_raises_oserror( fleet_module, monkeypatch, tmp_path ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path) monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) monkeypatch.setattr( fleet_module.os, "open", lambda *_args, **_kwargs: (_ for _ in ()).throw( PermissionError("state storage is unwritable") ), ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "failed", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "restart_intent_not_persisted", } def test_scheduler_recovery_keeps_intent_when_outcome_write_fails( fleet_module, monkeypatch, tmp_path ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] event_open_count = 0 real_open = fleet_module.os.open monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path) monkeypatch.setattr(fleet_module, "RECOVERY_BUDGET", 1) monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) def fail_outcome_open(path, flags, mode=0o777): nonlocal event_open_count if Path(path).name.startswith(".events."): event_open_count += 1 if event_open_count == 2: raise PermissionError("outcome storage is unwritable") return real_open(path, flags, mode) monkeypatch.setattr(fleet_module.os, "open", fail_outcome_open) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) result = service.automatic_cycle() assert commands == [ ["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"] ] assert result["schedulerRecovery"] == { "status": "requested", "target": "itworx-validation-runner-2", "jobId": 22931, } assert [event["action"] for event in service._events()] == ["restart"] assert service._recovery_available("itworx-validation-runner-2") is False @pytest.mark.parametrize( ("running", "health"), [ (True, "starting"), (True, "none"), (True, "unhealthy"), (False, "healthy"), ], ) def test_scheduler_recovery_rechecks_strict_runner_health_before_restart( fleet_module, monkeypatch, running, health ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": running, "health": health, }, ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) monkeypatch.setattr(service, "_record", lambda **_event: True) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "ineligible", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "docker_health_not_healthy", } def test_scheduler_recovery_rejects_duplicate_fresh_inspect_results( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() commands: list[tuple[list[str], int]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) def duplicate_inspect(args, timeout=20): commands.append((args, timeout)) return "[{}, {}]" monkeypatch.setattr(service, "_run", duplicate_inspect) monkeypatch.setattr(service, "_record", lambda **_event: True) result = service.automatic_cycle() assert commands == [ (["docker", "inspect", "itworx-validation-runner-2"], 20) ] assert result["schedulerRecovery"] == { "status": "ineligible", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "docker_health_not_healthy", } def test_scheduler_recovery_rejects_runner_without_healthy_docker_state( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["runners"][0]["health"] = "unhealthy" commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "ineligible", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "docker_health_not_healthy", } def test_scheduler_recovery_refuses_unknown_target(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "Gitea", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"]["status"] == "rejected" def test_scheduler_recovery_obeys_existing_recovery_budget(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: False) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"]["status"] == "budget_exhausted" def test_scheduler_recovery_does_nothing_without_stale_candidate(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == {"status": "not_needed"}