from __future__ import annotations import importlib.util import sys import types from datetime import UTC, datetime from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py" @pytest.fixture def fleet_module(monkeypatch): if sys.platform == "win32" and "fcntl" not in sys.modules: fcntl = types.ModuleType("fcntl") fcntl.LOCK_EX = 2 fcntl.LOCK_UN = 8 fcntl.LOCK_NB = 4 fcntl.flock = lambda *_args, **_kwargs: None monkeypatch.setitem(sys.modules, "fcntl", fcntl) spec = importlib.util.spec_from_file_location("appops_runner_fleet_scheduler_test", FLEET_PATH) assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module._host_guard_state_unpatched = module._host_guard_state monkeypatch.setitem( sys.modules, "runner_fleet_specialize", types.SimpleNamespace(specialization_state=lambda: "legacy"), ) monkeypatch.setattr(module, "_host_guard_state", lambda: ("clear", "")) monkeypatch.setattr( module.RunnerCapacityController, "snapshot", lambda _self: _healthy_capacity_snapshot(), ) return module def _healthy_capacity_snapshot(): return { "status": "ok", "decision": {"pressure": "headroom"}, "host": { "swapCritical": False, "loadRatio": 0.1, "availableMemoryBytes": 12 * 1024**3, "cacheFreeBytes": 100 * 1024**3, "swapSampleAt": datetime.now(UTC).timestamp(), }, } def _healthy_snapshot(): runners = [ { "name": name, "running": True, "health": "healthy", "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", "oomKilled": False, "restartPolicy": "unless-stopped", } for name in ( "itworx-security-runner", "itworx-validation-runner-2", ) ] return { "status": "healthy", "generatedAt": "2026-10-01T10:00:00Z", "capacity": _healthy_capacity_snapshot(), "networks": {"count": 0, "capacity": None, "cleanupCandidates": []}, "storage": { "status": "healthy", "freeBytes": 30 * 1024**3, "totalBytes": 40 * 1024**3, }, "scheduler": { "status": "healthy", "candidate": None, "stallThresholdSeconds": 600, }, "topology": { "dind": { "name": "gitea-runner-dind", "running": True, "health": "healthy", "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", "oomKilled": False, "restartPolicy": "unless-stopped", }, "runners": runners, "generalSlots": 2, "runnerCount": len(runners), "specializedRunners": [], "burstRunner": None, }, } def test_container_projection_includes_restart_policy(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() payload = [ { "Config": {"Labels": {"com.docker.compose.project": "gitea-deploy-runners"}}, "HostConfig": {"RestartPolicy": {"Name": "unless-stopped", "MaximumRetryCount": 0}}, "State": { "Running": True, "StartedAt": "2026-09-21T16:00:00Z", "OOMKilled": False, "Health": {"Status": "healthy"}, }, "RestartCount": 2, } ] monkeypatch.setattr(service, "_run", lambda _args, timeout=20: __import__("json").dumps(payload)) result = service._container("gitea-runner-dind") assert result["restartPolicy"] == "unless-stopped" assert result["running"] is True assert result["health"] == "healthy" assert result["restarts"] == 2 def test_snapshot_keeps_exited_dind_actionable_without_querying_dind_networks( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() states = { "gitea-runner-dind": { "name": "gitea-runner-dind", "running": False, "health": "none", "restarts": 0, "startedAt": "", "oomKilled": False, "restartPolicy": "no", }, "itworx-validation-runner-2": { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", "oomKilled": False, "restartPolicy": "unless-stopped", }, "itworx-security-runner": { "name": "itworx-security-runner", "running": True, "health": "healthy", "restarts": 0, "startedAt": "2026-09-16T18:00:00Z", "oomKilled": False, "restartPolicy": "unless-stopped", }, } monkeypatch.setattr(service, "_container", lambda name: states[name]) monkeypatch.setattr(service, "_optional_container", lambda _name: None) monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000}) monkeypatch.setattr( service, "_network_inventory", lambda: pytest.fail("network inventory must not query an exited DinD daemon"), ) monkeypatch.setattr( service, "cleanup_candidates", lambda: pytest.fail("cleanup must not query an exited DinD daemon"), ) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) result = service.snapshot() assert result["status"] == "critical" assert result["topology"]["dind"]["running"] is False assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []} assert result["policy"]["hostDockerMutationTargets"][0] == "gitea-runner-dind" def test_snapshot_does_not_query_dind_api_during_health_start_grace( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 monkeypatch.setattr(fleet_module.time, "time", lambda: now) states = { "gitea-runner-dind": { "name": "gitea-runner-dind", "running": True, "health": "starting", "restarts": 0, "startedAt": datetime.fromtimestamp(now - 10, UTC).isoformat(), "oomKilled": False, "restartPolicy": "unless-stopped", }, "itworx-validation-runner-2": { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", "restarts": 0, "startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(), "oomKilled": False, "restartPolicy": "unless-stopped", }, "itworx-security-runner": { "name": "itworx-security-runner", "running": True, "health": "healthy", "restarts": 0, "startedAt": datetime.fromtimestamp(now - 300, UTC).isoformat(), "oomKilled": False, "restartPolicy": "unless-stopped", }, } monkeypatch.setattr(service, "_container", lambda name: states[name]) monkeypatch.setattr(service, "_optional_container", lambda _name: None) monkeypatch.setattr(service, "_storage", lambda: {"status": "healthy", "freeBytes": 20_000_000_000, "totalBytes": 40_000_000_000}) monkeypatch.setattr( service, "_network_inventory", lambda: pytest.fail("DinD API must not be queried during health start grace"), ) monkeypatch.setattr( service, "cleanup_candidates", lambda: pytest.fail("cleanup must not query DinD during health start grace"), ) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) result = service.snapshot() assert result["status"] == "healthy" assert result["topology"]["dind"]["health"] == "starting" assert result["networks"] == {"count": 0, "capacity": None, "cleanupCandidates": []} def test_automatic_cycle_defers_every_other_recovery_while_dind_is_starting( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"] = { "name": "gitea-runner-dind", "running": True, "health": "starting", "startedAt": "2026-09-21T16:00:00Z", "restartPolicy": "unless-stopped", } commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [] assert events == [] assert result["dindRecovery"] == { "status": "starting", "target": "gitea-runner-dind", } assert result["schedulerRecovery"] == {"status": "deferred_for_dind_recovery"} def test_automatic_cycle_recovers_critical_exited_dind_with_safe_host_capacity( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["topology"]["dind"].update( { "running": False, "health": "none", "startedAt": "", "restartPolicy": "no", } ) commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_recovery_available", lambda target: target == "gitea-runner-dind") monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append((args, timeout)) or "", ) monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [ (["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30), (["docker", "start", "gitea-runner-dind"], 30), ] assert [event["action"] for event in events] == [ "restart-policy.update", "start", ] assert result["status"] == "ok" assert result["dindRecovery"] == { "status": "requested", "restartPolicy": "requested", "target": "gitea-runner-dind", } def test_automatic_cycle_repairs_fixed_dind_policy_when_healthy_and_guard_clear( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"]["restartPolicy"] = "no" commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append((args, timeout)) or "", ) monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [ (["docker", "update", "--restart", "unless-stopped", "gitea-runner-dind"], 30) ] assert result["dindRecovery"] == { "status": "not_needed", "restartPolicy": "requested", } assert len(events) == 1 assert events[0]["action"] == "restart-policy.update" def test_automatic_cycle_never_restarts_running_unhealthy_dind( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "healthy" snapshot["topology"]["dind"] = { "name": "gitea-runner-dind", "running": True, "health": "unhealthy", "startedAt": "2026-09-16T18:00:00Z", "restartPolicy": "unless-stopped", } commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [] assert result["dindRecovery"] == { "status": "deferred", "target": "gitea-runner-dind", } assert any( event.get("action") == "restart.deferred" and event.get("target") == "gitea-runner-dind" for event in events ) @pytest.mark.parametrize( ("guard_state", "guard_reason"), [ (("paused", "host_resource_guard_paused"), "host_resource_guard_paused"), (("unavailable", "host_guard_state_untrusted"), "host_guard_state_untrusted"), ], ) def test_automatic_cycle_never_mutates_when_host_guard_is_paused_or_untrusted( fleet_module, monkeypatch, guard_state, guard_reason ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"].update( {"running": False, "health": "none", "restartPolicy": "no"} ) commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: guard_state) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == guard_reason assert commands == [] assert events == [] @pytest.mark.parametrize( "action", [ "restart-policy-update", "dind-start", "scheduler-restart", "network-remove", "cache-prune", "runner-start", ], ) def test_guard_pause_appearing_after_cycle_entry_blocks_each_mutation( fleet_module, monkeypatch, action ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() dind = snapshot["topology"]["dind"] monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) if action == "restart-policy-update": dind["restartPolicy"] = "no" elif action == "dind-start": dind.update({"running": False, "health": "none"}) elif action == "scheduler-restart": snapshot["topology"]["runners"] = [ { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", } ] monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22, "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr( service, "_container", lambda name: { "name": name, "running": True, "health": "healthy", }, ) elif action == "network-remove": snapshot["networks"]["cleanupCandidates"] = [ { "id": "a" * 64, "name": "GITEA-ACTIONS-TASK-22-WORKFLOW-ci-JOB-test-network", "ageSeconds": 3600, "reason": "empty_gitea_actions_task_network_in_dedicated_dind", } ] elif action == "cache-prune": snapshot["storage"]["status"] = "critical" elif action == "runner-start": snapshot["topology"]["runners"] = [ { "name": "itworx-validation-runner-2", "running": False, "health": "unhealthy", } ] guard_states = iter( [ ("clear", ""), ("paused", "host_resource_guard_paused"), ] ) commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: next(guard_states)) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == "host_resource_guard_paused" assert commands == [] assert events == [] def test_critical_current_pressure_after_cycle_entry_blocks_mutation( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"]["restartPolicy"] = "no" current_samples = iter( [ { "status": "ok", "decision": {"pressure": "critical"}, "host": {"swapCritical": True}, } ] ) commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) monkeypatch.setattr( fleet_module.RunnerCapacityController, "snapshot", lambda _self: next(current_samples), ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "", ) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == "runner_capacity_pressure_critical" assert commands == [] def test_stale_capacity_sample_blocks_automatic_mutation(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"]["restartPolicy"] = "no" stale = _healthy_capacity_snapshot() stale["host"]["swapSampleAt"] = ( datetime.now(UTC).timestamp() - fleet_module.CAPACITY_SAMPLE_MAX_AGE_SECONDS - 1 ) commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) monkeypatch.setattr( fleet_module.RunnerCapacityController, "snapshot", lambda _self: stale, ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "", ) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == "runner_capacity_telemetry_unavailable" assert commands == [] @pytest.mark.parametrize("capacity", [{"status": "unavailable"}, {"status": "not_configured"}]) def test_unavailable_or_missing_current_pressure_blocks_mutation( fleet_module, monkeypatch, capacity ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["topology"]["dind"]["restartPolicy"] = "no" commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) monkeypatch.setattr( fleet_module.RunnerCapacityController, "snapshot", lambda _self: capacity, ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "", ) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == "runner_capacity_telemetry_unavailable" assert commands == [] def test_automatic_cycle_rejects_incomplete_critical_snapshot( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["topology"]["dind"].update( {"running": False, "health": "none", "restartPolicy": "no"} ) snapshot["topology"].pop("runnerCount") monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( service, "_run", lambda *_args, **_kwargs: pytest.fail("critical fleet must not mutate") ) monkeypatch.setattr( service, "_record", lambda **_event: pytest.fail("critical fleet must not audit a mutation") ) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == "runner_fleet_snapshot_untrusted" assert result["dindRecovery"]["status"] == "deferred" def test_automatic_cycle_recovers_stalled_scheduler_in_critical_snapshot( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["scheduler"] = { "status": "stalled", "candidate": { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, "stallThresholdSeconds": 600, } commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: snapshot["scheduler"]["candidate"], ) monkeypatch.setattr( service, "_container", lambda name: { "name": name, "running": True, "health": "healthy", "startedAt": "2026-09-16T18:00:00Z", "restarts": 0, "oomKilled": False, "restartPolicy": "unless-stopped", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append((args, timeout)) or "", ) monkeypatch.setattr(service, "_record", lambda **event: events.append(event) or True) result = service.automatic_cycle() assert commands == [ (["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45) ] assert result["status"] == "ok" assert result["schedulerRecovery"] == { "status": "requested", "target": "itworx-validation-runner-2", "jobId": 22931, } assert [event["action"] for event in events] == ["restart", "restart.result"] def test_automatic_cycle_allows_critical_storage_prune_with_safe_host_capacity( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["storage"] = { "status": "critical", "freeBytes": 8 * 1024**3, "totalBytes": 40 * 1024**3, } commands: list[list[str]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) monkeypatch.setattr(service, "_dind", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **event: events.append(event)) result = service.automatic_cycle() assert commands == [ ["image", "prune", "--force"], ["builder", "prune", "--force", "--filter", "until=168h"], ] assert result["status"] == "ok" assert events == [ { "action": "cache.prune", "target": "gitea-runner-dind", "reason": "ci_storage_critical", "result": "completed", } ] @pytest.mark.parametrize( ("capacity", "reason"), [ ({"status": "ok", "decision": {"pressure": "critical"}, "host": {"swapCritical": False}}, "runner_capacity_pressure_critical"), ({"status": "unavailable"}, "runner_capacity_telemetry_unavailable"), ({"status": "ok", "decision": {}, "host": {}}, "runner_capacity_telemetry_untrusted"), ], ) def test_automatic_cycle_blocks_critical_or_untrusted_capacity_snapshot( fleet_module, monkeypatch, capacity, reason ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() snapshot["status"] = "critical" snapshot["capacity"] = capacity snapshot["topology"]["dind"].update( {"running": False, "health": "none", "restartPolicy": "no"} ) monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( service, "_run", lambda *_args, **_kwargs: pytest.fail("unsafe capacity must not mutate") ) monkeypatch.setattr(fleet_module, "_host_guard_state", lambda: ("clear", "")) result = service.automatic_cycle() assert result["status"] == "deferred" assert result["reason"] == reason def test_host_guard_probe_fails_closed_when_path_is_not_posix_root(monkeypatch, fleet_module): monkeypatch.setattr(fleet_module.os, "name", "nt") assert fleet_module._host_guard_state_unpatched() == ( "unavailable", "host_guard_state_untrusted", ) def test_recovery_budget_counts_start_and_restart_attempts(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 monkeypatch.setattr(fleet_module.time, "time", lambda: now) recent = datetime.fromtimestamp(now - 60, UTC).isoformat() monkeypatch.setattr( service, "_events", lambda: [ {"at": recent, "action": "start", "target": "gitea-runner-dind"}, {"at": recent, "action": "restart", "target": "gitea-runner-dind"}, {"at": recent, "action": "start", "target": "gitea-runner-dind"}, ], ) assert service._recovery_available("gitea-runner-dind") is False def test_scheduler_candidate_uses_live_gitea_queue_and_fixed_label_target( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): return { "total_count": 1, "runners": [ { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation", "ubuntu-latest"], } ] } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() == { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", } def test_scheduler_candidate_never_targets_busy_runner(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): return { "total_count": 1, "runners": [ { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": False, "labels": ["linux-validation"], } ] } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_scheduler_candidate_never_restarts_runner_with_in_progress_job( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): return { "total_count": 1, "runners": [{ "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation"], }], } if params.get("status") == "queued": return {"jobs": [{"id": 22931, "created_at": created_at, "labels": ["linux-validation"]}]} if params.get("status") == "in_progress": return {"jobs": [{"id": 22932, "runner_id": 17, "status": "in_progress"}]} return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_scheduler_candidate_never_hides_disabled_busy_runner(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): if params.get("disabled") == "false": return {"total_count": 0, "runners": []} return { "total_count": 1, "runners": [ { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": True, "labels": ["linux-validation"], } ] } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_scheduler_candidate_reads_later_runner_inventory_pages( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) runner_calls: list[dict[str, object]] = [] def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): runner_calls.append(params) if params.get("page") == 2: return { "total_count": 2, "runners": [ { "id": 18, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": True, "labels": ["linux-validation"], } ], } return { "total_count": 2, "runners": [ { "id": 17, "name": "unrelated-runner", "status": "online", "busy": False, "disabled": False, "labels": ["unrelated"], } ], } if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None assert [call.get("page") for call in runner_calls] == [1, 2] def test_scheduler_candidate_fails_closed_on_malformed_runner_inventory( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() def fake_gitea(path, params=None): if path.endswith("/runners"): return {"total_count": 1} return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) with pytest.raises( fleet_module.RunnerFleetError, match="runner_fleet_gitea_invalid_runner_inventory", ): service._scheduler_recovery_candidate() @pytest.mark.parametrize("unsafe_first", [True, False]) def test_scheduler_candidate_vetoes_any_unsafe_duplicate_fixed_runner( fleet_module, monkeypatch, unsafe_first ): service = fleet_module.RunnerFleetService() now = 1_800_000_000 created_at = datetime.fromtimestamp(now - 900, UTC).isoformat() monkeypatch.setattr(fleet_module.time, "time", lambda: now) unsafe = { "id": 17, "name": "itworx-validation-runner-2", "status": "online", "busy": True, "disabled": True, "labels": ["linux-validation"], } idle = { "id": 18, "name": "itworx-validation-runner-2", "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation"], } def fake_gitea(path, params=None): params = params or {} if path.endswith("/runners"): runners = [unsafe, idle] if unsafe_first else [idle, unsafe] return {"total_count": 2, "runners": runners} if params.get("status") == "queued": return { "jobs": [ { "id": 22931, "created_at": created_at, "labels": ["linux-validation"], } ] } return {"jobs": []} monkeypatch.setattr(service, "_gitea_request", fake_gitea) assert service._scheduler_recovery_candidate() is None def test_automatic_cycle_restarts_only_exact_known_idle_runner_for_stale_scheduler( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() commands: list[tuple[list[str], int]] = [] events: list[dict[str, object]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append((args, timeout)) or "", ) monkeypatch.setattr( service, "_record", lambda **event: events.append(event) or True ) result = service.automatic_cycle() assert commands == [ (["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"], 45) ] assert any( event.get("action") == "restart" and event.get("target") == "itworx-validation-runner-2" and event.get("reason") == "stale_scheduler_assignment" for event in events ) assert result["schedulerRecovery"] == { "status": "requested", "target": "itworx-validation-runner-2", "jobId": 22931, } def test_scheduler_restart_persists_intent_before_invoking_docker( fleet_module, monkeypatch, tmp_path ): service = fleet_module.RunnerFleetService() monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path) monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) def observe_durable_intent(args, timeout=20): assert args == [ "docker", "restart", "--timeout", "30", "itworx-validation-runner-2", ] assert timeout == 45 persisted = service._events() assert persisted[-1] == { "at": persisted[-1]["at"], "action": "restart", "target": "itworx-validation-runner-2", "reason": "stale_scheduler_assignment", "result": "intent", "ageSeconds": 900, } return "" monkeypatch.setattr(service, "_run", observe_durable_intent) result = service.automatic_cycle() assert result["schedulerRecovery"]["status"] == "requested" def test_scheduler_recovery_refuses_restart_when_intent_cannot_be_persisted( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) monkeypatch.setattr(service, "_safe_state_root", lambda: False) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "failed", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "restart_intent_not_persisted", } def test_scheduler_recovery_refuses_restart_when_intent_write_raises_oserror( fleet_module, monkeypatch, tmp_path ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path) monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) monkeypatch.setattr( fleet_module.os, "open", lambda *_args, **_kwargs: (_ for _ in ()).throw( PermissionError("state storage is unwritable") ), ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "failed", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "restart_intent_not_persisted", } def test_scheduler_recovery_keeps_intent_when_outcome_write_fails( fleet_module, monkeypatch, tmp_path ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] event_open_count = 0 real_open = fleet_module.os.open monkeypatch.setattr(fleet_module, "STATE_ROOT", tmp_path) monkeypatch.setattr(fleet_module, "RECOVERY_BUDGET", 1) monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": True, "health": "healthy", }, ) def fail_outcome_open(path, flags, mode=0o777): nonlocal event_open_count if Path(path).name.startswith(".events."): event_open_count += 1 if event_open_count == 2: raise PermissionError("outcome storage is unwritable") return real_open(path, flags, mode) monkeypatch.setattr(fleet_module.os, "open", fail_outcome_open) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) result = service.automatic_cycle() assert commands == [ ["docker", "restart", "--timeout", "30", "itworx-validation-runner-2"] ] assert result["schedulerRecovery"] == { "status": "requested", "target": "itworx-validation-runner-2", "jobId": 22931, } assert [event["action"] for event in service._events()] == ["restart"] assert service._recovery_available("itworx-validation-runner-2") is False @pytest.mark.parametrize( ("running", "health"), [ (True, "starting"), (True, "none"), (True, "unhealthy"), (False, "healthy"), ], ) def test_scheduler_recovery_rechecks_strict_runner_health_before_restart( fleet_module, monkeypatch, running, health ): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_container", lambda _target: { "name": "itworx-validation-runner-2", "running": running, "health": health, }, ) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) monkeypatch.setattr(service, "_record", lambda **_event: True) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "ineligible", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "docker_health_not_healthy", } def test_scheduler_recovery_rejects_duplicate_fresh_inspect_results( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() commands: list[tuple[list[str], int]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) def duplicate_inspect(args, timeout=20): commands.append((args, timeout)) return "[{}, {}]" monkeypatch.setattr(service, "_run", duplicate_inspect) monkeypatch.setattr(service, "_record", lambda **_event: True) result = service.automatic_cycle() assert commands == [ (["docker", "inspect", "itworx-validation-runner-2"], 20) ] assert result["schedulerRecovery"] == { "status": "ineligible", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "docker_health_not_healthy", } def test_scheduler_recovery_rejects_runner_without_healthy_docker_state( fleet_module, monkeypatch ): service = fleet_module.RunnerFleetService() snapshot = _healthy_snapshot() next( runner for runner in snapshot["topology"]["runners"] if runner["name"] == "itworx-validation-runner-2" )["health"] = "unhealthy" commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", lambda: snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr( service, "_run", lambda args, timeout=20: commands.append(args) or "" ) monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == { "status": "ineligible", "target": "itworx-validation-runner-2", "jobId": 22931, "reason": "docker_health_not_healthy", } def test_scheduler_recovery_refuses_unknown_target(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "Gitea", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: True) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"]["status"] == "rejected" def test_scheduler_recovery_obeys_existing_recovery_budget(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr( service, "_scheduler_recovery_candidate", lambda: { "target": "itworx-validation-runner-2", "jobId": 22931, "label": "linux-validation", "ageSeconds": 900, "reason": "stale_scheduler_assignment", }, ) monkeypatch.setattr(service, "_recovery_available", lambda _target: False) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"]["status"] == "budget_exhausted" def test_scheduler_recovery_does_nothing_without_stale_candidate(fleet_module, monkeypatch): service = fleet_module.RunnerFleetService() commands: list[list[str]] = [] monkeypatch.setattr(service, "snapshot", _healthy_snapshot) monkeypatch.setattr(service, "_scheduler_recovery_candidate", lambda: None) monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "") monkeypatch.setattr(service, "_record", lambda **_event: None) result = service.automatic_cycle() assert commands == [] assert result["schedulerRecovery"] == {"status": "not_needed"}