from __future__ import annotations import copy import importlib.util import json import sys import time from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace import pytest from backend.app.services.runner_statistics import sanitize_statistics ROOT = Path(__file__).resolve().parents[2] CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker" def _load(name: str, path: Path): spec = importlib.util.spec_from_file_location(name, path) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module @pytest.fixture def controller_module(monkeypatch): profiles = _load( "appops_runner_profiles_for_admission_controller_test", CONTROLLER_ROOT / "runner_profiles.py", ) monkeypatch.setitem(sys.modules, "runner_profiles", profiles) policy = _load( "appops_runner_admission_policy_for_controller_test", CONTROLLER_ROOT / "runner_admission_policy.py", ) monkeypatch.setitem(sys.modules, "runner_admission_policy", policy) return _load( "appops_runner_admission_controller_test", CONTROLLER_ROOT / "runner_admission_controller.py", ) def _host_gate_sample(module, monkeypatch): now = [100.0] host = {"loadRatio": 0.62, "availableMemoryBytes": 19 * 1024**3, "cacheFreeBytes": 140 * 1024**3, "swapCritical": False, "swapSampleAt": now[0]} capacity = {"host": host, "decision": {"pressure": "headroom"}} controller = module.RunnerAdmissionController(clock=lambda: now[0]) monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, capacity)) return controller, now, host, capacity def test_durable_saves_preserve_pending_and_profile_references( controller_module, monkeypatch, tmp_path ): module = controller_module tmp_path.chmod(0o700) monkeypatch.setattr(module, "STATE_ROOT", tmp_path) monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json") controller = module.RunnerAdmissionController() state = module._initial_state() entry = state["profiles"]["frontend"] pending = {"profileId": "frontend", "stage": "reserved"} state["pending"] = pending controller._save_state(state) pending["stage"] = "enable_requested" controller._save_state(state) entry["enabledByController"] = True entry["observed"] = True state["pending"] = None controller._save_state(state) saved = controller._read_state() assert saved["revision"] == state["revision"] == 3 assert saved["profiles"]["frontend"]["enabledByController"] is True assert saved["profiles"]["frontend"]["observed"] is True assert saved["pending"] is None def test_durable_save_preserves_drain_entry_updates( controller_module, monkeypatch, tmp_path ): module = controller_module tmp_path.chmod(0o700) monkeypatch.setattr(module, "STATE_ROOT", tmp_path) monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json") controller = module.RunnerAdmissionController() state = module._initial_state() entry = state["profiles"]["frontend"] entry.update(enabledByController=True, drainOwned=True) controller._save_state(state) entry.update(enabledByController=False, drainOwned=False, idleSince=None) controller._save_state(state) saved = controller._read_state() assert saved["profiles"]["frontend"]["enabledByController"] is False assert saved["profiles"]["frontend"]["drainOwned"] is False def _operator_handover_fixture(module, monkeypatch): controller, state, observation, _, _ = _lease_fixture(module, enabled=("frontend",)) entry = state["profiles"]["frontend"] entry.update(enabledByController=False, operatorEnabled=True) state["bootstrapReceiptSha256"] = "a" * 64 observation.update(specialization="active", resourceProof={"ready": True}, taskImagesVerified=True, guardState="clear") observation["containers"][module.RUNNER_PROFILES["frontend"].container_name].update( pid=1234, startedAt=module._utc_iso()) monkeypatch.setattr(module.os, "geteuid", lambda: 0, raising=False) artifact = {"sourceRevision": "b" * 40, "artifactSha256": "c" * 64} monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: artifact)) monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) monkeypatch.setattr(controller, "_read_state", lambda: state) monkeypatch.setattr(controller, "_observe", lambda: observation) written = [] def persist(value, *, expected_revision): assert value["revision"] == expected_revision saved = json.loads(json.dumps(value)) saved["revision"] += 1 written.append(saved) return saved monkeypatch.setattr(controller, "_write_state", persist) monkeypatch.setattr(controller, "_set_disabled", lambda *_args: pytest.fail("handover must not patch Gitea")) monkeypatch.setattr(controller, "_docker_run", lambda *_args, **_kw: pytest.fail("handover must not mutate Docker")) return controller, state, observation, written, artifact def test_explicit_operator_handover_is_bound_and_only_transfers_metadata(controller_module, monkeypatch): controller, state, _, written, _ = _operator_handover_fixture(controller_module, monkeypatch) preflight = controller.operator_lease_handover("frontend") assert preflight["executed"] is False assert written == [] assert state["profiles"]["frontend"]["operatorEnabled"] is True result = controller.operator_lease_handover("frontend", expected_preflight=preflight["preflightSha256"]) assert result["status"] == "handed_over" and result["executed"] is True assert len(written) == 1 entry = written[0]["profiles"]["frontend"] assert entry["operatorEnabled"] is False and entry["enabledByController"] is True assert entry["lastOperatorHandover"]["containerId"] == preflight["containerId"] assert all(written[0]["profiles"][p] == state["profiles"][p] for p in controller_module.PROFILE_IDS if p != "frontend") with pytest.raises(controller_module.RunnerAdmissionError, match="preconditions_failed"): controller.operator_lease_handover("frontend", expected_preflight=preflight["preflightSha256"]) assert len(written) == 1 @pytest.mark.parametrize("change", ["busy", "active", "disabled", "operator_disabled", "controller_owned", "pending", "inactive", "bootstrap", "identity", "stopped", "pid", "started", "resource", "images", "guard"]) def test_operator_handover_refuses_unsafe_or_unowned_observations(controller_module, monkeypatch, change): controller, state, observation, written, _ = _operator_handover_fixture(controller_module, monkeypatch) entry = state["profiles"]["frontend"] row = observation["runnerRows"][controller_module.RUNNER_PROFILES["frontend"].container_name] container = observation["containers"][controller_module.RUNNER_PROFILES["frontend"].container_name] if change == "busy": row["busy"] = True elif change == "active": observation["activeJobs"] = {999: [{"id": "foreign-job"}]} elif change == "disabled": row["disabled"] = True elif change == "operator_disabled": entry["operatorDisabled"] = True elif change == "controller_owned": entry["enabledByController"] = True elif change == "pending": state["pending"] = {"profileId": "build"} elif change == "inactive": observation["specialization"] = "legacy" elif change == "bootstrap": state["bootstrapReceiptSha256"] = None elif change == "identity": row["id"] += 1 elif change == "stopped": container["running"] = False elif change == "pid": container["pid"] = 0 elif change == "started": container["startedAt"] = None elif change == "resource": observation["resourceProof"] = None elif change == "images": observation["taskImagesVerified"] = False else: observation["guardState"] = "paused" before = copy.deepcopy(state) with pytest.raises(controller_module.RunnerAdmissionError, match="preconditions_failed"): controller.operator_lease_handover("frontend") assert state == before and written == [] @pytest.mark.parametrize("change", ["state", "container", "artifact", "labels", "restart"]) def test_operator_handover_rechecks_current_fingerprint(controller_module, monkeypatch, change): controller, state, observation, written, artifact = _operator_handover_fixture(controller_module, monkeypatch) preflight = controller.operator_lease_handover("frontend") name = controller_module.RUNNER_PROFILES["frontend"].container_name if change == "state": state["revision"] += 1 elif change == "container": observation["containers"][name]["id"] = "f" * 64 elif change == "artifact": artifact["sourceRevision"] = "d" * 40 elif change == "restart": observation["containers"][name]["pid"] += 1 else: observation["runnerRows"][name]["labels"].add("self-hosted") with pytest.raises(controller_module.RunnerAdmissionError, match="preflight_changed"): controller.operator_lease_handover("frontend", expected_preflight=preflight["preflightSha256"]) assert written == [] def test_operator_handover_requires_root_before_reading_controller_or_credentials(controller_module, monkeypatch): controller = controller_module.RunnerAdmissionController() monkeypatch.setattr(controller_module.os, "geteuid", lambda: 10001, raising=False) monkeypatch.setattr(controller, "_read_state", lambda: pytest.fail("non-root must not read private state")) with pytest.raises(controller_module.RunnerAdmissionError, match="requires_root"): controller.operator_lease_handover("frontend") def test_operator_handover_cas_conflict_does_not_claim_publication(controller_module, monkeypatch): controller, _, _, written, _ = _operator_handover_fixture(controller_module, monkeypatch) preflight = controller.operator_lease_handover("frontend") def conflict(*_args, **_kwargs): raise controller_module.RunnerAdmissionError("runner_admission_state_conflict") monkeypatch.setattr(controller, "_write_state", conflict) with pytest.raises(controller_module.RunnerAdmissionError, match="state_conflict"): controller.operator_lease_handover("frontend", expected_preflight=preflight["preflightSha256"]) assert written == [] def test_operator_handover_reports_uncertain_durable_write(controller_module, monkeypatch): controller, _, _, _, _ = _operator_handover_fixture(controller_module, monkeypatch) preflight = controller.operator_lease_handover("frontend") def uncertain(*_args, **_kwargs): raise controller_module.RunnerAdmissionError("runner_admission_state_write_failed") monkeypatch.setattr(controller, "_write_state", uncertain) with pytest.raises(controller_module.RunnerAdmissionError, match="handover_outcome_unknown"): controller.operator_lease_handover("frontend", expected_preflight=preflight["preflightSha256"]) @pytest.mark.parametrize("args", [["--operator-handover", "frontend"], ["--preflight-operator-handover", "frontend", "--confirm", "RETURN_CI_RUNNER_TO_MANAGED_ADMISSION"]]) def test_handover_cli_refuses_missing_or_inappropriate_mutation_arguments(controller_module, args): with pytest.raises(SystemExit) as exc: controller_module.main(args) assert exc.value.code == 2 @pytest.mark.parametrize("error", [RuntimeError("fake-secret-never-output"), "uncertain"]) def test_handover_cli_redacts_errors_and_does_not_misreport_unknown_writes(controller_module, monkeypatch, capsys, error): controller = controller_module.RunnerAdmissionController() def refused(*_args, **_kwargs): if error == "uncertain": raise controller_module.RunnerAdmissionError("runner_admission_handover_outcome_unknown") raise error monkeypatch.setattr(controller, "operator_lease_handover", refused) monkeypatch.setattr(controller_module, "RunnerAdmissionController", lambda: controller) assert controller_module.main(["--preflight-operator-handover", "frontend"]) == 1 raw = capsys.readouterr().out assert "fake-secret-never-output" not in raw result = json.loads(raw) if error == "uncertain": assert result["status"] == "outcome_unknown" and result["executed"] is None assert result["noMutation"] is False else: assert result["status"] == "blocked" and result["executed"] is False def test_safe_live_metric_fluctuations_do_not_restart_stable_headroom(controller_module, monkeypatch): module = controller_module controller, now, host, _ = _host_gate_sample(module, monkeypatch) state = {} assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" for at, load, memory, cache in [(130, 0.58, 18.5, 139.4), (160, 0.53, 18.2, 138.9)]: now[0] = at host.update(loadRatio=load, availableMemoryBytes=int(memory * 1024**3), cacheFreeBytes=int(cache * 1024**3), swapSampleAt=at) reason, _ = controller._fresh_mutation_gate("backend", ["backend"], state) assert reason is None @pytest.mark.parametrize("change", ["load", "memory", "cache", "swap", "stale", "future", "pressure", "guard"]) def test_unsafe_sample_restarts_the_entire_stability_window(controller_module, monkeypatch, change): module = controller_module controller, now, host, capacity = _host_gate_sample(module, monkeypatch) state = {} controller._fresh_mutation_gate("backend", ["backend"], state) now[0] = 130 host["swapSampleAt"] = 130 original = copy.deepcopy(capacity) if change == "load": host["loadRatio"] = 0.71 elif change == "memory": host["availableMemoryBytes"] = 14 * 1024**3 elif change == "cache": host["cacheFreeBytes"] = 29 * 1024**3 elif change == "swap": host["swapCritical"] = True elif change == "stale": host["swapSampleAt"] = 1 elif change == "future": host["swapSampleAt"] = 131 elif change == "pressure": capacity["decision"]["pressure"] = "pressure" else: monkeypatch.setattr(controller, "_host_evidence", lambda: ("blocked", "host_guard_paused", capacity)) assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is not None capacity.clear() capacity.update(original) host = capacity["host"] now[0] = 160 host["swapSampleAt"] = 160 monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, capacity)) assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" now[0] = 220 host["swapSampleAt"] = 220 assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] is None def test_missing_observation_interval_cannot_count_as_stable_headroom(controller_module, monkeypatch): module = controller_module controller, now, host, _ = _host_gate_sample(module, monkeypatch) state = {} controller._fresh_mutation_gate("backend", ["backend"], state) now[0] = 200 host["swapSampleAt"] = 200 assert controller._fresh_mutation_gate("backend", ["backend"], state)[0] == "runner_admission_host_headroom_not_stable" @pytest.mark.parametrize("profile,claims", [("frontend", ["frontend"]), ("backend", ["backend", "checks"])]) def test_profile_or_reservation_change_requires_new_stability_window(controller_module, monkeypatch, profile, claims): module = controller_module controller, now, host, _ = _host_gate_sample(module, monkeypatch) state = {} controller._fresh_mutation_gate("backend", ["backend"], state) now[0] = 160 host["swapSampleAt"] = 160 assert controller._fresh_mutation_gate(profile, claims, state)[0] == "runner_admission_host_headroom_not_stable" def _public_observation(module, now: float): rows = {} containers = {} profile_runner_ids = {} runner_profile_ids = {} active = {} for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): runner_id = 100 + index rows[profile.container_name] = { "id": runner_id, "name": profile.container_name, "status": "offline", "busy": False, "disabled": True, "labels": {profile.runner_label, *profile.legacy_labels}, } containers[profile.container_name] = {"running": False, "health": "none"} profile_runner_ids[profile.id] = runner_id runner_profile_ids[runner_id] = profile.id active[runner_id] = [] return { "specialization": "active", "at": now, "runnerRows": rows, "profileRunnerIds": profile_runner_ids, "runnerProfileIds": runner_profile_ids, "containers": containers, "claims": [], "queue": [ { "id": "private-job-id", "labels": ["linux-validation"], "queued_at": now - 44, } ], "activeJobs": active, "resourceProof": { "ready": True, "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, "activeClaims": 0, "heavyClaims": 0, "runningRunnerContainers": 0, "cgroups": {}, "digest": "a" * 64, }, "resourceReason": None, "hostReason": None, "capacity": { "host": { "availableMemoryBytes": 12 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "loadRatio": 0.25, } }, } def _install_gitea_observation( monkeypatch, controller, module, *, queued_rows=(), pending_rows=(), active_rows=(), ): runner_rows = {} by_id = {} profile_runner_ids = {} runner_profile_ids = {} containers = {} for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): runner_id = 100 + index row = { "id": runner_id, "name": profile.container_name, "status": "offline", "busy": False, "disabled": True, "labels": {profile.runner_label, *profile.legacy_labels}, } runner_rows[profile.container_name] = row by_id[runner_id] = row profile_runner_ids[profile.id] = runner_id runner_profile_ids[runner_id] = profile.id containers[profile.container_name] = { "id": f"{index:064x}", "running": False, "health": "none", } containers[module.DIND_NAME] = { "id": "f" * 64, "running": True, "health": "healthy", } proof = { "ready": True, "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, "activeClaims": 0, "heavyClaims": 0, "runningRunnerContainers": 0, "cgroups": {}, "digest": "e" * 64, } def job_rows(status): if status == "queued": return list(queued_rows), True if status == "pending": return list(pending_rows), True if status == "in_progress": return list(active_rows), True return [], True monkeypatch.setattr(controller, "_specialization_state", lambda: "active") monkeypatch.setattr(controller, "_runner_inventory", lambda: []) monkeypatch.setattr(controller, "_normalized_runner_rows", lambda _rows: (runner_rows, by_id)) monkeypatch.setattr(controller, "_job_rows", job_rows) monkeypatch.setattr( controller, "_active_claims", lambda *_args: ([], profile_runner_ids, runner_profile_ids), ) monkeypatch.setattr(controller, "_container", lambda name: containers.get(name)) monkeypatch.setattr(controller, "_verify_profile_task_images", lambda *_args: True) monkeypatch.setattr(controller, "_resource_attestation", lambda *_args: proof) monkeypatch.setattr( controller, "_host_evidence", lambda: ( "clear", None, { "host": { "availableMemoryBytes": 8 * 1024**3, "cacheFreeBytes": 20 * 1024**3, "loadRatio": 0.25, } }, ), ) def _lease_fixture(module, *, enabled=("backend", "frontend"), busy=(), queue=()): controller = module.RunnerAdmissionController(clock=lambda: time.time()) state = module._initial_state() rows = {} containers = {} profile_runner_ids = {} active_jobs = {} inventory_rows = [] for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): runner_id = 100 + index is_enabled = profile.id in enabled is_busy = profile.id in busy labels = [profile.runner_label, *profile.legacy_labels] rows[profile.container_name] = { "id": runner_id, "name": profile.container_name, "status": "online" if is_enabled else "offline", "busy": is_busy, "disabled": not is_enabled, "labels": set(labels), } containers[profile.container_name] = { "id": f"{index:064x}", "running": is_enabled, "health": "healthy" if is_enabled else "none", } profile_runner_ids[profile.id] = runner_id state["registrationIds"][profile.id] = runner_id state["profiles"][profile.id].update( observed=True, enabledByController=is_enabled, operatorDisabled=False, operatorEnabled=False, ) inventory_rows.append( { "name": profile.display_name, "id": runner_id, "status": "online" if is_enabled else "offline", "busy": is_busy, "disabled": not is_enabled, "labels": labels, } ) if is_busy: active_jobs[runner_id] = [ { "id": f"active-{profile.id}", "labels": labels, "startedAt": module._utc_iso(), "attempt": 1, } ] by_id = {row["id"]: row for row in rows.values()} claims, _profile_ids, _runner_profiles = module.RunnerAdmissionController._active_claims( rows, by_id, active_jobs ) observation = { "runnerRows": rows, "containers": containers, "activeJobs": active_jobs, "profileRunnerIds": profile_runner_ids, "claims": claims, "queue": list(queue), } return controller, state, observation, inventory_rows, containers def _retired_security_inventory(module): rows = [ {"name": profile.display_name, "id": 101 + index, "status": "offline", "busy": False, "disabled": True, "labels": [profile.runner_label, *profile.legacy_labels]} for index, profile in enumerate(module.RUNNER_PROFILES.values()) ] rows.append({ "name": "itworx-security-runner", "id": 9, "status": "offline", "busy": False, "disabled": True, "labels": ["linux-validation", "docker-validation", "ubuntu-latest", "ubuntu-24.04", "ubuntu-22.04", "mcp-validation", "unraid"], }) return rows def test_exact_retired_v1_security_labels_do_not_block_new_profile_inventory(controller_module): module = controller_module rows, ids = module.RunnerAdmissionController._normalized_runner_rows( _retired_security_inventory(module) ) claims, profile_ids, _ = module.RunnerAdmissionController._active_claims(rows, ids, {}) assert claims == [] assert set(profile_ids) == set(module.PROFILE_IDS) assert rows["itworx-security-runner"]["disabled"] is True @pytest.mark.parametrize("change", ["enabled", "busy", "missing", "foreign", "replaced_id"]) def test_retired_v1_security_variant_refuses_live_or_changed_registration(controller_module, change): module = controller_module inventory = _retired_security_inventory(module) old = inventory[-1] if change == "enabled": old["disabled"] = False elif change == "busy": old["busy"] = True elif change == "missing": old["labels"].remove("mcp-validation") elif change == "replaced_id": old["id"] = 99 else: old["labels"].append("foreign-capability") with pytest.raises(module.RunnerAdmissionError, match="legacy_runner_labels_invalid"): module.RunnerAdmissionController._normalized_runner_rows(inventory) def test_retired_v1_security_active_job_still_refuses_admission(controller_module): module = controller_module rows, ids = module.RunnerAdmissionController._normalized_runner_rows( _retired_security_inventory(module) ) with pytest.raises(module.RunnerAdmissionError, match="legacy_claim_unmapped"): module.RunnerAdmissionController._active_claims(rows, ids, {9: [{"id": "foreign-active"}]}) def test_original_security_label_contract_is_preserved(controller_module): module = controller_module inventory = _retired_security_inventory(module) inventory[-1]["labels"] = sorted(module.LEGACY_REQUIRED_LABELS["itworx-security-runner"]) rows, _ = module.RunnerAdmissionController._normalized_runner_rows(inventory) assert rows["itworx-security-runner"]["labels"] == set(inventory[-1]["labels"]) def _migrated_tower_inventory(module): return [*_retired_security_inventory(module), {"name": "itworx-unraid-deploy", "id": 13, "status": "online", "busy": False, "disabled": False, "labels": ["unraid-deploy"]}, {"name": "itworx-windows11-native", "id": 14, "status": "online", "busy": False, "disabled": False, "labels": ["windows-native"]}] def test_migrated_tower_preserves_idle_native_registrations_without_family_claims(controller_module): module = controller_module rows, ids = module.RunnerAdmissionController._normalized_runner_rows(_migrated_tower_inventory(module)) claims, profiles, assigned = module.RunnerAdmissionController._active_claims(rows, ids, {}) assert claims == [] assert set(profiles) == set(module.PROFILE_IDS) assert 13 not in assigned and 14 not in assigned @pytest.mark.parametrize("index", [-2, -1]) @pytest.mark.parametrize("change", ["id", "labels", "name"]) def test_preserved_native_identity_or_capability_drift_refuses(controller_module, index, change): module = controller_module inventory = _migrated_tower_inventory(module) if change == "id": inventory[index]["id"] = 500 elif change == "labels": inventory[index]["labels"].append("linux-validation") else: inventory[index]["name"] = "foreign-native" with pytest.raises(module.RunnerAdmissionError, match="preserved_native_identity_invalid"): module.RunnerAdmissionController._normalized_runner_rows(inventory) @pytest.mark.parametrize("index", [-2, -1]) @pytest.mark.parametrize("busy", [True, False]) def test_active_preserved_native_blocks_family_admission_until_it_drains(controller_module, index, busy): module = controller_module inventory = _migrated_tower_inventory(module) row = inventory[index] row["busy"] = busy rows, ids = module.RunnerAdmissionController._normalized_runner_rows(inventory) active = {} if busy else {row["id"]: [{"id": "native-active"}]} with pytest.raises(module.RunnerAdmissionError, match="preserved_native_active"): module.RunnerAdmissionController._active_claims(rows, ids, active) def test_foreign_enabled_registration_still_refuses_with_preserved_natives(controller_module): module = controller_module inventory = _migrated_tower_inventory(module) inventory.append({"name": "foreign", "id": 500, "status": "online", "busy": False, "disabled": False, "labels": ["linux-validation"]}) rows, ids = module.RunnerAdmissionController._normalized_runner_rows(inventory) with pytest.raises(module.RunnerAdmissionError, match="enabled_registration_unmapped"): module.RunnerAdmissionController._active_claims(rows, ids, {}) def test_absent_retired_container_is_confirmed_by_successful_exact_inventory(controller_module, monkeypatch): module = controller_module calls = [] def run(args, **_kwargs): calls.append(args) if args[1] == "inspect": return SimpleNamespace(returncode=1, stdout="[]\n", stderr="opaque failure") return SimpleNamespace(returncode=0, stdout="", stderr="") monkeypatch.setattr(module.subprocess, "run", run) assert module.RunnerAdmissionController()._container("itworx-security-runner") is None assert all(args[1] != "inspect" for args in calls) @pytest.mark.parametrize("stdout,exit_code", [ ("", 1), ("foreign", 0), ("a" * 64 + " wrong-name\n", 0), (("a" * 64 + " itworx-security-runner\n") * 2, 0), ]) def test_retired_container_inventory_failure_or_ambiguity_refuses(controller_module, monkeypatch, stdout, exit_code): module = controller_module monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: SimpleNamespace(returncode=exit_code, stdout=stdout, stderr="opaque failure")) with pytest.raises(module.RunnerAdmissionError): module.RunnerAdmissionController()._container("itworx-security-runner") def test_present_retired_container_inspect_failure_is_not_absence(controller_module, monkeypatch): module = controller_module def run(args, **_kwargs): if args[1] == "inspect": return SimpleNamespace(returncode=1, stdout="[]\n", stderr="opaque failure") return SimpleNamespace(returncode=0, stdout="a" * 64 + " itworx-security-runner\n", stderr="") monkeypatch.setattr(module.subprocess, "run", run) with pytest.raises(module.RunnerAdmissionError, match="container_inspect_failed"): module.RunnerAdmissionController()._container("itworx-security-runner") def test_present_retired_container_replacement_refuses(controller_module, monkeypatch): module = controller_module def run(args, **_kwargs): if args[1] == "inspect": row = {"Id": "b" * 64, "Name": "/itworx-security-runner", "State": {"Running": False}, "Config": {"Labels": {"com.docker.compose.project": module.COMPOSE_PROJECT}}} return SimpleNamespace(returncode=0, stdout=json.dumps([row]), stderr="") return SimpleNamespace(returncode=0, stdout="a" * 64 + " itworx-security-runner\n", stderr="") monkeypatch.setattr(module.subprocess, "run", run) with pytest.raises(module.RunnerAdmissionError, match="container_identity_invalid"): module.RunnerAdmissionController()._container("itworx-security-runner") def _raw_job(module, job, *, runner_id=None, active=False): row = { "id": job["id"], "labels": list(job["labels"]), "attempt": 1, } if active: row.update( runner_id=runner_id, started_at=module._utc_iso(), status="in_progress", ) else: row["created_at"] = module._utc_iso(float(job["queued_at"])) return row def _install_lease_io( monkeypatch, controller, module, inventory_rows, containers, *, queue_rows=(), active_rows=(), saved=None, ): import copy def save(state): if saved is not None: saved.append(copy.deepcopy(state)) monkeypatch.setattr(controller, "_save_state", save) def job_rows(status): if status == "queued" or status == "pending": return list(queue_rows), True if status == "in_progress": return list(active_rows), True return [], True monkeypatch.setattr( controller, "_job_rows", job_rows, ) monkeypatch.setattr(controller, "_runner_inventory", lambda: [dict(row) for row in inventory_rows]) monkeypatch.setattr(controller, "_container", lambda name: dict(containers[name])) monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("clear", "")) def stop_container(args, **_kwargs): assert args[:2] == ["docker", "stop"] containers[args[2]]["running"] = False containers[args[2]]["health"] = "none" return "" monkeypatch.setattr(controller, "_docker_run", stop_container) def _complete_state(module, now: float): state = module._initial_state() state["terminalHistoryComplete"] = True state["terminalHistoryReason"] = None state["historySamples"] = [ { "at": module._utc_iso(now - 30), "activeJobs": 0, "queuedJobs": 2, "cpuUsed": 1.25, "memoryUsedBytes": 2048, }, { "at": module._utc_iso(now), "activeJobs": 0, "queuedJobs": 1, "cpuUsed": 1.5, "memoryUsedBytes": 4096, }, ] state["terminalHistory"] = [ { "jobId": str(index), "attempt": 1, "profileId": "checks", "result": "success", "createdAt": module._utc_iso(now - 300 + index * 30), "startedAt": module._utc_iso(now - 290 + index * 30), "finishedAt": module._utc_iso(now - 250 + index * 30), "waitSeconds": index * 10, "durationSeconds": index * 20, } for index in range(1, 4) ] return state def test_public_projection_matches_fixed_v1_and_uses_only_observed_statistics( controller_module, ): module = controller_module now = time.time() controller = module.RunnerAdmissionController(clock=lambda: now) payload = controller._public_snapshot( _public_observation(module, now), _complete_state(module, now) ) assert set(payload) == { "schemaVersion", "status", "generatedAt", "scope", "budget", "host", "profiles", "window", "history", } assert payload["scope"] == "ci-fleet" assert payload["status"] == "healthy" assert payload["budget"] == { "cpuLimit": 8, "memoryLimitBytes": 16 * 1024**3, "activeJobLimit": 2, "heavyJobLimit": 1, "reservedCpu": 1, "reservedMemoryBytes": 2 * 1024**3, "activeJobs": 0, "heavyJobs": 0, } assert payload["host"]["cpuBusyPercent"] == 18.75 assert payload["profiles"][0]["cpuLimit"] is None assert payload["profiles"][0]["memoryLimitBytes"] is None backend = next(row for row in payload["profiles"] if row["id"] == "backend") frontend = next(row for row in payload["profiles"] if row["id"] == "frontend") assert backend["queueDepth"] == frontend["queueDepth"] == 1 checks = payload["profiles"][0] assert checks["completedJobs"] == 3 assert checks["p50WaitSeconds"] == 20 assert checks["p95WaitSeconds"] == 30 assert checks["p50DurationSeconds"] == 40 assert checks["p95DurationSeconds"] == 60 assert checks["lastDurationSeconds"] == 60 assert payload["window"]["coverage"] == "bounded" assert "private-job-id" not in json.dumps(payload["history"]) assert all(set(row) == {"at", "activeJobs", "queuedJobs", "cpuUsed", "memoryUsedBytes"} for row in payload["history"]) assert sanitize_statistics(payload)["status"] == "healthy" def test_base_projection_does_not_claim_unattested_capacity(controller_module): payload = controller_module.RunnerAdmissionController._base_snapshot( "not_configured", "runner_admission_not_configured", time.time() ) assert payload["budget"] == { "cpuLimit": None, "memoryLimitBytes": None, "activeJobLimit": None, "heavyJobLimit": None, "reservedCpu": None, "reservedMemoryBytes": None, "activeJobs": None, "heavyJobs": None, } assert all(row["cpuLimit"] is None and row["memoryLimitBytes"] is None for row in payload["profiles"]) assert payload["window"]["coverage"] == "unavailable" def test_capacity_projection_uses_independent_host_sampler_without_burst_state( controller_module, monkeypatch ): module = controller_module calls = [] sample = { "cpuCount": 4, "loadOne": 1.0, "loadRatio": 0.25, "availableMemoryBytes": 16 * 1024**3, "cacheFreeBytes": 40 * 1024**3, "swapInKbps": 0.0, "swapOutKbps": 0.0, "swapPressureSamples": 0, "swapCritical": False, "swapSampleAt": time.time(), "swapInPages": 100, "swapOutPages": 200, } class HostSampler: def host_snapshot(self, previous=None): calls.append(previous) return {"status": "observed", "generatedAt": "2026-10-01T00:00:00Z", "host": sample} monkeypatch.setitem( sys.modules, "runner_capacity_controller", SimpleNamespace(RunnerCapacityController=HostSampler), ) monkeypatch.setitem( sys.modules, "runner_capacity_policy", SimpleNamespace( START_MAX_LOAD_RATIO=0.70, START_MIN_AVAILABLE_MEMORY_BYTES=12 * 1024**3, START_MIN_CACHE_FREE_BYTES=30 * 1024**3, STOP_LOAD_RATIO=0.90, STOP_MIN_AVAILABLE_MEMORY_BYTES=8 * 1024**3, STOP_MIN_CACHE_FREE_BYTES=15 * 1024**3, ), ) controller = module.RunnerAdmissionController() first = controller._capacity_snapshot() second = controller._capacity_snapshot() assert calls[0] is None assert calls[1] == sample assert first["status"] == second["status"] == "ok" assert first["decision"] == {"pressure": "headroom"} assert second["host"]["swapOutPages"] == 200 def test_capacity_projection_fails_closed_on_unavailable_host_sample( controller_module, monkeypatch ): module = controller_module class HostSampler: def host_snapshot(self, previous=None): return {"status": "unavailable", "host": None} monkeypatch.setitem( sys.modules, "runner_capacity_controller", SimpleNamespace(RunnerCapacityController=HostSampler), ) controller = module.RunnerAdmissionController() with pytest.raises(module.RunnerAdmissionError, match="host_telemetry_unavailable"): controller._capacity_snapshot() def test_runner_inventory_maps_exact_gitea_display_names_to_fixed_containers( controller_module, ): module = controller_module rows = [ { "name": profile.display_name, "id": index, "status": "offline", "busy": False, "disabled": True, "labels": [profile.runner_label, *profile.legacy_labels], } for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=100) ] by_name, by_id = module.RunnerAdmissionController._normalized_runner_rows(rows) for profile in module.RUNNER_PROFILES.values(): observed = by_name[profile.container_name] assert observed["displayName"] == profile.display_name assert observed["id"] in by_id canonical_alias = [dict(rows[0], name=module.RUNNER_PROFILES["checks"].container_name)] with pytest.raises( module.RunnerAdmissionError, match="runner_admission_registration_name_invalid", ): module.RunnerAdmissionController._normalized_runner_rows(canonical_alias) duplicate_alias = [ rows[0], dict(rows[1], name=module.RUNNER_PROFILES["checks"].display_name), ] with pytest.raises( module.RunnerAdmissionError, match="runner_admission_runner_inventory_duplicate", ): module.RunnerAdmissionController._normalized_runner_rows(duplicate_alias) def test_snapshot_reads_only_atomic_projection_and_downshifts_stale_health( controller_module, monkeypatch, tmp_path ): module = controller_module now = time.time() controller = module.RunnerAdmissionController(clock=lambda: now) projection = module.RunnerAdmissionController._base_snapshot( "healthy", "unused", now - 120 ) monkeypatch.setattr(module, "STATE_ROOT", tmp_path) monkeypatch.setattr(module, "PUBLIC_STATISTICS_FILE", tmp_path / "statistics.json") monkeypatch.setattr(module, "_safe_directory", lambda *_args, **_kwargs: True) monkeypatch.setattr( module.RunnerAdmissionController, "_safe_private_file", staticmethod(lambda *_args, **_kwargs: True), ) module.PUBLIC_STATISTICS_FILE.write_text(json.dumps(projection), encoding="utf-8") monkeypatch.setattr(controller, "_observe", lambda **_kwargs: pytest.fail("snapshot polled")) monkeypatch.setattr(controller, "_request", lambda *_args, **_kwargs: pytest.fail("snapshot called Gitea")) monkeypatch.setattr(controller, "_docker_run", lambda *_args, **_kwargs: pytest.fail("snapshot called Docker")) result = controller.snapshot() assert result["status"] == "degraded" assert result["generatedAt"] == projection["generatedAt"] assert result["budget"]["cpuLimit"] is None def test_projection_rejects_naive_future_and_out_of_order_history(controller_module): module = controller_module now = time.time() assert module._parse_time("2026-10-01T12:00:00") is None projection = module.RunnerAdmissionController._base_snapshot( "healthy", "unused", now ) projection["history"] = [ { "at": module._utc_iso(now), "activeJobs": 0, "queuedJobs": 0, "cpuUsed": None, "memoryUsedBytes": None, }, { "at": module._utc_iso(now - 30), "activeJobs": 0, "queuedJobs": 0, "cpuUsed": None, "memoryUsedBytes": None, }, ] assert not module.RunnerAdmissionController._valid_projection(projection) def test_resource_attestation_uses_fixed_outer_names_and_mapping_contract( controller_module, ): module = controller_module requests = [] proof = { "ready": True, "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, "activeClaims": 1, "heavyClaims": 0, "runningRunnerContainers": 1, "cgroups": {"familyPath": "/itworx-ci", "boundedGroups": 7}, "digest": "b" * 64, } controller = module.RunnerAdmissionController( resource_attestor=lambda **kwargs: requests.append(kwargs) or proof ) empty_legacy_inventory = { name: None for name in module.ALL_LEGACY_RUNNER_NAMES } result = controller._resource_attestation(["checks"], empty_legacy_inventory) assert result == proof assert requests[0]["active_profiles"] == ("checks",) assert requests[0]["expected_host_budget"] == {"cpuMilli": 8000, "memoryMiB": 16384} assert requests[0]["expected_outer_containers"] == ( *module.PROFILE_RUNNER_NAMES, module.DIND_NAME, ) @pytest.mark.parametrize("verified", [True, False]) def test_observation_requires_source_bound_profile_task_images( controller_module, monkeypatch, verified ): module = controller_module calls = [] fake_specializer = SimpleNamespace( verify_profile_task_images=lambda profile_ids: calls.append(tuple(profile_ids)) or verified ) monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer) attestation_requests = [] proof = { "ready": True, "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, "activeClaims": 0, "heavyClaims": 0, "runningRunnerContainers": 0, "cgroups": {}, "digest": "f" * 64, } controller = module.RunnerAdmissionController( resource_attestor=lambda **kwargs: attestation_requests.append(kwargs) or proof ) rows = {} by_id = {} profile_runner_ids = {} runner_profile_ids = {} for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): row = { "id": index, "name": profile.container_name, "disabled": True, "busy": False, "labels": {profile.runner_label, *profile.legacy_labels}, } rows[profile.container_name] = row by_id[index] = row profile_runner_ids[profile.id] = index runner_profile_ids[index] = profile.id monkeypatch.setattr(controller, "_specialization_state", lambda: "active") monkeypatch.setattr(controller, "_runner_inventory", lambda: []) monkeypatch.setattr(controller, "_normalized_runner_rows", lambda _rows: (rows, by_id)) monkeypatch.setattr(controller, "_job_rows", lambda _status: ([], True)) monkeypatch.setattr(controller, "_normalized_jobs", lambda *_args: ([], {})) monkeypatch.setattr( controller, "_active_claims", lambda *_args: ([], profile_runner_ids, runner_profile_ids), ) monkeypatch.setattr(controller, "_container", lambda name: {"name": name, "running": False, "health": "none"}) monkeypatch.setattr(controller, "_host_evidence", lambda: ("clear", None, {})) observation = controller._observe() assert calls == [tuple(module.PROFILE_IDS)] assert observation["taskImagesVerified"] is verified if verified: assert observation["resourceProof"] == proof assert observation["resourceReason"] is None assert len(attestation_requests) == 1 else: assert observation["resourceProof"] is None assert observation["resourceReason"] == "runner_admission_task_image_unverified" assert attestation_requests == [] def test_dependency_pending_job_does_not_count_or_admit_as_runnable( controller_module, monkeypatch ): module = controller_module now = time.time() controller = module.RunnerAdmissionController(clock=lambda: now) pending_build = { "id": 101, "labels": [module.RUNNER_PROFILES["build"].runner_label], "attempt": 1, "status": "requested", "created_at": module._utc_iso(now - 600), } queued_checks = { "id": 102, "labels": [module.RUNNER_PROFILES["checks"].runner_label], "attempt": 1, "created_at": module._utc_iso(now - 30), } _install_gitea_observation( monkeypatch, controller, module, queued_rows=[queued_checks], pending_rows=[pending_build], ) observation = controller._observe() state = module._initial_state() assert controller._record_aggregate_sample(state, observation) projection = controller._public_snapshot(observation, state) choice = module.select_admission( observation["queue"], available_profiles=controller._enabled_candidate_profiles(observation, state), active_claims=[], cursor=0, ) assert [job["id"] for job in observation["queue"]] == ["102"] assert next(row for row in projection["profiles"] if row["id"] == "checks")["queueDepth"] == 1 build = next(row for row in projection["profiles"] if row["id"] == "build") assert build["queueDepth"] == 0 assert build["oldestQueuedSeconds"] is None assert projection["history"][-1]["queuedJobs"] == 1 assert (choice.profile_id, choice.job_id) == ("checks", "102") def test_observation_still_rejects_malformed_pending_inventory( controller_module, monkeypatch ): module = controller_module controller = module.RunnerAdmissionController() _install_gitea_observation(monkeypatch, controller, module) original_job_rows = controller._job_rows def malformed_pending_rows(status): if status == "pending": return ([{"id": 103, "labels": [], "created_at": module._utc_iso()}], True) return original_job_rows(status) monkeypatch.setattr(controller, "_job_rows", malformed_pending_rows) with pytest.raises( module.RunnerAdmissionError, match="runner_admission_job_inventory_invalid", ): controller._observe() def test_pending_only_dependency_job_does_not_start_a_runner( controller_module, monkeypatch ): module = controller_module now = time.time() controller = module.RunnerAdmissionController(clock=lambda: now) pending_build = { "id": 101, "labels": [module.RUNNER_PROFILES["build"].runner_label], "attempt": 1, "status": "requested", "created_at": module._utc_iso(now - 600), } _install_gitea_observation( monkeypatch, controller, module, pending_rows=[pending_build], ) state = module._initial_state() state["terminalHistoryUpdatedAt"] = now starts = [] enables = [] projections = [] monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) monkeypatch.setattr(controller, "_read_state", lambda: state) monkeypatch.setattr(controller, "_save_state", lambda _state: None) monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False) monkeypatch.setattr(controller, "_reconcile_ownership", lambda *_args: False) monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False) monkeypatch.setattr(controller, "_write_projection", projections.append) monkeypatch.setattr(controller, "_docker_run", lambda args, **_kwargs: starts.append(args)) monkeypatch.setattr(controller, "_set_disabled", lambda *args: enables.append(args)) monkeypatch.setattr( controller, "_continue_pending", lambda *_args, **_kwargs: pytest.fail("no runner startup intent should be created"), ) result = controller.cycle() assert result["status"] == "runner_admission_no_compatible_queue" assert starts == [] assert enables == [] assert state["pending"] is None assert next(row for row in projections[0]["profiles"] if row["id"] == "build")["queueDepth"] == 0 def test_aggregate_history_uses_counter_deltas_and_real_memory(controller_module): module = controller_module now = time.time() controller = module.RunnerAdmissionController(clock=lambda: now) state = module._initial_state() state["lastFamilyCpuUsageUsec"] = 100_000_000 state["lastFamilyCpuSampleAt"] = now - 20 observation = { "activeJobs": {21: [{"id": "active-1"}]}, "queue": [{"id": "queued-1"}, {"id": "queued-2"}], "resourceProof": { "familyCpuUsageUsec": 110_000_000, "familyMemoryCurrentBytes": 512 * 1024**2, }, } assert controller._record_aggregate_sample(state, observation) assert state["historySamples"][-1] == { "at": module._utc_iso(now), "activeJobs": 1, "queuedJobs": 2, "cpuUsed": 0.5, "memoryUsedBytes": 512 * 1024**2, } state["lastFamilyCpuUsageUsec"] = 200_000_000 state["lastFamilyCpuSampleAt"] = now - 20 state["historySamples"][-1]["at"] = module._utc_iso(now - 30) observation["resourceProof"]["familyCpuUsageUsec"] = 100_000_000 now += 31 controller._clock = lambda: now assert controller._record_aggregate_sample(state, observation) assert state["historySamples"][-1]["cpuUsed"] is None def test_controller_owned_drain_rechecks_operator_and_allows_guard_pause( controller_module, monkeypatch ): module = controller_module now = time.time() controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("checks",) ) entry = state["profiles"]["checks"] entry.update( idleSince=now - module.IDLE_DRAIN_SECONDS - 1, ) profile = module.RUNNER_PROFILES["checks"] runner_id = observation["profileRunnerIds"]["checks"] disabled = [] container_running = {"value": True} row = next(item for item in inventory if item["id"] == runner_id) monkeypatch.setattr(controller, "_save_state", lambda _state: None) monkeypatch.setattr( controller, "_job_rows", lambda _status: ([], True), ) monkeypatch.setattr( controller, "_runner_inventory", lambda: [dict(item) for item in inventory], ) monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("paused", "guard")) monkeypatch.setattr( controller, "_set_disabled", lambda actual_id, value: ( disabled.append((actual_id, value)), row.update(disabled=value), ), ) monkeypatch.setattr( controller, "_container", lambda name: { "id": containers[name]["id"], "running": container_running["value"], "health": "healthy" if container_running["value"] else "none", }, ) def stop(args, **_kwargs): assert args == ["docker", "stop", profile.container_name] container_running["value"] = False return "" monkeypatch.setattr(controller, "_docker_run", stop) action = controller._drain_one_idle_profile(state, observation) assert action == "runner_admission_drained_idle_profile" assert disabled == [(runner_id, True)] assert entry["enabledByController"] is False assert entry["drainOwned"] is False assert container_running["value"] is False def test_drained_runner_requires_a_full_exact_container_id(controller_module): module = controller_module controller = module.RunnerAdmissionController() state = module._initial_state() state["profiles"]["checks"]["drainOwned"] = True assert controller._stop_drained_profile( state, profile_id="checks", runner_id=101, expected_container_id="", ) == "runner_admission_container_identity_invalid" def test_drain_does_not_stop_when_operator_reopens_registration_after_disable( controller_module, monkeypatch ): module = controller_module now = time.time() controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("checks",) ) entry = state["profiles"]["checks"] entry.update( idleSince=now - module.IDLE_DRAIN_SECONDS - 1, ) runner_id = observation["profileRunnerIds"]["checks"] row = next(item for item in inventory if item["id"] == runner_id) calls = 0 patches = [] monkeypatch.setattr(controller, "_save_state", lambda _state: None) monkeypatch.setattr(controller, "_job_rows", lambda _status: ([], True)) monkeypatch.setattr( controller, "_runner_inventory", lambda: [dict(item) for item in inventory], ) monkeypatch.setattr(controller, "_trusted_host_guard_state", lambda: ("clear", "")) def operator_reopens(actual_id, _value): patches.append((actual_id, True)) row.update(disabled=False) monkeypatch.setattr( controller, "_set_disabled", operator_reopens, ) monkeypatch.setattr(controller, "_container", lambda name: dict(containers[name])) def docker(args, **_kwargs): nonlocal calls calls += 1 return "" monkeypatch.setattr(controller, "_docker_run", docker) action = controller._drain_one_idle_profile(state, observation) assert action == "runner_admission_drain_stop_deferred" assert calls == 0 assert entry["drainOwned"] is True assert entry["enabledByController"] is False fresh_rows, _ = controller._normalized_runner_rows(inventory) observation["runnerRows"] = fresh_rows assert controller._reconcile_ownership(state, observation) assert entry["drainOwned"] is False assert entry["operatorEnabled"] is True assert controller._drain_one_idle_profile(state, observation) is None assert patches == [(runner_id, True)] def test_busy_controller_owned_runner_gets_a_durable_disable_but_job_is_not_stopped( controller_module, monkeypatch ): module = controller_module controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("backend",), busy=("backend",) ) profile = module.RUNNER_PROFILES["backend"] runner_id = observation["profileRunnerIds"]["backend"] active = _raw_job( module, {"id": "active-backend", "labels": [profile.runner_label]}, runner_id=runner_id, active=True, ) saved = [] patches = [] stops = [] _install_lease_io( monkeypatch, controller, module, inventory, containers, active_rows=[active], saved=saved, ) def disable(actual_id, disabled): patches.append((actual_id, disabled)) next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled monkeypatch.setattr(controller, "_set_disabled", disable) monkeypatch.setattr( controller, "_docker_run", lambda args, **_kwargs: stops.append(args), ) action = controller._drain_one_idle_profile(state, observation) assert action == "runner_admission_drain_stop_deferred" assert patches == [(runner_id, True)] assert stops == [] assert saved[-1]["profiles"]["backend"]["drainOwned"] is True assert state["profiles"]["backend"]["drainOwned"] is True assert state["profiles"]["backend"]["enabledByController"] is False fresh_rows, fresh_by_id = controller._normalized_runner_rows(inventory) claims, _, _ = controller._active_claims( fresh_rows, fresh_by_id, {runner_id: [{"id": "active-backend"}]} ) assert claims == ["backend"] def test_cycle_fences_a_busy_owned_lease_even_without_a_resource_proof( controller_module, monkeypatch ): module = controller_module now = time.time() controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("backend",), busy=("backend",) ) state["terminalHistoryUpdatedAt"] = module._utc_iso(now) observation.update(specialization="active", at=now) saved = [] patches = [] active = _raw_job( module, { "id": "active-backend", "labels": [module.RUNNER_PROFILES["backend"].runner_label], }, runner_id=observation["profileRunnerIds"]["backend"], active=True, ) _install_lease_io( monkeypatch, controller, module, inventory, containers, active_rows=[active], saved=saved, ) monkeypatch.setattr(controller, "_process_lock", lambda: nullcontext()) monkeypatch.setattr(controller, "_observe", lambda **_kwargs: observation) monkeypatch.setattr(controller, "_read_state", lambda: state) monkeypatch.setattr(controller, "_terminal_history", lambda: ([], True, None)) monkeypatch.setattr(controller, "_seed_or_consume_bootstrap_receipt", lambda *_args: False) monkeypatch.setattr(controller, "_record_aggregate_sample", lambda *_args: False) monkeypatch.setattr(controller, "_public_snapshot", lambda *_args, **_kwargs: {"status": "healthy"}) monkeypatch.setattr(controller, "_write_projection", lambda _projection: None) def disable(actual_id, disabled): patches.append((actual_id, disabled)) next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled monkeypatch.setattr(controller, "_set_disabled", disable) result = controller.cycle() assert result["status"] == "runner_admission_drain_stop_deferred" assert patches == [(observation["profileRunnerIds"]["backend"], True)] assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True assert saved[-1]["profiles"]["backend"]["drainOwned"] is True def test_oldest_other_profile_eventually_replaces_sustained_backend_frontend_claims( controller_module, monkeypatch ): module = controller_module now = time.time() older_build = { "id": "older-build", "labels": [module.RUNNER_PROFILES["build"].runner_label], "queued_at": now - 20, } newer_backend = { "id": "newer-backend", "labels": ["linux-validation"], "queued_at": now - 10, } controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("backend", "frontend"), queue=[older_build, newer_backend], ) for profile_id in ("backend", "frontend"): state["profiles"][profile_id]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1 raw_queue = [_raw_job(module, older_build), _raw_job(module, newer_backend)] patches = [] _install_lease_io( monkeypatch, controller, module, inventory, containers, queue_rows=raw_queue, ) def disable(actual_id, disabled): patches.append((actual_id, disabled)) next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled monkeypatch.setattr(controller, "_set_disabled", disable) first = controller._drain_one_idle_profile(state, observation) assert first == "runner_admission_drained_idle_profile" assert patches == [(observation["profileRunnerIds"]["backend"], True)] assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is False # The next poll can yield the other idle broad-label lease as the older # heavy job remains ahead of a newer backend job. rows, by_id = controller._normalized_runner_rows(inventory) observation["runnerRows"] = rows observation["claims"] = controller._active_claims(rows, by_id, {})[0] second = controller._drain_one_idle_profile(state, observation) assert second == "runner_admission_drained_idle_profile" assert patches == [ (observation["profileRunnerIds"]["backend"], True), (observation["profileRunnerIds"]["frontend"], True), ] assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is False rows, by_id = controller._normalized_runner_rows(inventory) claims, _, _ = controller._active_claims(rows, by_id, {}) choice = module.select_admission( [older_build, newer_backend], available_profiles=controller._enabled_candidate_profiles( { "runnerRows": rows, "containers": containers, "activeJobs": {}, "profileRunnerIds": observation["profileRunnerIds"], }, state, ), active_claims=claims, cursor=state["cursor"], ) assert (choice.profile_id, choice.job_id) == ("build", "older-build") def test_prepatch_drain_intent_survives_restart_and_retries_with_complete_inventory( controller_module, monkeypatch ): module = controller_module controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("backend",), busy=("backend",) ) runner_id = observation["profileRunnerIds"]["backend"] active = _raw_job( module, {"id": "active-backend", "labels": ["itworx-ci-backend"]}, runner_id=runner_id, active=True, ) saved = [] patches = [] _install_lease_io( monkeypatch, controller, module, inventory, containers, active_rows=[active], saved=saved, ) def incomplete(status): if status == "pending": raise module.RunnerAdmissionError("runner_admission_pending_inventory_incomplete") return ([active] if status == "in_progress" else [], True) monkeypatch.setattr(controller, "_job_rows", incomplete) assert controller._drain_one_idle_profile(state, observation) == ( "runner_admission_pending_inventory_incomplete" ) assert saved[-1]["profiles"]["backend"]["drainOwned"] is True assert patches == [] recovered = copy.deepcopy(saved[-1]) assert controller._reconcile_ownership(recovered, observation) is False assert recovered["profiles"]["backend"]["drainOwned"] is True assert recovered["profiles"]["backend"]["operatorEnabled"] is False def complete(status): if status in {"queued", "pending"}: return [], True return [active], True monkeypatch.setattr(controller, "_job_rows", complete) def disable(actual_id, disabled): patches.append((actual_id, disabled)) next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled monkeypatch.setattr(controller, "_set_disabled", disable) assert controller._drain_one_idle_profile(recovered, observation) == ( "runner_admission_drain_stop_deferred" ) assert patches == [(runner_id, True)] assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True def test_stale_idle_drain_intent_is_cleared_when_fresh_queue_no_longer_yields( controller_module, monkeypatch ): module = controller_module now = time.time() backend_job = { "id": "newer-backend", "labels": ["linux-validation"], "queued_at": now - 10, } controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("backend",), busy=(), queue=[backend_job], ) entry = state["profiles"]["backend"] entry.update(drainOwned=True, enabledByController=True, idleSince=None) saved = [] patches = [] _install_lease_io( monkeypatch, controller, module, inventory, containers, queue_rows=[_raw_job(module, backend_job)], saved=saved, ) monkeypatch.setattr( controller, "_set_disabled", lambda actual_id, disabled: patches.append((actual_id, disabled)), ) recovered = copy.deepcopy(state) assert controller._reconcile_ownership(recovered, observation) is False result = controller._drain_one_idle_profile(recovered, observation) assert result == "runner_admission_drain_deferred_queue" assert patches == [] assert containers[module.RUNNER_PROFILES["backend"].container_name]["running"] is True assert recovered["profiles"]["backend"]["drainOwned"] is False assert saved[-1]["profiles"]["backend"]["drainOwned"] is False def test_queue_identity_is_rechecked_before_yielding_and_busy_peer_is_never_stopped( controller_module, monkeypatch ): module = controller_module now = time.time() build = { "id": "build", "labels": [module.RUNNER_PROFILES["build"].runner_label], "queued_at": now - 20, } backend = { "id": "backend", "labels": ["linux-validation"], "queued_at": now - 10, } controller, state, observation, inventory, containers = _lease_fixture( module, enabled=("backend", "frontend"), busy=(), queue=[build, backend], ) state["profiles"]["backend"]["idleSince"] = now - module.IDLE_DRAIN_SECONDS - 1 raw_build = _raw_job(module, build) fresh_backend = {**backend, "queued_at": now - 30} fresh_build = {**build, "queued_at": now - 20} raw_backend = _raw_job(module, fresh_backend) raw_build = _raw_job(module, fresh_build) patches = [] stops = [] fresh_queue = [raw_backend, raw_build] fresh_active = [] _install_lease_io( monkeypatch, controller, module, inventory, containers, # Fresh inventory reverses the queue identity order; the controller # must not act on the stale observation's earlier build. queue_rows=fresh_queue, active_rows=fresh_active, ) def disable(actual_id, disabled): patches.append((actual_id, disabled)) next(row for row in inventory if row["id"] == actual_id)["disabled"] = disabled monkeypatch.setattr(controller, "_set_disabled", disable) def stop(args, **_kwargs): stops.append(args) containers[args[2]]["running"] = False containers[args[2]]["health"] = "none" return "" monkeypatch.setattr(controller, "_docker_run", stop) # Fresh backend demand is older, and Backend is still serving it. A drain # must not proceed just because a stale snapshot preferred Build. result = controller._drain_one_idle_profile(state, observation) assert result == "runner_admission_drain_deferred_queue" assert patches == [] assert stops == [] assert state["profiles"]["backend"]["drainOwned"] is False # Once the fresh queue again places Build first, the busy Frontend lease # gets its FetchTask fenced first, but its active process stays running. raw_build["created_at"] = module._utc_iso(now - 40) raw_backend["created_at"] = module._utc_iso(now - 30) fresh_queue[:] = [raw_build, raw_backend] frontend_id = observation["profileRunnerIds"]["frontend"] frontend_row = next(row for row in inventory if row["id"] == frontend_id) frontend_row["busy"] = True frontend_active = _raw_job( module, {"id": "active-frontend", "labels": ["itworx-ci-frontend"]}, runner_id=frontend_id, active=True, ) fresh_active.append(frontend_active) observation["runnerRows"][module.RUNNER_PROFILES["frontend"].container_name][ "busy" ] = True observation["activeJobs"][frontend_id] = [{"id": "active-frontend"}] observation["claims"] = ["backend", "frontend"] first = controller._drain_one_idle_profile(state, observation) assert first == "runner_admission_drain_stop_deferred" assert patches == [(frontend_id, True)] assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is True rows, by_id = controller._normalized_runner_rows(inventory) observation["runnerRows"] = rows observation["claims"] = controller._active_claims( rows, by_id, observation["activeJobs"] )[0] second = controller._drain_one_idle_profile(state, observation) assert second == "runner_admission_drained_idle_profile" assert patches == [ (frontend_id, True), (observation["profileRunnerIds"]["backend"], True), ] assert stops == [["docker", "stop", module.RUNNER_PROFILES["backend"].container_name]] assert containers[module.RUNNER_PROFILES["frontend"].container_name]["running"] is True def test_bootstrap_receipt_seeds_only_stopped_disabled_fixed_baseline( controller_module, monkeypatch, tmp_path ): module = controller_module controller = module.RunnerAdmissionController() state = module._initial_state() monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json") saved = [] consumed = [] rows = {} containers = {} registrations = [] for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): labels = [profile.runner_label, *profile.legacy_labels] rows[profile.container_name] = { "id": 100 + index, "displayName": profile.display_name, "disabled": True, "busy": False, "labels": set(labels), } containers[profile.container_name] = { "id": f"{index:064x}", "running": False, "health": "none", } registrations.append( { "profileId": profile.id, "displayName": profile.display_name, "containerName": profile.container_name, "id": 100 + index, "labels": labels, } ) receipt = { "status": "pending", "verifiedRunners": [ profile.container_name for profile in module.RUNNER_PROFILES.values() ], "registrations": registrations, } receipt_sha = "d" * 64 fake_specializer = SimpleNamespace( read_admission_bootstrap_receipt=lambda: (receipt, receipt_sha), consume_admission_bootstrap_receipt=lambda digest: ( consumed.append((digest, state["revision"])) or True ), ) monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer) monkeypatch.setattr(controller, "_save_state", lambda value: ( saved.append(value["revision"]), value.update(revision=value["revision"] + 1), )) seeded = controller._seed_or_consume_bootstrap_receipt( state, { "runnerRows": rows, "containers": containers, "resourceProof": {"ready": True, "runningRunnerContainers": 0}, "claims": [], }, ) assert seeded is True assert saved == [0] assert consumed == [(receipt_sha, 1)] assert state["bootstrapReceiptSha256"] == receipt_sha assert all(entry["observed"] for entry in state["profiles"].values()) assert all(not entry["operatorDisabled"] for entry in state["profiles"].values()) def test_bootstrap_receipt_rejects_any_running_initial_runner( controller_module, monkeypatch, tmp_path ): module = controller_module controller = module.RunnerAdmissionController() state = module._initial_state() monkeypatch.setattr(module, "STATE_FILE", tmp_path / "admission.json") rows = {} containers = {} registrations = [] for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): labels = [profile.runner_label, *profile.legacy_labels] rows[profile.container_name] = { "id": 100 + index, "displayName": profile.display_name, "disabled": True, "busy": False, "labels": set(labels), } containers[profile.container_name] = { "id": f"{index:064x}", "running": index == 1, "health": "healthy" if index == 1 else "none", } registrations.append( { "profileId": profile.id, "displayName": profile.display_name, "containerName": profile.container_name, "id": 100 + index, "labels": labels, } ) fake_specializer = SimpleNamespace( read_admission_bootstrap_receipt=lambda: ( { "status": "pending", "verifiedRunners": [ profile.container_name for profile in module.RUNNER_PROFILES.values() ], "registrations": registrations, }, "e" * 64, ), consume_admission_bootstrap_receipt=lambda _digest: pytest.fail( "receipt consumed before live baseline verification" ), ) monkeypatch.setitem(sys.modules, "runner_fleet_specialize", fake_specializer) saved = [] monkeypatch.setattr(controller, "_save_state", lambda value: saved.append(value)) with pytest.raises(module.RunnerAdmissionError): controller._seed_or_consume_bootstrap_receipt( state, { "runnerRows": rows, "containers": containers, "resourceProof": {"ready": True, "runningRunnerContainers": 1}, "claims": [], }, ) assert saved == [] def test_unattested_more_than_two_running_runner_outers_blocks_claims( controller_module, ): module = controller_module controller = module.RunnerAdmissionController( resource_attestor=lambda **_kwargs: { "ready": True, "effectiveCpuMilli": module.HOST_CPU_BUDGET_MILLI, "effectiveMemoryMiB": module.HOST_MEMORY_BUDGET_MIB, "activeClaims": 0, "heavyClaims": 0, "runningRunnerContainers": 3, "cgroups": {}, "digest": "c" * 64, } ) with pytest.raises( module.RunnerAdmissionError, match="runner_admission_running_outer_budget_exceeded", ): controller._resource_attestation( [], {name: None for name in module.ALL_LEGACY_RUNNER_NAMES}, ) def test_operator_open_then_disable_is_latched_from_bootstrap_baseline( controller_module, ): module = controller_module controller = module.RunnerAdmissionController() state = module._initial_state() entry = state["profiles"]["checks"] entry.update(observed=True, operatorDisabled=False, operatorEnabled=False) profile = module.RUNNER_PROFILES["checks"] rows = {} for index, item in enumerate(module.RUNNER_PROFILES.values(), start=1): rows[item.container_name] = { "id": 200 + index, "disabled": True, "busy": False, } state["registrationIds"][item.id] = 200 + index rows[profile.container_name]["disabled"] = False observation = { "runnerRows": rows } assert controller._reconcile_ownership(state, observation) assert entry["operatorEnabled"] is True assert entry["enabledByController"] is False rows[profile.container_name]["disabled"] = True assert controller._reconcile_ownership(state, observation) assert entry["operatorDisabled"] is True assert entry["operatorEnabled"] is False def test_registration_identity_change_fails_closed(controller_module): module = controller_module controller = module.RunnerAdmissionController() state = module._initial_state() rows = {} for index, profile in enumerate(module.RUNNER_PROFILES.values(), start=1): expected_id = 300 + index state["registrationIds"][profile.id] = expected_id rows[profile.container_name] = { "id": expected_id, "disabled": True, "busy": False, } rows[module.RUNNER_PROFILES["checks"].container_name]["id"] += 1 with pytest.raises( module.RunnerAdmissionError, match="runner_admission_registration_identity_changed", ): controller._reconcile_ownership(state, {"runnerRows": rows}) def test_pending_runner_start_does_not_enable_after_guard_pauses( controller_module, monkeypatch ): module = controller_module controller = module.RunnerAdmissionController(clock=lambda: time.time()) state = module._initial_state() profile = module.RUNNER_PROFILES["checks"] runner_id = 101 job_id = "job-42" state["pending"] = {"profileId": "checks", "runnerId": runner_id, "jobId": job_id} initial = { "runnerRows": {profile.container_name: {"id": runner_id, "disabled": True, "busy": False}}, "queue": [{"id": job_id, "labels": [profile.runner_label], "queued_at": time.time()}], "claims": ["checks"], "containers": {profile.container_name: {"running": False, "health": "none"}}, "resourceProof": {"ready": True}, } after_start = copy.deepcopy(initial) after_start["containers"][profile.container_name] = {"running": True, "health": "healthy"} observations = iter([after_start, after_start]) gates = iter([(None, {"host": {"swapSampleAt": time.time()}}), ("runner_admission_host_guard_paused", None)]) starts = [] enables = [] monkeypatch.setattr(controller, "_save_state", lambda _state: None) monkeypatch.setattr(controller, "_observe", lambda **_kwargs: next(observations)) monkeypatch.setattr(controller, "_fresh_mutation_gate", lambda *_args, **_kwargs: next(gates)) monkeypatch.setattr(controller, "_docker_run", lambda args, **_kwargs: starts.append(args)) monkeypatch.setattr(controller, "_set_disabled", lambda *args: enables.append(args)) result = controller._continue_pending(state, initial, reserved_profile_id="checks") assert result == "runner_admission_host_guard_paused" assert starts == [["docker", "start", profile.container_name]] assert enables == []