230 lines
13 KiB
Python
230 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import sys
|
|
from email.message import Message
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
CONTROLLER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker"
|
|
|
|
|
|
@pytest.fixture
|
|
def controller(monkeypatch):
|
|
monkeypatch.syspath_prepend(str(CONTROLLER))
|
|
spec = importlib.util.spec_from_file_location("specialization_surface_test", CONTROLLER / "status_server.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def handler(controller, monkeypatch, payload=None):
|
|
instance = object.__new__(controller.Handler)
|
|
body = json.dumps(payload or {"reason": "Specialize CI profiles"}).encode()
|
|
instance.headers = Message()
|
|
instance.headers["Content-Length"] = str(len(body))
|
|
instance.rfile = io.BytesIO(body)
|
|
instance.path = controller.RUNNER_FLEET_SPECIALIZE_PATH
|
|
responses = []
|
|
monkeypatch.setattr(instance, "_trusted_client", lambda: True)
|
|
monkeypatch.setattr(instance, "_send_json", lambda code, data: responses.append((code, json.loads(data))))
|
|
signatures = []
|
|
monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *args, **kwargs: signatures.append(kwargs) or True)
|
|
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {"revision": "a" * 40})
|
|
monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "a" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY})
|
|
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda revision: {"status": "already_deployed"})
|
|
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}})
|
|
monkeypatch.setattr(controller, "_repository_path_for_delivery", lambda *args: Path("fixed-repo.git"))
|
|
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: True)
|
|
monkeypatch.setattr(controller, "runner_fleet_specialized_topology", lambda: {"ready": False})
|
|
return instance, responses, signatures
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["blocked", "active", "unknown"])
|
|
def test_legacy_upgrade_cannot_recreate_runners_after_cutover(controller, monkeypatch, state):
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: state)
|
|
monkeypatch.setattr(controller, "execute_runner_fleet_v2", lambda _: pytest.fail("legacy executor must be fenced"))
|
|
instance._handle_runner_fleet_upgrade_v2()
|
|
assert responses == [(409, {"status": "legacy_fleet_upgrade_blocked"})]
|
|
|
|
|
|
def test_specialization_requires_matching_installed_controller_source(controller, monkeypatch):
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "b" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY})
|
|
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda _: pytest.fail("source mismatch must block mutation"))
|
|
instance.do_POST()
|
|
assert responses == [(409, {"status": "controller_source_mismatch"})]
|
|
|
|
|
|
def test_specialization_uses_only_signed_current_main_script(controller, monkeypatch):
|
|
instance, responses, signatures = handler(controller, monkeypatch)
|
|
commands = []
|
|
scripts = []
|
|
monkeypatch.setattr(controller.subprocess, "run", lambda command, **_: commands.append(command) or SimpleNamespace(returncode=0, stdout=b"exact source script"))
|
|
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda script: scripts.append(script) or {"status": "specialized", "verifiedRunners": [], "resourceEnvelope": {"ready": True}})
|
|
instance.do_POST()
|
|
assert signatures == [{"method": "POST", "path": controller.RUNNER_FLEET_SPECIALIZE_PATH}]
|
|
assert commands == [["git", "--git-dir=fixed-repo.git", "show", "a" * 40 + ":scripts/upgrade_ci_runner_fleet_specialized.sh"]]
|
|
assert scripts == [b"exact source script"]
|
|
assert responses[0][0] == 200
|
|
assert responses[0][1]["revision"] == "a" * 40
|
|
assert "burstRunning" not in responses[0][1]
|
|
|
|
|
|
def test_pending_protected_validation_blocks_specialization(controller, monkeypatch):
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _: {"status": "preflight_rejected"})
|
|
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", lambda _: pytest.fail("pending validation must block mutation"))
|
|
instance.do_POST()
|
|
assert responses[0][0] == 409
|
|
assert responses[0][1]["status"] == "preflight_rejected"
|
|
|
|
|
|
@pytest.mark.parametrize("private, expected", [
|
|
("runner_fleet_specialization_entrypoint_untrusted", "runner_fleet_specialization_entrypoint_untrusted"),
|
|
("private-token-value", "runner_fleet_specialization_script_execution_failed"),
|
|
("runner_fleet_specialization_entrypoint_untrusted\nprivate-token-value",
|
|
"runner_fleet_specialization_script_execution_failed"),
|
|
])
|
|
def test_specialization_failure_response_contains_only_finite_safe_reason(controller, monkeypatch, capsys, private, expected):
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
monkeypatch.setattr(controller.subprocess, "run", lambda *args, **kwargs:
|
|
SimpleNamespace(returncode=0, stdout=b"exact source script"))
|
|
|
|
def fail(_script):
|
|
raise controller.RunnerFleetSpecializeError(private)
|
|
|
|
monkeypatch.setattr(controller, "execute_runner_fleet_specialize", fail)
|
|
instance.do_POST()
|
|
assert capsys.readouterr().out == "runner_fleet_specialization_failure:" + expected + "\n"
|
|
assert responses == [(409, {"status": "upgrade_failed", "revision": "a" * 40, "reason": expected})]
|
|
|
|
|
|
def test_caller_cannot_choose_script_image_or_runner(controller, monkeypatch):
|
|
instance, responses, _ = handler(controller, monkeypatch, {"reason": "Specialize CI profiles", "script": "untrusted.sh"})
|
|
instance.do_POST()
|
|
assert responses == [(400, {"status": "invalid_request"})]
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["blocked", "unknown"])
|
|
def test_maintenance_never_falls_back_to_legacy_during_cutover(controller, monkeypatch, state):
|
|
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: state)
|
|
monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("legacy maintenance must be fenced"))
|
|
assert controller._runner_maintenance_cycle() == "deferred"
|
|
|
|
|
|
def test_cutover_and_maintenance_share_single_owner_lock(controller, monkeypatch):
|
|
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: pytest.fail("maintenance must not inspect during cutover"))
|
|
with controller._runner_fleet_upgrade_lock:
|
|
assert controller._runner_maintenance_cycle() == "migration_busy"
|
|
|
|
|
|
def test_active_maintenance_calls_only_admission_controller(controller, monkeypatch):
|
|
import runner_admission_controller
|
|
|
|
calls = []
|
|
monkeypatch.setattr(controller, "runner_fleet_specialization_state", lambda: "active")
|
|
monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "cycle", lambda _: calls.append("admission"))
|
|
monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("legacy worker must stop"))
|
|
assert controller._runner_maintenance_cycle() == "specialized"
|
|
assert calls == ["admission"]
|
|
|
|
|
|
def test_statistics_get_returns_persisted_projection_without_a_cycle(controller, monkeypatch):
|
|
import runner_admission_controller
|
|
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
instance.path = "/v1/runner-fleet/statistics"
|
|
monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "cycle", lambda _: pytest.fail("GET may not start collection or mutate fleet"))
|
|
monkeypatch.setattr(runner_admission_controller.RunnerAdmissionController, "snapshot", lambda _: {"schemaVersion": 1, "status": "unavailable"})
|
|
instance.do_GET()
|
|
assert responses == [(200, {"schemaVersion": 1, "status": "unavailable"})]
|
|
|
|
|
|
def test_statistics_configuration_uses_same_protected_source_and_fixed_signed_route(controller, monkeypatch):
|
|
instance, responses, signatures = handler(controller, monkeypatch)
|
|
instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH
|
|
calls = []
|
|
monkeypatch.setattr(controller, 'configure_runner_statistics_binding',
|
|
lambda **kwargs: calls.append(kwargs) or {'status': 'configured', 'credentialReturned': False})
|
|
monkeypatch.setattr(controller, 'runner_fleet_specialized_topology', lambda: pytest.fail('configuration must not mutate fleet'))
|
|
instance.do_POST()
|
|
assert signatures == [{'method': 'POST', 'path': controller.RUNNER_STATISTICS_CONFIGURE_PATH}]
|
|
assert calls == [{'revision': 'a' * 40, 'projects': {controller.SELF_UPGRADE_REPOSITORY: {'branch': 'main'}}}]
|
|
assert responses == [(200, {'status': 'configured', 'credentialReturned': False})]
|
|
|
|
|
|
def test_statistics_configuration_source_mismatch_writes_nothing(controller, monkeypatch):
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH
|
|
monkeypatch.setattr(controller, '_require_controller_release', lambda: {'sourceRevision': 'b' * 40, 'sourceRepository': controller.SELF_UPGRADE_REPOSITORY})
|
|
monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration'))
|
|
instance.do_POST()
|
|
assert responses == [(409, {'status': 'controller_source_mismatch'})]
|
|
|
|
|
|
def test_statistics_configuration_caller_cannot_select_credential_or_endpoint(controller, monkeypatch):
|
|
instance, responses, _ = handler(controller, monkeypatch, {'reason': 'configure stats', 'token': 'caller-credential'})
|
|
instance.path = controller.RUNNER_STATISTICS_CONFIGURE_PATH
|
|
monkeypatch.setattr(controller, 'configure_runner_statistics_binding', lambda **kwargs: pytest.fail('must not write configuration'))
|
|
instance.do_POST()
|
|
assert responses == [(400, {'status': 'invalid_request'})]
|
|
|
|
|
|
@pytest.mark.parametrize("stage", ["preflight", "quiesce"])
|
|
def test_acceptance_uses_fixed_signed_route_and_protected_source(controller, monkeypatch, stage):
|
|
import runner_acceptance_quiescence
|
|
instance, responses, signatures = handler(controller, monkeypatch)
|
|
instance.path = getattr(controller, "RUNNER_ACCEPTANCE_" + stage.upper() + "_PATH")
|
|
calls = []
|
|
monkeypatch.setattr(runner_acceptance_quiescence, stage,
|
|
lambda **kw: calls.append(kw) or {"status": "blocked", "blockers": ["terminal_origin_claim_missing"]})
|
|
instance.do_POST()
|
|
assert signatures == [{"method": "POST", "path": instance.path}]
|
|
assert calls == [{"revision": "a" * 40, "projects": {controller.SELF_UPGRADE_REPOSITORY: {"branch": "main"}}}]
|
|
assert responses == [(200, {"status": "blocked", "blockers": ["terminal_origin_claim_missing"]})]
|
|
|
|
|
|
@pytest.mark.parametrize("stage", ["preflight", "quiesce"])
|
|
@pytest.mark.parametrize("gate", ["untrusted", "invalid_signature", "source_mismatch", "push_pending", "wrong_head"])
|
|
def test_acceptance_security_gate_refuses_before_executor(controller, monkeypatch, stage, gate):
|
|
import runner_acceptance_quiescence
|
|
instance, responses, _ = handler(controller, monkeypatch)
|
|
instance.path = getattr(controller, "RUNNER_ACCEPTANCE_" + stage.upper() + "_PATH")
|
|
monkeypatch.setattr(runner_acceptance_quiescence, stage, lambda **_: pytest.fail("no executor before source/auth gate"))
|
|
if gate == "untrusted":
|
|
monkeypatch.setattr(instance, "_trusted_client", lambda: False)
|
|
elif gate == "invalid_signature":
|
|
monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_, **__: False)
|
|
elif gate == "source_mismatch":
|
|
monkeypatch.setattr(controller, "_require_controller_release", lambda: {"sourceRevision": "b" * 40, "sourceRepository": controller.SELF_UPGRADE_REPOSITORY})
|
|
elif gate == "push_pending":
|
|
monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _: {"status": "preflight_rejected"})
|
|
else:
|
|
monkeypatch.setattr(controller, "_is_current_branch_head", lambda *args: False)
|
|
instance.do_POST()
|
|
assert responses[0][0] in {403, 409}
|
|
|
|
|
|
@pytest.mark.parametrize("field", ["containerId", "project", "revision", "daemon", "command", "credential", "path"])
|
|
def test_acceptance_caller_cannot_choose_any_effect_identity(controller, monkeypatch, field):
|
|
instance, responses, _ = handler(controller, monkeypatch, {"reason": "inspect fixture", field: "SECRET-CANARY"})
|
|
instance.path = controller.RUNNER_ACCEPTANCE_QUIESCE_PATH
|
|
instance.do_POST()
|
|
assert responses == [(400, {"status": "invalid_request"})]
|
|
assert "SECRET-CANARY" not in str(responses)
|
|
|
|
|
|
def test_pending_acceptance_receipt_fences_legacy_maintenance(controller, monkeypatch):
|
|
import runner_acceptance_quiescence
|
|
monkeypatch.setattr(runner_acceptance_quiescence, "maintenance_fenced", lambda: True)
|
|
monkeypatch.setattr(controller.RunnerFleetService, "automatic_cycle", lambda _: pytest.fail("no re-enable during partial receipt"))
|
|
assert controller._runner_maintenance_cycle() == "acceptance_maintenance_owned"
|