116 lines
3.8 KiB
Python
116 lines
3.8 KiB
Python
"""CircuitBreaker: observable state transitions, cooldown and recovery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Callable
|
|
|
|
import pytest
|
|
|
|
from ragcore.infrastructure.models.resilience import (
|
|
CIRCUIT_CLOSED_EVENT,
|
|
CIRCUIT_HALF_OPEN_EVENT,
|
|
CIRCUIT_OPEN_EVENT,
|
|
CircuitBreaker,
|
|
CircuitOpenError,
|
|
CircuitState,
|
|
)
|
|
|
|
|
|
def _clock(value: dict[str, float]) -> Callable[[], float]:
|
|
return lambda: value["now"]
|
|
|
|
|
|
def test_circuit_starts_closed_and_allows_calls() -> None:
|
|
breaker = CircuitBreaker(provider="test-provider")
|
|
assert breaker.state is CircuitState.CLOSED
|
|
breaker.check() # must not raise
|
|
|
|
|
|
def test_circuit_opens_after_consecutive_failures_and_blocks_calls() -> None:
|
|
breaker = CircuitBreaker(provider="test-provider", failure_threshold=3)
|
|
for _ in range(2):
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.CLOSED
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
with pytest.raises(CircuitOpenError):
|
|
breaker.check()
|
|
|
|
|
|
def test_circuit_transitions_are_observable_via_state_and_logs(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
breaker = CircuitBreaker(provider="test-provider", failure_threshold=1)
|
|
with caplog.at_level(logging.WARNING, logger="ragcore.infrastructure.models.resilience"):
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
open_records = [r for r in caplog.records if r.getMessage() == CIRCUIT_OPEN_EVENT]
|
|
assert len(open_records) == 1
|
|
assert open_records[0].provider == "test-provider"
|
|
assert open_records[0].from_state == "closed"
|
|
assert open_records[0].to_state == "open"
|
|
|
|
|
|
def test_circuit_half_opens_after_cooldown_and_closes_on_success_recovery_probe() -> None:
|
|
time_value = {"now": 0.0}
|
|
breaker = CircuitBreaker(
|
|
provider="test-provider",
|
|
failure_threshold=1,
|
|
cooldown_seconds=10.0,
|
|
clock=_clock(time_value),
|
|
)
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
|
|
# Still within cooldown: blocked without a real recovery attempt.
|
|
with pytest.raises(CircuitOpenError):
|
|
breaker.check()
|
|
assert breaker.state is CircuitState.OPEN
|
|
|
|
# Cooldown elapsed: exactly one recovery probe is allowed through.
|
|
time_value["now"] = 10.1
|
|
breaker.check()
|
|
assert breaker.state is CircuitState.HALF_OPEN
|
|
|
|
breaker.record_success()
|
|
assert breaker.state is CircuitState.CLOSED
|
|
breaker.check() # closed again: no longer raises
|
|
|
|
|
|
def test_circuit_half_open_failure_reopens_immediately_without_a_fresh_threshold() -> None:
|
|
time_value = {"now": 0.0}
|
|
breaker = CircuitBreaker(
|
|
provider="test-provider",
|
|
failure_threshold=5,
|
|
cooldown_seconds=10.0,
|
|
clock=_clock(time_value),
|
|
)
|
|
for _ in range(5):
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
|
|
time_value["now"] = 10.1
|
|
breaker.check()
|
|
assert breaker.state is CircuitState.HALF_OPEN
|
|
|
|
# A single half-open failure reopens immediately -- it does not require
|
|
# accumulating the full failure_threshold all over again.
|
|
breaker.record_failure()
|
|
assert breaker.state is CircuitState.OPEN
|
|
|
|
|
|
def test_transition_events_are_named_consistently() -> None:
|
|
assert CIRCUIT_OPEN_EVENT == "provider_circuit_open"
|
|
assert CIRCUIT_HALF_OPEN_EVENT == "provider_circuit_half_open"
|
|
assert CIRCUIT_CLOSED_EVENT == "provider_circuit_closed"
|
|
|
|
|
|
def test_invalid_construction_arguments_are_rejected() -> None:
|
|
with pytest.raises(ValueError, match="provider"):
|
|
CircuitBreaker(provider="")
|
|
with pytest.raises(ValueError, match="failure_threshold"):
|
|
CircuitBreaker(provider="p", failure_threshold=0)
|
|
with pytest.raises(ValueError, match="cooldown_seconds"):
|
|
CircuitBreaker(provider="p", cooldown_seconds=0)
|