Skip to content

Commit be60e06

Browse files
committed
fix(circuit_breaker): await async coroutines inside circuit breaker failure tracking
1 parent 13dbc55 commit be60e06

3 files changed

Lines changed: 279 additions & 1 deletion

File tree

‎aws_lambda_powertools/utilities/circuit_breaker/base.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from __future__ import annotations
1010

1111
import datetime as dt
12+
import inspect
1213
import logging
1314
import os
1415
import threading
@@ -271,6 +272,121 @@ def _open_response(self, circuit: CircuitInfo) -> Any:
271272
circuit=circuit,
272273
)
273274

275+
async def _open_response_async(self, circuit: CircuitInfo) -> Any:
276+
"""Produce the response for an open circuit in an async context."""
277+
if self.on_circuit_open is not None:
278+
res = self.on_circuit_open(*self.fn_args, **self.fn_kwargs, circuit=circuit)
279+
if inspect.iscoroutine(res):
280+
return await res
281+
return res
282+
raise CircuitBreakerOpenError(
283+
f"Circuit '{self.name}' is open.",
284+
circuit=circuit,
285+
)
286+
287+
async def handle_async(self) -> Any:
288+
"""
289+
Evaluate the circuit and route the call asynchronously.
290+
291+
Returns
292+
-------
293+
Any
294+
The protected async function's result when the call runs, or the
295+
``on_circuit_open`` callback's return value when the circuit is open.
296+
297+
Raises
298+
------
299+
CircuitBreakerOpenError
300+
If the circuit is open and no callback is registered.
301+
"""
302+
record = self.persistence_store.get_state(self.name)
303+
304+
if record.state == CircuitState.CLOSED:
305+
with _COUNTERS_LOCK:
306+
prev = _LAST_OBSERVED_STATE.get(self.name)
307+
if prev is not None and prev != CircuitState.CLOSED:
308+
_LOCAL_FAILURES[self.name] = 0
309+
_LAST_OBSERVED_STATE[self.name] = CircuitState.CLOSED
310+
return await self._call_closed_async()
311+
312+
if record.state == CircuitState.OPEN:
313+
with _COUNTERS_LOCK:
314+
_LAST_OBSERVED_STATE[self.name] = CircuitState.OPEN
315+
opened_at = record.opened_at if record.opened_at is not None else self._now()
316+
if self._now() >= opened_at + self.config.recovery_timeout:
317+
if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), opened_at):
318+
self._reset_probe_successes()
319+
self._notify(CircuitState.OPEN, CircuitState.HALF_OPEN, opened_at=opened_at)
320+
return await self._call_probe_async()
321+
return await self._open_response_async(record.to_circuit_info())
322+
323+
with _COUNTERS_LOCK:
324+
_LAST_OBSERVED_STATE[self.name] = CircuitState.HALF_OPEN
325+
if record.half_open_owner == _probe_owner_id():
326+
return await self._call_probe_async()
327+
328+
if record.probe_lease_expiry is not None and self._now() >= record.probe_lease_expiry:
329+
logger.debug("Circuit '%s' probe lease expired; attempting takeover.", self.name)
330+
if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), record.opened_at or 0):
331+
self._reset_probe_successes()
332+
return await self._call_probe_async()
333+
334+
return await self._open_response_async(record.to_circuit_info())
335+
336+
async def _call_closed_async(self) -> Any:
337+
"""Run the protected async call while the circuit is closed, tracking failures."""
338+
try:
339+
result = await self.function(*self.fn_args, **self.fn_kwargs)
340+
except Exception as exc:
341+
if not self.config.counts_as_failure(exc):
342+
raise
343+
with _COUNTERS_LOCK:
344+
failures = _LOCAL_FAILURES.get(self.name, 0) + 1
345+
tripped = failures >= self.config.failure_threshold
346+
_LOCAL_FAILURES[self.name] = 0 if tripped else failures
347+
if tripped:
348+
logger.debug("Circuit '%s' tripping CLOSED to OPEN after %d failures.", self.name, failures)
349+
opened_at = self._now()
350+
self._safe_persist(
351+
self.persistence_store.save_open,
352+
self.name,
353+
failure_count=failures,
354+
opened_at=opened_at,
355+
)
356+
self._notify(CircuitState.CLOSED, CircuitState.OPEN, opened_at=opened_at)
357+
raise
358+
else:
359+
with _COUNTERS_LOCK:
360+
_LOCAL_FAILURES[self.name] = 0
361+
return result
362+
363+
async def _call_probe_async(self) -> Any:
364+
"""Run an async probe during half-open, closing or reopening based on the outcome."""
365+
try:
366+
result = await self.function(*self.fn_args, **self.fn_kwargs)
367+
except Exception as exc:
368+
if not self.config.counts_as_failure(exc):
369+
raise
370+
logger.debug("Circuit '%s' probe failed; reopening.", self.name)
371+
opened_at = self._now()
372+
self._safe_persist(self.persistence_store.save_reopen, self.name, opened_at=opened_at)
373+
with _COUNTERS_LOCK:
374+
_LOCAL_SUCCESSES[self.name] = 0
375+
self._notify(CircuitState.HALF_OPEN, CircuitState.OPEN, opened_at=opened_at)
376+
raise
377+
else:
378+
with _COUNTERS_LOCK:
379+
successes = _LOCAL_SUCCESSES.get(self.name, 0) + 1
380+
closed = successes >= self.config.success_threshold
381+
_LOCAL_SUCCESSES[self.name] = 0 if closed else successes
382+
if closed:
383+
_LOCAL_FAILURES[self.name] = 0
384+
if closed:
385+
logger.debug("Circuit '%s' closing after %d probe successes.", self.name, successes)
386+
self._safe_persist(self.persistence_store.save_closed, self.name)
387+
self._notify(CircuitState.HALF_OPEN, CircuitState.CLOSED)
388+
return result
389+
274390
def _notify(self, from_state: CircuitState, to_state: CircuitState, opened_at: int | None = None) -> None:
275391
"""
276392
Fire the ``on_transition`` hook for a state change.

‎aws_lambda_powertools/utilities/circuit_breaker/circuit_breaker.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from __future__ import annotations
66

77
import functools
8+
import inspect
89
import logging
910
import os
1011
import warnings
@@ -90,6 +91,31 @@ def charge(order: dict) -> dict:
9091
config = config or CircuitBreakerConfig()
9192

9293
def decorator(function: Callable) -> Callable:
94+
is_async = inspect.iscoroutinefunction(function)
95+
96+
@functools.wraps(function)
97+
async def async_wrapper(*args, **kwargs) -> Any:
98+
if strtobool(os.getenv(constants.CIRCUIT_BREAKER_DISABLED_ENV, "false")):
99+
warnings.warn(
100+
message="Disabling the circuit breaker is intended for development environments only "
101+
"and should not be used in production.",
102+
category=PowertoolsUserWarning,
103+
stacklevel=2,
104+
)
105+
return await function(*args, **kwargs)
106+
107+
handler = CircuitBreakerHandler(
108+
function=function,
109+
name=name,
110+
config=config,
111+
persistence_store=persistence_store,
112+
on_circuit_open=on_circuit_open,
113+
on_transition=on_transition,
114+
function_args=args,
115+
function_kwargs=kwargs,
116+
)
117+
return await handler.handle_async()
118+
93119
@functools.wraps(function)
94120
def wrapper(*args, **kwargs) -> Any:
95121
# Skip the circuit entirely when disabled (development only).
@@ -114,6 +140,6 @@ def wrapper(*args, **kwargs) -> Any:
114140
)
115141
return handler.handle()
116142

117-
return wrapper
143+
return async_wrapper if is_async else wrapper
118144

119145
return decorator

‎tests/functional/circuit_breaker/test_circuit_breaker.py‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -888,3 +888,139 @@ def test_settings_are_kept_per_circuit_on_a_shared_layer(store, now):
888888
lease = store.db["a"].probe_lease_expiry
889889
assert lease is not None
890890
assert lease <= now + 30 + 2, "the lease must derive from circuit 'a' recovery_timeout, not 'b'"
891+
892+
893+
@pytest.mark.asyncio
894+
async def test_async_closed_circuit_returns_result_and_writes_nothing(store):
895+
@circuit_breaker(name="async-c", persistence_store=store, config=CircuitBreakerConfig(local_cache_max_age=0))
896+
async def call(value):
897+
return value * 2
898+
899+
assert await call(21) == 42
900+
assert store.db == {}, "healthy path must not write to the store"
901+
902+
903+
@pytest.mark.asyncio
904+
async def test_async_trips_open_after_failure_threshold(store):
905+
config = CircuitBreakerConfig(failure_threshold=2, local_cache_max_age=0)
906+
907+
@circuit_breaker(name="async-trip", persistence_store=store, config=config)
908+
async def failing_call():
909+
raise ConnectionError("downstream down")
910+
911+
for _ in range(2):
912+
with pytest.raises(ConnectionError):
913+
await failing_call()
914+
915+
assert store.db["async-trip"].state == CircuitState.OPEN
916+
assert store.db["async-trip"].opened_at is not None
917+
918+
919+
@pytest.mark.asyncio
920+
async def test_async_open_with_callback_returns_callback_value(store, now):
921+
store.db["async-open"] = CircuitStateRecord(
922+
name="async-open",
923+
state=CircuitState.OPEN,
924+
failure_count=5,
925+
opened_at=now,
926+
)
927+
928+
def sync_on_open(order, circuit):
929+
return {"buffered": order, "state": str(circuit.state)}
930+
931+
config = CircuitBreakerConfig(recovery_timeout=9999, local_cache_max_age=0)
932+
933+
@circuit_breaker(name="async-open", persistence_store=store, on_circuit_open=sync_on_open, config=config)
934+
async def charge(order):
935+
return f"charged {order}"
936+
937+
result = await charge({"id": 1})
938+
assert result == {"buffered": {"id": 1}, "state": "OPEN"}
939+
940+
941+
@pytest.mark.asyncio
942+
async def test_async_open_with_async_callback_returns_callback_value(store, now):
943+
store.db["async-open-2"] = CircuitStateRecord(
944+
name="async-open-2",
945+
state=CircuitState.OPEN,
946+
failure_count=5,
947+
opened_at=now,
948+
)
949+
950+
async def async_on_open(order, circuit):
951+
return {"async_buffered": order, "state": str(circuit.state)}
952+
953+
config = CircuitBreakerConfig(recovery_timeout=9999, local_cache_max_age=0)
954+
955+
@circuit_breaker(name="async-open-2", persistence_store=store, on_circuit_open=async_on_open, config=config)
956+
async def charge(order):
957+
return f"charged {order}"
958+
959+
result = await charge({"id": 2})
960+
assert result == {"async_buffered": {"id": 2}, "state": "OPEN"}
961+
962+
963+
@pytest.mark.asyncio
964+
async def test_async_open_without_callback_raises_with_circuit_info(store, now):
965+
store.db["async-err"] = CircuitStateRecord(
966+
name="async-err",
967+
state=CircuitState.OPEN,
968+
failure_count=3,
969+
opened_at=now,
970+
)
971+
config = CircuitBreakerConfig(recovery_timeout=9999, local_cache_max_age=0)
972+
973+
@circuit_breaker(name="async-err", persistence_store=store, config=config)
974+
async def call():
975+
return "success"
976+
977+
with pytest.raises(CircuitBreakerOpenError) as exc_info:
978+
await call()
979+
980+
assert exc_info.value.circuit.name == "async-err"
981+
982+
983+
@pytest.mark.asyncio
984+
async def test_async_half_open_probe_success_closes(store, now):
985+
store.db["async-probe"] = CircuitStateRecord(name="async-probe", state=CircuitState.OPEN, opened_at=now - 100)
986+
config = CircuitBreakerConfig(recovery_timeout=30, success_threshold=1, local_cache_max_age=0)
987+
988+
@circuit_breaker(name="async-probe", persistence_store=store, config=config)
989+
async def call():
990+
return "healed"
991+
992+
res = await call()
993+
assert res == "healed"
994+
assert store.db["async-probe"].state == CircuitState.CLOSED
995+
996+
997+
@pytest.mark.asyncio
998+
async def test_async_half_open_probe_failure_reopens(store, now):
999+
store.db["async-reopen"] = CircuitStateRecord(name="async-reopen", state=CircuitState.OPEN, opened_at=now - 100)
1000+
config = CircuitBreakerConfig(recovery_timeout=30, success_threshold=1, local_cache_max_age=0)
1001+
1002+
@circuit_breaker(name="async-reopen", persistence_store=store, config=config)
1003+
async def call():
1004+
raise ConnectionError("still down")
1005+
1006+
with pytest.raises(ConnectionError):
1007+
await call()
1008+
1009+
assert store.db["async-reopen"].state == CircuitState.OPEN
1010+
1011+
1012+
@pytest.mark.asyncio
1013+
async def test_async_disabled_env_bypasses_circuit(store, monkeypatch):
1014+
monkeypatch.setenv("POWERTOOLS_CIRCUIT_BREAKER_DISABLED", "true")
1015+
config = CircuitBreakerConfig(failure_threshold=1, local_cache_max_age=0)
1016+
1017+
@circuit_breaker(name="async-disabled", persistence_store=store, config=config)
1018+
async def call():
1019+
return "bypassed"
1020+
1021+
with warnings.catch_warnings(record=True) as recorded:
1022+
warnings.simplefilter("always")
1023+
result = await call()
1024+
1025+
assert result == "bypassed"
1026+
assert any("Disabling the circuit breaker is intended for development" in str(w.message) for w in recorded)

0 commit comments

Comments
 (0)