Skip to content

Commit dd58df6

Browse files
refactor(circuit_breaker): share the state machine between sync and async paths
_route() decides CLOSED/PROBE/OPEN once; _on_closed_*/_on_probe_* hold the bookkeeping; the sync and async call methods only differ in how they invoke the protected function. Also dedupes the decorator wrappers and documents async support.
1 parent be60e06 commit dd58df6

4 files changed

Lines changed: 193 additions & 196 deletions

File tree

‎aws_lambda_powertools/utilities/circuit_breaker/base.py‎

Lines changed: 119 additions & 161 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import os
1515
import threading
1616
import uuid
17+
from enum import Enum
1718
from typing import TYPE_CHECKING, Any
1819

1920
from aws_lambda_powertools.utilities.circuit_breaker.exceptions import CircuitBreakerOpenError
@@ -51,6 +52,14 @@
5152
_PROBE_OWNER = threading.local()
5253

5354

55+
class _Route(Enum):
56+
"""How a call is dispatched after reading the persisted circuit state."""
57+
58+
CLOSED = "closed"
59+
PROBE = "probe"
60+
OPEN = "open"
61+
62+
5463
def _probe_owner_id() -> str:
5564
"""
5665
Return this thread's stable probe-owner identifier, minting it on first use.
@@ -131,6 +140,30 @@ def handle(self) -> Any:
131140
CircuitBreakerOpenError
132141
If the circuit is open and no callback is registered.
133142
"""
143+
route, circuit = self._route()
144+
if route is _Route.CLOSED:
145+
return self._call_closed()
146+
if route is _Route.PROBE:
147+
return self._call_probe()
148+
return self._open_response(circuit)
149+
150+
async def handle_async(self) -> Any:
151+
"""Async counterpart of :meth:`handle` for coroutine functions; same routing and bookkeeping."""
152+
route, circuit = self._route()
153+
if route is _Route.CLOSED:
154+
return await self._call_closed_async()
155+
if route is _Route.PROBE:
156+
return await self._call_probe_async()
157+
return await self._open_response_async(circuit)
158+
159+
def _route(self) -> tuple[_Route, CircuitInfo]:
160+
"""
161+
Read the persisted state and decide how this call runs.
162+
163+
Shared by the sync and async paths so the state machine lives in one place. Side
164+
effects (probe election, OPEN to HALF_OPEN notification) happen here; only the
165+
protected call itself differs between the two paths.
166+
"""
134167
record = self.persistence_store.get_state(self.name)
135168

136169
if record.state == CircuitState.CLOSED:
@@ -142,7 +175,7 @@ def handle(self) -> Any:
142175
if prev is not None and prev != CircuitState.CLOSED:
143176
_LOCAL_FAILURES[self.name] = 0
144177
_LAST_OBSERVED_STATE[self.name] = CircuitState.CLOSED
145-
return self._call_closed()
178+
return _Route.CLOSED, record.to_circuit_info()
146179

147180
if record.state == CircuitState.OPEN:
148181
with _COUNTERS_LOCK:
@@ -154,23 +187,23 @@ def handle(self) -> Any:
154187
if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), opened_at):
155188
self._reset_probe_successes()
156189
self._notify(CircuitState.OPEN, CircuitState.HALF_OPEN, opened_at=opened_at)
157-
return self._call_probe()
158-
return self._open_response(record.to_circuit_info())
190+
return _Route.PROBE, record.to_circuit_info()
191+
return _Route.OPEN, record.to_circuit_info()
159192

160193
# HALF_OPEN: only the thread that owns the probe lock runs.
161194
with _COUNTERS_LOCK:
162195
_LAST_OBSERVED_STATE[self.name] = CircuitState.HALF_OPEN
163196
if record.half_open_owner == _probe_owner_id():
164-
return self._call_probe()
197+
return _Route.PROBE, record.to_circuit_info()
165198

166199
# If the probe lease has expired (owner recycled mid-probe), take over.
167200
if record.probe_lease_expiry is not None and self._now() >= record.probe_lease_expiry:
168201
logger.debug("Circuit '%s' probe lease expired; attempting takeover.", self.name)
169202
if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), record.opened_at or 0):
170203
self._reset_probe_successes()
171-
return self._call_probe()
204+
return _Route.PROBE, record.to_circuit_info()
172205

173-
return self._open_response(record.to_circuit_info())
206+
return _Route.OPEN, record.to_circuit_info()
174207

175208
def _reset_probe_successes(self) -> None:
176209
"""
@@ -189,57 +222,90 @@ def _call_closed(self) -> Any:
189222
try:
190223
result = self.function(*self.fn_args, **self.fn_kwargs)
191224
except Exception as exc:
192-
if not self.config.counts_as_failure(exc):
193-
raise
194-
# Increment and reset atomically so exactly one thread observes the threshold
195-
# crossing; racing threads would otherwise lose increments (tripping late) or
196-
# each persist the same transition.
197-
with _COUNTERS_LOCK:
198-
failures = _LOCAL_FAILURES.get(self.name, 0) + 1
199-
tripped = failures >= self.config.failure_threshold
200-
_LOCAL_FAILURES[self.name] = 0 if tripped else failures
201-
if tripped:
202-
logger.debug("Circuit '%s' tripping CLOSED to OPEN after %d failures.", self.name, failures)
203-
opened_at = self._now()
204-
self._safe_persist(
205-
self.persistence_store.save_open,
206-
self.name,
207-
failure_count=failures,
208-
opened_at=opened_at,
209-
)
210-
self._notify(CircuitState.CLOSED, CircuitState.OPEN, opened_at=opened_at)
225+
self._on_closed_failure(exc)
211226
raise
212-
else:
213-
with _COUNTERS_LOCK:
214-
_LOCAL_FAILURES[self.name] = 0
215-
return result
227+
self._on_closed_success()
228+
return result
229+
230+
async def _call_closed_async(self) -> Any:
231+
"""Await the protected coroutine while the circuit is closed, tracking failures."""
232+
try:
233+
result = await self.function(*self.fn_args, **self.fn_kwargs)
234+
except Exception as exc:
235+
self._on_closed_failure(exc)
236+
raise
237+
self._on_closed_success()
238+
return result
239+
240+
def _on_closed_failure(self, exc: Exception) -> None:
241+
"""Count a qualifying failure while closed and trip the circuit at the threshold."""
242+
if not self.config.counts_as_failure(exc):
243+
return
244+
# Increment and reset atomically so exactly one thread observes the threshold
245+
# crossing; racing threads would otherwise lose increments (tripping late) or
246+
# each persist the same transition.
247+
with _COUNTERS_LOCK:
248+
failures = _LOCAL_FAILURES.get(self.name, 0) + 1
249+
tripped = failures >= self.config.failure_threshold
250+
_LOCAL_FAILURES[self.name] = 0 if tripped else failures
251+
if tripped:
252+
logger.debug("Circuit '%s' tripping CLOSED to OPEN after %d failures.", self.name, failures)
253+
opened_at = self._now()
254+
self._safe_persist(
255+
self.persistence_store.save_open,
256+
self.name,
257+
failure_count=failures,
258+
opened_at=opened_at,
259+
)
260+
self._notify(CircuitState.CLOSED, CircuitState.OPEN, opened_at=opened_at)
261+
262+
def _on_closed_success(self) -> None:
263+
with _COUNTERS_LOCK:
264+
_LOCAL_FAILURES[self.name] = 0
216265

217266
def _call_probe(self) -> Any:
218267
"""Run a probe during half-open, closing or reopening based on the outcome."""
219268
try:
220269
result = self.function(*self.fn_args, **self.fn_kwargs)
221270
except Exception as exc:
222-
if not self.config.counts_as_failure(exc):
223-
raise
224-
logger.debug("Circuit '%s' probe failed; reopening.", self.name)
225-
opened_at = self._now()
226-
self._safe_persist(self.persistence_store.save_reopen, self.name, opened_at=opened_at)
227-
with _COUNTERS_LOCK:
228-
_LOCAL_SUCCESSES[self.name] = 0
229-
self._notify(CircuitState.HALF_OPEN, CircuitState.OPEN, opened_at=opened_at)
271+
self._on_probe_failure(exc)
230272
raise
231-
else:
232-
with _COUNTERS_LOCK:
233-
successes = _LOCAL_SUCCESSES.get(self.name, 0) + 1
234-
closed = successes >= self.config.success_threshold
235-
_LOCAL_SUCCESSES[self.name] = 0 if closed else successes
236-
if closed:
237-
_LOCAL_FAILURES[self.name] = 0
273+
self._on_probe_success()
274+
return result
275+
276+
async def _call_probe_async(self) -> Any:
277+
"""Await a probe during half-open, closing or reopening based on the outcome."""
278+
try:
279+
result = await self.function(*self.fn_args, **self.fn_kwargs)
280+
except Exception as exc:
281+
self._on_probe_failure(exc)
282+
raise
283+
self._on_probe_success()
284+
return result
285+
286+
def _on_probe_failure(self, exc: Exception) -> None:
287+
"""A qualifying probe failure reopens the circuit."""
288+
if not self.config.counts_as_failure(exc):
289+
return
290+
logger.debug("Circuit '%s' probe failed; reopening.", self.name)
291+
opened_at = self._now()
292+
self._safe_persist(self.persistence_store.save_reopen, self.name, opened_at=opened_at)
293+
with _COUNTERS_LOCK:
294+
_LOCAL_SUCCESSES[self.name] = 0
295+
self._notify(CircuitState.HALF_OPEN, CircuitState.OPEN, opened_at=opened_at)
296+
297+
def _on_probe_success(self) -> None:
298+
"""Count a probe success and close the circuit at the threshold."""
299+
with _COUNTERS_LOCK:
300+
successes = _LOCAL_SUCCESSES.get(self.name, 0) + 1
301+
closed = successes >= self.config.success_threshold
302+
_LOCAL_SUCCESSES[self.name] = 0 if closed else successes
238303
if closed:
239-
logger.debug("Circuit '%s' closing after %d probe successes.", self.name, successes)
240-
self._safe_persist(self.persistence_store.save_closed, self.name)
241-
self._notify(CircuitState.HALF_OPEN, CircuitState.CLOSED)
242-
return result
304+
_LOCAL_FAILURES[self.name] = 0
305+
if closed:
306+
logger.debug("Circuit '%s' closing after %d probe successes.", self.name, successes)
307+
self._safe_persist(self.persistence_store.save_closed, self.name)
308+
self._notify(CircuitState.HALF_OPEN, CircuitState.CLOSED)
243309

244310
def _safe_persist(self, fn: Callable, *args: Any, **kwargs: Any) -> None:
245311
"""
@@ -273,119 +339,11 @@ def _open_response(self, circuit: CircuitInfo) -> Any:
273339
)
274340

275341
async def _open_response_async(self, circuit: CircuitInfo) -> Any:
276-
"""Produce the response for an open circuit in an async context."""
277-
if self.on_circuit_open is not None:
278-
res = self.on_circuit_open(*self.fn_args, **self.fn_kwargs, circuit=circuit)
279-
if inspect.iscoroutine(res):
280-
return await res
281-
return res
282-
raise CircuitBreakerOpenError(
283-
f"Circuit '{self.name}' is open.",
284-
circuit=circuit,
285-
)
286-
287-
async def handle_async(self) -> Any:
288-
"""
289-
Evaluate the circuit and route the call asynchronously.
290-
291-
Returns
292-
-------
293-
Any
294-
The protected async function's result when the call runs, or the
295-
``on_circuit_open`` callback's return value when the circuit is open.
296-
297-
Raises
298-
------
299-
CircuitBreakerOpenError
300-
If the circuit is open and no callback is registered.
301-
"""
302-
record = self.persistence_store.get_state(self.name)
303-
304-
if record.state == CircuitState.CLOSED:
305-
with _COUNTERS_LOCK:
306-
prev = _LAST_OBSERVED_STATE.get(self.name)
307-
if prev is not None and prev != CircuitState.CLOSED:
308-
_LOCAL_FAILURES[self.name] = 0
309-
_LAST_OBSERVED_STATE[self.name] = CircuitState.CLOSED
310-
return await self._call_closed_async()
311-
312-
if record.state == CircuitState.OPEN:
313-
with _COUNTERS_LOCK:
314-
_LAST_OBSERVED_STATE[self.name] = CircuitState.OPEN
315-
opened_at = record.opened_at if record.opened_at is not None else self._now()
316-
if self._now() >= opened_at + self.config.recovery_timeout:
317-
if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), opened_at):
318-
self._reset_probe_successes()
319-
self._notify(CircuitState.OPEN, CircuitState.HALF_OPEN, opened_at=opened_at)
320-
return await self._call_probe_async()
321-
return await self._open_response_async(record.to_circuit_info())
322-
323-
with _COUNTERS_LOCK:
324-
_LAST_OBSERVED_STATE[self.name] = CircuitState.HALF_OPEN
325-
if record.half_open_owner == _probe_owner_id():
326-
return await self._call_probe_async()
327-
328-
if record.probe_lease_expiry is not None and self._now() >= record.probe_lease_expiry:
329-
logger.debug("Circuit '%s' probe lease expired; attempting takeover.", self.name)
330-
if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), record.opened_at or 0):
331-
self._reset_probe_successes()
332-
return await self._call_probe_async()
333-
334-
return await self._open_response_async(record.to_circuit_info())
335-
336-
async def _call_closed_async(self) -> Any:
337-
"""Run the protected async call while the circuit is closed, tracking failures."""
338-
try:
339-
result = await self.function(*self.fn_args, **self.fn_kwargs)
340-
except Exception as exc:
341-
if not self.config.counts_as_failure(exc):
342-
raise
343-
with _COUNTERS_LOCK:
344-
failures = _LOCAL_FAILURES.get(self.name, 0) + 1
345-
tripped = failures >= self.config.failure_threshold
346-
_LOCAL_FAILURES[self.name] = 0 if tripped else failures
347-
if tripped:
348-
logger.debug("Circuit '%s' tripping CLOSED to OPEN after %d failures.", self.name, failures)
349-
opened_at = self._now()
350-
self._safe_persist(
351-
self.persistence_store.save_open,
352-
self.name,
353-
failure_count=failures,
354-
opened_at=opened_at,
355-
)
356-
self._notify(CircuitState.CLOSED, CircuitState.OPEN, opened_at=opened_at)
357-
raise
358-
else:
359-
with _COUNTERS_LOCK:
360-
_LOCAL_FAILURES[self.name] = 0
361-
return result
362-
363-
async def _call_probe_async(self) -> Any:
364-
"""Run an async probe during half-open, closing or reopening based on the outcome."""
365-
try:
366-
result = await self.function(*self.fn_args, **self.fn_kwargs)
367-
except Exception as exc:
368-
if not self.config.counts_as_failure(exc):
369-
raise
370-
logger.debug("Circuit '%s' probe failed; reopening.", self.name)
371-
opened_at = self._now()
372-
self._safe_persist(self.persistence_store.save_reopen, self.name, opened_at=opened_at)
373-
with _COUNTERS_LOCK:
374-
_LOCAL_SUCCESSES[self.name] = 0
375-
self._notify(CircuitState.HALF_OPEN, CircuitState.OPEN, opened_at=opened_at)
376-
raise
377-
else:
378-
with _COUNTERS_LOCK:
379-
successes = _LOCAL_SUCCESSES.get(self.name, 0) + 1
380-
closed = successes >= self.config.success_threshold
381-
_LOCAL_SUCCESSES[self.name] = 0 if closed else successes
382-
if closed:
383-
_LOCAL_FAILURES[self.name] = 0
384-
if closed:
385-
logger.debug("Circuit '%s' closing after %d probe successes.", self.name, successes)
386-
self._safe_persist(self.persistence_store.save_closed, self.name)
387-
self._notify(CircuitState.HALF_OPEN, CircuitState.CLOSED)
388-
return result
342+
"""Produce the response for an open circuit, awaiting an async ``on_circuit_open`` callback."""
343+
response = self._open_response(circuit)
344+
if inspect.isawaitable(response):
345+
return await response
346+
return response
389347

390348
def _notify(self, from_state: CircuitState, to_state: CircuitState, opened_at: int | None = None) -> None:
391349
"""

0 commit comments

Comments
 (0)