1414import os
1515import threading
1616import uuid
17+ from enum import Enum
1718from typing import TYPE_CHECKING , Any
1819
1920from aws_lambda_powertools .utilities .circuit_breaker .exceptions import CircuitBreakerOpenError
5152_PROBE_OWNER = threading .local ()
5253
5354
55+ class _Route (Enum ):
56+ """How a call is dispatched after reading the persisted circuit state."""
57+
58+ CLOSED = "closed"
59+ PROBE = "probe"
60+ OPEN = "open"
61+
62+
5463def _probe_owner_id () -> str :
5564 """
5665 Return this thread's stable probe-owner identifier, minting it on first use.
@@ -131,6 +140,30 @@ def handle(self) -> Any:
131140 CircuitBreakerOpenError
132141 If the circuit is open and no callback is registered.
133142 """
143+ route , circuit = self ._route ()
144+ if route is _Route .CLOSED :
145+ return self ._call_closed ()
146+ if route is _Route .PROBE :
147+ return self ._call_probe ()
148+ return self ._open_response (circuit )
149+
150+ async def handle_async (self ) -> Any :
151+ """Async counterpart of :meth:`handle` for coroutine functions; same routing and bookkeeping."""
152+ route , circuit = self ._route ()
153+ if route is _Route .CLOSED :
154+ return await self ._call_closed_async ()
155+ if route is _Route .PROBE :
156+ return await self ._call_probe_async ()
157+ return await self ._open_response_async (circuit )
158+
159+ def _route (self ) -> tuple [_Route , CircuitInfo ]:
160+ """
161+ Read the persisted state and decide how this call runs.
162+
163+ Shared by the sync and async paths so the state machine lives in one place. Side
164+ effects (probe election, OPEN to HALF_OPEN notification) happen here; only the
165+ protected call itself differs between the two paths.
166+ """
134167 record = self .persistence_store .get_state (self .name )
135168
136169 if record .state == CircuitState .CLOSED :
@@ -142,7 +175,7 @@ def handle(self) -> Any:
142175 if prev is not None and prev != CircuitState .CLOSED :
143176 _LOCAL_FAILURES [self .name ] = 0
144177 _LAST_OBSERVED_STATE [self .name ] = CircuitState .CLOSED
145- return self . _call_closed ()
178+ return _Route . CLOSED , record . to_circuit_info ()
146179
147180 if record .state == CircuitState .OPEN :
148181 with _COUNTERS_LOCK :
@@ -154,23 +187,23 @@ def handle(self) -> Any:
154187 if self .persistence_store .try_acquire_half_open (self .name , _probe_owner_id (), opened_at ):
155188 self ._reset_probe_successes ()
156189 self ._notify (CircuitState .OPEN , CircuitState .HALF_OPEN , opened_at = opened_at )
157- return self . _call_probe ()
158- return self . _open_response ( record .to_circuit_info () )
190+ return _Route . PROBE , record . to_circuit_info ()
191+ return _Route . OPEN , record .to_circuit_info ()
159192
160193 # HALF_OPEN: only the thread that owns the probe lock runs.
161194 with _COUNTERS_LOCK :
162195 _LAST_OBSERVED_STATE [self .name ] = CircuitState .HALF_OPEN
163196 if record .half_open_owner == _probe_owner_id ():
164- return self . _call_probe ()
197+ return _Route . PROBE , record . to_circuit_info ()
165198
166199 # If the probe lease has expired (owner recycled mid-probe), take over.
167200 if record .probe_lease_expiry is not None and self ._now () >= record .probe_lease_expiry :
168201 logger .debug ("Circuit '%s' probe lease expired; attempting takeover." , self .name )
169202 if self .persistence_store .try_acquire_half_open (self .name , _probe_owner_id (), record .opened_at or 0 ):
170203 self ._reset_probe_successes ()
171- return self . _call_probe ()
204+ return _Route . PROBE , record . to_circuit_info ()
172205
173- return self . _open_response ( record .to_circuit_info () )
206+ return _Route . OPEN , record .to_circuit_info ()
174207
175208 def _reset_probe_successes (self ) -> None :
176209 """
@@ -189,57 +222,90 @@ def _call_closed(self) -> Any:
189222 try :
190223 result = self .function (* self .fn_args , ** self .fn_kwargs )
191224 except Exception as exc :
192- if not self .config .counts_as_failure (exc ):
193- raise
194- # Increment and reset atomically so exactly one thread observes the threshold
195- # crossing; racing threads would otherwise lose increments (tripping late) or
196- # each persist the same transition.
197- with _COUNTERS_LOCK :
198- failures = _LOCAL_FAILURES .get (self .name , 0 ) + 1
199- tripped = failures >= self .config .failure_threshold
200- _LOCAL_FAILURES [self .name ] = 0 if tripped else failures
201- if tripped :
202- logger .debug ("Circuit '%s' tripping CLOSED to OPEN after %d failures." , self .name , failures )
203- opened_at = self ._now ()
204- self ._safe_persist (
205- self .persistence_store .save_open ,
206- self .name ,
207- failure_count = failures ,
208- opened_at = opened_at ,
209- )
210- self ._notify (CircuitState .CLOSED , CircuitState .OPEN , opened_at = opened_at )
225+ self ._on_closed_failure (exc )
211226 raise
212- else :
213- with _COUNTERS_LOCK :
214- _LOCAL_FAILURES [self .name ] = 0
215- return result
227+ self ._on_closed_success ()
228+ return result
229+
230+ async def _call_closed_async (self ) -> Any :
231+ """Await the protected coroutine while the circuit is closed, tracking failures."""
232+ try :
233+ result = await self .function (* self .fn_args , ** self .fn_kwargs )
234+ except Exception as exc :
235+ self ._on_closed_failure (exc )
236+ raise
237+ self ._on_closed_success ()
238+ return result
239+
240+ def _on_closed_failure (self , exc : Exception ) -> None :
241+ """Count a qualifying failure while closed and trip the circuit at the threshold."""
242+ if not self .config .counts_as_failure (exc ):
243+ return
244+ # Increment and reset atomically so exactly one thread observes the threshold
245+ # crossing; racing threads would otherwise lose increments (tripping late) or
246+ # each persist the same transition.
247+ with _COUNTERS_LOCK :
248+ failures = _LOCAL_FAILURES .get (self .name , 0 ) + 1
249+ tripped = failures >= self .config .failure_threshold
250+ _LOCAL_FAILURES [self .name ] = 0 if tripped else failures
251+ if tripped :
252+ logger .debug ("Circuit '%s' tripping CLOSED to OPEN after %d failures." , self .name , failures )
253+ opened_at = self ._now ()
254+ self ._safe_persist (
255+ self .persistence_store .save_open ,
256+ self .name ,
257+ failure_count = failures ,
258+ opened_at = opened_at ,
259+ )
260+ self ._notify (CircuitState .CLOSED , CircuitState .OPEN , opened_at = opened_at )
261+
262+ def _on_closed_success (self ) -> None :
263+ with _COUNTERS_LOCK :
264+ _LOCAL_FAILURES [self .name ] = 0
216265
217266 def _call_probe (self ) -> Any :
218267 """Run a probe during half-open, closing or reopening based on the outcome."""
219268 try :
220269 result = self .function (* self .fn_args , ** self .fn_kwargs )
221270 except Exception as exc :
222- if not self .config .counts_as_failure (exc ):
223- raise
224- logger .debug ("Circuit '%s' probe failed; reopening." , self .name )
225- opened_at = self ._now ()
226- self ._safe_persist (self .persistence_store .save_reopen , self .name , opened_at = opened_at )
227- with _COUNTERS_LOCK :
228- _LOCAL_SUCCESSES [self .name ] = 0
229- self ._notify (CircuitState .HALF_OPEN , CircuitState .OPEN , opened_at = opened_at )
271+ self ._on_probe_failure (exc )
230272 raise
231- else :
232- with _COUNTERS_LOCK :
233- successes = _LOCAL_SUCCESSES .get (self .name , 0 ) + 1
234- closed = successes >= self .config .success_threshold
235- _LOCAL_SUCCESSES [self .name ] = 0 if closed else successes
236- if closed :
237- _LOCAL_FAILURES [self .name ] = 0
273+ self ._on_probe_success ()
274+ return result
275+
276+ async def _call_probe_async (self ) -> Any :
277+ """Await a probe during half-open, closing or reopening based on the outcome."""
278+ try :
279+ result = await self .function (* self .fn_args , ** self .fn_kwargs )
280+ except Exception as exc :
281+ self ._on_probe_failure (exc )
282+ raise
283+ self ._on_probe_success ()
284+ return result
285+
286+ def _on_probe_failure (self , exc : Exception ) -> None :
287+ """A qualifying probe failure reopens the circuit."""
288+ if not self .config .counts_as_failure (exc ):
289+ return
290+ logger .debug ("Circuit '%s' probe failed; reopening." , self .name )
291+ opened_at = self ._now ()
292+ self ._safe_persist (self .persistence_store .save_reopen , self .name , opened_at = opened_at )
293+ with _COUNTERS_LOCK :
294+ _LOCAL_SUCCESSES [self .name ] = 0
295+ self ._notify (CircuitState .HALF_OPEN , CircuitState .OPEN , opened_at = opened_at )
296+
297+ def _on_probe_success (self ) -> None :
298+ """Count a probe success and close the circuit at the threshold."""
299+ with _COUNTERS_LOCK :
300+ successes = _LOCAL_SUCCESSES .get (self .name , 0 ) + 1
301+ closed = successes >= self .config .success_threshold
302+ _LOCAL_SUCCESSES [self .name ] = 0 if closed else successes
238303 if closed :
239- logger .debug ("Circuit '%s' closing after %d probe successes." , self .name , successes )
240- self ._safe_persist (self .persistence_store .save_closed , self .name )
241- self ._notify (CircuitState .HALF_OPEN , CircuitState .CLOSED )
242- return result
304+ _LOCAL_FAILURES [self .name ] = 0
305+ if closed :
306+ logger .debug ("Circuit '%s' closing after %d probe successes." , self .name , successes )
307+ self ._safe_persist (self .persistence_store .save_closed , self .name )
308+ self ._notify (CircuitState .HALF_OPEN , CircuitState .CLOSED )
243309
244310 def _safe_persist (self , fn : Callable , * args : Any , ** kwargs : Any ) -> None :
245311 """
@@ -273,119 +339,11 @@ def _open_response(self, circuit: CircuitInfo) -> Any:
273339 )
274340
275341 async def _open_response_async (self , circuit : CircuitInfo ) -> Any :
276- """Produce the response for an open circuit in an async context."""
277- if self .on_circuit_open is not None :
278- res = self .on_circuit_open (* self .fn_args , ** self .fn_kwargs , circuit = circuit )
279- if inspect .iscoroutine (res ):
280- return await res
281- return res
282- raise CircuitBreakerOpenError (
283- f"Circuit '{ self .name } ' is open." ,
284- circuit = circuit ,
285- )
286-
287- async def handle_async (self ) -> Any :
288- """
289- Evaluate the circuit and route the call asynchronously.
290-
291- Returns
292- -------
293- Any
294- The protected async function's result when the call runs, or the
295- ``on_circuit_open`` callback's return value when the circuit is open.
296-
297- Raises
298- ------
299- CircuitBreakerOpenError
300- If the circuit is open and no callback is registered.
301- """
302- record = self .persistence_store .get_state (self .name )
303-
304- if record .state == CircuitState .CLOSED :
305- with _COUNTERS_LOCK :
306- prev = _LAST_OBSERVED_STATE .get (self .name )
307- if prev is not None and prev != CircuitState .CLOSED :
308- _LOCAL_FAILURES [self .name ] = 0
309- _LAST_OBSERVED_STATE [self .name ] = CircuitState .CLOSED
310- return await self ._call_closed_async ()
311-
312- if record .state == CircuitState .OPEN :
313- with _COUNTERS_LOCK :
314- _LAST_OBSERVED_STATE [self .name ] = CircuitState .OPEN
315- opened_at = record .opened_at if record .opened_at is not None else self ._now ()
316- if self ._now () >= opened_at + self .config .recovery_timeout :
317- if self .persistence_store .try_acquire_half_open (self .name , _probe_owner_id (), opened_at ):
318- self ._reset_probe_successes ()
319- self ._notify (CircuitState .OPEN , CircuitState .HALF_OPEN , opened_at = opened_at )
320- return await self ._call_probe_async ()
321- return await self ._open_response_async (record .to_circuit_info ())
322-
323- with _COUNTERS_LOCK :
324- _LAST_OBSERVED_STATE [self .name ] = CircuitState .HALF_OPEN
325- if record .half_open_owner == _probe_owner_id ():
326- return await self ._call_probe_async ()
327-
328- if record .probe_lease_expiry is not None and self ._now () >= record .probe_lease_expiry :
329- logger .debug ("Circuit '%s' probe lease expired; attempting takeover." , self .name )
330- if self .persistence_store .try_acquire_half_open (self .name , _probe_owner_id (), record .opened_at or 0 ):
331- self ._reset_probe_successes ()
332- return await self ._call_probe_async ()
333-
334- return await self ._open_response_async (record .to_circuit_info ())
335-
336- async def _call_closed_async (self ) -> Any :
337- """Run the protected async call while the circuit is closed, tracking failures."""
338- try :
339- result = await self .function (* self .fn_args , ** self .fn_kwargs )
340- except Exception as exc :
341- if not self .config .counts_as_failure (exc ):
342- raise
343- with _COUNTERS_LOCK :
344- failures = _LOCAL_FAILURES .get (self .name , 0 ) + 1
345- tripped = failures >= self .config .failure_threshold
346- _LOCAL_FAILURES [self .name ] = 0 if tripped else failures
347- if tripped :
348- logger .debug ("Circuit '%s' tripping CLOSED to OPEN after %d failures." , self .name , failures )
349- opened_at = self ._now ()
350- self ._safe_persist (
351- self .persistence_store .save_open ,
352- self .name ,
353- failure_count = failures ,
354- opened_at = opened_at ,
355- )
356- self ._notify (CircuitState .CLOSED , CircuitState .OPEN , opened_at = opened_at )
357- raise
358- else :
359- with _COUNTERS_LOCK :
360- _LOCAL_FAILURES [self .name ] = 0
361- return result
362-
363- async def _call_probe_async (self ) -> Any :
364- """Run an async probe during half-open, closing or reopening based on the outcome."""
365- try :
366- result = await self .function (* self .fn_args , ** self .fn_kwargs )
367- except Exception as exc :
368- if not self .config .counts_as_failure (exc ):
369- raise
370- logger .debug ("Circuit '%s' probe failed; reopening." , self .name )
371- opened_at = self ._now ()
372- self ._safe_persist (self .persistence_store .save_reopen , self .name , opened_at = opened_at )
373- with _COUNTERS_LOCK :
374- _LOCAL_SUCCESSES [self .name ] = 0
375- self ._notify (CircuitState .HALF_OPEN , CircuitState .OPEN , opened_at = opened_at )
376- raise
377- else :
378- with _COUNTERS_LOCK :
379- successes = _LOCAL_SUCCESSES .get (self .name , 0 ) + 1
380- closed = successes >= self .config .success_threshold
381- _LOCAL_SUCCESSES [self .name ] = 0 if closed else successes
382- if closed :
383- _LOCAL_FAILURES [self .name ] = 0
384- if closed :
385- logger .debug ("Circuit '%s' closing after %d probe successes." , self .name , successes )
386- self ._safe_persist (self .persistence_store .save_closed , self .name )
387- self ._notify (CircuitState .HALF_OPEN , CircuitState .CLOSED )
388- return result
342+ """Produce the response for an open circuit, awaiting an async ``on_circuit_open`` callback."""
343+ response = self ._open_response (circuit )
344+ if inspect .isawaitable (response ):
345+ return await response
346+ return response
389347
390348 def _notify (self , from_state : CircuitState , to_state : CircuitState , opened_at : int | None = None ) -> None :
391349 """
0 commit comments