|
9 | 9 | from __future__ import annotations |
10 | 10 |
|
11 | 11 | import datetime as dt |
| 12 | +import inspect |
12 | 13 | import logging |
13 | 14 | import os |
14 | 15 | import threading |
@@ -271,6 +272,121 @@ def _open_response(self, circuit: CircuitInfo) -> Any: |
271 | 272 | circuit=circuit, |
272 | 273 | ) |
273 | 274 |
|
| 275 | + async def _open_response_async(self, circuit: CircuitInfo) -> Any: |
| 276 | + """Produce the response for an open circuit in an async context.""" |
| 277 | + if self.on_circuit_open is not None: |
| 278 | + res = self.on_circuit_open(*self.fn_args, **self.fn_kwargs, circuit=circuit) |
| 279 | + if inspect.iscoroutine(res): |
| 280 | + return await res |
| 281 | + return res |
| 282 | + raise CircuitBreakerOpenError( |
| 283 | + f"Circuit '{self.name}' is open.", |
| 284 | + circuit=circuit, |
| 285 | + ) |
| 286 | + |
| 287 | + async def handle_async(self) -> Any: |
| 288 | + """ |
| 289 | + Evaluate the circuit and route the call asynchronously. |
| 290 | +
|
| 291 | + Returns |
| 292 | + ------- |
| 293 | + Any |
| 294 | + The protected async function's result when the call runs, or the |
| 295 | + ``on_circuit_open`` callback's return value when the circuit is open. |
| 296 | +
|
| 297 | + Raises |
| 298 | + ------ |
| 299 | + CircuitBreakerOpenError |
| 300 | + If the circuit is open and no callback is registered. |
| 301 | + """ |
| 302 | + record = self.persistence_store.get_state(self.name) |
| 303 | + |
| 304 | + if record.state == CircuitState.CLOSED: |
| 305 | + with _COUNTERS_LOCK: |
| 306 | + prev = _LAST_OBSERVED_STATE.get(self.name) |
| 307 | + if prev is not None and prev != CircuitState.CLOSED: |
| 308 | + _LOCAL_FAILURES[self.name] = 0 |
| 309 | + _LAST_OBSERVED_STATE[self.name] = CircuitState.CLOSED |
| 310 | + return await self._call_closed_async() |
| 311 | + |
| 312 | + if record.state == CircuitState.OPEN: |
| 313 | + with _COUNTERS_LOCK: |
| 314 | + _LAST_OBSERVED_STATE[self.name] = CircuitState.OPEN |
| 315 | + opened_at = record.opened_at if record.opened_at is not None else self._now() |
| 316 | + if self._now() >= opened_at + self.config.recovery_timeout: |
| 317 | + if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), opened_at): |
| 318 | + self._reset_probe_successes() |
| 319 | + self._notify(CircuitState.OPEN, CircuitState.HALF_OPEN, opened_at=opened_at) |
| 320 | + return await self._call_probe_async() |
| 321 | + return await self._open_response_async(record.to_circuit_info()) |
| 322 | + |
| 323 | + with _COUNTERS_LOCK: |
| 324 | + _LAST_OBSERVED_STATE[self.name] = CircuitState.HALF_OPEN |
| 325 | + if record.half_open_owner == _probe_owner_id(): |
| 326 | + return await self._call_probe_async() |
| 327 | + |
| 328 | + if record.probe_lease_expiry is not None and self._now() >= record.probe_lease_expiry: |
| 329 | + logger.debug("Circuit '%s' probe lease expired; attempting takeover.", self.name) |
| 330 | + if self.persistence_store.try_acquire_half_open(self.name, _probe_owner_id(), record.opened_at or 0): |
| 331 | + self._reset_probe_successes() |
| 332 | + return await self._call_probe_async() |
| 333 | + |
| 334 | + return await self._open_response_async(record.to_circuit_info()) |
| 335 | + |
| 336 | + async def _call_closed_async(self) -> Any: |
| 337 | + """Run the protected async call while the circuit is closed, tracking failures.""" |
| 338 | + try: |
| 339 | + result = await self.function(*self.fn_args, **self.fn_kwargs) |
| 340 | + except Exception as exc: |
| 341 | + if not self.config.counts_as_failure(exc): |
| 342 | + raise |
| 343 | + with _COUNTERS_LOCK: |
| 344 | + failures = _LOCAL_FAILURES.get(self.name, 0) + 1 |
| 345 | + tripped = failures >= self.config.failure_threshold |
| 346 | + _LOCAL_FAILURES[self.name] = 0 if tripped else failures |
| 347 | + if tripped: |
| 348 | + logger.debug("Circuit '%s' tripping CLOSED to OPEN after %d failures.", self.name, failures) |
| 349 | + opened_at = self._now() |
| 350 | + self._safe_persist( |
| 351 | + self.persistence_store.save_open, |
| 352 | + self.name, |
| 353 | + failure_count=failures, |
| 354 | + opened_at=opened_at, |
| 355 | + ) |
| 356 | + self._notify(CircuitState.CLOSED, CircuitState.OPEN, opened_at=opened_at) |
| 357 | + raise |
| 358 | + else: |
| 359 | + with _COUNTERS_LOCK: |
| 360 | + _LOCAL_FAILURES[self.name] = 0 |
| 361 | + return result |
| 362 | + |
| 363 | + async def _call_probe_async(self) -> Any: |
| 364 | + """Run an async probe during half-open, closing or reopening based on the outcome.""" |
| 365 | + try: |
| 366 | + result = await self.function(*self.fn_args, **self.fn_kwargs) |
| 367 | + except Exception as exc: |
| 368 | + if not self.config.counts_as_failure(exc): |
| 369 | + raise |
| 370 | + logger.debug("Circuit '%s' probe failed; reopening.", self.name) |
| 371 | + opened_at = self._now() |
| 372 | + self._safe_persist(self.persistence_store.save_reopen, self.name, opened_at=opened_at) |
| 373 | + with _COUNTERS_LOCK: |
| 374 | + _LOCAL_SUCCESSES[self.name] = 0 |
| 375 | + self._notify(CircuitState.HALF_OPEN, CircuitState.OPEN, opened_at=opened_at) |
| 376 | + raise |
| 377 | + else: |
| 378 | + with _COUNTERS_LOCK: |
| 379 | + successes = _LOCAL_SUCCESSES.get(self.name, 0) + 1 |
| 380 | + closed = successes >= self.config.success_threshold |
| 381 | + _LOCAL_SUCCESSES[self.name] = 0 if closed else successes |
| 382 | + if closed: |
| 383 | + _LOCAL_FAILURES[self.name] = 0 |
| 384 | + if closed: |
| 385 | + logger.debug("Circuit '%s' closing after %d probe successes.", self.name, successes) |
| 386 | + self._safe_persist(self.persistence_store.save_closed, self.name) |
| 387 | + self._notify(CircuitState.HALF_OPEN, CircuitState.CLOSED) |
| 388 | + return result |
| 389 | + |
274 | 390 | def _notify(self, from_state: CircuitState, to_state: CircuitState, opened_at: int | None = None) -> None: |
275 | 391 | """ |
276 | 392 | Fire the ``on_transition`` hook for a state change. |
|
0 commit comments