Repository navigation
feat(crt): add maxPendingConnectionAcquires #7522
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| { | ||
| "type": "feature", | ||
| "category": "AWS CRT HTTP Client", | ||
| "contributor": "", | ||
| "description": "Add `maxPendingConnectionAcquires` to `AwsCrtHttpClient.Builder` and `AwsCrtAsyncHttpClient.Builder`. Unbounded by default." | ||
| } |
| Original file line number | Diff line number | Diff line change | ||
|---|---|---|---|---|
|
|
@@ -45,6 +45,36 @@ public final class CrtUtils { | |||
| * Corresponds to CRT error: AWS_IO_SOCKET_TIMEOUT | ||||
| */ | ||||
| public static final int CRT_SOCKET_TIMEOUT = 1048; | ||||
| /** | ||||
| * Corresponds to CRT error: AWS_ERROR_HTTP_CONNECTION_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED | ||||
| */ | ||||
| public static final int CRT_CONNECTION_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED = 2094; | ||||
| /** | ||||
| * Corresponds to CRT error: AWS_ERROR_HTTP_STREAM_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED | ||||
| */ | ||||
| public static final int CRT_STREAM_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED = 2096; | ||||
|
|
||||
| /** | ||||
| * Message for a request rejected because the endpoint's pending acquisitions reached | ||||
| * {@code maxPendingConnectionAcquires}. The first sentence matches | ||||
| * {@code NettyUtils#getMessageForTooManyAcquireOperationsError} in netty-nio-client; the rest is adapted to the CRT | ||||
| * builder options. | ||||
| */ | ||||
| private static final String MAX_PENDING_ACQUISITIONS_EXCEEDED_MESSAGE = | ||||
| "Maximum pending connection acquisitions exceeded. The request rate is too high for the client to keep up.\n" | ||||
|
|
||||
| + "Consider taking any of the following actions to mitigate the issue: increase maxConcurrency, " | ||||
| + "increase maxPendingConnectionAcquires, or slow the request rate.\n" | ||||
|
|
||||
| + "Increasing maxConcurrency can increase client throughput (unless the network interface is already fully " | ||||
| + "utilized), but can eventually start to hit operating system limitations on the number of file descriptors " | ||||
| + "used by the process. If you are already fully utilizing your network interface or cannot further increase " | ||||
| + "maxConcurrency, increasing maxPendingConnectionAcquires allows extra requests to be buffered by the client, " | ||||
| + "but can cause additional request latency and higher memory usage.\n" | ||||
|
|
||||
| + "If the above mechanisms are not able to fix the issue, try smoothing out your requests so that large " | ||||
| + "traffic bursts cannot overload the client, being more efficient with the number of times you need to call " | ||||
| + "AWS, or by increasing the number of hosts sending requests."; | ||||
|
|
||||
| // HTTP error codes that the native CRT classifier (CRT.awsIsTransientError) does NOT mark as transient | ||||
| // but that the SDK considers recoverable. See enum aws_http_errors in aws-c-http/include/aws/http/http.h | ||||
|
|
@@ -65,8 +95,6 @@ public final class CrtUtils { | |||
| codes.add(2092); | ||||
| // AWS_ERROR_HTTP_CONNECTION_MANAGER_ACQUISITION_TIMEOUT | ||||
| codes.add(2093); | ||||
| // AWS_ERROR_HTTP_CONNECTION_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED | ||||
| codes.add(2094); | ||||
| ADDITIONAL_RETRYABLE_ERROR_CODES = unmodifiableSet(codes); | ||||
| } | ||||
|
|
||||
|
|
@@ -105,9 +133,20 @@ public static Throwable wrapCrtException(Throwable throwable) { | |||
| return connectException; | ||||
| } | ||||
|
|
||||
| if (httpErrorCode == CRT_CONNECTION_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED) { | ||||
| return maxPendingAcquisitionsExceeded(httpException); | ||||
| } | ||||
|
|
||||
| return wrapWithIoExceptionIfRetryable((HttpException) throwable); | ||||
| } | ||||
|
|
||||
| // The HTTP/2 stream manager reports its over-limit rejection as a CrtRuntimeException. Handle it like the HTTP/1.1 | ||||
| // rejection, instead of wrapping it in the retryable IOException below. | ||||
| if (throwable instanceof CrtRuntimeException | ||||
| && ((CrtRuntimeException) throwable).errorCode == CRT_STREAM_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED) { | ||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we consider handle 2094 and 2096 error here with a more clear error message? In netty we have Line 143 in c259777
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good call! Updated |
||||
| return maxPendingAcquisitionsExceeded(throwable); | ||||
| } | ||||
|
|
||||
| if (throwable instanceof IllegalStateException || throwable instanceof CrtRuntimeException) { | ||||
| // CRT throws IllegalStateException if the connection is closed | ||||
| return new IOException("An exception occurred when making the request", throwable); | ||||
|
|
@@ -116,6 +155,33 @@ public static Throwable wrapCrtException(Throwable throwable) { | |||
| return throwable; | ||||
| } | ||||
|
|
||||
| /** | ||||
| * Returns true if the given exception is the rejection {@link #wrapCrtException} creates when a request is over the | ||||
| * pending acquisition limit. | ||||
| */ | ||||
| public static boolean isMaxPendingAcquisitionsExceeded(Throwable throwable) { | ||||
| if (!(throwable instanceof IllegalStateException)) { | ||||
| return false; | ||||
| } | ||||
| Throwable cause = throwable.getCause(); | ||||
| return (cause instanceof HttpException | ||||
| && ((HttpException) cause).getErrorCode() == CRT_CONNECTION_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED) | ||||
| || (cause instanceof CrtRuntimeException | ||||
| && ((CrtRuntimeException) cause).errorCode == CRT_STREAM_MANAGER_MAX_PENDING_ACQUISITIONS_EXCEEDED); | ||||
| } | ||||
|
|
||||
| /** | ||||
| * Creates the exception for a request rejected because the endpoint's pending acquisitions reached | ||||
| * {@code maxPendingConnectionAcquires}. It is unchecked and not an {@link IOException}, so the SDK does not retry it, | ||||
| * matching the Netty client. It is the same type Netty's connection pool raises for this condition. | ||||
| * | ||||
| * @param crtException the CRT exception that reported the rejection | ||||
| * @return the exception to surface, with the CRT exception as its cause | ||||
| */ | ||||
| private static IllegalStateException maxPendingAcquisitionsExceeded(Throwable crtException) { | ||||
| return new IllegalStateException(MAX_PENDING_ACQUISITIONS_EXCEEDED_MESSAGE, crtException); | ||||
| } | ||||
|
|
||||
| public static void reportMetrics(HttpStreamManager connManager, MetricCollector metricCollector, | ||||
| long acquireStartTime) { | ||||
| long acquireCompletionTime = System.nanoTime(); | ||||
|
|
||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Removing 2094 from the retryable codes doesn't change behavior for existing users because it was basically unreachable until now. aws-c-http only raises 2094 when max_pending_connection_acquisitions is set, and the SDK never set it (HttpClientConnectionManagerOptions.withMaxPendingConnectionAcquisitions was never called).
The reason we removed it here is to match netty behavior; we don't retry max pending acquisitions error for netty