From c382d7b4e40eae8de791ebe9b985422362f62259 Mon Sep 17 00:00:00 2001 From: Saranya Somepalli Date: Wed, 18 Nov 2026 11:14:51 -0800 Subject: [PATCH] Wire aws-chunked response decoder into sync and async response paths --- .../SdkInternalExecutionAttribute.java | 15 ++ .../AwsChunkedResponseDecodingUtils.java | 169 ++++++++++++++++ .../handler/BaseAsyncClientHandler.java | 9 +- .../internal/http/AmazonSyncHttpClient.java | 2 + ...syncAwsChunkedResponseDecodingHandler.java | 96 ++++++++++ .../AwsChunkedResponseDecodingStage.java | 93 +++++++++ .../AwsChunkedResponseDecodingUtilsTest.java | 181 ++++++++++++++++++ ...ntHandlerResponseInterceptorChainTest.java | 50 ++++- ...nSyncHttpClientAwsChunkedDecodingTest.java | 148 ++++++++++++++ ...AwsChunkedResponseDecodingHandlerTest.java | 162 ++++++++++++++++ .../AwsChunkedResponseDecodingStageTest.java | 152 +++++++++++++++ 11 files changed, 1071 insertions(+), 6 deletions(-) create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtils.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandler.java create mode 100644 core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStage.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtilsTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClientAwsChunkedDecodingTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandlerTest.java create mode 100644 core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStageTest.java diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java index 20336369cf71..8004701ebe9b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.core.ClientEndpointProvider; import software.amazon.awssdk.core.SdkClient; @@ -304,6 +305,20 @@ public final class SdkInternalExecutionAttribute extends SdkExecutionAttribute { */ public static final ExecutionAttribute IS_LONG_POLLING = new ExecutionAttribute<>("IsLongPolling"); + /** + * Whether the operation's response body may be aws-chunked encoded, as declared by the {@code awsChunked} trait on its + * output blob. When true and the response {@code Content-Encoding} ends with {@code aws-chunked}, the SDK decodes the body. + */ + public static final ExecutionAttribute AWS_CHUNKED_RESPONSE_DECODING = + new ExecutionAttribute<>("AwsChunkedResponseDecoding"); + + /** + * Returns the trailers of a decoded aws-chunked response body, keyed by lower-case name. Throws + * {@link IllegalStateException} until the body has been fully and successfully read. + */ + public static final ExecutionAttribute>> AWS_CHUNKED_RESPONSE_TRAILERS = + new ExecutionAttribute<>("AwsChunkedResponseTrailers"); + /** * Indicates whether retries v2.1 is enabled. * diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtils.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtils.java new file mode 100644 index 000000000000..a606613a1b77 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtils.java @@ -0,0 +1,169 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.chunked; + +import static software.amazon.awssdk.core.HttpChecksumConstant.AWS_CHUNKED_HEADER; +import static software.amazon.awssdk.core.HttpChecksumConstant.X_AMZ_TRAILER; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.SdkHttpResponse; + +/** + * Header handling for aws-chunked response decoding, shared by the sync and async response paths. + */ +@SdkInternalApi +public final class AwsChunkedResponseDecodingUtils { + + private static final String CONTENT_ENCODING = "Content-Encoding"; + private static final String CONTENT_LENGTH = "Content-Length"; + private static final String DECODED_CONTENT_LENGTH = "x-amz-decoded-content-length"; + + private AwsChunkedResponseDecodingUtils() { + } + + /** + * Returns whether the response body must be aws-chunked decoded: the operation supports it and {@code aws-chunked} is + * the last {@code Content-Encoding} value. + * + * @throws SdkClientException if {@code aws-chunked} appears more than once or is not the last encoding + */ + public static boolean shouldDecode(SdkHttpResponse response, ExecutionAttributes executionAttributes) { + if (!Boolean.TRUE.equals(executionAttributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING))) { + return false; + } + List encodings = contentEncodings(response); + long awsChunkedCount = encodings.stream().filter(AwsChunkedResponseDecodingUtils::isAwsChunked).count(); + if (awsChunkedCount == 0) { + return false; + } + if (awsChunkedCount > 1 || !isAwsChunked(encodings.get(encodings.size() - 1))) { + throw malformed("aws-chunked must appear exactly once, as the last Content-Encoding value"); + } + return true; + } + + /** + * Returns the {@code x-amz-decoded-content-length} value. + * + * @throws SdkClientException if the header is missing or not a non-negative integer + */ + public static long decodedContentLength(SdkHttpResponse response) { + String value = response.firstMatchingHeader(DECODED_CONTENT_LENGTH) + .orElseThrow(() -> malformed(DECODED_CONTENT_LENGTH + " header is missing")); + try { + long length = Long.parseLong(value.trim()); + if (length >= 0) { + return length; + } + } catch (NumberFormatException e) { + // Fall through + } + throw malformed(DECODED_CONTENT_LENGTH + " header is not a valid non-negative integer"); + } + + /** + * Returns the lower-case trailer names declared in {@code x-amz-trailer}, or an empty set if the header is absent. + */ + public static Set declaredTrailers(SdkHttpResponse response) { + Set names = new HashSet<>(); + for (String header : response.matchingHeaders(X_AMZ_TRAILER)) { + for (String name : header.split(",")) { + String trimmed = name.trim(); + if (!trimmed.isEmpty()) { + names.add(trimmed.toLowerCase(Locale.ROOT)); + } + } + } + return Collections.unmodifiableSet(names); + } + + /** + * Rewrites the headers that describe the aws-chunked framing so they describe the decoded body: + * {@code Content-Length} becomes the decoded length, {@code aws-chunked} is removed from {@code Content-Encoding} + * (removing the header if nothing remains), and {@code x-amz-decoded-content-length} and {@code x-amz-trailer} are + * removed. + */ + public static SdkHttpFullResponse rewriteHeaders(SdkHttpFullResponse response, long decodedContentLength) { + SdkHttpFullResponse.Builder builder = response.toBuilder(); + rewriteHeaders(builder, contentEncodings(response), decodedContentLength); + return builder.build(); + } + + /** + * Same as {@link #rewriteHeaders(SdkHttpFullResponse, long)}, for the async path where the body arrives separately. + */ + public static SdkHttpResponse rewriteHeaders(SdkHttpResponse response, long decodedContentLength) { + SdkHttpResponse.Builder builder = response.toBuilder(); + rewriteHeaders(builder, contentEncodings(response), decodedContentLength); + return builder.build(); + } + + private static void rewriteHeaders(SdkHttpResponse.Builder builder, List encodings, long decodedContentLength) { + encodings.removeIf(AwsChunkedResponseDecodingUtils::isAwsChunked); + builder.removeHeader(CONTENT_ENCODING) + .removeHeader(DECODED_CONTENT_LENGTH) + .removeHeader(X_AMZ_TRAILER) + .putHeader(CONTENT_LENGTH, Long.toString(decodedContentLength)); + if (!encodings.isEmpty()) { + builder.putHeader(CONTENT_ENCODING, String.join(",", encodings)); + } + } + + /** + * Returns the trailers from a completed trailers future. + * + * @throws IllegalStateException if the future has not completed successfully + */ + public static Map completedTrailers(CompletableFuture> trailersFuture) { + if (!trailersFuture.isDone() || trailersFuture.isCompletedExceptionally()) { + throw new IllegalStateException("Trailers are only available after the stream has been fully decoded"); + } + return trailersFuture.join(); + } + + private static List contentEncodings(SdkHttpResponse response) { + List encodings = new ArrayList<>(); + for (String header : response.matchingHeaders(CONTENT_ENCODING)) { + for (String encoding : header.split(",")) { + String trimmed = encoding.trim(); + if (!trimmed.isEmpty()) { + encodings.add(trimmed); + } + } + } + return encodings; + } + + private static boolean isAwsChunked(String encoding) { + return AWS_CHUNKED_HEADER.equalsIgnoreCase(encoding); + } + + private static SdkClientException malformed(String message) { + return SdkClientException.builder().message("Malformed aws-chunked response: " + message).build(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java index 782398be5421..66a5a3475b06 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandler.java @@ -42,6 +42,7 @@ import software.amazon.awssdk.core.internal.http.IdempotentAsyncResponseHandler; import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler; import software.amazon.awssdk.core.internal.http.async.AsyncAfterTransmissionInterceptorCallingResponseHandler; +import software.amazon.awssdk.core.internal.http.async.AsyncAwsChunkedResponseDecodingHandler; import software.amazon.awssdk.core.internal.http.async.AsyncModifyHttpResponseInterceptorCallingResponseHandler; import software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler; import software.amazon.awssdk.core.internal.http.async.AsyncStreamingResponseHandler; @@ -260,13 +261,15 @@ public void close() { } /** - * Wraps the response handler with the response interceptor steps, in order: read-only {@code afterTransmission} on - * the raw response, then the interceptors that can modify it. + * Wraps the response handler with the response interceptor and decoding steps, in order: read-only + * {@code afterTransmission} on the raw response, aws-chunked decoding, then the interceptors that can modify it. */ static TransformingAsyncResponseHandler interceptorCallingResponseHandler( TransformingAsyncResponseHandler responseHandler, ExecutionContext executionContext) { return new AsyncAfterTransmissionInterceptorCallingResponseHandler<>( - new AsyncModifyHttpResponseInterceptorCallingResponseHandler<>(responseHandler, executionContext), + new AsyncAwsChunkedResponseDecodingHandler<>( + new AsyncModifyHttpResponseInterceptorCallingResponseHandler<>(responseHandler, executionContext), + executionContext), executionContext); } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java index 7ff37a78c59b..d522aec85c82 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClient.java @@ -34,6 +34,7 @@ import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyTransactionIdStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyUserAgentStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.AuthSchemeResolutionStage; +import software.amazon.awssdk.core.internal.http.pipeline.stages.AwsChunkedResponseDecodingStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeTransmissionExecutionInterceptorsStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeUnmarshallingExecutionInterceptorsStage; import software.amazon.awssdk.core.internal.http.pipeline.stages.CompressRequestStage; @@ -198,6 +199,7 @@ public OutputT execute(HttpResponseHandler> response .then(BeforeTransmissionExecutionInterceptorsStage::new) .then(MakeHttpRequestStage::new) .then(AfterTransmissionExecutionInterceptorsStage::new) + .then(AwsChunkedResponseDecodingStage::new) .then(ModifyHttpResponseExecutionInterceptorsStage::new) .then(BeforeUnmarshallingExecutionInterceptorsStage::new) .then(() -> new HandleResponseStage<>(responseHandler)) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandler.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandler.java new file mode 100644 index 000000000000..1060543d1202 --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandler.java @@ -0,0 +1,96 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.async; + +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.UnaryOperator; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.internal.async.AwsChunkedResponseDecodingPublisher; +import software.amazon.awssdk.core.internal.chunked.AwsChunkedResponseDecodingUtils; +import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler; +import software.amazon.awssdk.http.SdkHttpResponse; + +/** + * Async response handler decorator that decodes an aws-chunked response body and rewrites its framing headers. Sits + * between the read-only {@code afterTransmission} interceptors and any interceptor that can modify the response, so those + * only see the decoded body and headers. + * + * @param the type of the result + */ +@SdkInternalApi +public final class AsyncAwsChunkedResponseDecodingHandler implements TransformingAsyncResponseHandler { + private final TransformingAsyncResponseHandler delegate; + private final ExecutionContext context; + + /** + * Set in onHeaders when the body must be decoded, and applied to the body in onStream. Volatile because the two + * callbacks may run on different threads. + */ + private volatile UnaryOperator> bodyDecoder; + + public AsyncAwsChunkedResponseDecodingHandler(TransformingAsyncResponseHandler delegate, ExecutionContext context) { + this.delegate = delegate; + this.context = context; + } + + @Override + public void onHeaders(SdkHttpResponse response) { + bodyDecoder = null; + ExecutionAttributes executionAttributes = context.executionAttributes(); + // Clear trailers from an earlier attempt, so they are never read for this response + executionAttributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, null); + if (!AwsChunkedResponseDecodingUtils.shouldDecode(response, executionAttributes)) { + delegate.onHeaders(response); + return; + } + + long decodedContentLength = AwsChunkedResponseDecodingUtils.decodedContentLength(response); + Set declaredTrailers = AwsChunkedResponseDecodingUtils.declaredTrailers(response); + bodyDecoder = body -> { + AwsChunkedResponseDecodingPublisher decoded = + new AwsChunkedResponseDecodingPublisher(body, decodedContentLength, declaredTrailers); + executionAttributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, + () -> AwsChunkedResponseDecodingUtils.completedTrailers(decoded.trailersFuture())); + return decoded; + }; + + SdkHttpResponse rewritten = AwsChunkedResponseDecodingUtils.rewriteHeaders(response, decodedContentLength); + context.interceptorContext(context.interceptorContext().copy(b -> b.httpResponse(rewritten))); + delegate.onHeaders(rewritten); + } + + @Override + public void onError(Throwable error) { + delegate.onError(error); + } + + @Override + public void onStream(Publisher publisher) { + UnaryOperator> decoder = bodyDecoder; + delegate.onStream(decoder == null ? publisher : decoder.apply(publisher)); + } + + @Override + public CompletableFuture prepare() { + return delegate.prepare(); + } +} diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStage.java new file mode 100644 index 000000000000..0b8deb09261f --- /dev/null +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStage.java @@ -0,0 +1,93 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.internal.chunked.AwsChunkedResponseDecodingUtils; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.core.internal.http.pipeline.RequestPipeline; +import software.amazon.awssdk.core.internal.io.AwsChunkedResponseDecoder; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.utils.IoUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Pair; + +/** + * Decodes an aws-chunked response body and rewrites its framing headers. Runs after the read-only + * {@code afterTransmission} interceptors and before any interceptor that can modify the response, so those only see the + * decoded body and headers. + */ +@SdkInternalApi +public class AwsChunkedResponseDecodingStage + implements RequestPipeline, Pair> { + private static final Logger LOG = Logger.loggerFor(AwsChunkedResponseDecodingStage.class); + + @Override + public Pair execute(Pair input, + RequestExecutionContext context) { + ExecutionAttributes executionAttributes = context.executionAttributes(); + // Clear trailers from an earlier attempt, so they are never read for this response + executionAttributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, null); + try { + return decode(input, context); + } catch (RuntimeException e) { + // The body has not been handed to a later stage yet, so release the connection here + input.right().content().ifPresent(AwsChunkedResponseDecodingStage::abortQuietly); + throw e; + } + } + + private static Pair decode(Pair input, + RequestExecutionContext context) { + SdkHttpFullResponse response = input.right(); + ExecutionAttributes executionAttributes = context.executionAttributes(); + if (!AwsChunkedResponseDecodingUtils.shouldDecode(response, executionAttributes)) { + return input; + } + + long decodedContentLength = AwsChunkedResponseDecodingUtils.decodedContentLength(response); + SdkHttpFullResponse decoded = AwsChunkedResponseDecodingUtils.rewriteHeaders(response, decodedContentLength); + + Optional body = response.content(); + if (body.isPresent()) { + AwsChunkedResponseDecoder decoder = + new AwsChunkedResponseDecoder(body.get(), decodedContentLength, + AwsChunkedResponseDecodingUtils.declaredTrailers(response)); + executionAttributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, decoder::trailers); + decoded = decoded.toBuilder().content(AbortableInputStream.create(decoder)).build(); + } + + SdkHttpFullResponse result = decoded; + context.executionContext().interceptorContext( + context.executionContext().interceptorContext().copy(b -> b.httpResponse(result) + .responseBody(result.content().orElse(null)))); + return Pair.of(input.left(), result); + } + + private static void abortQuietly(AbortableInputStream body) { + try { + body.abort(); + } catch (Throwable e) { + LOG.debug(() -> "Failed to abort the response body after an aws-chunked decoding failure.", e); + } + IoUtils.closeQuietlyV2(body, LOG); + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtilsTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtilsTest.java new file mode 100644 index 000000000000..5eeea909c2b4 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/chunked/AwsChunkedResponseDecodingUtilsTest.java @@ -0,0 +1,181 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.chunked; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.http.SdkHttpFullResponse; + +class AwsChunkedResponseDecodingUtilsTest { + + @ParameterizedTest(name = "Content-Encoding: {0}") + @CsvSource(delimiter = '|', value = { + "aws-chunked | true", + "gzip,aws-chunked | true", + "gzip, aws-chunked | true", + "AWS-CHUNKED | true", + "gzip | false" + }) + void shouldDecode_whenEnabled_dependsOnContentEncoding(String contentEncoding, boolean expected) { + SdkHttpFullResponse response = response().putHeader("Content-Encoding", contentEncoding).build(); + + assertThat(AwsChunkedResponseDecodingUtils.shouldDecode(response, enabled())).isEqualTo(expected); + } + + @Test + void shouldDecode_whenEncodingsSplitAcrossHeaders_usesLastValue() { + SdkHttpFullResponse response = response().appendHeader("Content-Encoding", "gzip") + .appendHeader("Content-Encoding", "aws-chunked") + .build(); + + assertThat(AwsChunkedResponseDecodingUtils.shouldDecode(response, enabled())).isTrue(); + } + + @ParameterizedTest(name = "Content-Encoding: {0}") + @ValueSource(strings = {"aws-chunked,gzip", "aws-chunked,aws-chunked"}) + void shouldDecode_whenAwsChunkedNotLastOrRepeated_throws(String contentEncoding) { + SdkHttpFullResponse response = response().putHeader("Content-Encoding", contentEncoding).build(); + + assertThatThrownBy(() -> AwsChunkedResponseDecodingUtils.shouldDecode(response, enabled())) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("aws-chunked must appear exactly once, as the last Content-Encoding value"); + } + + @Test + void shouldDecode_whenNotEnabledForOperation_returnsFalse() { + SdkHttpFullResponse response = response().putHeader("Content-Encoding", "aws-chunked").build(); + + assertThat(AwsChunkedResponseDecodingUtils.shouldDecode(response, new ExecutionAttributes())).isFalse(); + } + + @Test + void shouldDecode_whenNoContentEncoding_returnsFalse() { + assertThat(AwsChunkedResponseDecodingUtils.shouldDecode(response().build(), enabled())).isFalse(); + } + + @Test + void decodedContentLength_whenValid_returnsValue() { + SdkHttpFullResponse response = response().putHeader("x-amz-decoded-content-length", "11").build(); + + assertThat(AwsChunkedResponseDecodingUtils.decodedContentLength(response)).isEqualTo(11); + } + + @Test + void decodedContentLength_whenMissing_throws() { + assertThatThrownBy(() -> AwsChunkedResponseDecodingUtils.decodedContentLength(response().build())) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("x-amz-decoded-content-length header is missing"); + } + + @ParameterizedTest(name = "x-amz-decoded-content-length: \"{0}\"") + @ValueSource(strings = {"abc", "-1", "1.5"}) + void decodedContentLength_whenInvalid_throws(String value) { + SdkHttpFullResponse response = response().putHeader("x-amz-decoded-content-length", value).build(); + + assertThatThrownBy(() -> AwsChunkedResponseDecodingUtils.decodedContentLength(response)) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("not a valid non-negative integer"); + } + + @Test + void declaredTrailers_whenPresent_returnsLowerCaseNames() { + SdkHttpFullResponse response = response().appendHeader("x-amz-trailer", "X-Amz-Checksum-CRC32, other ,") + .appendHeader("x-amz-trailer", "third") + .build(); + + assertThat(AwsChunkedResponseDecodingUtils.declaredTrailers(response)) + .containsExactlyInAnyOrder("x-amz-checksum-crc32", "other", "third"); + } + + @Test + void declaredTrailers_whenAbsent_returnsEmptySet() { + assertThat(AwsChunkedResponseDecodingUtils.declaredTrailers(response().build())).isEmpty(); + } + + @Test + void rewriteHeaders_whenOnlyAwsChunked_removesFramingHeaders() { + SdkHttpFullResponse response = response().putHeader("Content-Encoding", "aws-chunked") + .putHeader("Content-Length", "64") + .putHeader("x-amz-decoded-content-length", "11") + .putHeader("x-amz-trailer", "x-amz-checksum-crc32") + .putHeader("ETag", "abc") + .build(); + + SdkHttpFullResponse rewritten = AwsChunkedResponseDecodingUtils.rewriteHeaders(response, 11); + + assertThat(rewritten.statusCode()).isEqualTo(200); + assertThat(rewritten.firstMatchingHeader("Content-Length")).hasValue("11"); + assertThat(rewritten.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(rewritten.firstMatchingHeader("x-amz-decoded-content-length")).isEmpty(); + assertThat(rewritten.firstMatchingHeader("x-amz-trailer")).isEmpty(); + assertThat(rewritten.firstMatchingHeader("ETag")).hasValue("abc"); + } + + @Test + void rewriteHeaders_whenOtherEncodingsPresent_keepsThem() { + SdkHttpFullResponse response = response().putHeader("Content-Encoding", "gzip, aws-chunked").build(); + + SdkHttpFullResponse rewritten = AwsChunkedResponseDecodingUtils.rewriteHeaders(response, 11); + + assertThat(rewritten.matchingHeaders("Content-Encoding")).containsExactly("gzip"); + } + + @Test + void completedTrailers_whenFutureNotDone_throws() { + assertThatThrownBy(() -> AwsChunkedResponseDecodingUtils.completedTrailers(new CompletableFuture<>())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("only available after"); + } + + @Test + void completedTrailers_whenFutureFailed_throws() { + CompletableFuture> future = new CompletableFuture<>(); + future.completeExceptionally(new RuntimeException("boom")); + + assertThatThrownBy(() -> AwsChunkedResponseDecodingUtils.completedTrailers(future)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("only available after"); + } + + @Test + void completedTrailers_whenFutureCompleted_returnsTrailers() { + Map trailers = Collections.singletonMap("x-amz-checksum-crc32", "abc=="); + + assertThat(AwsChunkedResponseDecodingUtils.completedTrailers(CompletableFuture.completedFuture(trailers))) + .isSameAs(trailers); + } + + private static SdkHttpFullResponse.Builder response() { + return SdkHttpFullResponse.builder().statusCode(200); + } + + private static ExecutionAttributes enabled() { + ExecutionAttributes attributes = new ExecutionAttributes(); + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + return attributes; + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandlerResponseInterceptorChainTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandlerResponseInterceptorChainTest.java index e19af5c6cd45..5fbaf5ecdff8 100644 --- a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandlerResponseInterceptorChainTest.java +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/handler/BaseAsyncClientHandlerResponseInterceptorChainTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import io.reactivex.Flowable; +import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -34,6 +35,8 @@ import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.internal.async.AwsChunkedResponseDecodingPublisher; import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler; import software.amazon.awssdk.http.SdkHttpFullResponse; import software.amazon.awssdk.http.SdkHttpResponse; @@ -73,6 +76,33 @@ void responseChain_whenInterceptorDoesNotModifyResponse_delegateSeesOriginal() { assertThat(delegate.stream).isSameAs(ORIGINAL_BODY); } + @Test + void responseChain_whenAwsChunkedDecodingEnabled_afterTransmissionSeesRawAndModifyingInterceptorsSeeDecoded() { + RecordingInterceptor interceptor = new RecordingInterceptor(false); + CapturingHandler delegate = new CapturingHandler(); + ExecutionAttributes attributes = new ExecutionAttributes(); + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + SdkHttpFullResponse response = SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Encoding", "aws-chunked") + .putHeader("x-amz-decoded-content-length", "11") + .putHeader("x-amz-trailer", "x-amz-checksum-crc32") + .build(); + String encoded = "5\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:abc==\r\n\r\n"; + + TransformingAsyncResponseHandler chain = + BaseAsyncClientHandler.interceptorCallingResponseHandler(delegate, context(interceptor, attributes)); + chain.onHeaders(response); + chain.onStream(Flowable.just(utf8(encoded))); + + assertThat(interceptor.afterTransmissionResponse.firstMatchingHeader("Content-Encoding")).hasValue("aws-chunked"); + assertThat(interceptor.beforeUnmarshallingResponse.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(interceptor.beforeUnmarshallingResponse.firstMatchingHeader("Content-Length")).hasValue("11"); + assertThat(interceptor.modifyAsyncHttpResponsePublisher).isInstanceOf(AwsChunkedResponseDecodingPublisher.class); + assertThat(delegate.headers.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(readAll(delegate.stream)).isEqualTo("hello world"); + } + @Test void responseChain_whenModifyHttpResponseThrows_afterTransmissionRanAndExceptionPropagates() { RuntimeException failure = new RuntimeException("modifyHttpResponse boom"); @@ -113,15 +143,29 @@ private static void run(ExecutionInterceptor interceptor, CapturingHandler deleg } private static ExecutionContext contextFor(ExecutionInterceptor interceptor) { + return context(interceptor, new ExecutionAttributes()); + } + + private static SdkHttpFullResponse httpResponse() { + return SdkHttpFullResponse.builder().statusCode(200).putHeader("x-test", "original").build(); + } + + private static ExecutionContext context(ExecutionInterceptor interceptor, ExecutionAttributes attributes) { return ExecutionContext.builder() .interceptorChain(new ExecutionInterceptorChain(Collections.singletonList(interceptor))) .interceptorContext(InterceptorContext.builder().request(ValidSdkObjects.sdkRequest()).build()) - .executionAttributes(new ExecutionAttributes()) + .executionAttributes(attributes) .build(); } - private static SdkHttpFullResponse httpResponse() { - return SdkHttpFullResponse.builder().statusCode(200).putHeader("x-test", "original").build(); + private static String readAll(Publisher publisher) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (ByteBuffer buffer : Flowable.fromPublisher(publisher).blockingIterable()) { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + out.write(bytes, 0, bytes.length); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); } private static ByteBuffer utf8(String s) { diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClientAwsChunkedDecodingTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClientAwsChunkedDecodingTest.java new file mode 100644 index 000000000000..a5c398833c00 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/AmazonSyncHttpClientAwsChunkedDecodingTest.java @@ -0,0 +1,148 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static software.amazon.awssdk.core.internal.util.ResponseHandlerTestUtils.combinedSyncResponseHandler; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.http.NoopTestRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain; +import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.ExecutableHttpRequest; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.utils.IoUtils; +import utils.HttpTestUtils; +import utils.ValidSdkObjects; + +/** + * Runs a request through the full sync pipeline to verify where aws-chunked decoding happens relative to interceptors. + */ +class AmazonSyncHttpClientAwsChunkedDecodingTest { + + private static final String ENCODED = "5\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:abc==\r\n\r\n"; + + private final RecordingInterceptor interceptor = new RecordingInterceptor(); + private final ExecutionAttributes attributes = new ExecutionAttributes(); + private AmazonSyncHttpClient client; + + @BeforeEach + void setUp() throws Exception { + SdkHttpClient httpClient = mock(SdkHttpClient.class); + ExecutableHttpRequest call = mock(ExecutableHttpRequest.class); + when(httpClient.prepareRequest(any())).thenReturn(call); + when(httpClient.clientName()).thenReturn("UNKNOWN"); + when(call.call()).thenReturn(awsChunkedResponse()); + client = HttpTestUtils.testClientBuilder().httpClient(httpClient).build(); + } + + @AfterEach + void tearDown() { + client.close(); + } + + @Test + void execute_whenDecodingEnabled_afterTransmissionSeesRawAndModifyHttpResponseSeesDecoded() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + + String body = execute(); + + assertThat(interceptor.afterTransmissionResponse.firstMatchingHeader("Content-Encoding")).hasValue("aws-chunked"); + assertThat(interceptor.modifyHttpResponseResponse.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(interceptor.modifyHttpResponseResponse.firstMatchingHeader("Content-Length")).hasValue("11"); + assertThat(body).isEqualTo("hello world"); + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS).get()) + .containsEntry("x-amz-checksum-crc32", "abc=="); + } + + @Test + void execute_whenDecodingNotEnabled_bodyIsUnchanged() { + String body = execute(); + + assertThat(interceptor.modifyHttpResponseResponse.firstMatchingHeader("Content-Encoding")).hasValue("aws-chunked"); + assertThat(body).isEqualTo(ENCODED); + } + + private String execute() { + SdkHttpFullRequest request = ValidSdkObjects.sdkHttpFullRequest().build(); + ExecutionContext executionContext = + ExecutionContext.builder() + .signer(new NoOpSigner()) + .interceptorChain(new ExecutionInterceptorChain(Collections.singletonList(interceptor))) + .executionAttributes(attributes) + .interceptorContext(InterceptorContext.builder() + .request(NoopTestRequest.builder().build()) + .httpRequest(request) + .build()) + .metricCollector(MetricCollector.create("ApiCall")) + .build(); + + return client.requestExecutionBuilder() + .request(request) + .originalRequest(NoopTestRequest.builder().build()) + .executionContext(executionContext) + .execute(combinedSyncResponseHandler( + (response, executionAttributes) -> IoUtils.toUtf8String(response.content().get()), null)); + } + + private static HttpExecuteResponse awsChunkedResponse() { + SdkHttpResponse response = SdkHttpResponse.builder() + .statusCode(200) + .putHeader("Content-Encoding", "aws-chunked") + .putHeader("x-amz-decoded-content-length", "11") + .putHeader("x-amz-trailer", "x-amz-checksum-crc32") + .build(); + AbortableInputStream body = + AbortableInputStream.create(new ByteArrayInputStream(ENCODED.getBytes(StandardCharsets.UTF_8))); + return HttpExecuteResponse.builder().response(response).responseBody(body).build(); + } + + private static final class RecordingInterceptor implements ExecutionInterceptor { + private SdkHttpResponse afterTransmissionResponse; + private SdkHttpResponse modifyHttpResponseResponse; + + @Override + public void afterTransmission(Context.AfterTransmission context, ExecutionAttributes executionAttributes) { + afterTransmissionResponse = context.httpResponse(); + } + + @Override + public SdkHttpResponse modifyHttpResponse(Context.ModifyHttpResponse context, + ExecutionAttributes executionAttributes) { + modifyHttpResponseResponse = context.httpResponse(); + return context.httpResponse(); + } + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandlerTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandlerTest.java new file mode 100644 index 000000000000..2934748b19c2 --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/async/AsyncAwsChunkedResponseDecodingHandlerTest.java @@ -0,0 +1,162 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.async; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.reactivex.Flowable; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.http.SdkHttpResponse; +import utils.ValidSdkObjects; + +class AsyncAwsChunkedResponseDecodingHandlerTest { + + private static final String ENCODED = "5\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:abc==\r\n\r\n"; + + private final CapturingHandler delegate = new CapturingHandler(); + private final ExecutionAttributes attributes = new ExecutionAttributes(); + private final ExecutionContext context = + ExecutionContext.builder() + .interceptorContext(InterceptorContext.builder().request(ValidSdkObjects.sdkRequest()).build()) + .executionAttributes(attributes) + .build(); + private final AsyncAwsChunkedResponseDecodingHandler handler = + new AsyncAwsChunkedResponseDecodingHandler<>(delegate, context); + + @Test + void onStream_whenAwsChunkedResponse_delegateReceivesDecodedBodyAndRewrittenHeaders() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + + handler.onHeaders(awsChunkedResponse().build()); + handler.onStream(Flowable.just(utf8(ENCODED))); + + assertThat(delegate.headers.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(delegate.headers.firstMatchingHeader("Content-Length")).hasValue("11"); + assertThat(delegate.headers.firstMatchingHeader("x-amz-trailer")).isEmpty(); + assertThat(context.interceptorContext().httpResponse()).isSameAs(delegate.headers); + + assertThat(readAll(delegate.stream)).isEqualTo("hello world"); + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS).get()) + .containsEntry("x-amz-checksum-crc32", "abc=="); + } + + @Test + void onStream_whenDecodingNotEnabled_passesHeadersAndPublisherThrough() { + SdkHttpFullResponse response = awsChunkedResponse().build(); + Publisher raw = Flowable.just(utf8(ENCODED)); + + handler.onHeaders(response); + handler.onStream(raw); + + assertThat(delegate.headers).isSameAs(response); + assertThat(delegate.stream).isSameAs(raw); + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS)).isNull(); + } + + @Test + void trailers_whenBodyNotYetConsumed_throws() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + + handler.onHeaders(awsChunkedResponse().build()); + handler.onStream(Flowable.just(utf8(ENCODED))); + + Supplier trailers = attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS); + assertThatThrownBy(trailers::get) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("only available after"); + } + + @Test + void onHeaders_whenAwsChunkedIsNotLastEncoding_throws() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + SdkHttpFullResponse response = awsChunkedResponse().putHeader("Content-Encoding", "aws-chunked,gzip").build(); + + assertThatThrownBy(() -> handler.onHeaders(response)) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("as the last Content-Encoding value"); + assertThat(delegate.headers).isNull(); + } + + @Test + void onHeaders_whenEarlierAttemptLeftTrailers_clearsThem() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, Collections::emptyMap); + + handler.onHeaders(SdkHttpFullResponse.builder().statusCode(200).build()); + + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS)).isNull(); + } + + static SdkHttpFullResponse.Builder awsChunkedResponse() { + return SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Encoding", "aws-chunked") + .putHeader("x-amz-decoded-content-length", "11") + .putHeader("x-amz-trailer", "x-amz-checksum-crc32"); + } + + static ByteBuffer utf8(String s) { + return ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)); + } + + static String readAll(Publisher publisher) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (ByteBuffer buffer : Flowable.fromPublisher(publisher).blockingIterable()) { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + out.write(bytes, 0, bytes.length); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + + static final class CapturingHandler implements TransformingAsyncResponseHandler { + SdkHttpResponse headers; + Publisher stream; + + @Override + public CompletableFuture prepare() { + return new CompletableFuture<>(); + } + + @Override + public void onHeaders(SdkHttpResponse headers) { + this.headers = headers; + } + + @Override + public void onStream(Publisher stream) { + this.stream = stream; + } + + @Override + public void onError(Throwable error) { + } + } +} diff --git a/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStageTest.java b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStageTest.java new file mode 100644 index 000000000000..0e163b967c8c --- /dev/null +++ b/core/sdk-core/src/test/java/software/amazon/awssdk/core/internal/http/pipeline/stages/AwsChunkedResponseDecodingStageTest.java @@ -0,0 +1,152 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.core.internal.http.pipeline.stages; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.http.ExecutionContext; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.InterceptorContext; +import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute; +import software.amazon.awssdk.core.internal.http.RequestExecutionContext; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpFullResponse; +import software.amazon.awssdk.utils.IoUtils; +import software.amazon.awssdk.utils.Pair; +import utils.ValidSdkObjects; + +class AwsChunkedResponseDecodingStageTest { + + private static final String ENCODED = "5\r\nhello\r\n6\r\n world\r\n0\r\nx-amz-checksum-crc32:abc==\r\n\r\n"; + + private final AwsChunkedResponseDecodingStage stage = new AwsChunkedResponseDecodingStage(); + private final ExecutionAttributes attributes = new ExecutionAttributes(); + private final RequestExecutionContext context = context(attributes); + + @Test + void execute_whenAwsChunkedResponse_decodesBodyRewritesHeadersAndExposesTrailers() throws Exception { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + + SdkHttpFullResponse result = execute(awsChunkedResponse().content(body(ENCODED)).build()); + + assertThat(result.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(result.firstMatchingHeader("Content-Length")).hasValue("11"); + assertThat(result.firstMatchingHeader("x-amz-decoded-content-length")).isEmpty(); + assertThat(result.firstMatchingHeader("x-amz-trailer")).isEmpty(); + assertThat(context.executionContext().interceptorContext().httpResponse()).isSameAs(result); + assertThat(context.executionContext().interceptorContext().responseBody()).containsSame(result.content().get()); + + assertThat(IoUtils.toUtf8String(result.content().get())).isEqualTo("hello world"); + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS).get()) + .containsEntry("x-amz-checksum-crc32", "abc=="); + } + + @Test + void execute_whenDecodingNotEnabled_returnsInputUnchanged() throws Exception { + Pair input = + Pair.of(ValidSdkObjects.sdkHttpFullRequest().build(), awsChunkedResponse().content(body(ENCODED)).build()); + + assertThat(stage.execute(input, context)).isSameAs(input); + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS)).isNull(); + } + + @Test + void execute_whenAwsChunkedResponseHasNoBody_onlyRewritesHeaders() throws Exception { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + + SdkHttpFullResponse result = execute(awsChunkedResponse().build()); + + assertThat(result.firstMatchingHeader("Content-Encoding")).isEmpty(); + assertThat(result.content()).isEmpty(); + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS)).isNull(); + } + + @Test + void execute_whenDecodedContentLengthMissing_throwsAndAbortsBody() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + AtomicBoolean aborted = new AtomicBoolean(); + SdkHttpFullResponse response = awsChunkedResponse().removeHeader("x-amz-decoded-content-length") + .content(trackedBody(ENCODED, aborted)) + .build(); + + assertThatThrownBy(() -> execute(response)) + .isInstanceOf(SdkClientException.class) + .hasMessageContaining("x-amz-decoded-content-length header is missing"); + assertThat(aborted).isTrue(); + } + + @Test + void execute_whenAwsChunkedIsNotLastEncoding_throwsAndAbortsBody() { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING, true); + AtomicBoolean aborted = new AtomicBoolean(); + SdkHttpFullResponse response = awsChunkedResponse().putHeader("Content-Encoding", "aws-chunked,gzip") + .content(trackedBody(ENCODED, aborted)) + .build(); + + assertThatThrownBy(() -> execute(response)).isInstanceOf(SdkClientException.class); + assertThat(aborted).isTrue(); + } + + @Test + void execute_whenEarlierAttemptLeftTrailers_clearsThem() throws Exception { + attributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, Collections::emptyMap); + + execute(SdkHttpFullResponse.builder().statusCode(200).content(body("plain")).build()); + + assertThat(attributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS)).isNull(); + } + + private SdkHttpFullResponse execute(SdkHttpFullResponse response) throws Exception { + return stage.execute(Pair.of(ValidSdkObjects.sdkHttpFullRequest().build(), response), context).right(); + } + + private static RequestExecutionContext context(ExecutionAttributes attributes) { + ExecutionContext executionContext = + ExecutionContext.builder() + .interceptorContext(InterceptorContext.builder().request(ValidSdkObjects.sdkRequest()).build()) + .executionAttributes(attributes) + .build(); + return RequestExecutionContext.builder() + .originalRequest(ValidSdkObjects.sdkRequest()) + .executionContext(executionContext) + .build(); + } + + private static SdkHttpFullResponse.Builder awsChunkedResponse() { + return SdkHttpFullResponse.builder() + .statusCode(200) + .putHeader("Content-Encoding", "aws-chunked") + .putHeader("x-amz-decoded-content-length", "11") + .putHeader("x-amz-trailer", "x-amz-checksum-crc32"); + } + + private static AbortableInputStream body(String content) { + return AbortableInputStream.create(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))); + } + + private static AbortableInputStream trackedBody(String content, AtomicBoolean aborted) { + return AbortableInputStream.create(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), + () -> aborted.set(true)); + } +}