Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Supplier;
import software.amazon.awssdk.annotations.SdkProtectedApi;
import software.amazon.awssdk.core.ClientEndpointProvider;
import software.amazon.awssdk.core.SdkClient;
Expand Down Expand Up @@ -304,6 +305,20 @@ public final class SdkInternalExecutionAttribute extends SdkExecutionAttribute {
*/
public static final ExecutionAttribute<Boolean> IS_LONG_POLLING = new ExecutionAttribute<>("IsLongPolling");

/**
* Whether the operation's response body may be aws-chunked encoded, as declared by the {@code awsChunked} trait on its
* output blob. When true and the response {@code Content-Encoding} ends with {@code aws-chunked}, the SDK decodes the body.
*/
public static final ExecutionAttribute<Boolean> AWS_CHUNKED_RESPONSE_DECODING =
new ExecutionAttribute<>("AwsChunkedResponseDecoding");

/**
* Returns the trailers of a decoded aws-chunked response body, keyed by lower-case name. Throws
* {@link IllegalStateException} until the body has been fully and successfully read.
*/
public static final ExecutionAttribute<Supplier<Map<String, String>>> AWS_CHUNKED_RESPONSE_TRAILERS =
new ExecutionAttribute<>("AwsChunkedResponseTrailers");

/**
* Indicates whether retries v2.1 is enabled.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.awssdk.core.internal.chunked;

import static software.amazon.awssdk.core.HttpChecksumConstant.AWS_CHUNKED_HEADER;
import static software.amazon.awssdk.core.HttpChecksumConstant.X_AMZ_TRAILER;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute;
import software.amazon.awssdk.http.SdkHttpFullResponse;
import software.amazon.awssdk.http.SdkHttpResponse;

/**
* Header handling for aws-chunked response decoding, shared by the sync and async response paths.
*/
@SdkInternalApi
public final class AwsChunkedResponseDecodingUtils {

private static final String CONTENT_ENCODING = "Content-Encoding";
private static final String CONTENT_LENGTH = "Content-Length";
private static final String DECODED_CONTENT_LENGTH = "x-amz-decoded-content-length";

private AwsChunkedResponseDecodingUtils() {
}

/**
* Returns whether the response body must be aws-chunked decoded: the operation supports it and {@code aws-chunked} is
* the last {@code Content-Encoding} value.
*
* @throws SdkClientException if {@code aws-chunked} appears more than once or is not the last encoding
*/
public static boolean shouldDecode(SdkHttpResponse response, ExecutionAttributes executionAttributes) {
if (!Boolean.TRUE.equals(executionAttributes.getAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_DECODING))) {
return false;
}
List<String> encodings = contentEncodings(response);
long awsChunkedCount = encodings.stream().filter(AwsChunkedResponseDecodingUtils::isAwsChunked).count();
if (awsChunkedCount == 0) {
return false;
}
if (awsChunkedCount > 1 || !isAwsChunked(encodings.get(encodings.size() - 1))) {
throw malformed("aws-chunked must appear exactly once, as the last Content-Encoding value");
}
return true;
}

/**
* Returns the {@code x-amz-decoded-content-length} value.
*
* @throws SdkClientException if the header is missing or not a non-negative integer
*/
public static long decodedContentLength(SdkHttpResponse response) {
String value = response.firstMatchingHeader(DECODED_CONTENT_LENGTH)
.orElseThrow(() -> malformed(DECODED_CONTENT_LENGTH + " header is missing"));
try {
long length = Long.parseLong(value.trim());
if (length >= 0) {
return length;
}
} catch (NumberFormatException e) {
// Fall through
}
throw malformed(DECODED_CONTENT_LENGTH + " header is not a valid non-negative integer");
}

/**
* Returns the lower-case trailer names declared in {@code x-amz-trailer}, or an empty set if the header is absent.
*/
public static Set<String> declaredTrailers(SdkHttpResponse response) {
Set<String> names = new HashSet<>();
for (String header : response.matchingHeaders(X_AMZ_TRAILER)) {
for (String name : header.split(",")) {
String trimmed = name.trim();
if (!trimmed.isEmpty()) {
names.add(trimmed.toLowerCase(Locale.ROOT));
}
}
}
return Collections.unmodifiableSet(names);
}

/**
* Rewrites the headers that describe the aws-chunked framing so they describe the decoded body:
* {@code Content-Length} becomes the decoded length, {@code aws-chunked} is removed from {@code Content-Encoding}
* (removing the header if nothing remains), and {@code x-amz-decoded-content-length} and {@code x-amz-trailer} are
* removed.
*/
public static SdkHttpFullResponse rewriteHeaders(SdkHttpFullResponse response, long decodedContentLength) {
SdkHttpFullResponse.Builder builder = response.toBuilder();
rewriteHeaders(builder, contentEncodings(response), decodedContentLength);
return builder.build();
}

/**
* Same as {@link #rewriteHeaders(SdkHttpFullResponse, long)}, for the async path where the body arrives separately.
*/
public static SdkHttpResponse rewriteHeaders(SdkHttpResponse response, long decodedContentLength) {
SdkHttpResponse.Builder builder = response.toBuilder();
rewriteHeaders(builder, contentEncodings(response), decodedContentLength);
return builder.build();
}

private static void rewriteHeaders(SdkHttpResponse.Builder builder, List<String> encodings, long decodedContentLength) {
encodings.removeIf(AwsChunkedResponseDecodingUtils::isAwsChunked);
builder.removeHeader(CONTENT_ENCODING)
.removeHeader(DECODED_CONTENT_LENGTH)
.removeHeader(X_AMZ_TRAILER)
.putHeader(CONTENT_LENGTH, Long.toString(decodedContentLength));
if (!encodings.isEmpty()) {
builder.putHeader(CONTENT_ENCODING, String.join(",", encodings));
}
}

/**
* Returns the trailers from a completed trailers future.
*
* @throws IllegalStateException if the future has not completed successfully
*/
public static Map<String, String> completedTrailers(CompletableFuture<Map<String, String>> trailersFuture) {
if (!trailersFuture.isDone() || trailersFuture.isCompletedExceptionally()) {
throw new IllegalStateException("Trailers are only available after the stream has been fully decoded");
}
return trailersFuture.join();
}

private static List<String> contentEncodings(SdkHttpResponse response) {
List<String> encodings = new ArrayList<>();
for (String header : response.matchingHeaders(CONTENT_ENCODING)) {
for (String encoding : header.split(",")) {
String trimmed = encoding.trim();
if (!trimmed.isEmpty()) {
encodings.add(trimmed);
}
}
}
return encodings;
}

private static boolean isAwsChunked(String encoding) {
return AWS_CHUNKED_HEADER.equalsIgnoreCase(encoding);
}

private static SdkClientException malformed(String message) {
return SdkClientException.builder().message("Malformed aws-chunked response: " + message).build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import software.amazon.awssdk.core.internal.http.IdempotentAsyncResponseHandler;
import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler;
import software.amazon.awssdk.core.internal.http.async.AsyncAfterTransmissionInterceptorCallingResponseHandler;
import software.amazon.awssdk.core.internal.http.async.AsyncAwsChunkedResponseDecodingHandler;
import software.amazon.awssdk.core.internal.http.async.AsyncModifyHttpResponseInterceptorCallingResponseHandler;
import software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler;
import software.amazon.awssdk.core.internal.http.async.AsyncStreamingResponseHandler;
Expand Down Expand Up @@ -260,13 +261,15 @@ public void close() {
}

/**
* Wraps the response handler with the response interceptor steps, in order: read-only {@code afterTransmission} on
* the raw response, then the interceptors that can modify it.
* Wraps the response handler with the response interceptor and decoding steps, in order: read-only
* {@code afterTransmission} on the raw response, aws-chunked decoding, then the interceptors that can modify it.
*/
static <T> TransformingAsyncResponseHandler<T> interceptorCallingResponseHandler(
TransformingAsyncResponseHandler<T> responseHandler, ExecutionContext executionContext) {
return new AsyncAfterTransmissionInterceptorCallingResponseHandler<>(
new AsyncModifyHttpResponseInterceptorCallingResponseHandler<>(responseHandler, executionContext),
new AsyncAwsChunkedResponseDecodingHandler<>(
new AsyncModifyHttpResponseInterceptorCallingResponseHandler<>(responseHandler, executionContext),
executionContext),
executionContext);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyTransactionIdStage;
import software.amazon.awssdk.core.internal.http.pipeline.stages.ApplyUserAgentStage;
import software.amazon.awssdk.core.internal.http.pipeline.stages.AuthSchemeResolutionStage;
import software.amazon.awssdk.core.internal.http.pipeline.stages.AwsChunkedResponseDecodingStage;
import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeTransmissionExecutionInterceptorsStage;
import software.amazon.awssdk.core.internal.http.pipeline.stages.BeforeUnmarshallingExecutionInterceptorsStage;
import software.amazon.awssdk.core.internal.http.pipeline.stages.CompressRequestStage;
Expand Down Expand Up @@ -198,6 +199,7 @@ public <OutputT> OutputT execute(HttpResponseHandler<Response<OutputT>> response
.then(BeforeTransmissionExecutionInterceptorsStage::new)
.then(MakeHttpRequestStage::new)
.then(AfterTransmissionExecutionInterceptorsStage::new)
.then(AwsChunkedResponseDecodingStage::new)
.then(ModifyHttpResponseExecutionInterceptorsStage::new)
.then(BeforeUnmarshallingExecutionInterceptorsStage::new)
.then(() -> new HandleResponseStage<>(responseHandler))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.awssdk.core.internal.http.async;

import java.nio.ByteBuffer;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.UnaryOperator;
import org.reactivestreams.Publisher;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.core.http.ExecutionContext;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute;
import software.amazon.awssdk.core.internal.async.AwsChunkedResponseDecodingPublisher;
import software.amazon.awssdk.core.internal.chunked.AwsChunkedResponseDecodingUtils;
import software.amazon.awssdk.core.internal.http.TransformingAsyncResponseHandler;
import software.amazon.awssdk.http.SdkHttpResponse;

/**
* Async response handler decorator that decodes an aws-chunked response body and rewrites its framing headers. Sits
* between the read-only {@code afterTransmission} interceptors and any interceptor that can modify the response, so those
* only see the decoded body and headers.
*
* @param <T> the type of the result
*/
@SdkInternalApi
public final class AsyncAwsChunkedResponseDecodingHandler<T> implements TransformingAsyncResponseHandler<T> {
private final TransformingAsyncResponseHandler<T> delegate;
private final ExecutionContext context;

/**
* Set in onHeaders when the body must be decoded, and applied to the body in onStream. Volatile because the two
* callbacks may run on different threads.
*/
private volatile UnaryOperator<Publisher<ByteBuffer>> bodyDecoder;

public AsyncAwsChunkedResponseDecodingHandler(TransformingAsyncResponseHandler<T> delegate, ExecutionContext context) {
this.delegate = delegate;
this.context = context;
}

@Override
public void onHeaders(SdkHttpResponse response) {
bodyDecoder = null;
ExecutionAttributes executionAttributes = context.executionAttributes();
// Clear trailers from an earlier attempt, so they are never read for this response
executionAttributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS, null);
if (!AwsChunkedResponseDecodingUtils.shouldDecode(response, executionAttributes)) {
delegate.onHeaders(response);
return;
}

long decodedContentLength = AwsChunkedResponseDecodingUtils.decodedContentLength(response);
Set<String> declaredTrailers = AwsChunkedResponseDecodingUtils.declaredTrailers(response);
bodyDecoder = body -> {
AwsChunkedResponseDecodingPublisher decoded =
new AwsChunkedResponseDecodingPublisher(body, decodedContentLength, declaredTrailers);
executionAttributes.putAttribute(SdkInternalExecutionAttribute.AWS_CHUNKED_RESPONSE_TRAILERS,
() -> AwsChunkedResponseDecodingUtils.completedTrailers(decoded.trailersFuture()));
return decoded;
};

SdkHttpResponse rewritten = AwsChunkedResponseDecodingUtils.rewriteHeaders(response, decodedContentLength);
context.interceptorContext(context.interceptorContext().copy(b -> b.httpResponse(rewritten)));
delegate.onHeaders(rewritten);
}

@Override
public void onError(Throwable error) {
delegate.onError(error);
}

@Override
public void onStream(Publisher<ByteBuffer> publisher) {
UnaryOperator<Publisher<ByteBuffer>> decoder = bodyDecoder;
delegate.onStream(decoder == null ? publisher : decoder.apply(publisher));
}

@Override
public CompletableFuture<T> prepare() {
return delegate.prepare();
}
}
Loading
Loading