Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"type": "bugfix",
"category": "Netty NIO HTTP Client",
"contributor": "",
"description": "Fixed three HTTP/2 issues: (1) A read timeout or other I/O error on one request ended the other requests on its connection. Some of them returned a truncated response with no error. With this fix, the other requests continue. (2) A server GOAWAY that rejected a new request also ended the requests the server had accepted. With this fix, the accepted requests complete. (3) Closing the client during requests sometimes returned a truncated response with no error. With this fix, those requests fail with an IOException."
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,14 @@
*/
@SdkInternalApi
public class Http2MultiplexedChannelPool implements SdkChannelPool {
private static final NettyClientLogger log = NettyClientLogger.getLogger(Http2MultiplexedChannelPool.class);

/**
* Reference to the {@link MultiplexedChannelRecord} on a channel.
*/
private static final AttributeKey<MultiplexedChannelRecord> MULTIPLEXED_CHANNEL = NettyUtils.getOrCreateAttributeKey(
static final AttributeKey<MultiplexedChannelRecord> MULTIPLEXED_CHANNEL = NettyUtils.getOrCreateAttributeKey(
"software.amazon.awssdk.http.nio.netty.internal.http2.Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL");

private static final NettyClientLogger log = NettyClientLogger.getLogger(Http2MultiplexedChannelPool.class);

/**
* Whether a parent channel has been released yet. This guards against double-releasing to the delegate connection pool.
*/
Expand Down Expand Up @@ -174,7 +174,8 @@ private void acquireStreamOnFreshConnection(Promise<Channel> promise, Channel pa
Validate.isTrue(maxStreams > 0, "Maximum streams were not positive on channel (%s).", parentChannel);

MultiplexedChannelRecord multiplexedChannel = new MultiplexedChannelRecord(parentChannel, maxStreams,
idleConnectionTimeout);
idleConnectionTimeout,
ch -> closeAndReleaseParent(ch));
parentChannel.attr(MULTIPLEXED_CHANNEL).set(multiplexedChannel);

Promise<Channel> streamPromise = parentChannel.eventLoop().newPromise();
Expand Down Expand Up @@ -316,12 +317,6 @@ public Future<Void> release(Channel childChannel, Promise<Void> promise) {
}

multiplexedChannel.closeAndReleaseChild(childChannel);

if (multiplexedChannel.canBeClosedAndReleased()) {
// We just closed the last stream in a connection that has reached the end of its life.
return closeAndReleaseParent(parentChannel, null, promise);
}

return promise.setSuccess(null);
}

Expand All @@ -347,11 +342,8 @@ private Future<Void> closeAndReleaseParent(Channel parentChannel, Throwable caus

// We may not have a multiplexed channel if the parent channel hasn't been fully initialized.
if (multiplexedChannel != null) {
if (cause == null) {
multiplexedChannel.closeChildChannels();
} else {
multiplexedChannel.closeChildChannels(cause);
}
Throwable closeCause = cause != null ? cause : new IOException("The HTTP client closed the connection");
multiplexedChannel.closeChildChannels(closeCause);
connections.remove(multiplexedChannel);
}

Expand Down Expand Up @@ -488,20 +480,7 @@ public void channelInactive(ChannelHandlerContext ctx) {

@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
if (cause instanceof Http2ConnectionTerminatingException) {
closeConnectionToNewRequests(ctx, cause);
} else {
closeAndReleaseParent(ctx, cause);
}
}

void closeConnectionToNewRequests(ChannelHandlerContext ctx, Throwable cause) {
MultiplexedChannelRecord multiplexedChannel = ctx.channel().attr(MULTIPLEXED_CHANNEL).get();
if (multiplexedChannel != null) {
multiplexedChannel.closeToNewStreams();
} else {
closeAndReleaseParent(ctx, cause);
}
closeAndReleaseParent(ctx, cause);
}

private void closeAndReleaseParent(ChannelHandlerContext ctx, Throwable cause) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,15 @@ public static Http2StreamExceptionHandler create() {

@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
if (isIoError(cause) && ctx.channel().parent() != null) {
Channel parent = ctx.channel().parent();
log.debug(parent, () -> "An I/O error occurred on an Http2 stream, notifying the connection channel " + parent);
parent.pipeline().fireExceptionCaught(new Http2ConnectionTerminatingException("An I/O error occurred on an "
+ "associated Http2 "
+ "stream " + ctx.channel()));
Channel parent = ctx.channel().parent();
if (isIoError(cause) && parent != null) {
MultiplexedChannelRecord record = parent.attr(Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL).get();
if (record != null) {
log.debug(parent, () -> "An I/O error occurred on HTTP/2 stream " + ctx.channel()
+ ". Connection " + parent + " will take no new streams.");
record.closeToNewStreams();
}
}

ctx.fireExceptionCaught(cause);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import io.netty.channel.Channel;
import io.netty.channel.ChannelId;
import io.netty.channel.ChannelOutboundInvoker;
import io.netty.handler.codec.http2.Http2GoAwayFrame;
import io.netty.handler.codec.http2.Http2StreamChannel;
import io.netty.handler.codec.http2.Http2StreamChannelBootstrap;
Expand All @@ -39,6 +38,7 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.annotations.SdkTestInternalApi;
import software.amazon.awssdk.http.nio.netty.internal.ChannelAttributeKey;
import software.amazon.awssdk.http.nio.netty.internal.ChannelDiagnostics;
import software.amazon.awssdk.http.nio.netty.internal.UnusedChannelExceptionHandler;
Expand All @@ -55,6 +55,7 @@ public class MultiplexedChannelRecord {
private final Channel connection;
private final long maxConcurrencyPerConnection;
private final Long allowedIdleConnectionTimeMillis;
private final Consumer<Channel> closeAndReleaseConnection;

private final AtomicLong availableChildChannels;
private volatile long lastReserveAttemptTimeMillis;
Expand All @@ -66,13 +67,21 @@ public class MultiplexedChannelRecord {
// Only write in the connection.eventLoop()
private volatile RecordState state = RecordState.OPEN;

private volatile int lastStreamId;
// Stored so a failed stream acquire can report why this connection takes no new streams.
private volatile String closedToNewReason;

MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime) {
MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime,
Consumer<Channel> closeAndReleaseConnection) {
this.connection = connection;
this.maxConcurrencyPerConnection = maxConcurrencyPerConnection;
this.availableChildChannels = new AtomicLong(maxConcurrencyPerConnection);
this.allowedIdleConnectionTimeMillis = allowedIdleConnectionTime == null ? null : allowedIdleConnectionTime.toMillis();
this.closeAndReleaseConnection = closeAndReleaseConnection;
}

@SdkTestInternalApi
MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime) {
this(connection, maxConcurrencyPerConnection, allowedIdleConnectionTime, Channel::close);
}

boolean acquireStream(Promise<Channel> promise) {
Expand All @@ -88,10 +97,9 @@ void acquireClaimedStream(Promise<Channel> promise) {
doInEventLoop(connection.eventLoop(), () -> {
if (state != RecordState.OPEN) {
String message;
// GOAWAY
if (state == RecordState.CLOSED_TO_NEW) {
message = String.format("Connection %s received GOAWAY with Last Stream ID %d. Unable to open new "
+ "streams on this connection.", connection, lastStreamId);
message = String.format("Connection %s %s. Unable to open new streams on this connection.", connection,
closedToNewReason);
} else {
message = String.format("Connection %s was closed while acquiring new stream.", connection);
}
Expand Down Expand Up @@ -142,7 +150,7 @@ private void releaseClaimOnFailure(Promise<Channel> promise) {
}
});
} catch (Throwable e) {
releaseClaim();
doInEventLoop(connection.eventLoop(), this::releaseClaim);
throw e;
}
}
Expand All @@ -154,20 +162,36 @@ private void releaseClaim() {
+ "Please report this issue to the AWS SDK for Java team.");
availableChildChannels.decrementAndGet();
}
closeIfRetiredAndDrained();
}

private void closeIfRetiredAndDrained() {
warnIfNotInEventLoop(connection.eventLoop());
if (state != RecordState.CLOSED_TO_NEW
|| !childChannels.isEmpty()
|| availableChildChannels.get() != maxConcurrencyPerConnection) {
return;
}
log.debug(connection, () -> "Closing connection " + connection + " after its last stream; it takes no new streams.");
state = RecordState.CLOSED;
closeAndReleaseConnection.accept(connection);
}

/**
* Handle a {@link Http2GoAwayFrame} on this connection, preventing new streams from being created on it, and closing any
* streams newer than the last-stream-id on the go-away frame.
* streams newer than the last-stream-id on the go-away frame. The connection closes once its last stream and pending
* acquires end.
*/
void handleGoAway(int lastStreamId, GoAwayException exception) {
doInEventLoop(connection.eventLoop(), () -> {
this.lastStreamId = lastStreamId;

if (state == RecordState.CLOSED) {
return;
}

// A server can send more than one GOAWAY, and each later one can lower the Last Stream ID. Setting the
// reason on every GOAWAY keeps the current Last Stream ID in the acquire failure message.
closedToNewReason = "received GOAWAY with Last Stream ID " + lastStreamId;

if (state == RecordState.OPEN) {
state = RecordState.CLOSED_TO_NEW;
}
Expand All @@ -177,27 +201,31 @@ void handleGoAway(int lastStreamId, GoAwayException exception) {
childrenToClose.stream()
.filter(cc -> cc.stream().id() > lastStreamId)
.forEach(cc -> cc.pipeline().fireExceptionCaught(exception));

// Queued, so the drained check runs after Netty's GOAWAY listener returns and closes the connection if it has no
// streams.
if (state == RecordState.CLOSED_TO_NEW) {
connection.eventLoop().execute(this::closeIfRetiredAndDrained);
}
});
}

/**
* Prevent new streams from being acquired from the existing connection.
* Prevent new streams from being acquired from the existing connection. The connection closes once its last stream and
* pending acquires end.
*/
void closeToNewStreams() {
doInEventLoop(connection.eventLoop(), () -> {
if (state == RecordState.OPEN) {
state = RecordState.CLOSED_TO_NEW;
closedToNewReason = "stopped taking new streams after an I/O error on another stream";
// Queued, so the drained check runs after the caller's exception handler returns and closes the connection if
// it has no streams.
connection.eventLoop().execute(this::closeIfRetiredAndDrained);
}
});
}

/**
* Close all registered child channels, and prohibit new streams from being created on this connection.
*/
void closeChildChannels() {
closeAndExecuteOnChildChannels(ChannelOutboundInvoker::close);
}

/**
* Delivers the exception to all registered child channels, and prohibits new streams being created on this connection.
*/
Expand Down Expand Up @@ -247,6 +275,17 @@ private void closeIfIdle() {
return;
}

// Check before the compareAndSet below: setting the count of a retired connection to 0 would stop it from closing.
if (state == RecordState.CLOSED_TO_NEW) {
closeIfRetiredAndDrained();
return;
}

// If we've been closed, no need to shut down.
if (state != RecordState.OPEN) {
return;
}

// Don't close if there have been any reserves attempted since the idle connection time.
long nonVolatileLastReserveAttemptTimeMillis = lastReserveAttemptTimeMillis;
if (nonVolatileLastReserveAttemptTimeMillis > System.currentTimeMillis() - allowedIdleConnectionTimeMillis) {
Expand All @@ -259,11 +298,6 @@ private void closeIfIdle() {
return;
}

// If we've been closed, no need to shut down.
if (state != RecordState.OPEN) {
return;
}

log.debug(connection, () -> "Connection " + connection + " has been idle for " +
(System.currentTimeMillis() - nonVolatileLastReserveAttemptTimeMillis) +
"ms and will be shut down.");
Expand Down Expand Up @@ -300,15 +334,11 @@ private boolean claimStream() {
return false;
}

boolean canBeClosedAndReleased() {
return state != RecordState.OPEN && availableChildChannels.get() == maxConcurrencyPerConnection;
}

CompletableFuture<Metrics> getMetrics() {
CompletableFuture<Metrics> result = new CompletableFuture<>();
doInEventLoop(connection.eventLoop(), () -> {
int streamCount = childChannels.size();
result.complete(new Metrics().setAvailableStreams(maxConcurrencyPerConnection - streamCount));
long availableStreams = state == RecordState.OPEN ? maxConcurrencyPerConnection - childChannels.size() : 0;
result.complete(new Metrics().setAvailableStreams(availableStreams));
});
return result;
}
Expand All @@ -320,8 +350,9 @@ private enum RecordState {
OPEN,

/**
* The connection is open, but new streams may not be acquired from it. This occurs when a connection is being
* shut down (e.g. after it has received a GOAWAY frame), but all streams haven't been closed yet.
* The connection is open, but new streams may not be acquired from it. This occurs after the connection receives a
* GOAWAY frame or has an I/O error on one of its streams. The connection closes once its last stream and pending
* acquires end.
*/
CLOSED_TO_NEW,

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.awssdk.http.nio.netty;

import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES;

import io.netty.handler.timeout.ReadTimeoutException;
import java.time.Duration;
import software.amazon.awssdk.http.Protocol;
import software.amazon.awssdk.http.ProtocolNegotiation;
import software.amazon.awssdk.http.SdkAsyncHttpClientH2StreamErrorTestSuite;
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
import software.amazon.awssdk.utils.AttributeMap;

public class NettyNioAsyncHttpClientH2StreamErrorTest extends SdkAsyncHttpClientH2StreamErrorTestSuite {
@Override
protected SdkAsyncHttpClient createH2Client(int eventLoopThreads, Duration readTimeout) {
return NettyNioAsyncHttpClient.builder()
.protocol(Protocol.HTTP2)
.protocolNegotiation(ProtocolNegotiation.ALPN)
.readTimeout(readTimeout)
.http2Configuration(b -> b.maxStreams(10L))
.eventLoopGroupBuilder(SdkEventLoopGroup.builder().numberOfThreads(eventLoopThreads))
.buildWithDefaults(AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, true).build());
}

@Override
protected Class<? extends Exception> expectedReadTimeoutCause() {
return ReadTimeoutException.class;
}
}
Loading
Loading