From 32a96567e20b749ad7df47d0442bd9c3b5d8ae82 Mon Sep 17 00:00:00 2001 From: John Viegas Date: Tue, 17 Nov 2026 08:55:22 -0800 Subject: [PATCH 1/3] Fix an HTTP/2 stream I/O error ending the other streams on its connection --- .../bugfix-NettyNIOHTTPClient-3d76772.json | 6 + .../Http2ConnectionTerminatingException.java | 29 -- .../http2/Http2MultiplexedChannelPool.java | 36 +- .../http2/Http2StreamExceptionHandler.java | 15 +- .../http2/MultiplexedChannelRecord.java | 87 ++-- .../http/nio/netty/fault/GoAwayTest.java | 124 +++++- .../netty/fault/Http2StreamIoErrorTest.java | 376 ++++++++++++++++++ .../Http2MultiplexedChannelPoolTest.java | 29 ++ .../Http2StreamExceptionHandlerTest.java | 33 +- .../http2/MultiplexedChannelRecordTest.java | 136 ++++++- 10 files changed, 754 insertions(+), 117 deletions(-) create mode 100644 .changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json delete mode 100644 http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2ConnectionTerminatingException.java create mode 100644 http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java diff --git a/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json b/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json new file mode 100644 index 000000000000..1af317cb1a68 --- /dev/null +++ b/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "Netty NIO HTTP Client", + "contributor": "", + "description": "Fixed an issue where an I/O error, such as a read timeout, on one HTTP/2 stream, or a GOAWAY that refused a newer stream, ended the other requests on the same connection. Requests reading a response body with no Content-Length completed with a partial response and no error. Closing the client with HTTP/2 requests in flight now fails those requests with an IOException instead of returning a partial response." +} diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2ConnectionTerminatingException.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2ConnectionTerminatingException.java deleted file mode 100644 index e241eb98c572..000000000000 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2ConnectionTerminatingException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.http.nio.netty.internal.http2; - -import software.amazon.awssdk.annotations.SdkInternalApi; - -/** - * Exception indicating a connection is terminating - */ -@SdkInternalApi -final class Http2ConnectionTerminatingException extends RuntimeException { - - Http2ConnectionTerminatingException(String message) { - super(message); - } -} diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java index cbdbf4fe78ff..160aa5eeca20 100644 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java +++ b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java @@ -74,14 +74,14 @@ */ @SdkInternalApi public class Http2MultiplexedChannelPool implements SdkChannelPool { - private static final NettyClientLogger log = NettyClientLogger.getLogger(Http2MultiplexedChannelPool.class); - /** * Reference to the {@link MultiplexedChannelRecord} on a channel. */ - private static final AttributeKey MULTIPLEXED_CHANNEL = NettyUtils.getOrCreateAttributeKey( + static final AttributeKey MULTIPLEXED_CHANNEL = NettyUtils.getOrCreateAttributeKey( "software.amazon.awssdk.http.nio.netty.internal.http2.Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL"); + private static final NettyClientLogger log = NettyClientLogger.getLogger(Http2MultiplexedChannelPool.class); + /** * Whether a parent channel has been released yet. This guards against double-releasing to the delegate connection pool. */ @@ -174,7 +174,8 @@ private void acquireStreamOnFreshConnection(Promise promise, Channel pa Validate.isTrue(maxStreams > 0, "Maximum streams were not positive on channel (%s).", parentChannel); MultiplexedChannelRecord multiplexedChannel = new MultiplexedChannelRecord(parentChannel, maxStreams, - idleConnectionTimeout); + idleConnectionTimeout, + ch -> closeAndReleaseParent(ch)); parentChannel.attr(MULTIPLEXED_CHANNEL).set(multiplexedChannel); Promise streamPromise = parentChannel.eventLoop().newPromise(); @@ -316,12 +317,6 @@ public Future release(Channel childChannel, Promise promise) { } multiplexedChannel.closeAndReleaseChild(childChannel); - - if (multiplexedChannel.canBeClosedAndReleased()) { - // We just closed the last stream in a connection that has reached the end of its life. - return closeAndReleaseParent(parentChannel, null, promise); - } - return promise.setSuccess(null); } @@ -347,11 +342,7 @@ private Future closeAndReleaseParent(Channel parentChannel, Throwable caus // We may not have a multiplexed channel if the parent channel hasn't been fully initialized. if (multiplexedChannel != null) { - if (cause == null) { - multiplexedChannel.closeChildChannels(); - } else { - multiplexedChannel.closeChildChannels(cause); - } + multiplexedChannel.closeChildChannels(cause != null ? cause : new ClosedChannelException()); connections.remove(multiplexedChannel); } @@ -488,20 +479,7 @@ public void channelInactive(ChannelHandlerContext ctx) { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - if (cause instanceof Http2ConnectionTerminatingException) { - closeConnectionToNewRequests(ctx, cause); - } else { - closeAndReleaseParent(ctx, cause); - } - } - - void closeConnectionToNewRequests(ChannelHandlerContext ctx, Throwable cause) { - MultiplexedChannelRecord multiplexedChannel = ctx.channel().attr(MULTIPLEXED_CHANNEL).get(); - if (multiplexedChannel != null) { - multiplexedChannel.closeToNewStreams(); - } else { - closeAndReleaseParent(ctx, cause); - } + closeAndReleaseParent(ctx, cause); } private void closeAndReleaseParent(ChannelHandlerContext ctx, Throwable cause) { diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandler.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandler.java index 0b4a246e7074..8c1e8f171a87 100644 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandler.java +++ b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandler.java @@ -42,14 +42,15 @@ public static Http2StreamExceptionHandler create() { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - if (isIoError(cause) && ctx.channel().parent() != null) { - Channel parent = ctx.channel().parent(); - log.debug(parent, () -> "An I/O error occurred on an Http2 stream, notifying the connection channel " + parent); - parent.pipeline().fireExceptionCaught(new Http2ConnectionTerminatingException("An I/O error occurred on an " - + "associated Http2 " - + "stream " + ctx.channel())); + Channel parent = ctx.channel().parent(); + if (isIoError(cause) && parent != null) { + MultiplexedChannelRecord record = parent.attr(Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL).get(); + if (record != null) { + log.debug(parent, () -> "An I/O error occurred on HTTP/2 stream " + ctx.channel() + + ". Connection " + parent + " will take no new streams."); + record.closeToNewStreams(); + } } - ctx.fireExceptionCaught(cause); } diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java index 785b2ce8a114..7b4aeb15ef17 100644 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java +++ b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java @@ -20,7 +20,6 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelId; -import io.netty.channel.ChannelOutboundInvoker; import io.netty.handler.codec.http2.Http2GoAwayFrame; import io.netty.handler.codec.http2.Http2StreamChannel; import io.netty.handler.codec.http2.Http2StreamChannelBootstrap; @@ -39,6 +38,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; import software.amazon.awssdk.http.nio.netty.internal.ChannelAttributeKey; import software.amazon.awssdk.http.nio.netty.internal.ChannelDiagnostics; import software.amazon.awssdk.http.nio.netty.internal.UnusedChannelExceptionHandler; @@ -55,6 +55,7 @@ public class MultiplexedChannelRecord { private final Channel connection; private final long maxConcurrencyPerConnection; private final Long allowedIdleConnectionTimeMillis; + private final Consumer closeAndReleaseConnection; private final AtomicLong availableChildChannels; private volatile long lastReserveAttemptTimeMillis; @@ -66,13 +67,20 @@ public class MultiplexedChannelRecord { // Only write in the connection.eventLoop() private volatile RecordState state = RecordState.OPEN; - private volatile int lastStreamId; + private volatile String closedToNewReason; - MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime) { + MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime, + Consumer closeAndReleaseConnection) { this.connection = connection; this.maxConcurrencyPerConnection = maxConcurrencyPerConnection; this.availableChildChannels = new AtomicLong(maxConcurrencyPerConnection); this.allowedIdleConnectionTimeMillis = allowedIdleConnectionTime == null ? null : allowedIdleConnectionTime.toMillis(); + this.closeAndReleaseConnection = closeAndReleaseConnection; + } + + @SdkTestInternalApi + MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime) { + this(connection, maxConcurrencyPerConnection, allowedIdleConnectionTime, Channel::close); } boolean acquireStream(Promise promise) { @@ -88,10 +96,9 @@ void acquireClaimedStream(Promise promise) { doInEventLoop(connection.eventLoop(), () -> { if (state != RecordState.OPEN) { String message; - // GOAWAY if (state == RecordState.CLOSED_TO_NEW) { - message = String.format("Connection %s received GOAWAY with Last Stream ID %d. Unable to open new " - + "streams on this connection.", connection, lastStreamId); + message = String.format("Connection %s %s. Unable to open new streams on this connection.", connection, + closedToNewReason); } else { message = String.format("Connection %s was closed while acquiring new stream.", connection); } @@ -142,7 +149,7 @@ private void releaseClaimOnFailure(Promise promise) { } }); } catch (Throwable e) { - releaseClaim(); + doInEventLoop(connection.eventLoop(), this::releaseClaim); throw e; } } @@ -154,20 +161,35 @@ private void releaseClaim() { + "Please report this issue to the AWS SDK for Java team."); availableChildChannels.decrementAndGet(); } + closeIfRetiredAndDrained(); + } + + private void closeIfRetiredAndDrained() { + warnIfNotInEventLoop(connection.eventLoop()); + if (state != RecordState.CLOSED_TO_NEW + || !childChannels.isEmpty() + || availableChildChannels.get() != maxConcurrencyPerConnection) { + return; + } + log.debug(connection, () -> "Closing connection " + connection + " after its last stream; it takes no new streams."); + state = RecordState.CLOSED; + closeAndReleaseConnection.accept(connection); } /** * Handle a {@link Http2GoAwayFrame} on this connection, preventing new streams from being created on it, and closing any - * streams newer than the last-stream-id on the go-away frame. + * streams newer than the last-stream-id on the go-away frame. The connection closes once its last stream and pending + * acquires end. */ void handleGoAway(int lastStreamId, GoAwayException exception) { doInEventLoop(connection.eventLoop(), () -> { - this.lastStreamId = lastStreamId; - if (state == RecordState.CLOSED) { return; } + // A GOAWAY reason replaces an earlier I/O error reason. + closedToNewReason = "received GOAWAY with Last Stream ID " + lastStreamId; + if (state == RecordState.OPEN) { state = RecordState.CLOSED_TO_NEW; } @@ -177,27 +199,31 @@ void handleGoAway(int lastStreamId, GoAwayException exception) { childrenToClose.stream() .filter(cc -> cc.stream().id() > lastStreamId) .forEach(cc -> cc.pipeline().fireExceptionCaught(exception)); + + // Queued, so the drained check runs after Netty's GOAWAY listener returns and closes the connection if it has no + // streams. + if (state == RecordState.CLOSED_TO_NEW) { + connection.eventLoop().execute(this::closeIfRetiredAndDrained); + } }); } /** - * Prevent new streams from being acquired from the existing connection. + * Prevent new streams from being acquired from the existing connection. The connection closes once its last stream and + * pending acquires end. */ void closeToNewStreams() { doInEventLoop(connection.eventLoop(), () -> { if (state == RecordState.OPEN) { state = RecordState.CLOSED_TO_NEW; + closedToNewReason = "stopped taking new streams after an I/O error on another stream"; + // Queued, so the drained check runs after the caller's exception handler returns and closes the connection if + // it has no streams. + connection.eventLoop().execute(this::closeIfRetiredAndDrained); } }); } - /** - * Close all registered child channels, and prohibit new streams from being created on this connection. - */ - void closeChildChannels() { - closeAndExecuteOnChildChannels(ChannelOutboundInvoker::close); - } - /** * Delivers the exception to all registered child channels, and prohibits new streams being created on this connection. */ @@ -247,6 +273,17 @@ private void closeIfIdle() { return; } + // Check before the compareAndSet below: setting the count of a retired connection to 0 would stop it from closing. + if (state == RecordState.CLOSED_TO_NEW) { + closeIfRetiredAndDrained(); + return; + } + + // If we've been closed, no need to shut down. + if (state != RecordState.OPEN) { + return; + } + // Don't close if there have been any reserves attempted since the idle connection time. long nonVolatileLastReserveAttemptTimeMillis = lastReserveAttemptTimeMillis; if (nonVolatileLastReserveAttemptTimeMillis > System.currentTimeMillis() - allowedIdleConnectionTimeMillis) { @@ -259,11 +296,6 @@ private void closeIfIdle() { return; } - // If we've been closed, no need to shut down. - if (state != RecordState.OPEN) { - return; - } - log.debug(connection, () -> "Connection " + connection + " has been idle for " + (System.currentTimeMillis() - nonVolatileLastReserveAttemptTimeMillis) + "ms and will be shut down."); @@ -300,10 +332,6 @@ private boolean claimStream() { return false; } - boolean canBeClosedAndReleased() { - return state != RecordState.OPEN && availableChildChannels.get() == maxConcurrencyPerConnection; - } - CompletableFuture getMetrics() { CompletableFuture result = new CompletableFuture<>(); doInEventLoop(connection.eventLoop(), () -> { @@ -320,8 +348,9 @@ private enum RecordState { OPEN, /** - * The connection is open, but new streams may not be acquired from it. This occurs when a connection is being - * shut down (e.g. after it has received a GOAWAY frame), but all streams haven't been closed yet. + * The connection is open, but new streams may not be acquired from it. This occurs after the connection receives a + * GOAWAY frame or has an I/O error on one of its streams. The connection closes once its last stream and pending + * acquires end. */ CLOSED_TO_NEW, diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/GoAwayTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/GoAwayTest.java index 9d13d98c3cdd..633df4b17dbb 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/GoAwayTest.java +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/GoAwayTest.java @@ -20,6 +20,7 @@ import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInitializer; @@ -200,6 +201,86 @@ private void onHeadersReadDelegator(ChannelHandlerContext ctx, int streamId) { // ID' given in the GOAWAY frame since it means they were processed fully @Test public void execute_goAwayReceived_lastStreamId_lowerStreamsNotClosed() throws InterruptedException { + GoAwayAfterTwoStreams scenario = goAwayBetweenStream3AndStream5(); + + waitForFuture(scenario.stream3Cf); + waitForFuture(scenario.stream5Cf); + + // Note: asserting on the future itself rather than isCompletedExceptionally() so that the failure message carries + // the cause if stream 3 is unexpectedly closed. + assertThat(scenario.stream3Cf).isCompleted() + .isNotCompletedExceptionally(); + + assertThat(scenario.stream5Cf).isCompletedExceptionally(); + assertThatThrownBy(scenario.stream5Cf::join).hasCauseInstanceOf(IOException.class) + .hasMessageContaining("GOAWAY received from service"); + } + + @Test + public void execute_goAwayReceived_shouldCloseConnectionAfterLowerStreamCompletes() throws InterruptedException { + GoAwayAfterTwoStreams scenario = goAwayBetweenStream3AndStream5(); + + waitForFuture(scenario.stream3Cf); + + assertThat(scenario.stream3Cf).isCompleted(); + assertThat(scenario.stream3ServerChannel.closeFuture().await(2, TimeUnit.SECONDS)) + .withFailMessage("The client did not close the connection within 2 seconds after its last stream completed.") + .isTrue(); + } + + @Test + public void execute_goAwayReceivedOnIdleConnection_shouldCloseConnection() throws Exception { + CompletableFuture servingChannel = new CompletableFuture<>(); + byte[] payload = "hello".getBytes(StandardCharsets.UTF_8); + Supplier frameListenerSupplier = () -> new TestFrameListener() { + @Override + public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers, int padding, boolean endStream) { + onHeadersReadDelegator(ctx, streamId); + } + + @Override + public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers, int streamDependency, short weight, boolean exclusive, int padding, boolean endStream) { + onHeadersReadDelegator(ctx, streamId); + } + + private void onHeadersReadDelegator(ChannelHandlerContext ctx, int streamId) { + Http2Headers outboundHeaders = new DefaultHttp2Headers() + .status("200") + .add("content-type", "text/plain") + .addInt("content-length", payload.length); + + frameWriter().writeHeaders(ctx, streamId, outboundHeaders, 0, false, ctx.newPromise()); + frameWriter().writeData(ctx, streamId, Unpooled.wrappedBuffer(payload), 0, true, ctx.newPromise()); + ctx.flush(); + + servingChannel.complete((SocketChannel) ctx.channel()); + } + }; + + endpointDriver = new SimpleEndpointDriver(frameListenerSupplier); + endpointDriver.init(); + + netty = NettyNioAsyncHttpClient.builder() + .protocol(Protocol.HTTP2) + .build(); + + sendGetRequest().get(5, TimeUnit.SECONDS); // stream ID 3 + SocketChannel serverChannel = servingChannel.get(5, TimeUnit.SECONDS); + + // The connection has no streams left. The server keeps TCP open after the GOAWAY. + endpointDriver.goAway(serverChannel, 3); + + assertThat(serverChannel.closeFuture().await(2, TimeUnit.SECONDS)) + .withFailMessage("The client did not close a connection with no streams within 2 seconds after it received " + + "GOAWAY.") + .isTrue(); + } + + /** + * Sends stream 3, then stream 5, then GOAWAY with last stream ID 3, then the DATA for stream 3 500 ms later. The server + * answers stream 3 with HEADERS right away and never answers stream 5. + */ + private GoAwayAfterTwoStreams goAwayBetweenStream3AndStream5() throws InterruptedException { ConcurrentHashMap> channelToStreams = new ConcurrentHashMap<>(); CompletableFuture stream3Received = new CompletableFuture<>(); @@ -260,32 +341,41 @@ private void onHeadersReadDelegator(ChannelHandlerContext ctx, int streamId) { .withFailMessage("Server did not receive both requests within 10 seconds.") .isTrue(); + SocketChannel stream3ServerChannel = null; + // send the GOAWAY first, specifying that everything after 3 is not processed - endpointDriver.channels.forEach(ch -> { + for (SocketChannel ch : endpointDriver.channels) { Set streams = channelToStreams.getOrDefault(ch.id().asShortText(), Collections.emptySet()); if (streams.contains(3)) { + stream3ServerChannel = ch; endpointDriver.goAway(ch, 3); - } - }); - // now send the DATA for stream 3, which should still be valid - endpointDriver.channels.forEach(ch -> { - Set streams = channelToStreams.getOrDefault(ch.id().asShortText(), Collections.emptySet()); - if (streams.contains(3)) { - endpointDriver.data(ch, 3, getPayload); + // now send the DATA for stream 3, which should still be valid. Send it 500 ms after the GOAWAY: without + // the delay the client reads the GOAWAY and the DATA in one batch, so stream 3 finished before the close + // queued after the GOAWAY ran. That is why this test passed while that close was still there. + ch.eventLoop().schedule(() -> endpointDriver.data(ch, 3, getPayload), 500, TimeUnit.MILLISECONDS); } - }); + } - waitForFuture(stream3Cf); - waitForFuture(stream5Cf); + assertThat(stream3ServerChannel) + .withFailMessage("No server connection received stream ID 3.") + .isNotNull(); - // Note: asserting on the future itself rather than isCompletedExceptionally() so that the failure message carries - // the cause if stream 3 is unexpectedly closed. - assertThat(stream3Cf).isCompleted() - .isNotCompletedExceptionally(); + return new GoAwayAfterTwoStreams(stream3Cf, stream5Cf, stream3ServerChannel); + } - assertThat(stream5Cf).isCompletedExceptionally(); - assertThatThrownBy(stream5Cf::join).hasCauseInstanceOf(IOException.class); + private static final class GoAwayAfterTwoStreams { + private final CompletableFuture stream3Cf; + private final CompletableFuture stream5Cf; + private final SocketChannel stream3ServerChannel; + + private GoAwayAfterTwoStreams(CompletableFuture stream3Cf, + CompletableFuture stream5Cf, + SocketChannel stream3ServerChannel) { + this.stream3Cf = stream3Cf; + this.stream5Cf = stream5Cf; + this.stream3ServerChannel = stream3ServerChannel; + } } private CompletableFuture sendGetRequest() { diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java new file mode 100644 index 000000000000..92f3d050137c --- /dev/null +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java @@ -0,0 +1,376 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.nio.netty.fault; + +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.ServerSocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http2.DefaultHttp2DataFrame; +import io.netty.handler.codec.http2.DefaultHttp2Headers; +import io.netty.handler.codec.http2.DefaultHttp2HeadersFrame; +import io.netty.handler.codec.http2.Http2DataFrame; +import io.netty.handler.codec.http2.Http2FrameCodecBuilder; +import io.netty.handler.codec.http2.Http2Headers; +import io.netty.handler.codec.http2.Http2HeadersFrame; +import io.netty.handler.codec.http2.Http2MultiplexHandler; +import io.netty.handler.codec.http2.Http2StreamFrame; +import io.netty.handler.ssl.ApplicationProtocolConfig; +import io.netty.handler.ssl.ApplicationProtocolNames; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslProvider; +import io.netty.handler.ssl.util.SelfSignedCertificate; +import io.netty.handler.timeout.ReadTimeoutException; +import io.netty.util.AttributeKey; +import io.reactivex.Flowable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.reactivestreams.Publisher; +import software.amazon.awssdk.http.EmptyPublisher; +import software.amazon.awssdk.http.Protocol; +import software.amazon.awssdk.http.ProtocolNegotiation; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup; +import software.amazon.awssdk.http.nio.netty.internal.utils.NettyUtils; +import software.amazon.awssdk.utils.AttributeMap; + +/** + * A read timeout on one HTTP/2 stream must not end the other streams on its connection. The connection takes no new + * streams and closes after its last stream ends. Closing the client fails the streams still in flight with an + * IOException. Uses TLS with ALPN, the default for H2 service clients. + */ +@EnabledIf("alpnSupported") +public class Http2StreamIoErrorTest { + private static final String REQUEST_ID = "x-request-id"; + private static final int FRAME_COUNT = 20; + private static final int FRAME_BYTES = 10; + private static final Duration WAIT = Duration.ofSeconds(5); + + private final Server server = new Server(); + private SdkEventLoopGroup clientEventLoopGroup; + private SdkAsyncHttpClient client; + + @BeforeEach + public void setup() throws Exception { + server.start(); + } + + @AfterEach + public void teardown() { + if (client != null) { + client.close(); + } + if (clientEventLoopGroup != null) { + clientEventLoopGroup.eventLoopGroup().shutdownGracefully(0, 1, TimeUnit.SECONDS) + .awaitUninterruptibly(10, TimeUnit.SECONDS); + } + server.shutdown(); + } + + @ParameterizedTest(name = "eventLoopThreads={0}") + @ValueSource(ints = {1, 3}) + public void execute_readTimeoutOnOneStream_shouldCompleteSiblingStreams(int eventLoopThreads) { + List siblings = streamSiblingsAndTimeOutStall(eventLoopThreads); + + Call afterTimeout = send("/quick", "after-timeout"); + + assertThat(afterTimeout.execute).succeedsWithin(WAIT); + assertFullBodies(siblings); + assertThat(server.requestConnections).as("connection index that served each request") + .containsOnly(entry("warmup", 1), entry("sibling-0", 1), entry("sibling-1", 1), + entry("sibling-2", 1), entry("stall", 1), entry("after-timeout", 2)); + } + + @ParameterizedTest(name = "eventLoopThreads={0}") + @ValueSource(ints = {1, 3}) + public void execute_readTimeoutOnOneStream_shouldCloseConnectionAfterLastSibling(int eventLoopThreads) { + List siblings = streamSiblingsAndTimeOutStall(eventLoopThreads); + + assertFullBodies(siblings); + long lastSiblingDone = siblings.stream().mapToLong(c -> c.bodyCompletedNanos).max().getAsLong(); + assertThat(server.closedNanos(1)).as("connection 1 closed") + .succeedsWithin(Duration.ofSeconds(2)) + .satisfies(closed -> assertThat(closed).isBetween( + lastSiblingDone, lastSiblingDone + TimeUnit.SECONDS.toNanos(2))); + } + + @Test + public void close_withStreamsInFlight_shouldFailStreamsWithIOException() { + initClient(3); + assertThat(send("/quick", "warmup").execute).succeedsWithin(WAIT); + List streams = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + streams.add(send("/stream", "stream-" + i)); + } + streams.forEach(s -> assertThat(s.firstFrameReceived).succeedsWithin(WAIT)); + + client.close(); + client = null; + + for (Call stream : streams) { + assertThat(stream.execute).as("%s, %s body bytes received", stream.requestId, stream.bodyBytes) + .failsWithin(WAIT) + .withThrowableOfType(ExecutionException.class) + .withCauseInstanceOf(IOException.class) + .withMessageContaining("An error occurred on the connection") + .withRootCauseInstanceOf(ClosedChannelException.class); + assertThat(stream.bodyError).as("%s body error", stream.requestId) + .isInstanceOf(IOException.class) + .hasMessageContaining("An error occurred on the connection"); + assertThat(stream.bodyCompletedNanos).as("%s body completed", stream.requestId).isZero(); + } + } + + /** + * Opens connection 1 with one request, starts three streaming siblings on it, then a stalled request that hits the + * read timeout. Returns the siblings, still streaming. + */ + private List streamSiblingsAndTimeOutStall(int eventLoopThreads) { + initClient(eventLoopThreads); + // One request first: concurrent first requests can each open a connection. + assertThat(send("/quick", "warmup").execute).succeedsWithin(WAIT); + List siblings = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + siblings.add(send("/stream", "sibling-" + i)); + } + siblings.forEach(s -> assertThat(server.firstDataSent(s.requestId)).succeedsWithin(WAIT)); + + Call stall = send("/stall", "stall"); + + assertThat(stall.execute).failsWithin(WAIT) + .withThrowableOfType(ExecutionException.class) + .withMessageContaining("Read timed out") + .withRootCauseInstanceOf(ReadTimeoutException.class); + return siblings; + } + + private static void assertFullBodies(List siblings) { + for (Call sibling : siblings) { + assertThat(sibling.execute).as(sibling.requestId).succeedsWithin(WAIT); + assertThat(sibling.bodyBytes).as("%s body bytes", sibling.requestId).hasValue(FRAME_COUNT * FRAME_BYTES); + } + } + + private void initClient(int eventLoopThreads) { + clientEventLoopGroup = SdkEventLoopGroup.builder().numberOfThreads(eventLoopThreads).build(); + client = NettyNioAsyncHttpClient.builder() + .protocol(Protocol.HTTP2) + .protocolNegotiation(ProtocolNegotiation.ALPN) + .readTimeout(Duration.ofSeconds(1)) + .http2Configuration(b -> b.maxStreams(10L)) + .eventLoopGroup(clientEventLoopGroup) + .buildWithDefaults(AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, true).build()); + } + + private Call send(String path, String requestId) { + Call call = new Call(requestId); + SdkHttpFullRequest request = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.GET) + .protocol("https") + .host("127.0.0.1") + .port(server.port()) + .encodedPath(path) + .putHeader(REQUEST_ID, requestId) + .build(); + call.execute = client.execute(AsyncExecuteRequest.builder() + .request(request) + .requestContentPublisher(new EmptyPublisher()) + .responseHandler(call) + .build()); + return call; + } + + private static boolean alpnSupported() { + return NettyUtils.isAlpnSupported(SslProvider.JDK); + } + + /** + * Counts body bytes, not onNext calls, because Netty may coalesce DATA frames. + */ + private static final class Call implements SdkAsyncHttpResponseHandler { + private final String requestId; + private final AtomicLong bodyBytes = new AtomicLong(); + private final CompletableFuture firstFrameReceived = new CompletableFuture<>(); + private volatile long bodyCompletedNanos; + private volatile Throwable bodyError; + private CompletableFuture execute; + + private Call(String requestId) { + this.requestId = requestId; + } + + @Override + public void onHeaders(SdkHttpResponse headers) { + } + + @Override + public void onStream(Publisher stream) { + Flowable.fromPublisher(stream).subscribe(b -> { + if (bodyBytes.addAndGet(b.remaining()) >= FRAME_BYTES) { + firstFrameReceived.complete(null); + } + }, + t -> bodyError = t, + () -> bodyCompletedNanos = System.nanoTime()); + } + + @Override + public void onError(Throwable error) { + } + } + + /** + * Answers {@code /quick} with one DATA frame, {@code /stream} with {@value #FRAME_COUNT} DATA frames 100 ms apart and no + * content-length, and {@code /stall} with nothing. Records which connection served each request and when each + * connection closed. + */ + private static final class Server extends ChannelInitializer { + private static final AttributeKey CONNECTION_INDEX = + AttributeKey.valueOf(Http2StreamIoErrorTest.class.getName() + ".CONNECTION_INDEX"); + + private final NioEventLoopGroup group = new NioEventLoopGroup(); + private final AtomicInteger connectionCount = new AtomicInteger(); + private final Map> closedNanos = new ConcurrentHashMap<>(); + private final Map requestConnections = new ConcurrentHashMap<>(); + private final Map> firstDataSent = new ConcurrentHashMap<>(); + private SslContext sslContext; + private ServerSocketChannel serverSocket; + + void start() throws Exception { + SelfSignedCertificate certificate = new SelfSignedCertificate(); + sslContext = SslContextBuilder.forServer(certificate.certificate(), certificate.privateKey()) + .applicationProtocolConfig(new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2)) + .build(); + serverSocket = (ServerSocketChannel) new ServerBootstrap().channel(NioServerSocketChannel.class) + .group(group) + .childHandler(this) + .bind("127.0.0.1", 0).sync().channel(); + } + + void shutdown() { + if (serverSocket != null) { + serverSocket.close().awaitUninterruptibly(5, TimeUnit.SECONDS); + } + group.shutdownGracefully(0, 1, TimeUnit.SECONDS).awaitUninterruptibly(10, TimeUnit.SECONDS); + } + + int port() { + return serverSocket.localAddress().getPort(); + } + + CompletableFuture closedNanos(int connectionIndex) { + return closedNanos.computeIfAbsent(connectionIndex, i -> new CompletableFuture<>()); + } + + CompletableFuture firstDataSent(String requestId) { + return firstDataSent.computeIfAbsent(requestId, id -> new CompletableFuture<>()); + } + + @Override + protected void initChannel(Channel ch) { + int index = connectionCount.incrementAndGet(); + ch.attr(CONNECTION_INDEX).set(index); + ch.closeFuture().addListener(f -> closedNanos(index).complete(System.nanoTime())); + ch.pipeline().addLast(sslContext.newHandler(ch.alloc()), + Http2FrameCodecBuilder.forServer().build(), + new Http2MultiplexHandler(new ChannelInitializer() { + @Override + protected void initChannel(Channel stream) { + stream.pipeline().addLast(new StreamHandler()); + } + })); + } + + private final class StreamHandler extends SimpleChannelInboundHandler { + private Http2Headers request; + + @Override + protected void channelRead0(ChannelHandlerContext ctx, Http2StreamFrame frame) { + boolean requestEnded = false; + if (frame instanceof Http2HeadersFrame) { + request = ((Http2HeadersFrame) frame).headers(); + requestConnections.put(requestId(), ctx.channel().parent().attr(CONNECTION_INDEX).get()); + requestEnded = ((Http2HeadersFrame) frame).isEndStream(); + } else if (frame instanceof Http2DataFrame) { + requestEnded = ((Http2DataFrame) frame).isEndStream(); + } + if (!requestEnded) { + return; + } + + String path = request.path().toString(); + if ("/quick".equals(path)) { + ctx.write(new DefaultHttp2HeadersFrame(new DefaultHttp2Headers().status(OK.codeAsText()))); + ctx.writeAndFlush(dataFrame(true)); + } else if ("/stream".equals(path)) { + ctx.write(new DefaultHttp2HeadersFrame(new DefaultHttp2Headers().status(OK.codeAsText()))); + ctx.writeAndFlush(dataFrame(false)).addListener(f -> firstDataSent(requestId()).complete(null)); + for (int i = 1; i < FRAME_COUNT; i++) { + boolean last = i == FRAME_COUNT - 1; + ctx.channel().eventLoop().schedule(() -> ctx.writeAndFlush(dataFrame(last)), + i * 100L, TimeUnit.MILLISECONDS); + } + } + } + + private String requestId() { + return String.valueOf(request.get(REQUEST_ID)); + } + + private DefaultHttp2DataFrame dataFrame(boolean endStream) { + return new DefaultHttp2DataFrame(Unpooled.wrappedBuffer(new byte[FRAME_BYTES]), endStream); + } + } + } +} diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPoolTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPoolTest.java index b53835800fe1..d4079cc6f7bd 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPoolTest.java +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPoolTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.http.nio.netty.internal.http2; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; @@ -40,6 +41,7 @@ import java.io.IOException; import java.util.Collections; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -260,6 +262,33 @@ public void acquire_shouldExpandConnectionWindowSizeProportionally() { } } + @Test + public void release_lastChildOfGoAwayConnection_shouldReleaseParentOnce() { + EmbeddedChannel channel = newHttp2Channel(); + channel.attr(ChannelAttributeKey.MAX_CONCURRENT_STREAMS).set(3L); + try { + ChannelPool connectionPool = Mockito.mock(ChannelPool.class); + Mockito.when(connectionPool.acquire()).thenReturn(new DefaultPromise(loopGroup.next()).setSuccess(channel)); + Mockito.when(connectionPool.release(eq(channel), any(Promise.class))) + .thenAnswer(invocation -> invocation.>getArgument(1).setSuccess(null)); + Http2MultiplexedChannelPool h2Pool = new Http2MultiplexedChannelPool(connectionPool, loopGroup, + Collections.emptySet(), null); + Future acquire = h2Pool.acquire(); + assertThat(acquire.awaitUninterruptibly(5, TimeUnit.SECONDS)).isTrue(); + channel.runPendingTasks(); + // The child has not written HEADERS, so its stream ID is below 1000 and the GOAWAY leaves it open. + h2Pool.handleGoAway(channel, 1000, new GoAwayException(0, "test")); + + h2Pool.release(acquire.getNow()); + channel.runPendingTasks(); + + Mockito.verify(connectionPool, Mockito.times(1)).release(eq(channel), any(Promise.class)); + assertThat(channel.isOpen()).isFalse(); + } finally { + channel.close(); + } + } + @Test public void metricsShouldSumAllChildChannels() throws InterruptedException { int maxConcurrentStream = 2; diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandlerTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandlerTest.java index d4d93ff9e425..0c0b7bacd707 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandlerTest.java +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2StreamExceptionHandlerTest.java @@ -18,6 +18,9 @@ import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static software.amazon.awssdk.http.nio.netty.internal.ChannelAttributeKey.PING_TRACKER; @@ -81,26 +84,45 @@ public static void teardown() { } @Test - public void timeoutException_shouldFireExceptionAndPropagateException() { + public void exceptionCaught_timeoutException_shouldRetireConnectionAndPropagateException() { + MultiplexedChannelRecord record = mock(MultiplexedChannelRecord.class); + embeddedParentChannel.attr(Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL).set(record); when(streamChannel.parent()).thenReturn(embeddedParentChannel); handler.exceptionCaught(context, ReadTimeoutException.INSTANCE); - assertThat(verifyExceptionHandler.exceptionCaught).isExactlyInstanceOf(Http2ConnectionTerminatingException.class); + verify(record).closeToNewStreams(); + assertThat(verifyExceptionHandler.exceptionCaught).isNull(); verify(context).fireExceptionCaught(ReadTimeoutException.INSTANCE); } @Test - public void ioException_shouldFireExceptionAndPropagateException() { + public void exceptionCaught_ioException_shouldRetireConnectionAndPropagateException() { + MultiplexedChannelRecord record = mock(MultiplexedChannelRecord.class); + embeddedParentChannel.attr(Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL).set(record); IOException ioException = new IOException("yolo"); when(streamChannel.parent()).thenReturn(embeddedParentChannel); handler.exceptionCaught(context, ioException); - assertThat(verifyExceptionHandler.exceptionCaught).isExactlyInstanceOf(Http2ConnectionTerminatingException.class); + verify(record).closeToNewStreams(); + assertThat(verifyExceptionHandler.exceptionCaught).isNull(); + verify(context).fireExceptionCaught(ioException); + } + + @Test + public void exceptionCaught_ioExceptionWithNoRecordOnParent_shouldJustPropagateException() { + IOException ioException = new IOException("yolo"); + when(streamChannel.parent()).thenReturn(embeddedParentChannel); + + assertThatCode(() -> handler.exceptionCaught(context, ioException)).doesNotThrowAnyException(); + + assertThat(verifyExceptionHandler.exceptionCaught).isNull(); verify(context).fireExceptionCaught(ioException); } @Test - public void otherException_shouldJustPropagateException() { + public void exceptionCaught_otherException_shouldJustPropagateException() { + MultiplexedChannelRecord record = mock(MultiplexedChannelRecord.class); + embeddedParentChannel.attr(Http2MultiplexedChannelPool.MULTIPLEXED_CHANNEL).set(record); when(streamChannel.parent()).thenReturn(embeddedParentChannel); RuntimeException otherException = new RuntimeException("test"); @@ -110,6 +132,7 @@ public void otherException_shouldJustPropagateException() { verify(context).fireExceptionCaught(otherException); assertThat(verifyExceptionHandler.exceptionCaught).isNull(); + verify(record, never()).closeToNewStreams(); } private static final class TestVerifyExceptionHandler extends ChannelInboundHandlerAdapter { diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java index 5e9359be28cb..7f4d443e3162 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java @@ -28,13 +28,22 @@ import io.netty.handler.codec.http2.Http2FrameCodecBuilder; import io.netty.handler.codec.http2.Http2MultiplexHandler; import io.netty.util.concurrent.DefaultPromise; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; import io.netty.util.concurrent.Promise; import java.io.IOException; +import java.nio.channels.ClosedChannelException; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import software.amazon.awssdk.http.Protocol; import software.amazon.awssdk.http.nio.netty.internal.ChannelAttributeKey; import software.amazon.awssdk.http.nio.netty.internal.MockChannel; @@ -189,7 +198,7 @@ public void acquireClaimedConnection_channelClosed_shouldThrowIOException() { MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 1, Duration.ofSeconds(10)); - record.closeChildChannels(); + record.closeChildChannels(new ClosedChannelException()); record.acquireClaimedStream(channelPromise); @@ -235,6 +244,131 @@ public void closeToNewStreams_AcquireStreamShouldReturnFalse() { assertThat(record.acquireStream(streamPromise)).isFalse(); } + @ParameterizedTest(name = "{0} with {1} open streams") + @CsvSource({"CLOSE_TO_NEW_STREAMS, 0", "CLOSE_TO_NEW_STREAMS, 2", "GO_AWAY, 0", "GO_AWAY, 2"}) + public void retire_whenStreamsDrain_shouldCloseConnectionAfterLastStream(Retirement retirement, int openStreams) { + EmbeddedChannel channel = newHttp2Channel(); + MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 2, null); + record.closeAndReleaseChild(acquireStream(channel, record)); + channel.runPendingTasks(); + List streams = new ArrayList<>(); + for (int i = 0; i < openStreams; i++) { + streams.add(acquireStream(channel, record)); + } + + retirement.retire(record); + channel.runPendingTasks(); + + for (Channel stream : streams) { + assertThat(channel.isOpen()).as("connection open while a stream remains").isTrue(); + record.closeAndReleaseChild(stream); + channel.runPendingTasks(); + } + assertThat(channel.isOpen()).as("connection open after its last stream ended").isFalse(); + } + + @Test + public void acquireStream_whenRetiredAfterClaimBeforeStreamOpens_shouldFailWithIoErrorReasonAndCloseConnection() { + EmbeddedChannel channel = newHttp2Channel(); + MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 1, null); + RetireOnFirstListenerPromise streamPromise = new RetireOnFirstListenerPromise(channel, record); + + assertThat(record.acquireStream(streamPromise)).isTrue(); + + assertThat(streamPromise.connectionOpenAfterDrainedCheck).as("connection open while the claim is outstanding").isTrue(); + assertThat(streamPromise.cause()).isInstanceOf(IOException.class) + .hasMessageContaining("I/O error on another stream") + .hasMessageNotContaining("GOAWAY"); + assertThat(channel.isOpen()).as("connection open after the failed claim returned").isFalse(); + } + + @Test + public void closeIfIdle_retiredRecordWithNoStreams_shouldCloseConnection() { + EmbeddedChannel channel = newHttp2Channel(); + MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 1, Duration.ofMillis(1000)); + record.closeAndReleaseChild(acquireStream(channel, record)); + channel.runPendingTasks(); + // Make the 1 second idle task due. + channel.advanceTimeBy(1, TimeUnit.SECONDS); + + record.closeToNewStreams(); + // Run only the due idle task, so it runs before the drained check that the retire queued. + channel.runScheduledPendingTasks(); + + assertThat(channel.isOpen()).as("connection open after the idle task ran").isFalse(); + } + + @Test + public void retire_whenRepeatedAfterLastStream_shouldInvokeCloseCallbackOnce() { + EmbeddedChannel channel = newHttp2Channel(); + List closedConnections = new ArrayList<>(); + MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 2, null, closedConnections::add); + Channel stream = acquireStream(channel, record); + record.closeToNewStreams(); + + record.closeAndReleaseChild(stream); + record.closeToNewStreams(); + record.handleGoAway(3, new GoAwayException(0, "test")); + channel.runPendingTasks(); + + assertThat(closedConnections).containsExactly(channel); + } + + private static Channel acquireStream(EmbeddedChannel channel, MultiplexedChannelRecord record) { + Promise streamPromise = channel.eventLoop().newPromise(); + assertThat(record.acquireStream(streamPromise)).isTrue(); + channel.runPendingTasks(); + assertThat(streamPromise.isSuccess()).isTrue(); + return streamPromise.getNow(); + } + + private enum Retirement { + CLOSE_TO_NEW_STREAMS(MultiplexedChannelRecord::closeToNewStreams), + // The test streams have not written HEADERS, so their stream IDs are below 3 and the GOAWAY leaves them open. + GO_AWAY(record -> record.handleGoAway(3, new GoAwayException(0, "test"))); + + private final Consumer action; + + Retirement(Consumer action) { + this.action = action; + } + + void retire(MultiplexedChannelRecord record) { + action.accept(record); + } + } + + /** + * {@link MultiplexedChannelRecord#acquireStream} claims a stream, adds a listener to this promise, and then runs the stream + * open task. On the first listener, this promise retires the record and runs the drained check that the retire queued, so + * both run after the claim and before the stream open task. The claim is still taken, so the drained check does not close + * the connection; only the claim's return can. + */ + private static final class RetireOnFirstListenerPromise extends DefaultPromise { + private final EmbeddedChannel connection; + private final MultiplexedChannelRecord record; + private boolean retired; + private boolean connectionOpenAfterDrainedCheck; + + RetireOnFirstListenerPromise(EmbeddedChannel connection, MultiplexedChannelRecord record) { + super(connection.eventLoop()); + this.connection = connection; + this.record = record; + } + + @Override + public Promise addListener(GenericFutureListener> listener) { + if (!retired) { + retired = true; + // EmbeddedEventLoop.inEventLoop() is always true, so doInEventLoop runs closeToNewStreams inline. + record.closeToNewStreams(); + connection.runPendingTasks(); + connectionOpenAfterDrainedCheck = connection.isOpen(); + } + return super.addListener(listener); + } + } + private static final class VerifyExceptionHandler extends ChannelInboundHandlerAdapter { private Throwable exceptionCaught; @Override From 4302557f47cd435608380d2720e1c511814c668d Mon Sep 17 00:00:00 2001 From: John Viegas Date: Wed, 18 Nov 2026 09:40:44 -0800 Subject: [PATCH 2/3] handled review comments --- .../bugfix-NettyNIOHTTPClient-3d76772.json | 2 +- .../internal/http2/MultiplexedChannelRecord.java | 4 ++-- .../http2/MultiplexedChannelRecordTest.java | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json b/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json index 1af317cb1a68..b9d4292b60c7 100644 --- a/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json +++ b/.changes/next-release/bugfix-NettyNIOHTTPClient-3d76772.json @@ -2,5 +2,5 @@ "type": "bugfix", "category": "Netty NIO HTTP Client", "contributor": "", - "description": "Fixed an issue where an I/O error, such as a read timeout, on one HTTP/2 stream, or a GOAWAY that refused a newer stream, ended the other requests on the same connection. Requests reading a response body with no Content-Length completed with a partial response and no error. Closing the client with HTTP/2 requests in flight now fails those requests with an IOException instead of returning a partial response." + "description": "Fixed three HTTP/2 issues: (1) A read timeout or other I/O error on one request ended the other requests on its connection. Some of them returned a truncated response with no error. With this fix, the other requests continue. (2) A server GOAWAY that rejected a new request also ended the requests the server had accepted. With this fix, the accepted requests complete. (3) Closing the client during requests sometimes returned a truncated response with no error. With this fix, those requests fail with an IOException." } diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java index 7b4aeb15ef17..8820c9ffc68d 100644 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java +++ b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java @@ -335,8 +335,8 @@ private boolean claimStream() { CompletableFuture getMetrics() { CompletableFuture result = new CompletableFuture<>(); doInEventLoop(connection.eventLoop(), () -> { - int streamCount = childChannels.size(); - result.complete(new Metrics().setAvailableStreams(maxConcurrencyPerConnection - streamCount)); + long availableStreams = state == RecordState.OPEN ? maxConcurrencyPerConnection - childChannels.size() : 0; + result.complete(new Metrics().setAvailableStreams(availableStreams)); }); return result; } diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java index 7f4d443e3162..7257a1ae11dd 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java @@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; import software.amazon.awssdk.http.Protocol; import software.amazon.awssdk.http.nio.netty.internal.ChannelAttributeKey; import software.amazon.awssdk.http.nio.netty.internal.MockChannel; @@ -314,6 +315,21 @@ public void retire_whenRepeatedAfterLastStream_shouldInvokeCloseCallbackOnce() { assertThat(closedConnections).containsExactly(channel); } + @ParameterizedTest(name = "{0}") + @EnumSource(Retirement.class) + public void getMetrics_whenRetiredWithOpenStream_shouldReportNoAvailableStreams(Retirement retirement) throws Exception { + EmbeddedChannel channel = newHttp2Channel(); + MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 2, null); + acquireStream(channel, record); + assertThat(record.getMetrics().get(5, TimeUnit.SECONDS).getAvailableStreams()).as("before retire").isEqualTo(1); + + retirement.retire(record); + channel.runPendingTasks(); + + assertThat(channel.isOpen()).as("connection open while a stream remains").isTrue(); + assertThat(record.getMetrics().get(5, TimeUnit.SECONDS).getAvailableStreams()).as("after retire").isZero(); + } + private static Channel acquireStream(EmbeddedChannel channel, MultiplexedChannelRecord record) { Promise streamPromise = channel.eventLoop().newPromise(); assertThat(record.acquireStream(streamPromise)).isTrue(); From f3365a17d58cdfd966ee4d82969c6cecdf2cad1a Mon Sep 17 00:00:00 2001 From: John Viegas Date: Wed, 18 Nov 2026 12:48:00 -0800 Subject: [PATCH 3/3] handled Zoe's review comments --- .../http2/Http2MultiplexedChannelPool.java | 3 +- .../http2/MultiplexedChannelRecord.java | 4 +- ...tyNioAsyncHttpClientH2StreamErrorTest.java | 44 ++++++++++ .../http2/MultiplexedChannelRecordTest.java | 16 ++++ test/http-client-tests/pom.xml | 4 + ...AsyncHttpClientH2StreamErrorTestSuite.java | 85 +++++++++---------- 6 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/NettyNioAsyncHttpClientH2StreamErrorTest.java rename http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java => test/http-client-tests/src/main/java/software/amazon/awssdk/http/SdkAsyncHttpClientH2StreamErrorTestSuite.java (83%) diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java index 160aa5eeca20..6d900562c5a9 100644 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java +++ b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/Http2MultiplexedChannelPool.java @@ -342,7 +342,8 @@ private Future closeAndReleaseParent(Channel parentChannel, Throwable caus // We may not have a multiplexed channel if the parent channel hasn't been fully initialized. if (multiplexedChannel != null) { - multiplexedChannel.closeChildChannels(cause != null ? cause : new ClosedChannelException()); + Throwable closeCause = cause != null ? cause : new IOException("The HTTP client closed the connection"); + multiplexedChannel.closeChildChannels(closeCause); connections.remove(multiplexedChannel); } diff --git a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java index 8820c9ffc68d..9bd4c5fda94f 100644 --- a/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java +++ b/http-clients/netty-nio-client/src/main/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecord.java @@ -67,6 +67,7 @@ public class MultiplexedChannelRecord { // Only write in the connection.eventLoop() private volatile RecordState state = RecordState.OPEN; + // Stored so a failed stream acquire can report why this connection takes no new streams. private volatile String closedToNewReason; MultiplexedChannelRecord(Channel connection, long maxConcurrencyPerConnection, Duration allowedIdleConnectionTime, @@ -187,7 +188,8 @@ void handleGoAway(int lastStreamId, GoAwayException exception) { return; } - // A GOAWAY reason replaces an earlier I/O error reason. + // A server can send more than one GOAWAY, and each later one can lower the Last Stream ID. Setting the + // reason on every GOAWAY keeps the current Last Stream ID in the acquire failure message. closedToNewReason = "received GOAWAY with Last Stream ID " + lastStreamId; if (state == RecordState.OPEN) { diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/NettyNioAsyncHttpClientH2StreamErrorTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/NettyNioAsyncHttpClientH2StreamErrorTest.java new file mode 100644 index 000000000000..9daed6caa95d --- /dev/null +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/NettyNioAsyncHttpClientH2StreamErrorTest.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http.nio.netty; + +import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; + +import io.netty.handler.timeout.ReadTimeoutException; +import java.time.Duration; +import software.amazon.awssdk.http.Protocol; +import software.amazon.awssdk.http.ProtocolNegotiation; +import software.amazon.awssdk.http.SdkAsyncHttpClientH2StreamErrorTestSuite; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.utils.AttributeMap; + +public class NettyNioAsyncHttpClientH2StreamErrorTest extends SdkAsyncHttpClientH2StreamErrorTestSuite { + @Override + protected SdkAsyncHttpClient createH2Client(int eventLoopThreads, Duration readTimeout) { + return NettyNioAsyncHttpClient.builder() + .protocol(Protocol.HTTP2) + .protocolNegotiation(ProtocolNegotiation.ALPN) + .readTimeout(readTimeout) + .http2Configuration(b -> b.maxStreams(10L)) + .eventLoopGroupBuilder(SdkEventLoopGroup.builder().numberOfThreads(eventLoopThreads)) + .buildWithDefaults(AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, true).build()); + } + + @Override + protected Class expectedReadTimeoutCause() { + return ReadTimeoutException.class; + } +} diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java index 7257a1ae11dd..8285093291b1 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java +++ b/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/internal/http2/MultiplexedChannelRecordTest.java @@ -283,6 +283,22 @@ public void acquireStream_whenRetiredAfterClaimBeforeStreamOpens_shouldFailWithI assertThat(channel.isOpen()).as("connection open after the failed claim returned").isFalse(); } + @Test + public void acquireClaimedStream_whenTwoGoAwaysReceived_shouldFailWithLatestLastStreamId() { + EmbeddedChannel channel = newHttp2Channel(); + MultiplexedChannelRecord record = new MultiplexedChannelRecord(channel, 2, null); + acquireStream(channel, record); + record.handleGoAway(Integer.MAX_VALUE, new GoAwayException(0, "test")); + record.handleGoAway(3, new GoAwayException(0, "test")); + channel.runPendingTasks(); + + Promise streamPromise = channel.eventLoop().newPromise(); + record.acquireClaimedStream(streamPromise); + + assertThat(streamPromise.cause()).isInstanceOf(IOException.class) + .hasMessageContaining("received GOAWAY with Last Stream ID 3."); + } + @Test public void closeIfIdle_retiredRecordWithNoStreams_shouldCloseConnection() { EmbeddedChannel channel = newHttp2Channel(); diff --git a/test/http-client-tests/pom.xml b/test/http-client-tests/pom.xml index 128fe92c928b..2b5f705e5d51 100644 --- a/test/http-client-tests/pom.xml +++ b/test/http-client-tests/pom.xml @@ -102,6 +102,10 @@ io.netty netty-codec-http + + io.netty + netty-codec-http2 + io.netty netty-transport diff --git a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java b/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SdkAsyncHttpClientH2StreamErrorTestSuite.java similarity index 83% rename from http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java rename to test/http-client-tests/src/main/java/software/amazon/awssdk/http/SdkAsyncHttpClientH2StreamErrorTestSuite.java index 92f3d050137c..106fa021dec1 100644 --- a/http-clients/netty-nio-client/src/test/java/software/amazon/awssdk/http/nio/netty/fault/Http2StreamIoErrorTest.java +++ b/test/http-client-tests/src/main/java/software/amazon/awssdk/http/SdkAsyncHttpClientH2StreamErrorTestSuite.java @@ -13,12 +13,11 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.http.nio.netty.fault; +package software.amazon.awssdk.http; import static io.netty.handler.codec.http.HttpResponseStatus.OK; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; -import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.Unpooled; @@ -44,12 +43,10 @@ import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.SslProvider; import io.netty.handler.ssl.util.SelfSignedCertificate; -import io.netty.handler.timeout.ReadTimeoutException; import io.netty.util.AttributeKey; import io.reactivex.Flowable; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.ClosedChannelException; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -67,36 +64,43 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.reactivestreams.Publisher; -import software.amazon.awssdk.http.EmptyPublisher; -import software.amazon.awssdk.http.Protocol; -import software.amazon.awssdk.http.ProtocolNegotiation; -import software.amazon.awssdk.http.SdkHttpFullRequest; -import software.amazon.awssdk.http.SdkHttpMethod; -import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.http.async.AsyncExecuteRequest; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; -import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; -import software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup; -import software.amazon.awssdk.http.nio.netty.internal.utils.NettyUtils; -import software.amazon.awssdk.utils.AttributeMap; /** - * A read timeout on one HTTP/2 stream must not end the other streams on its connection. The connection takes no new - * streams and closes after its last stream ends. Closing the client fails the streams still in flight with an - * IOException. Uses TLS with ALPN, the default for H2 service clients. + * Reusable suite for HTTP/2 stream errors on an async HTTP client. A read timeout on one HTTP/2 stream must not end the + * other streams on its connection. The connection takes no new streams and closes after its last stream ends. Closing the + * client fails the streams still in flight with an IOException. Uses TLS with ALPN, the default for H2 service clients. + * + *

Each async HTTP client module subclasses this and provides its client via {@link #createH2Client(int, Duration)}. A + * client that does not support a tested behavior overrides that test with an empty body. */ @EnabledIf("alpnSupported") -public class Http2StreamIoErrorTest { +public abstract class SdkAsyncHttpClientH2StreamErrorTestSuite { private static final String REQUEST_ID = "x-request-id"; private static final int FRAME_COUNT = 20; private static final int FRAME_BYTES = 10; + private static final Duration READ_TIMEOUT = Duration.ofSeconds(1); private static final Duration WAIT = Duration.ofSeconds(5); private final Server server = new Server(); - private SdkEventLoopGroup clientEventLoopGroup; private SdkAsyncHttpClient client; + /** + * Creates a client that uses HTTP/2 over TLS with ALPN, trusts all certificates, and allows at least 10 streams per + * connection. + * + * @param eventLoopThreads the number of event loop threads the client uses + * @param readTimeout the read timeout, for clients that support one + */ + protected abstract SdkAsyncHttpClient createH2Client(int eventLoopThreads, Duration readTimeout); + + /** + * Returns the exception the underlying HTTP library raises when a stream hits the read timeout. + */ + protected abstract Class expectedReadTimeoutCause(); + @BeforeEach public void setup() throws Exception { server.start(); @@ -107,10 +111,6 @@ public void teardown() { if (client != null) { client.close(); } - if (clientEventLoopGroup != null) { - clientEventLoopGroup.eventLoopGroup().shutdownGracefully(0, 1, TimeUnit.SECONDS) - .awaitUninterruptibly(10, TimeUnit.SECONDS); - } server.shutdown(); } @@ -142,8 +142,8 @@ public void execute_readTimeoutOnOneStream_shouldCloseConnectionAfterLastSibling } @Test - public void close_withStreamsInFlight_shouldFailStreamsWithIOException() { - initClient(3); + public void close_withStreamsInFlight_shouldFailStreamsWithIoException() { + client = createH2Client(3, READ_TIMEOUT); assertThat(send("/quick", "warmup").execute).succeedsWithin(WAIT); List streams = new ArrayList<>(); for (int i = 0; i < 2; i++) { @@ -160,7 +160,7 @@ public void close_withStreamsInFlight_shouldFailStreamsWithIOException() { .withThrowableOfType(ExecutionException.class) .withCauseInstanceOf(IOException.class) .withMessageContaining("An error occurred on the connection") - .withRootCauseInstanceOf(ClosedChannelException.class); + .withMessageContaining("The HTTP client closed the connection"); assertThat(stream.bodyError).as("%s body error", stream.requestId) .isInstanceOf(IOException.class) .hasMessageContaining("An error occurred on the connection"); @@ -173,7 +173,7 @@ public void close_withStreamsInFlight_shouldFailStreamsWithIOException() { * read timeout. Returns the siblings, still streaming. */ private List streamSiblingsAndTimeOutStall(int eventLoopThreads) { - initClient(eventLoopThreads); + client = createH2Client(eventLoopThreads, READ_TIMEOUT); // One request first: concurrent first requests can each open a connection. assertThat(send("/quick", "warmup").execute).succeedsWithin(WAIT); List siblings = new ArrayList<>(); @@ -187,7 +187,7 @@ private List streamSiblingsAndTimeOutStall(int eventLoopThreads) { assertThat(stall.execute).failsWithin(WAIT) .withThrowableOfType(ExecutionException.class) .withMessageContaining("Read timed out") - .withRootCauseInstanceOf(ReadTimeoutException.class); + .withRootCauseInstanceOf(expectedReadTimeoutCause()); return siblings; } @@ -198,17 +198,6 @@ private static void assertFullBodies(List siblings) { } } - private void initClient(int eventLoopThreads) { - clientEventLoopGroup = SdkEventLoopGroup.builder().numberOfThreads(eventLoopThreads).build(); - client = NettyNioAsyncHttpClient.builder() - .protocol(Protocol.HTTP2) - .protocolNegotiation(ProtocolNegotiation.ALPN) - .readTimeout(Duration.ofSeconds(1)) - .http2Configuration(b -> b.maxStreams(10L)) - .eventLoopGroup(clientEventLoopGroup) - .buildWithDefaults(AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, true).build()); - } - private Call send(String path, String requestId) { Call call = new Call(requestId); SdkHttpFullRequest request = SdkHttpFullRequest.builder() @@ -228,11 +217,11 @@ private Call send(String path, String requestId) { } private static boolean alpnSupported() { - return NettyUtils.isAlpnSupported(SslProvider.JDK); + return SslProvider.isAlpnSupported(SslProvider.JDK); } /** - * Counts body bytes, not onNext calls, because Netty may coalesce DATA frames. + * Counts body bytes, not onNext calls, because a client may coalesce DATA frames. */ private static final class Call implements SdkAsyncHttpResponseHandler { private final String requestId; @@ -252,15 +241,17 @@ public void onHeaders(SdkHttpResponse headers) { @Override public void onStream(Publisher stream) { - Flowable.fromPublisher(stream).subscribe(b -> { - if (bodyBytes.addAndGet(b.remaining()) >= FRAME_BYTES) { - firstFrameReceived.complete(null); - } - }, + Flowable.fromPublisher(stream).subscribe(this::onBody, t -> bodyError = t, () -> bodyCompletedNanos = System.nanoTime()); } + private void onBody(ByteBuffer b) { + if (bodyBytes.addAndGet(b.remaining()) >= FRAME_BYTES) { + firstFrameReceived.complete(null); + } + } + @Override public void onError(Throwable error) { } @@ -273,7 +264,7 @@ public void onError(Throwable error) { */ private static final class Server extends ChannelInitializer { private static final AttributeKey CONNECTION_INDEX = - AttributeKey.valueOf(Http2StreamIoErrorTest.class.getName() + ".CONNECTION_INDEX"); + AttributeKey.valueOf(SdkAsyncHttpClientH2StreamErrorTestSuite.class.getName() + ".CONNECTION_INDEX"); private final NioEventLoopGroup group = new NioEventLoopGroup(); private final AtomicInteger connectionCount = new AtomicInteger();